2023-09-30T08:49:36.226 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-30T08:49:36.226 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --owner scheduled_yuriw@teuthology --verbose --archive /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221 --name yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi -- /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221/orig.config.yaml 2023-09-30T08:49:36.287 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-30T08:49:36.384 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221 branch: main description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: yweinste@redhat.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7407221' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 1d10b71792f3be8887a7631e69851ac2df3585af ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 1d10b71792f3be8887a7631e69851ac2df3585af selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 1d10b71792f3be8887a7631e69851ac2df3585af owner: scheduled_yuriw@teuthology priority: 76 repo: https://github.com/ceph/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 6770 sha1: 1d10b71792f3be8887a7631e69851ac2df3585af sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 111/120000 suite: upgrade:quincy-x suite_branch: main suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_1d10b71792f3be8887a7631e69851ac2df3585af/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: 1d10b71792f3be8887a7631e69851ac2df3585af targets: smithi019.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi167.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-29_19:44:19 tube: smithi user: yuriw verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-30T08:49:36.384 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_1d10b71792f3be8887a7631e69851ac2df3585af/qa; will attempt to use it 2023-09-30T08:49:36.385 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_1d10b71792f3be8887a7631e69851ac2df3585af/qa/tasks 2023-09-30T08:49:36.385 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-30T08:49:36.387 INFO:teuthology.task.internal:Checking packages... 2023-09-30T08:49:36.411 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash '1d10b71792f3be8887a7631e69851ac2df3585af' 2023-09-30T08:49:36.411 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-30T08:49:36.412 INFO:teuthology.packaging:ref: None 2023-09-30T08:49:36.412 INFO:teuthology.packaging:tag: None 2023-09-30T08:49:36.412 INFO:teuthology.packaging:branch: main 2023-09-30T08:49:36.412 INFO:teuthology.packaging:sha1: 1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T08:49:36.412 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=main 2023-09-30T08:49:36.753 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-6447.g1d10b717 2023-09-30T08:49:36.753 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-30T08:49:36.762 INFO:teuthology.task.internal:no buildpackages task found 2023-09-30T08:49:36.762 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-30T08:49:36.778 INFO:teuthology.task.internal:Saving configuration 2023-09-30T08:49:36.800 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-30T08:49:36.808 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-30T08:49:36.831 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi019.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-30 08:44:44.310717', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-30T08:49:36.853 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi167.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-30 08:44:44.313552', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-30T08:49:36.854 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-30T08:49:36.861 INFO:teuthology.task.internal:roles: ubuntu@smithi019.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-30T08:49:36.861 INFO:teuthology.task.internal:roles: ubuntu@smithi167.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-30T08:49:36.861 INFO:teuthology.run_tasks:Running task console_log... 2023-09-30T08:49:36.936 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe298ca4af0>, signals=[15]) 2023-09-30T08:49:36.936 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-30T08:49:36.943 INFO:teuthology.task.internal:Opening connections... 2023-09-30T08:49:36.944 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi019.front.sepia.ceph.com 2023-09-30T08:49:36.945 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi019.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-30T08:49:37.013 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi167.front.sepia.ceph.com 2023-09-30T08:49:37.013 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-30T08:49:37.080 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-30T08:49:37.088 DEBUG:teuthology.orchestra.run.smithi019:> uname -m 2023-09-30T08:49:37.110 INFO:teuthology.orchestra.run.smithi019.stdout:x86_64 2023-09-30T08:49:37.110 DEBUG:teuthology.orchestra.run.smithi019:> cat /etc/os-release 2023-09-30T08:49:37.169 INFO:teuthology.orchestra.run.smithi019.stdout:NAME="Red Hat Enterprise Linux" 2023-09-30T08:49:37.170 INFO:teuthology.orchestra.run.smithi019.stdout:VERSION="8.6 (Ootpa)" 2023-09-30T08:49:37.170 INFO:teuthology.orchestra.run.smithi019.stdout:ID="rhel" 2023-09-30T08:49:37.170 INFO:teuthology.orchestra.run.smithi019.stdout:ID_LIKE="fedora" 2023-09-30T08:49:37.170 INFO:teuthology.orchestra.run.smithi019.stdout:VERSION_ID="8.6" 2023-09-30T08:49:37.170 INFO:teuthology.orchestra.run.smithi019.stdout:PLATFORM_ID="platform:el8" 2023-09-30T08:49:37.170 INFO:teuthology.orchestra.run.smithi019.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-30T08:49:37.170 INFO:teuthology.orchestra.run.smithi019.stdout:ANSI_COLOR="0;31" 2023-09-30T08:49:37.170 INFO:teuthology.orchestra.run.smithi019.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-30T08:49:37.171 INFO:teuthology.orchestra.run.smithi019.stdout:HOME_URL="https://www.redhat.com/" 2023-09-30T08:49:37.171 INFO:teuthology.orchestra.run.smithi019.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-30T08:49:37.171 INFO:teuthology.orchestra.run.smithi019.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-30T08:49:37.171 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T08:49:37.171 INFO:teuthology.orchestra.run.smithi019.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-30T08:49:37.171 INFO:teuthology.orchestra.run.smithi019.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-30T08:49:37.171 INFO:teuthology.orchestra.run.smithi019.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-30T08:49:37.171 INFO:teuthology.orchestra.run.smithi019.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-30T08:49:37.172 INFO:teuthology.lock.ops:Updating smithi019.front.sepia.ceph.com on lock server 2023-09-30T08:49:37.203 DEBUG:teuthology.orchestra.run.smithi167:> uname -m 2023-09-30T08:49:37.223 INFO:teuthology.orchestra.run.smithi167.stdout:x86_64 2023-09-30T08:49:37.223 DEBUG:teuthology.orchestra.run.smithi167:> cat /etc/os-release 2023-09-30T08:49:37.281 INFO:teuthology.orchestra.run.smithi167.stdout:NAME="Red Hat Enterprise Linux" 2023-09-30T08:49:37.282 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION="8.6 (Ootpa)" 2023-09-30T08:49:37.282 INFO:teuthology.orchestra.run.smithi167.stdout:ID="rhel" 2023-09-30T08:49:37.282 INFO:teuthology.orchestra.run.smithi167.stdout:ID_LIKE="fedora" 2023-09-30T08:49:37.282 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION_ID="8.6" 2023-09-30T08:49:37.282 INFO:teuthology.orchestra.run.smithi167.stdout:PLATFORM_ID="platform:el8" 2023-09-30T08:49:37.282 INFO:teuthology.orchestra.run.smithi167.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-30T08:49:37.282 INFO:teuthology.orchestra.run.smithi167.stdout:ANSI_COLOR="0;31" 2023-09-30T08:49:37.282 INFO:teuthology.orchestra.run.smithi167.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-30T08:49:37.283 INFO:teuthology.orchestra.run.smithi167.stdout:HOME_URL="https://www.redhat.com/" 2023-09-30T08:49:37.283 INFO:teuthology.orchestra.run.smithi167.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-30T08:49:37.283 INFO:teuthology.orchestra.run.smithi167.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-30T08:49:37.283 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T08:49:37.283 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-30T08:49:37.283 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-30T08:49:37.283 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-30T08:49:37.284 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-30T08:49:37.284 INFO:teuthology.lock.ops:Updating smithi167.front.sepia.ceph.com on lock server 2023-09-30T08:49:37.312 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-30T08:49:37.323 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-30T08:49:37.330 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-30T08:49:37.331 DEBUG:teuthology.orchestra.run.smithi019:> test '!' -e /home/ubuntu/cephtest 2023-09-30T08:49:37.333 DEBUG:teuthology.orchestra.run.smithi167:> test '!' -e /home/ubuntu/cephtest 2023-09-30T08:49:37.352 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-30T08:49:37.359 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-30T08:49:37.359 DEBUG:teuthology.orchestra.run.smithi019:> test -z $(ls -A /var/lib/ceph) 2023-09-30T08:49:37.394 DEBUG:teuthology.orchestra.run.smithi167:> test -z $(ls -A /var/lib/ceph) 2023-09-30T08:49:37.425 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-30T08:49:37.483 INFO:teuthology.run_tasks:Running task kernel... 2023-09-30T08:49:37.498 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-30T08:49:37.498 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-30T08:49:37.499 DEBUG:teuthology.orchestra.run.smithi019:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-30T08:49:37.499 DEBUG:teuthology.orchestra.run.smithi167:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-30T08:49:37.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T08:49:37.517 DEBUG:teuthology.orchestra.run.smithi167:> uname -r 2023-09-30T08:49:37.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T08:49:37.519 DEBUG:teuthology.orchestra.run.smithi019:> uname -r 2023-09-30T08:49:37.575 INFO:teuthology.orchestra.run.smithi167.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-30T08:49:37.575 INFO:teuthology.task.kernel:Running kernel on smithi167: 4.18.0-372.9.1.el8.x86_64 2023-09-30T08:49:37.576 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum install -y kernel 2023-09-30T08:49:37.579 INFO:teuthology.orchestra.run.smithi019.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-30T08:49:37.579 INFO:teuthology.task.kernel:Running kernel on smithi019: 4.18.0-372.9.1.el8.x86_64 2023-09-30T08:49:37.579 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum install -y kernel 2023-09-30T08:49:38.760 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T08:49:38.760 INFO:teuthology.orchestra.run.smithi167.stdout:Unable to read consumer identity 2023-09-30T08:49:38.895 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T08:49:38.896 INFO:teuthology.orchestra.run.smithi019.stdout:Unable to read consumer identity 2023-09-30T08:49:40.271 INFO:teuthology.orchestra.run.smithi019.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-09-30T08:49:41.144 INFO:teuthology.orchestra.run.smithi019.stdout:Extra Packages for Enterprise Linux 19 MB/s | 16 MB 00:00 2023-09-30T08:49:41.377 INFO:teuthology.orchestra.run.smithi167.stdout:Copr repo for python3-asyncssh owned by ceph 2.6 kB/s | 3.5 kB 00:01 2023-09-30T08:49:42.070 INFO:teuthology.orchestra.run.smithi167.stdout:Extra Packages for Enterprise Linux 24 MB/s | 16 MB 00:00 2023-09-30T08:49:45.022 INFO:teuthology.orchestra.run.smithi019.stdout:lab-extras 445 kB/s | 24 kB 00:00 2023-09-30T08:49:45.570 INFO:teuthology.orchestra.run.smithi167.stdout:lab-extras 547 kB/s | 24 kB 00:00 2023-09-30T08:49:45.783 INFO:teuthology.orchestra.run.smithi019.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-30T08:49:45.851 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T08:49:45.852 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T08:49:45.852 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T08:49:45.969 DEBUG:teuthology.orchestra.run.smithi019:> echo no | sudo yum reinstall kernel || true 2023-09-30T08:49:46.222 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-30T08:49:46.276 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T08:49:46.277 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T08:49:46.278 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T08:49:46.360 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T08:49:46.360 INFO:teuthology.orchestra.run.smithi019.stdout:Unable to read consumer identity 2023-09-30T08:49:46.395 DEBUG:teuthology.orchestra.run.smithi167:> echo no | sudo yum reinstall kernel || true 2023-09-30T08:49:46.644 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:00:02 ago on Sat 30 Sep 2023 08:49:44 AM UTC. 2023-09-30T08:49:46.689 INFO:teuthology.orchestra.run.smithi019.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-30T08:49:46.718 INFO:teuthology.orchestra.run.smithi019.stderr:Error: No packages marked for reinstall. 2023-09-30T08:49:46.725 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T08:49:46.726 INFO:teuthology.orchestra.run.smithi167.stdout:Unable to read consumer identity 2023-09-30T08:49:46.765 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum reinstall -y kernel || true 2023-09-30T08:49:46.996 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:02 ago on Sat 30 Sep 2023 08:49:45 AM UTC. 2023-09-30T08:49:47.037 INFO:teuthology.orchestra.run.smithi167.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-30T08:49:47.086 INFO:teuthology.orchestra.run.smithi167.stderr:Error: No packages marked for reinstall. 2023-09-30T08:49:47.116 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T08:49:47.116 INFO:teuthology.orchestra.run.smithi019.stdout:Unable to read consumer identity 2023-09-30T08:49:47.134 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum reinstall -y kernel || true 2023-09-30T08:49:47.391 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:00:03 ago on Sat 30 Sep 2023 08:49:44 AM UTC. 2023-09-30T08:49:47.436 INFO:teuthology.orchestra.run.smithi019.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-30T08:49:47.461 INFO:teuthology.orchestra.run.smithi019.stderr:Error: No packages marked for reinstall. 2023-09-30T08:49:47.465 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T08:49:47.465 INFO:teuthology.orchestra.run.smithi167.stdout:Unable to read consumer identity 2023-09-30T08:49:47.513 DEBUG:teuthology.orchestra.run.smithi019:> rpm -q kernel | sort -rV | head -n 1 2023-09-30T08:49:47.596 INFO:teuthology.orchestra.run.smithi019.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-30T08:49:47.596 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-30T08:49:47.596 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-30T08:49:47.596 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-30T08:49:47.597 DEBUG:teuthology.orchestra.run.smithi019:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-30T08:49:47.623 INFO:teuthology.orchestra.run.smithi019.stdout:ttyS1 2023-09-30T08:49:47.638 DEBUG:teuthology.parallel:result is None 2023-09-30T08:49:47.731 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:02 ago on Sat 30 Sep 2023 08:49:45 AM UTC. 2023-09-30T08:49:47.772 INFO:teuthology.orchestra.run.smithi167.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-30T08:49:47.801 INFO:teuthology.orchestra.run.smithi167.stderr:Error: No packages marked for reinstall. 2023-09-30T08:49:47.850 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q kernel | sort -rV | head -n 1 2023-09-30T08:49:47.930 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-30T08:49:47.930 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-30T08:49:47.930 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-30T08:49:47.931 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-30T08:49:47.931 DEBUG:teuthology.orchestra.run.smithi167:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-30T08:49:47.957 INFO:teuthology.orchestra.run.smithi167.stdout:ttyS1 2023-09-30T08:49:47.969 DEBUG:teuthology.parallel:result is None 2023-09-30T08:49:47.969 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-30T08:49:47.983 INFO:teuthology.task.internal:Creating test directory... 2023-09-30T08:49:47.983 DEBUG:teuthology.orchestra.run.smithi019:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-30T08:49:47.986 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-30T08:49:48.026 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-30T08:49:48.037 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-30T08:49:48.050 INFO:teuthology.task.internal:Creating archive directory... 2023-09-30T08:49:48.050 DEBUG:teuthology.orchestra.run.smithi019:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-30T08:49:48.052 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-30T08:49:48.101 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-30T08:49:48.112 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-30T08:49:48.112 DEBUG:teuthology.orchestra.run.smithi019:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-30T08:49:48.116 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-30T08:49:48.153 INFO:teuthology.orchestra.run.smithi019.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-30T08:49:48.164 INFO:teuthology.orchestra.run.smithi019.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-30T08:49:48.174 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-30T08:49:48.185 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-30T08:49:48.188 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-30T08:49:48.199 INFO:teuthology.task.internal:Configuring sudo... 2023-09-30T08:49:48.199 DEBUG:teuthology.orchestra.run.smithi019:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-30T08:49:48.208 DEBUG:teuthology.orchestra.run.smithi167:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-30T08:49:48.255 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-30T08:49:48.263 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-30T08:49:48.264 DEBUG:teuthology.orchestra.run.smithi019:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-30T08:49:48.277 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-30T08:49:48.311 DEBUG:teuthology.orchestra.run.smithi019:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-30T08:49:48.376 DEBUG:teuthology.orchestra.run.smithi019:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-30T08:49:48.431 DEBUG:teuthology.orchestra.run.smithi019:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-30T08:49:48.515 DEBUG:teuthology.orchestra.run.smithi019:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-30T08:49:48.558 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T08:49:48.558 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-30T08:49:48.627 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-30T08:49:48.668 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-30T08:49:48.720 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-30T08:49:48.805 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-30T08:49:48.846 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T08:49:48.846 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-30T08:49:48.914 DEBUG:teuthology.orchestra.run.smithi019:> sudo service rsyslog restart 2023-09-30T08:49:48.916 DEBUG:teuthology.orchestra.run.smithi167:> sudo service rsyslog restart 2023-09-30T08:49:48.967 INFO:teuthology.orchestra.run.smithi019.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-30T08:49:49.045 INFO:teuthology.orchestra.run.smithi167.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-30T08:49:49.488 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-30T08:49:49.497 INFO:teuthology.task.internal:Starting timer... 2023-09-30T08:49:49.497 INFO:teuthology.run_tasks:Running task pcp... 2023-09-30T08:49:49.590 INFO:teuthology.run_tasks:Running task selinux... 2023-09-30T08:49:49.598 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-30T08:49:49.598 DEBUG:teuthology.orchestra.run.smithi019:> sudo service auditd rotate 2023-09-30T08:49:49.662 INFO:teuthology.orchestra.run.smithi019.stdout:Rotating logs: 2023-09-30T08:49:49.666 DEBUG:teuthology.orchestra.run.smithi167:> sudo service auditd rotate 2023-09-30T08:49:49.741 INFO:teuthology.orchestra.run.smithi167.stdout:Rotating logs: 2023-09-30T08:49:49.743 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-30T08:49:49.743 DEBUG:teuthology.orchestra.run.smithi019:> /usr/sbin/getenforce 2023-09-30T08:49:49.769 INFO:teuthology.orchestra.run.smithi019.stdout:Permissive 2023-09-30T08:49:49.769 DEBUG:teuthology.orchestra.run.smithi167:> /usr/sbin/getenforce 2023-09-30T08:49:49.794 INFO:teuthology.orchestra.run.smithi167.stdout:Permissive 2023-09-30T08:49:49.795 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi019.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi167.front.sepia.ceph.com': 'permissive'} 2023-09-30T08:49:49.795 DEBUG:teuthology.orchestra.run.smithi019:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-30T08:49:49.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T08:49:49.837 DEBUG:teuthology.orchestra.run.smithi167:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-30T08:49:49.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T08:49:49.864 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-30T08:49:49.864 DEBUG:teuthology.orchestra.run.smithi019:> sudo /usr/sbin/setenforce permissive 2023-09-30T08:49:49.905 DEBUG:teuthology.orchestra.run.smithi167:> sudo /usr/sbin/setenforce permissive 2023-09-30T08:49:49.933 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-30T08:49:49.943 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-09-30T08:49:49.943 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-30T08:49:49.966 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-30T08:49:49.967 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi019.front.sepia.ceph.com,smithi167.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-30T08:58:55.769 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi019.front.sepia.ceph.com'), Remote(name='ubuntu@smithi167.front.sepia.ceph.com')] 2023-09-30T08:58:55.770 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi019.front.sepia.ceph.com' 2023-09-30T08:58:55.771 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi019.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-30T08:58:55.840 DEBUG:teuthology.orchestra.run.smithi019:> true 2023-09-30T08:58:55.910 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi019.front.sepia.ceph.com' 2023-09-30T08:58:55.911 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-09-30T08:58:55.911 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-30T08:58:55.980 DEBUG:teuthology.orchestra.run.smithi167:> true 2023-09-30T08:58:56.048 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-09-30T08:58:56.049 INFO:teuthology.run_tasks:Running task clock... 2023-09-30T08:58:56.059 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-30T08:58:56.059 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-30T08:58:56.060 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-30T08:58:56.062 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-30T08:58:56.062 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-30T08:58:56.098 INFO:teuthology.orchestra.run.smithi019.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-30T08:58:56.114 INFO:teuthology.orchestra.run.smithi019.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-30T08:58:56.120 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-30T08:58:56.135 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-30T08:58:56.188 INFO:teuthology.orchestra.run.smithi019.stderr:sudo: ntpd: command not found 2023-09-30T08:58:56.202 INFO:teuthology.orchestra.run.smithi019.stdout:506 Cannot talk to daemon 2023-09-30T08:58:56.213 INFO:teuthology.orchestra.run.smithi167.stderr:sudo: ntpd: command not found 2023-09-30T08:58:56.218 INFO:teuthology.orchestra.run.smithi019.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-30T08:58:56.225 INFO:teuthology.orchestra.run.smithi167.stdout:506 Cannot talk to daemon 2023-09-30T08:58:56.233 INFO:teuthology.orchestra.run.smithi019.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-30T08:58:56.239 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-30T08:58:56.253 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-30T08:58:56.311 INFO:teuthology.orchestra.run.smithi019.stderr:bash: ntpq: command not found 2023-09-30T08:58:56.315 INFO:teuthology.orchestra.run.smithi019.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-30T08:58:56.315 INFO:teuthology.orchestra.run.smithi019.stdout:=============================================================================== 2023-09-30T08:58:56.315 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T08:58:56.315 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T08:58:56.315 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T08:58:56.315 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T08:58:56.320 INFO:teuthology.orchestra.run.smithi167.stderr:bash: ntpq: command not found 2023-09-30T08:58:56.324 INFO:teuthology.orchestra.run.smithi167.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-30T08:58:56.324 INFO:teuthology.orchestra.run.smithi167.stdout:=============================================================================== 2023-09-30T08:58:56.325 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T08:58:56.325 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T08:58:56.325 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T08:58:56.325 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T08:58:56.326 INFO:teuthology.run_tasks:Running task pexec... 2023-09-30T08:58:56.336 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-30T08:58:56.336 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi019.front.sepia.ceph.com 2023-09-30T08:58:56.337 DEBUG:teuthology.orchestra.run.smithi019:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-30T08:58:56.337 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi167.front.sepia.ceph.com 2023-09-30T08:58:56.337 DEBUG:teuthology.orchestra.run.smithi167:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-30T08:58:56.736 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T08:58:56.750 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T08:58:57.545 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:52 ago on Sat 30 Sep 2023 08:57:05 AM UTC. 2023-09-30T08:58:57.595 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:29 ago on Sat 30 Sep 2023 08:57:28 AM UTC. 2023-09-30T08:58:59.428 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout:Resetting modules: 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: container-tools 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T08:58:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T08:58:59.532 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T08:58:59.533 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T08:58:59.533 INFO:teuthology.orchestra.run.smithi019.stdout: Package Architecture Version Repository Size 2023-09-30T08:58:59.533 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T08:58:59.533 INFO:teuthology.orchestra.run.smithi019.stdout:Resetting modules: 2023-09-30T08:58:59.534 INFO:teuthology.orchestra.run.smithi019.stdout: container-tools 2023-09-30T08:58:59.534 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T08:58:59.534 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T08:58:59.534 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T08:58:59.534 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T08:59:00.410 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:54 ago on Sat 30 Sep 2023 08:57:05 AM UTC. 2023-09-30T08:59:00.544 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:32 ago on Sat 30 Sep 2023 08:57:28 AM UTC. 2023-09-30T08:59:03.052 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T08:59:03.195 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T08:59:03.509 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T08:59:03.667 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T08:59:04.286 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:58 ago on Sat 30 Sep 2023 08:57:05 AM UTC. 2023-09-30T08:59:04.582 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:36 ago on Sat 30 Sep 2023 08:57:28 AM UTC. 2023-09-30T08:59:05.989 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T08:59:06.081 INFO:teuthology.orchestra.run.smithi167.stdout:========================================================================================================================== 2023-09-30T08:59:06.081 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T08:59:06.081 INFO:teuthology.orchestra.run.smithi167.stdout:========================================================================================================================== 2023-09-30T08:59:06.081 INFO:teuthology.orchestra.run.smithi167.stdout:Installing group/module packages: 2023-09-30T08:59:06.081 INFO:teuthology.orchestra.run.smithi167.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-30T08:59:06.081 INFO:teuthology.orchestra.run.smithi167.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-30T08:59:06.081 INFO:teuthology.orchestra.run.smithi167.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout:Downgrading: 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-30T08:59:06.082 INFO:teuthology.orchestra.run.smithi167.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout:Installing module profiles: 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: container-tools/common 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout:Enabling module streams: 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: container-tools 3.0 2023-09-30T08:59:06.083 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T08:59:06.084 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T08:59:06.084 INFO:teuthology.orchestra.run.smithi167.stdout:========================================================================================================================== 2023-09-30T08:59:06.084 INFO:teuthology.orchestra.run.smithi167.stdout:Install 7 Packages 2023-09-30T08:59:06.084 INFO:teuthology.orchestra.run.smithi167.stdout:Downgrade 12 Packages 2023-09-30T08:59:06.084 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T08:59:06.084 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 51 M 2023-09-30T08:59:06.085 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T08:59:06.143 INFO:teuthology.orchestra.run.smithi167.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 906 kB/s | 53 kB 00:00 2023-09-30T08:59:06.168 INFO:teuthology.orchestra.run.smithi167.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 613 kB/s | 51 kB 00:00 2023-09-30T08:59:06.219 INFO:teuthology.orchestra.run.smithi167.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 679 kB/s | 91 kB 00:00 2023-09-30T08:59:06.244 INFO:teuthology.orchestra.run.smithi167.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 917 kB/s | 69 kB 00:00 2023-09-30T08:59:06.278 INFO:teuthology.orchestra.run.smithi167.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 3.7 MB/s | 511 kB 00:00 2023-09-30T08:59:06.295 INFO:teuthology.orchestra.run.smithi167.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.4 MB/s | 72 kB 00:00 2023-09-30T08:59:06.336 INFO:teuthology.orchestra.run.smithi167.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.3 MB/s | 56 kB 00:00 2023-09-30T08:59:06.403 INFO:teuthology.orchestra.run.smithi167.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 16 MB/s | 2.9 MB 00:00 2023-09-30T08:59:06.429 INFO:teuthology.orchestra.run.smithi167.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 555 kB/s | 51 kB 00:00 2023-09-30T08:59:06.471 INFO:teuthology.orchestra.run.smithi167.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 7.5 MB/s | 322 kB 00:00 2023-09-30T08:59:06.512 INFO:teuthology.orchestra.run.smithi167.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 979 kB/s | 41 kB 00:00 2023-09-30T08:59:06.548 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T08:59:06.621 INFO:teuthology.orchestra.run.smithi167.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 20 MB/s | 2.2 MB 00:00 2023-09-30T08:59:06.648 INFO:teuthology.orchestra.run.smithi019.stdout:========================================================================================================================== 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout:========================================================================================================================== 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout:Installing group/module packages: 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-30T08:59:06.649 INFO:teuthology.orchestra.run.smithi019.stdout:Downgrading: 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-30T08:59:06.650 INFO:teuthology.orchestra.run.smithi019.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout:Installing module profiles: 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout: container-tools/common 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout:Enabling module streams: 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout: container-tools 3.0 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout:========================================================================================================================== 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout:Install 7 Packages 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout:Downgrade 12 Packages 2023-09-30T08:59:06.651 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T08:59:06.652 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 51 M 2023-09-30T08:59:06.652 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T08:59:06.760 INFO:teuthology.orchestra.run.smithi019.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 852 kB/s | 91 kB 00:00 2023-09-30T08:59:06.785 INFO:teuthology.orchestra.run.smithi019.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 401 kB/s | 53 kB 00:00 2023-09-30T08:59:06.788 INFO:teuthology.orchestra.run.smithi167.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 21 MB/s | 11 MB 00:00 2023-09-30T08:59:06.810 INFO:teuthology.orchestra.run.smithi019.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 326 kB/s | 51 kB 00:00 2023-09-30T08:59:06.843 INFO:teuthology.orchestra.run.smithi019.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 6.0 MB/s | 511 kB 00:00 2023-09-30T08:59:06.869 INFO:teuthology.orchestra.run.smithi019.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.2 MB/s | 69 kB 00:00 2023-09-30T08:59:06.927 INFO:teuthology.orchestra.run.smithi019.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 862 kB/s | 72 kB 00:00 2023-09-30T08:59:06.947 INFO:teuthology.orchestra.run.smithi167.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 20 MB/s | 6.4 MB 00:00 2023-09-30T08:59:06.961 INFO:teuthology.orchestra.run.smithi019.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.6 MB/s | 56 kB 00:00 2023-09-30T08:59:07.003 INFO:teuthology.orchestra.run.smithi019.stdout:(8/19): conmon-2.0.26-1.module+el8.6.0+14874+64 1.2 MB/s | 51 kB 00:00 2023-09-30T08:59:07.005 INFO:teuthology.orchestra.run.smithi167.stdout:(15/19): cockpit-podman-29-2.module+el8.6.0+148 18 MB/s | 1.1 MB 00:00 2023-09-30T08:59:07.061 INFO:teuthology.orchestra.run.smithi019.stdout:(9/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 12 MB/s | 2.9 MB 00:00 2023-09-30T08:59:07.098 INFO:teuthology.orchestra.run.smithi167.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 22 MB/s | 6.8 MB 00:00 2023-09-30T08:59:07.112 INFO:teuthology.orchestra.run.smithi019.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-09-30T08:59:07.123 INFO:teuthology.orchestra.run.smithi167.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 1.5 MB/s | 185 kB 00:00 2023-09-30T08:59:07.140 INFO:teuthology.orchestra.run.smithi167.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.2 MB/s | 51 kB 00:00 2023-09-30T08:59:07.145 INFO:teuthology.orchestra.run.smithi019.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 1.2 MB/s | 41 kB 00:00 2023-09-30T08:59:07.340 INFO:teuthology.orchestra.run.smithi167.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 20 MB/s | 19 MB 00:00 2023-09-30T08:59:07.341 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T08:59:07.341 INFO:teuthology.orchestra.run.smithi167.stdout:Total 40 MB/s | 51 MB 00:01 2023-09-30T08:59:07.370 INFO:teuthology.orchestra.run.smithi019.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-09-30T08:59:07.554 INFO:teuthology.orchestra.run.smithi019.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 16 MB/s | 11 MB 00:00 2023-09-30T08:59:07.721 INFO:teuthology.orchestra.run.smithi019.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 18 MB/s | 6.4 MB 00:00 2023-09-30T08:59:07.780 INFO:teuthology.orchestra.run.smithi019.stdout:(15/19): cockpit-podman-29-2.module+el8.6.0+148 18 MB/s | 1.1 MB 00:00 2023-09-30T08:59:07.820 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T08:59:07.863 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T08:59:07.864 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T08:59:07.888 INFO:teuthology.orchestra.run.smithi019.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 20 MB/s | 6.8 MB 00:00 2023-09-30T08:59:07.914 INFO:teuthology.orchestra.run.smithi019.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 1.4 MB/s | 185 kB 00:00 2023-09-30T08:59:07.931 INFO:teuthology.orchestra.run.smithi019.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.2 MB/s | 51 kB 00:00 2023-09-30T08:59:08.297 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T08:59:08.301 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T08:59:08.339 INFO:teuthology.orchestra.run.smithi019.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 14 MB/s | 19 MB 00:01 2023-09-30T08:59:08.340 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T08:59:08.340 INFO:teuthology.orchestra.run.smithi019.stdout:Total 30 MB/s | 51 MB 00:01 2023-09-30T08:59:08.832 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T08:59:08.878 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T08:59:08.878 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T08:59:09.323 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T08:59:09.327 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T08:59:09.451 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T08:59:09.507 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-30T08:59:09.676 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-30T08:59:09.685 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-30T08:59:09.709 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-30T08:59:09.901 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-30T08:59:10.525 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T08:59:10.576 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-30T08:59:10.770 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-30T08:59:10.780 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-30T08:59:10.803 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-30T08:59:10.979 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-30T08:59:27.042 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-30T08:59:28.806 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-30T08:59:28.814 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-30T08:59:29.074 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-30T08:59:29.242 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-30T08:59:29.396 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-30T08:59:29.633 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-30T08:59:29.633 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-30T08:59:29.633 INFO:teuthology.orchestra.run.smithi167.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-30T08:59:29.633 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T08:59:30.429 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-30T08:59:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-30T08:59:30.668 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-30T08:59:30.885 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-30T08:59:31.097 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-30T08:59:31.299 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-30T08:59:31.495 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-30T08:59:31.496 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-30T08:59:31.496 INFO:teuthology.orchestra.run.smithi019.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-30T08:59:31.496 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T08:59:32.138 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-30T08:59:32.288 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-30T08:59:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-30T08:59:32.486 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-30T08:59:32.842 INFO:teuthology.orchestra.run.smithi167.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-30T08:59:33.096 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-30T08:59:33.825 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-30T08:59:34.029 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-30T08:59:34.316 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-30T08:59:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-30T08:59:34.707 INFO:teuthology.orchestra.run.smithi019.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-30T08:59:34.746 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-30T08:59:34.954 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-30T08:59:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-30T08:59:35.148 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-30T08:59:35.270 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-30T08:59:35.271 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-30T08:59:35.571 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-30T08:59:35.729 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-30T08:59:35.729 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-30T08:59:35.732 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-30T08:59:35.874 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-30T08:59:36.048 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-30T08:59:36.199 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-30T08:59:36.340 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-30T08:59:36.496 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-30T08:59:36.496 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-30T08:59:36.648 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-30T08:59:36.709 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-30T08:59:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-30T08:59:36.901 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-30T08:59:36.938 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-30T08:59:36.938 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-30T08:59:37.084 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-30T08:59:37.122 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-30T08:59:37.233 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-30T08:59:37.234 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-30T08:59:37.530 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-30T08:59:37.661 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-30T08:59:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-30T08:59:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-30T08:59:37.877 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-30T08:59:38.001 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-30T08:59:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-30T08:59:38.083 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-30T08:59:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-30T08:59:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-30T08:59:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-30T08:59:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-30T08:59:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-30T08:59:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-30T08:59:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-30T08:59:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-30T08:59:38.309 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-30T08:59:38.529 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-30T08:59:38.651 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-30T08:59:38.776 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-30T08:59:38.937 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-30T08:59:38.937 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-30T08:59:39.126 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-30T08:59:39.507 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-30T08:59:39.508 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:02:34 ago on Sat 30 Sep 2023 08:57:05 AM UTC. 2023-09-30T08:59:39.672 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-30T08:59:40.098 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-30T08:59:40.098 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-30T08:59:40.098 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-30T08:59:40.098 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-30T08:59:40.098 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-30T08:59:40.098 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-30T08:59:40.098 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-30T08:59:40.098 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-30T08:59:40.099 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-30T08:59:40.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-30T08:59:40.101 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-30T08:59:40.101 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-30T08:59:41.583 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-30T08:59:41.583 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:13 ago on Sat 30 Sep 2023 08:57:28 AM UTC. 2023-09-30T08:59:41.891 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T08:59:42.088 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout:Downgraded: 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-30T08:59:42.089 INFO:teuthology.orchestra.run.smithi167.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T08:59:42.090 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T08:59:42.291 DEBUG:teuthology.parallel:result is None 2023-09-30T08:59:44.400 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T08:59:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T08:59:44.607 INFO:teuthology.orchestra.run.smithi019.stdout:Downgraded: 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T08:59:44.609 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T08:59:44.791 DEBUG:teuthology.parallel:result is None 2023-09-30T08:59:44.792 INFO:teuthology.run_tasks:Running task install... 2023-09-30T08:59:44.802 DEBUG:teuthology.task.install:project ceph 2023-09-30T08:59:44.802 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '1d10b71792f3be8887a7631e69851ac2df3585af'}} 2023-09-30T08:59:44.802 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '1d10b71792f3be8887a7631e69851ac2df3585af'} 2023-09-30T08:59:44.802 INFO:teuthology.task.install:Using flavor: default 2023-09-30T08:59:44.808 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-30T08:59:44.808 INFO:teuthology.task.install:extra packages: [] 2023-09-30T08:59:44.808 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1d10b71792f3be8887a7631e69851ac2df3585af', 'tag': None, 'wait_for_package': False} 2023-09-30T08:59:44.809 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-30T08:59:44.809 INFO:teuthology.packaging:ref: None 2023-09-30T08:59:44.809 INFO:teuthology.packaging:tag: None 2023-09-30T08:59:44.809 INFO:teuthology.packaging:branch: quincy 2023-09-30T08:59:44.809 INFO:teuthology.packaging:sha1: 1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T08:59:44.809 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-30T08:59:44.811 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1d10b71792f3be8887a7631e69851ac2df3585af', 'tag': None, 'wait_for_package': False} 2023-09-30T08:59:44.811 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-30T08:59:44.811 INFO:teuthology.packaging:ref: None 2023-09-30T08:59:44.811 INFO:teuthology.packaging:tag: None 2023-09-30T08:59:44.811 INFO:teuthology.packaging:branch: quincy 2023-09-30T08:59:44.811 INFO:teuthology.packaging:sha1: 1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T08:59:44.812 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-30T08:59:45.196 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/191c18699e384ce04d90fb4045d6dce3223c4681/centos/8/flavors/default/ 2023-09-30T08:59:45.196 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1161.g191c1869 2023-09-30T08:59:45.233 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/191c18699e384ce04d90fb4045d6dce3223c4681/centos/8/flavors/default/ 2023-09-30T08:59:45.234 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1161.g191c1869 2023-09-30T08:59:45.336 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/191c18699e384ce04d90fb4045d6dce3223c4681/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/191c18699e384ce04d90fb4045d6dce3223c4681/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/191c18699e384ce04d90fb4045d6dce3223c4681/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-30T08:59:45.337 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T08:59:45.337 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-30T08:59:45.365 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/191c18699e384ce04d90fb4045d6dce3223c4681/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/191c18699e384ce04d90fb4045d6dce3223c4681/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/191c18699e384ce04d90fb4045d6dce3223c4681/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-30T08:59:45.365 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T08:59:45.365 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-30T08:59:45.369 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-30T08:59:45.369 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-30T08:59:45.369 INFO:teuthology.packaging:ref: None 2023-09-30T08:59:45.369 INFO:teuthology.packaging:tag: None 2023-09-30T08:59:45.369 INFO:teuthology.packaging:branch: quincy 2023-09-30T08:59:45.369 INFO:teuthology.packaging:sha1: 1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T08:59:45.370 DEBUG:teuthology.orchestra.run.smithi019:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-30T08:59:45.398 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-30T08:59:45.398 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-30T08:59:45.398 INFO:teuthology.packaging:ref: None 2023-09-30T08:59:45.398 INFO:teuthology.packaging:tag: None 2023-09-30T08:59:45.398 INFO:teuthology.packaging:branch: quincy 2023-09-30T08:59:45.398 INFO:teuthology.packaging:sha1: 1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T08:59:45.399 DEBUG:teuthology.orchestra.run.smithi167:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-30T08:59:45.441 DEBUG:teuthology.orchestra.run.smithi019:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-30T08:59:45.470 DEBUG:teuthology.orchestra.run.smithi167:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-30T08:59:45.524 DEBUG:teuthology.orchestra.run.smithi019:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-30T08:59:45.550 DEBUG:teuthology.orchestra.run.smithi167:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-30T08:59:45.592 INFO:teuthology.orchestra.run.smithi019.stdout:check_obsoletes = 1 2023-09-30T08:59:45.594 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum clean all 2023-09-30T08:59:45.617 INFO:teuthology.orchestra.run.smithi167.stdout:check_obsoletes = 1 2023-09-30T08:59:45.618 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum clean all 2023-09-30T08:59:45.988 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T08:59:45.990 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T08:59:46.382 INFO:teuthology.orchestra.run.smithi167.stdout:44 files removed 2023-09-30T08:59:46.429 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-radosgw 2023-09-30T08:59:46.444 INFO:teuthology.orchestra.run.smithi019.stdout:44 files removed 2023-09-30T08:59:46.493 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-radosgw 2023-09-30T08:59:46.760 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T08:59:46.846 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T08:59:47.561 INFO:teuthology.orchestra.run.smithi167.stdout:ceph packages for x86_64 247 kB/s | 78 kB 00:00 2023-09-30T08:59:47.730 INFO:teuthology.orchestra.run.smithi019.stdout:ceph packages for x86_64 204 kB/s | 78 kB 00:00 2023-09-30T08:59:47.929 INFO:teuthology.orchestra.run.smithi167.stdout:ceph noarch packages 27 kB/s | 9.1 kB 00:00 2023-09-30T08:59:48.106 INFO:teuthology.orchestra.run.smithi019.stdout:ceph noarch packages 27 kB/s | 9.1 kB 00:00 2023-09-30T08:59:48.346 INFO:teuthology.orchestra.run.smithi167.stdout:ceph source packages 3.6 kB/s | 1.4 kB 00:00 2023-09-30T08:59:48.423 INFO:teuthology.orchestra.run.smithi019.stdout:ceph source packages 4.9 kB/s | 1.4 kB 00:00 2023-09-30T08:59:48.497 INFO:teuthology.orchestra.run.smithi167.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2023-09-30T08:59:48.566 INFO:teuthology.orchestra.run.smithi019.stdout:Copr repo for python3-asyncssh owned by ceph 29 kB/s | 3.5 kB 00:00 2023-09-30T08:59:49.547 INFO:teuthology.orchestra.run.smithi167.stdout:Extra Packages for Enterprise Linux 15 MB/s | 16 MB 00:01 2023-09-30T08:59:50.360 INFO:teuthology.orchestra.run.smithi019.stdout:Extra Packages for Enterprise Linux 9.0 MB/s | 16 MB 00:01 2023-09-30T08:59:53.114 INFO:teuthology.orchestra.run.smithi167.stdout:lab-extras 399 kB/s | 24 kB 00:00 2023-09-30T08:59:54.083 INFO:teuthology.orchestra.run.smithi167.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 59 MB/s | 53 MB 00:00 2023-09-30T08:59:54.099 INFO:teuthology.orchestra.run.smithi019.stdout:lab-extras 385 kB/s | 24 kB 00:00 2023-09-30T08:59:55.068 INFO:teuthology.orchestra.run.smithi019.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 59 MB/s | 53 MB 00:00 2023-09-30T09:00:02.315 INFO:teuthology.orchestra.run.smithi167.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 53 MB/s | 47 MB 00:00 2023-09-30T09:00:03.736 INFO:teuthology.orchestra.run.smithi019.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 52 MB/s | 47 MB 00:00 2023-09-30T09:00:13.368 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout:==================================================================================================== 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout:==================================================================================================== 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-radosgw x86_64 2:17.2.6-1161.g191c1869.el8 ceph 17 M 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout:Upgrading: 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout: librados2 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 3.5 M 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout: librbd1 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 3.6 M 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-base x86_64 2:17.2.6-1161.g191c1869.el8 ceph 5.2 M 2023-09-30T09:00:13.371 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-common x86_64 2:17.2.6-1161.g191c1869.el8 ceph 23 M 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-selinux x86_64 2:17.2.6-1161.g191c1869.el8 ceph 24 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: libcephfs2 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 795 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: libradosstriper1 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 507 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: librgw2 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 5.5 M 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ceph-argparse x86_64 2:17.2.6-1161.g191c1869.el8 ceph 45 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ceph-common x86_64 2:17.2.6-1161.g191c1869.el8 ceph 115 k 2023-09-30T09:00:13.372 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cephfs x86_64 2:17.2.6-1161.g191c1869.el8 ceph 190 k 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rados x86_64 2:17.2.6-1161.g191c1869.el8 ceph 371 k 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rbd x86_64 2:17.2.6-1161.g191c1869.el8 ceph 358 k 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rgw x86_64 2:17.2.6-1161.g191c1869.el8 ceph 106 k 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout:==================================================================================================== 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout:Install 21 Packages 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout:Upgrade 2 Packages 2023-09-30T09:00:13.373 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:00:13.374 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 63 M 2023-09-30T09:00:13.374 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:00:15.327 INFO:teuthology.orchestra.run.smithi167.stdout:(1/23): ceph-base-17.2.6-1161.g191c1869.el8.x86 2.7 MB/s | 5.2 MB 00:01 2023-09-30T09:00:15.344 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:00:15.347 INFO:teuthology.orchestra.run.smithi019.stdout:==================================================================================================== 2023-09-30T09:00:15.347 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:00:15.347 INFO:teuthology.orchestra.run.smithi019.stdout:==================================================================================================== 2023-09-30T09:00:15.347 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:00:15.347 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-radosgw x86_64 2:17.2.6-1161.g191c1869.el8 ceph 17 M 2023-09-30T09:00:15.347 INFO:teuthology.orchestra.run.smithi019.stdout:Upgrading: 2023-09-30T09:00:15.347 INFO:teuthology.orchestra.run.smithi019.stdout: librados2 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 3.5 M 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: librbd1 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 3.6 M 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-base x86_64 2:17.2.6-1161.g191c1869.el8 ceph 5.2 M 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-common x86_64 2:17.2.6-1161.g191c1869.el8 ceph 23 M 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-selinux x86_64 2:17.2.6-1161.g191c1869.el8 ceph 24 k 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: libcephfs2 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 795 k 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: libradosstriper1 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 507 k 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-30T09:00:15.348 INFO:teuthology.orchestra.run.smithi019.stdout: librgw2 x86_64 2:17.2.6-1161.g191c1869.el8 ceph 5.5 M 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: python3-ceph-argparse x86_64 2:17.2.6-1161.g191c1869.el8 ceph 45 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: python3-ceph-common x86_64 2:17.2.6-1161.g191c1869.el8 ceph 115 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cephfs x86_64 2:17.2.6-1161.g191c1869.el8 ceph 190 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: python3-rados x86_64 2:17.2.6-1161.g191c1869.el8 ceph 371 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: python3-rbd x86_64 2:17.2.6-1161.g191c1869.el8 ceph 358 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: python3-rgw x86_64 2:17.2.6-1161.g191c1869.el8 ceph 106 k 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:00:15.349 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:00:15.350 INFO:teuthology.orchestra.run.smithi019.stdout:==================================================================================================== 2023-09-30T09:00:15.350 INFO:teuthology.orchestra.run.smithi019.stdout:Install 21 Packages 2023-09-30T09:00:15.350 INFO:teuthology.orchestra.run.smithi019.stdout:Upgrade 2 Packages 2023-09-30T09:00:15.350 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:00:15.350 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 63 M 2023-09-30T09:00:15.350 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:00:15.402 INFO:teuthology.orchestra.run.smithi167.stdout:(2/23): ceph-selinux-17.2.6-1161.g191c1869.el8. 327 kB/s | 24 kB 00:00 2023-09-30T09:00:15.577 INFO:teuthology.orchestra.run.smithi167.stdout:(3/23): libcephfs2-17.2.6-1161.g191c1869.el8.x8 4.4 MB/s | 795 kB 00:00 2023-09-30T09:00:15.719 INFO:teuthology.orchestra.run.smithi167.stdout:(4/23): libradosstriper1-17.2.6-1161.g191c1869. 3.5 MB/s | 507 kB 00:00 2023-09-30T09:00:16.178 INFO:teuthology.orchestra.run.smithi019.stdout:(1/23): ceph-base-17.2.6-1161.g191c1869.el8.x86 7.7 MB/s | 5.2 MB 00:00 2023-09-30T09:00:16.245 INFO:teuthology.orchestra.run.smithi019.stdout:(2/23): ceph-selinux-17.2.6-1161.g191c1869.el8. 366 kB/s | 24 kB 00:00 2023-09-30T09:00:16.379 INFO:teuthology.orchestra.run.smithi019.stdout:(3/23): libcephfs2-17.2.6-1161.g191c1869.el8.x8 5.8 MB/s | 795 kB 00:00 2023-09-30T09:00:16.462 INFO:teuthology.orchestra.run.smithi019.stdout:(4/23): libradosstriper1-17.2.6-1161.g191c1869. 5.9 MB/s | 507 kB 00:00 2023-09-30T09:00:17.096 INFO:teuthology.orchestra.run.smithi019.stdout:(5/23): librgw2-17.2.6-1161.g191c1869.el8.x86_6 8.8 MB/s | 5.5 MB 00:00 2023-09-30T09:00:17.163 INFO:teuthology.orchestra.run.smithi019.stdout:(6/23): python3-ceph-argparse-17.2.6-1161.g191c 672 kB/s | 45 kB 00:00 2023-09-30T09:00:17.371 INFO:teuthology.orchestra.run.smithi019.stdout:(7/23): ceph-radosgw-17.2.6-1161.g191c1869.el8. 9.0 MB/s | 17 MB 00:01 2023-09-30T09:00:17.397 INFO:teuthology.orchestra.run.smithi019.stdout:(8/23): python3-ceph-common-17.2.6-1161.g191c18 493 kB/s | 115 kB 00:00 2023-09-30T09:00:17.422 INFO:teuthology.orchestra.run.smithi019.stdout:(9/23): python3-cephfs-17.2.6-1161.g191c1869.el 3.7 MB/s | 190 kB 00:00 2023-09-30T09:00:17.455 INFO:teuthology.orchestra.run.smithi019.stdout:(10/23): python3-rados-17.2.6-1161.g191c1869.el 6.2 MB/s | 371 kB 00:00 2023-09-30T09:00:17.481 INFO:teuthology.orchestra.run.smithi019.stdout:(11/23): python3-rbd-17.2.6-1161.g191c1869.el8. 6.0 MB/s | 358 kB 00:00 2023-09-30T09:00:17.506 INFO:teuthology.orchestra.run.smithi019.stdout:(12/23): python3-rgw-17.2.6-1161.g191c1869.el8. 2.1 MB/s | 106 kB 00:00 2023-09-30T09:00:17.594 INFO:teuthology.orchestra.run.smithi167.stdout:(5/23): librgw2-17.2.6-1161.g191c1869.el8.x86_6 3.0 MB/s | 5.5 MB 00:01 2023-09-30T09:00:17.653 INFO:teuthology.orchestra.run.smithi167.stdout:(6/23): python3-ceph-argparse-17.2.6-1161.g191c 769 kB/s | 45 kB 00:00 2023-09-30T09:00:17.681 INFO:teuthology.orchestra.run.smithi019.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 336 kB/s | 59 kB 00:00 2023-09-30T09:00:17.712 INFO:teuthology.orchestra.run.smithi167.stdout:(7/23): python3-ceph-common-17.2.6-1161.g191c18 1.9 MB/s | 115 kB 00:00 2023-09-30T09:00:17.723 INFO:teuthology.orchestra.run.smithi019.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.2 MB/s | 306 kB 00:00 2023-09-30T09:00:17.774 INFO:teuthology.orchestra.run.smithi019.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 818 kB/s | 75 kB 00:00 2023-09-30T09:00:17.787 INFO:teuthology.orchestra.run.smithi167.stdout:(8/23): python3-cephfs-17.2.6-1161.g191c1869.el 2.5 MB/s | 190 kB 00:00 2023-09-30T09:00:17.841 INFO:teuthology.orchestra.run.smithi019.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 703 kB/s | 47 kB 00:00 2023-09-30T09:00:17.883 INFO:teuthology.orchestra.run.smithi019.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.1 MB/s | 259 kB 00:00 2023-09-30T09:00:17.895 INFO:teuthology.orchestra.run.smithi167.stdout:(9/23): python3-rados-17.2.6-1161.g191c1869.el8 3.4 MB/s | 371 kB 00:00 2023-09-30T09:00:17.924 INFO:teuthology.orchestra.run.smithi019.stdout:(18/23): thrift-0.13.0-2.el8.x86_64.rpm 8.6 MB/s | 1.7 MB 00:00 2023-09-30T09:00:17.950 INFO:teuthology.orchestra.run.smithi019.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 657 kB/s | 44 kB 00:00 2023-09-30T09:00:17.996 INFO:teuthology.orchestra.run.smithi167.stdout:(10/23): python3-rbd-17.2.6-1161.g191c1869.el8. 3.5 MB/s | 358 kB 00:00 2023-09-30T09:00:18.008 INFO:teuthology.orchestra.run.smithi019.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 4.1 MB/s | 354 kB 00:00 2023-09-30T09:00:18.054 INFO:teuthology.orchestra.run.smithi167.stdout:(11/23): python3-rgw-17.2.6-1161.g191c1869.el8. 1.8 MB/s | 106 kB 00:00 2023-09-30T09:00:18.317 INFO:teuthology.orchestra.run.smithi019.stdout:(21/23): librados2-17.2.6-1161.g191c1869.el8.x8 9.7 MB/s | 3.5 MB 00:00 2023-09-30T09:00:18.321 INFO:teuthology.orchestra.run.smithi167.stdout:(12/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.1 MB/s | 306 kB 00:00 2023-09-30T09:00:18.380 INFO:teuthology.orchestra.run.smithi167.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 1.0 MB/s | 59 kB 00:00 2023-09-30T09:00:18.442 INFO:teuthology.orchestra.run.smithi019.stdout:(22/23): librbd1-17.2.6-1161.g191c1869.el8.x86_ 8.3 MB/s | 3.6 MB 00:00 2023-09-30T09:00:18.614 INFO:teuthology.orchestra.run.smithi167.stdout:(14/23): ceph-radosgw-17.2.6-1161.g191c1869.el8 3.2 MB/s | 17 MB 00:05 2023-09-30T09:00:18.639 INFO:teuthology.orchestra.run.smithi167.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 290 kB/s | 75 kB 00:00 2023-09-30T09:00:18.898 INFO:teuthology.orchestra.run.smithi167.stdout:(16/23): ceph-common-17.2.6-1161.g191c1869.el8. 4.3 MB/s | 23 MB 00:05 2023-09-30T09:00:18.948 INFO:teuthology.orchestra.run.smithi167.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 152 kB/s | 47 kB 00:00 2023-09-30T09:00:18.973 INFO:teuthology.orchestra.run.smithi167.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 3.6 MB/s | 259 kB 00:00 2023-09-30T09:00:18.990 INFO:teuthology.orchestra.run.smithi167.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 1.0 MB/s | 44 kB 00:00 2023-09-30T09:00:19.024 INFO:teuthology.orchestra.run.smithi167.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 6.9 MB/s | 354 kB 00:00 2023-09-30T09:00:19.233 INFO:teuthology.orchestra.run.smithi167.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 2.8 MB/s | 1.7 MB 00:00 2023-09-30T09:00:19.525 INFO:teuthology.orchestra.run.smithi167.stdout:(22/23): librbd1-17.2.6-1161.g191c1869.el8.x86_ 7.2 MB/s | 3.6 MB 00:00 2023-09-30T09:00:19.658 INFO:teuthology.orchestra.run.smithi167.stdout:(23/23): librados2-17.2.6-1161.g191c1869.el8.x8 5.2 MB/s | 3.5 MB 00:00 2023-09-30T09:00:19.660 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:00:19.661 INFO:teuthology.orchestra.run.smithi167.stdout:Total 10 MB/s | 63 MB 00:06 2023-09-30T09:00:19.709 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:00:19.754 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:00:19.754 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:00:20.100 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:00:20.104 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:00:21.163 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:00:21.374 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-30T09:00:21.738 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-30T09:00:21.915 INFO:teuthology.orchestra.run.smithi167.stdout: Upgrading : librados2-2:17.2.6-1161.g191c1869.el8.x86_64 2/25 2023-09-30T09:00:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librados2-2:17.2.6-1161.g191c1869.el8.x86_64 2/25 2023-09-30T09:00:22.179 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-rados-2:17.2.6-1161.g191c1869.el8.x86_64 3/25 2023-09-30T09:00:22.414 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-30T09:00:22.607 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 5/25 2023-09-30T09:00:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 5/25 2023-09-30T09:00:23.182 INFO:teuthology.orchestra.run.smithi167.stdout: Upgrading : librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 6/25 2023-09-30T09:00:23.281 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 6/25 2023-09-30T09:00:23.432 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-30T09:00:23.498 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-30T09:00:23.876 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-30T09:00:24.334 INFO:teuthology.orchestra.run.smithi019.stdout:(23/23): ceph-common-17.2.6-1161.g191c1869.el8. 2.7 MB/s | 23 MB 00:08 2023-09-30T09:00:24.336 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:00:24.337 INFO:teuthology.orchestra.run.smithi019.stdout:Total 7.0 MB/s | 63 MB 00:08 2023-09-30T09:00:24.386 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:00:24.435 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:00:24.435 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:00:24.581 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-30T09:00:24.732 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : librgw2-2:17.2.6-1161.g191c1869.el8.x86_64 10/25 2023-09-30T09:00:24.796 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:00:24.800 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:00:24.802 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librgw2-2:17.2.6-1161.g191c1869.el8.x86_64 10/25 2023-09-30T09:00:24.925 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-ceph-argparse-2:17.2.6-1161.g191c1869.el8. 11/25 2023-09-30T09:00:25.102 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cephfs-2:17.2.6-1161.g191c1869.el8.x86_64 12/25 2023-09-30T09:00:25.291 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-rgw-2:17.2.6-1161.g191c1869.el8.x86_64 13/25 2023-09-30T09:00:25.493 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-rbd-2:17.2.6-1161.g191c1869.el8.x86_64 14/25 2023-09-30T09:00:25.665 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libradosstriper1-2:17.2.6-1161.g191c1869.el8.x86_6 15/25 2023-09-30T09:00:25.732 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1161.g191c1869.el8.x86_6 15/25 2023-09-30T09:00:25.865 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-30T09:00:26.047 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-30T09:00:26.089 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:00:26.285 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-30T09:00:26.290 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-30T09:00:26.474 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-ceph-common-2:17.2.6-1161.g191c1869.el8.x8 19/25 2023-09-30T09:00:26.694 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-30T09:00:26.868 INFO:teuthology.orchestra.run.smithi019.stdout: Upgrading : librados2-2:17.2.6-1161.g191c1869.el8.x86_64 2/25 2023-09-30T09:00:26.952 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: librados2-2:17.2.6-1161.g191c1869.el8.x86_64 2/25 2023-09-30T09:00:27.145 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-rados-2:17.2.6-1161.g191c1869.el8.x86_64 3/25 2023-09-30T09:00:27.398 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-30T09:00:27.560 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 5/25 2023-09-30T09:00:27.977 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 5/25 2023-09-30T09:00:28.152 INFO:teuthology.orchestra.run.smithi019.stdout: Upgrading : librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 6/25 2023-09-30T09:00:28.252 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 6/25 2023-09-30T09:00:28.460 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-30T09:00:28.540 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-30T09:00:28.865 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-30T09:00:29.562 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-30T09:00:29.735 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : librgw2-2:17.2.6-1161.g191c1869.el8.x86_64 10/25 2023-09-30T09:00:29.803 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: librgw2-2:17.2.6-1161.g191c1869.el8.x86_64 10/25 2023-09-30T09:00:29.939 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-ceph-argparse-2:17.2.6-1161.g191c1869.el8. 11/25 2023-09-30T09:00:30.114 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-cephfs-2:17.2.6-1161.g191c1869.el8.x86_64 12/25 2023-09-30T09:00:30.212 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 20/25 2023-09-30T09:00:30.331 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-rgw-2:17.2.6-1161.g191c1869.el8.x86_64 13/25 2023-09-30T09:00:30.473 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 20/25 2023-09-30T09:00:30.542 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-rbd-2:17.2.6-1161.g191c1869.el8.x86_64 14/25 2023-09-30T09:00:30.685 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : libradosstriper1-2:17.2.6-1161.g191c1869.el8.x86_6 15/25 2023-09-30T09:00:30.757 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1161.g191c1869.el8.x86_6 15/25 2023-09-30T09:00:31.054 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 20/25 2023-09-30T09:00:31.057 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-30T09:00:31.279 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 21/25 2023-09-30T09:00:31.365 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 21/25 2023-09-30T09:00:31.531 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-selinux-2:17.2.6-1161.g191c1869.el8.x86_64 22/25 2023-09-30T09:00:31.534 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-30T09:00:31.809 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-30T09:00:31.977 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-ceph-common-2:17.2.6-1161.g191c1869.el8.x8 19/25 2023-09-30T09:00:35.978 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 20/25 2023-09-30T09:00:36.251 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 20/25 2023-09-30T09:00:36.958 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 20/25 2023-09-30T09:00:37.212 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 21/25 2023-09-30T09:00:37.309 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 21/25 2023-09-30T09:00:37.473 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-selinux-2:17.2.6-1161.g191c1869.el8.x86_64 22/25 2023-09-30T09:00:50.104 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1161.g191c1869.el8.x86_64 22/25 2023-09-30T09:00:50.104 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /sys 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /proc 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /dev 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /run 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /mnt 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /var/tmp 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /home 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /tmp 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /dev 2023-09-30T09:00:50.105 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:00:51.784 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-radosgw-2:17.2.6-1161.g191c1869.el8.x86_64 23/25 2023-09-30T09:00:51.865 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1161.g191c1869.el8.x86_64 23/25 2023-09-30T09:00:51.865 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-30T09:00:52.105 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-30T09:00:52.105 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-30T09:00:52.704 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-30T09:00:52.704 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 1/25 2023-09-30T09:00:52.704 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 2/25 2023-09-30T09:00:52.704 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-radosgw-2:17.2.6-1161.g191c1869.el8.x86_64 3/25 2023-09-30T09:00:52.704 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-selinux-2:17.2.6-1161.g191c1869.el8.x86_64 4/25 2023-09-30T09:00:52.704 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 5/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libradosstriper1-2:17.2.6-1161.g191c1869.el8.x86_6 6/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librgw2-2:17.2.6-1161.g191c1869.el8.x86_64 7/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1161.g191c1869.el8. 8/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-ceph-common-2:17.2.6-1161.g191c1869.el8.x8 9/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cephfs-2:17.2.6-1161.g191c1869.el8.x86_64 10/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-rados-2:17.2.6-1161.g191c1869.el8.x86_64 11/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-rbd-2:17.2.6-1161.g191c1869.el8.x86_64 12/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-rgw-2:17.2.6-1161.g191c1869.el8.x86_64 13/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-30T09:00:52.705 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-30T09:00:52.706 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-30T09:00:52.706 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-30T09:00:52.706 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-30T09:00:52.706 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-30T09:00:52.706 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-30T09:00:52.706 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librados2-2:17.2.6-1161.g191c1869.el8.x86_64 22/25 2023-09-30T09:00:52.706 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-30T09:00:52.706 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 24/25 2023-09-30T09:00:54.080 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-30T09:00:54.080 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:53 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:00:56.574 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:00:56.767 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:00:56.767 INFO:teuthology.orchestra.run.smithi167.stdout:Upgraded: 2023-09-30T09:00:56.767 INFO:teuthology.orchestra.run.smithi167.stdout: librados2-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.767 INFO:teuthology.orchestra.run.smithi167.stdout: librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.767 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:00:56.767 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-radosgw-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-selinux-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: libradosstriper1-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: librgw2-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.768 INFO:teuthology.orchestra.run.smithi167.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ceph-argparse-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cephfs-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rados-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rbd-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rgw-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-30T09:00:56.769 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:00:56.770 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:00:56.925 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-test 2023-09-30T09:00:56.963 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1161.g191c1869.el8.x86_64 22/25 2023-09-30T09:00:56.963 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /sys 2023-09-30T09:00:56.963 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /proc 2023-09-30T09:00:56.964 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /dev 2023-09-30T09:00:56.964 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /run 2023-09-30T09:00:56.964 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /mnt 2023-09-30T09:00:56.964 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /var/tmp 2023-09-30T09:00:56.964 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /home 2023-09-30T09:00:56.964 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /tmp 2023-09-30T09:00:56.964 INFO:teuthology.orchestra.run.smithi019.stdout:skipping the directory /dev 2023-09-30T09:00:56.964 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:00:57.277 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:00:58.135 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:57 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:00:58.759 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-radosgw-2:17.2.6-1161.g191c1869.el8.x86_64 23/25 2023-09-30T09:00:58.838 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1161.g191c1869.el8.x86_64 23/25 2023-09-30T09:00:58.865 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-30T09:00:59.098 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-30T09:00:59.098 INFO:teuthology.orchestra.run.smithi019.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-30T09:00:59.535 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:00:59.536 INFO:teuthology.orchestra.run.smithi167.stdout:============================================================================================ 2023-09-30T09:00:59.536 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:00:59.536 INFO:teuthology.orchestra.run.smithi167.stdout:============================================================================================ 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-test x86_64 2:17.2.6-1161.g191c1869.el8 ceph 52 M 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout: libcephsqlite x86_64 2:17.2.6-1161.g191c1869.el8 ceph 174 k 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:00:59.537 INFO:teuthology.orchestra.run.smithi167.stdout:============================================================================================ 2023-09-30T09:00:59.538 INFO:teuthology.orchestra.run.smithi167.stdout:Install 6 Packages 2023-09-30T09:00:59.538 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:00:59.538 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 53 M 2023-09-30T09:00:59.538 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 233 M 2023-09-30T09:00:59.538 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:00:59.615 INFO:teuthology.orchestra.run.smithi167.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 916 kB/s | 70 kB 00:00 2023-09-30T09:00:59.692 INFO:teuthology.orchestra.run.smithi167.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.6 MB/s | 323 kB 00:00 2023-09-30T09:00:59.715 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 1/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 2/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-radosgw-2:17.2.6-1161.g191c1869.el8.x86_64 3/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-selinux-2:17.2.6-1161.g191c1869.el8.x86_64 4/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 5/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libradosstriper1-2:17.2.6-1161.g191c1869.el8.x86_6 6/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : librgw2-2:17.2.6-1161.g191c1869.el8.x86_64 7/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1161.g191c1869.el8. 8/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-ceph-common-2:17.2.6-1161.g191c1869.el8.x8 9/25 2023-09-30T09:00:59.716 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-cephfs-2:17.2.6-1161.g191c1869.el8.x86_64 10/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-rados-2:17.2.6-1161.g191c1869.el8.x86_64 11/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-rbd-2:17.2.6-1161.g191c1869.el8.x86_64 12/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-rgw-2:17.2.6-1161.g191c1869.el8.x86_64 13/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-30T09:00:59.717 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : librados2-2:17.2.6-1161.g191c1869.el8.x86_64 22/25 2023-09-30T09:00:59.718 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-30T09:00:59.718 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 24/25 2023-09-30T09:00:59.725 INFO:teuthology.orchestra.run.smithi167.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 5.5 MB/s | 187 kB 00:00 2023-09-30T09:00:59.767 INFO:teuthology.orchestra.run.smithi167.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-09-30T09:00:59.801 INFO:teuthology.orchestra.run.smithi167.stdout:(5/6): libcephsqlite-17.2.6-1161.g191c1869.el8. 664 kB/s | 174 kB 00:00 2023-09-30T09:01:01.100 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-30T09:01:01.100 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:00:58 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:02.743 INFO:teuthology.orchestra.run.smithi167.stdout:(6/6): ceph-test-17.2.6-1161.g191c1869.el8.x86_ 16 MB/s | 52 MB 00:03 2023-09-30T09:01:02.745 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:01:02.745 INFO:teuthology.orchestra.run.smithi167.stdout:Total 16 MB/s | 53 MB 00:03 2023-09-30T09:01:02.783 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:01:02.804 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:01:02.804 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:01:03.078 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:01:03.082 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:01:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:01:03.627 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:01:03.701 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-30T09:01:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-30T09:01:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:03.832 INFO:teuthology.orchestra.run.smithi019.stdout:Upgraded: 2023-09-30T09:01:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: librados2-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.832 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:01:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-radosgw-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-selinux-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: libradosstriper1-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: librgw2-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: python3-ceph-argparse-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: python3-ceph-common-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cephfs-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: python3-rados-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: python3-rbd-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: python3-rgw-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:03.834 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:01:03.984 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-test 2023-09-30T09:01:03.996 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-30T09:01:04.151 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-30T09:01:04.300 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-30T09:01:04.355 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:04.468 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libcephsqlite-2:17.2.6-1161.g191c1869.el8.x86_64 5/6 2023-09-30T09:01:05.196 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:02 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:06.733 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:01:06.734 INFO:teuthology.orchestra.run.smithi019.stdout:============================================================================================ 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout:============================================================================================ 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-test x86_64 2:17.2.6-1161.g191c1869.el8 ceph 52 M 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout: libcephsqlite x86_64 2:17.2.6-1161.g191c1869.el8 ceph 174 k 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-30T09:01:06.735 INFO:teuthology.orchestra.run.smithi019.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-30T09:01:06.736 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:06.736 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:01:06.736 INFO:teuthology.orchestra.run.smithi019.stdout:============================================================================================ 2023-09-30T09:01:06.736 INFO:teuthology.orchestra.run.smithi019.stdout:Install 6 Packages 2023-09-30T09:01:06.736 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:06.736 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 53 M 2023-09-30T09:01:06.737 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 233 M 2023-09-30T09:01:06.737 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:01:06.801 INFO:teuthology.orchestra.run.smithi019.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.1 MB/s | 70 kB 00:00 2023-09-30T09:01:06.887 INFO:teuthology.orchestra.run.smithi019.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.1 MB/s | 323 kB 00:00 2023-09-30T09:01:06.929 INFO:teuthology.orchestra.run.smithi019.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-09-30T09:01:06.971 INFO:teuthology.orchestra.run.smithi019.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.8 MB/s | 202 kB 00:00 2023-09-30T09:01:07.096 INFO:teuthology.orchestra.run.smithi019.stdout:(5/6): libcephsqlite-17.2.6-1161.g191c1869.el8. 485 kB/s | 174 kB 00:00 2023-09-30T09:01:09.313 INFO:teuthology.orchestra.run.smithi019.stdout:(6/6): ceph-test-17.2.6-1161.g191c1869.el8.x86_ 20 MB/s | 52 MB 00:02 2023-09-30T09:01:09.315 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:01:09.315 INFO:teuthology.orchestra.run.smithi019.stdout:Total 20 MB/s | 53 MB 00:02 2023-09-30T09:01:09.353 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:01:09.376 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:01:09.376 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:01:09.389 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1161.g191c1869.el8.x86_64 5/6 2023-09-30T09:01:09.597 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-test-2:17.2.6-1161.g191c1869.el8.x86_64 6/6 2023-09-30T09:01:09.665 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:01:09.669 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:01:09.844 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-test-2:17.2.6-1161.g191c1869.el8.x86_64 6/6 2023-09-30T09:01:09.844 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-test-2:17.2.6-1161.g191c1869.el8.x86_64 1/6 2023-09-30T09:01:09.844 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libcephsqlite-2:17.2.6-1161.g191c1869.el8.x86_64 2/6 2023-09-30T09:01:09.844 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-30T09:01:09.844 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-30T09:01:09.844 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-30T09:01:10.151 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:01:10.279 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-30T09:01:10.382 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-30T09:01:10.578 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-30T09:01:10.764 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-30T09:01:10.903 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-30T09:01:10.903 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:09 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:10.937 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-30T09:01:11.071 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : libcephsqlite-2:17.2.6-1161.g191c1869.el8.x86_64 5/6 2023-09-30T09:01:13.578 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-test-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout: jq-1.6-3.el8.x86_64 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout: libcephsqlite-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:13.759 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:01:13.892 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph 2023-09-30T09:01:14.228 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:15.071 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:14 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:16.334 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1161.g191c1869.el8.x86_64 5/6 2023-09-30T09:01:16.442 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:01:16.446 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================= 2023-09-30T09:01:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:01:16.446 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================= 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: ceph x86_64 2:17.2.6-1161.g191c1869.el8 ceph 6.4 k 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mds x86_64 2:17.2.6-1161.g191c1869.el8 ceph 2.3 M 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr x86_64 2:17.2.6-1161.g191c1869.el8 ceph 1.6 M 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 238 k 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mon x86_64 2:17.2.6-1161.g191c1869.el8 ceph 3.9 M 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-osd x86_64 2:17.2.6-1161.g191c1869.el8 ceph 17 M 2023-09-30T09:01:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-30T09:01:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-30T09:01:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-30T09:01:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-30T09:01:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-30T09:01:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-30T09:01:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-30T09:01:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-30T09:01:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-30T09:01:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-30T09:01:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-30T09:01:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-30T09:01:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-30T09:01:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-30T09:01:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-30T09:01:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-30T09:01:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:16.453 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:01:16.453 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================= 2023-09-30T09:01:16.453 INFO:teuthology.orchestra.run.smithi167.stdout:Install 32 Packages 2023-09-30T09:01:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:16.454 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 28 M 2023-09-30T09:01:16.454 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 110 M 2023-09-30T09:01:16.455 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:01:16.593 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-test-2:17.2.6-1161.g191c1869.el8.x86_64 6/6 2023-09-30T09:01:16.830 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-test-2:17.2.6-1161.g191c1869.el8.x86_64 6/6 2023-09-30T09:01:16.830 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-test-2:17.2.6-1161.g191c1869.el8.x86_64 1/6 2023-09-30T09:01:16.830 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libcephsqlite-2:17.2.6-1161.g191c1869.el8.x86_64 2/6 2023-09-30T09:01:16.830 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-30T09:01:16.830 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-30T09:01:16.830 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-30T09:01:17.071 INFO:teuthology.orchestra.run.smithi167.stdout:(1/32): ceph-17.2.6-1161.g191c1869.el8.x86_64.r 31 kB/s | 6.4 kB 00:00 2023-09-30T09:01:17.221 INFO:teuthology.orchestra.run.smithi167.stdout:(2/32): ceph-mds-17.2.6-1161.g191c1869.el8.x86_ 6.4 MB/s | 2.3 MB 00:00 2023-09-30T09:01:17.438 INFO:teuthology.orchestra.run.smithi167.stdout:(3/32): ceph-mgr-17.2.6-1161.g191c1869.el8.x86_ 2.7 MB/s | 1.6 MB 00:00 2023-09-30T09:01:17.513 INFO:teuthology.orchestra.run.smithi167.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1161.g191c 3.1 MB/s | 238 kB 00:00 2023-09-30T09:01:17.789 INFO:teuthology.orchestra.run.smithi167.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 159 kB/s | 44 kB 00:00 2023-09-30T09:01:17.889 INFO:teuthology.orchestra.run.smithi167.stdout:(6/32): ceph-mon-17.2.6-1161.g191c1869.el8.x86_ 4.8 MB/s | 3.9 MB 00:00 2023-09-30T09:01:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-30T09:01:17.932 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:14 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:17.989 INFO:teuthology.orchestra.run.smithi167.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 924 kB/s | 185 kB 00:00 2023-09-30T09:01:18.156 INFO:teuthology.orchestra.run.smithi167.stdout:(8/32): ceph-osd-17.2.6-1161.g191c1869.el8.x86_ 18 MB/s | 17 MB 00:00 2023-09-30T09:01:18.181 INFO:teuthology.orchestra.run.smithi167.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 2.0 MB/s | 384 kB 00:00 2023-09-30T09:01:18.257 INFO:teuthology.orchestra.run.smithi167.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 234 kB/s | 18 kB 00:00 2023-09-30T09:01:18.332 INFO:teuthology.orchestra.run.smithi167.stdout:(11/32): python3-logutils-0.3.5-11.el8.noarch.r 650 kB/s | 49 kB 00:00 2023-09-30T09:01:18.357 INFO:teuthology.orchestra.run.smithi167.stdout:(12/32): python3-jaraco-6.2-6.el8.noarch.rpm 55 kB/s | 11 kB 00:00 2023-09-30T09:01:18.408 INFO:teuthology.orchestra.run.smithi167.stdout:(13/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 333 kB/s | 173 kB 00:00 2023-09-30T09:01:18.433 INFO:teuthology.orchestra.run.smithi167.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 585 kB/s | 59 kB 00:00 2023-09-30T09:01:18.508 INFO:teuthology.orchestra.run.smithi167.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 208 kB/s | 16 kB 00:00 2023-09-30T09:01:18.533 INFO:teuthology.orchestra.run.smithi167.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 187 kB/s | 19 kB 00:00 2023-09-30T09:01:18.584 INFO:teuthology.orchestra.run.smithi167.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 318 kB/s | 24 kB 00:00 2023-09-30T09:01:18.609 INFO:teuthology.orchestra.run.smithi167.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 388 kB/s | 29 kB 00:00 2023-09-30T09:01:18.634 INFO:teuthology.orchestra.run.smithi167.stdout:(19/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.0 MB/s | 283 kB 00:00 2023-09-30T09:01:18.660 INFO:teuthology.orchestra.run.smithi167.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 364 kB/s | 27 kB 00:00 2023-09-30T09:01:18.743 INFO:teuthology.orchestra.run.smithi167.stdout:(21/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.0 MB/s | 85 kB 00:00 2023-09-30T09:01:18.768 INFO:teuthology.orchestra.run.smithi167.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.6 MB/s | 255 kB 00:00 2023-09-30T09:01:18.793 INFO:teuthology.orchestra.run.smithi167.stdout:(23/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.5 MB/s | 251 kB 00:00 2023-09-30T09:01:18.811 INFO:teuthology.orchestra.run.smithi167.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 341 kB/s | 23 kB 00:00 2023-09-30T09:01:18.894 INFO:teuthology.orchestra.run.smithi167.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 868 kB/s | 109 kB 00:00 2023-09-30T09:01:18.920 INFO:teuthology.orchestra.run.smithi167.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 2.4 MB/s | 238 kB 00:00 2023-09-30T09:01:18.937 INFO:teuthology.orchestra.run.smithi167.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.3 MB/s | 54 kB 00:00 2023-09-30T09:01:18.953 INFO:teuthology.orchestra.run.smithi167.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 1.1 MB/s | 39 kB 00:00 2023-09-30T09:01:18.979 INFO:teuthology.orchestra.run.smithi167.stdout:(29/32): python3-mako-1.0.6-13.el8.noarch.rpm 3.6 MB/s | 157 kB 00:00 2023-09-30T09:01:19.004 INFO:teuthology.orchestra.run.smithi167.stdout:(30/32): python3-cryptography-3.2.1-5.el8.x86_6 2.6 MB/s | 559 kB 00:00 2023-09-30T09:01:19.029 INFO:teuthology.orchestra.run.smithi167.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-09-30T09:01:19.063 INFO:teuthology.orchestra.run.smithi167.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.1 MB/s | 457 kB 00:00 2023-09-30T09:01:19.067 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:01:19.067 INFO:teuthology.orchestra.run.smithi167.stdout:Total 11 MB/s | 28 MB 00:02 2023-09-30T09:01:19.154 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:01:19.191 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:01:19.191 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:01:19.396 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:01:19.400 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:01:20.459 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:01:20.578 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:01:20.658 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:20.658 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:01:20.658 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-test-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:20.658 INFO:teuthology.orchestra.run.smithi019.stdout: jq-1.6-3.el8.x86_64 2023-09-30T09:01:20.659 INFO:teuthology.orchestra.run.smithi019.stdout: libcephsqlite-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:20.659 INFO:teuthology.orchestra.run.smithi019.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-30T09:01:20.659 INFO:teuthology.orchestra.run.smithi019.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-30T09:01:20.659 INFO:teuthology.orchestra.run.smithi019.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-30T09:01:20.659 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:20.659 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:01:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-30T09:01:20.791 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph 2023-09-30T09:01:21.065 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-30T09:01:21.146 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:21.286 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-30T09:01:21.475 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-30T09:01:21.677 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-30T09:01:21.877 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-30T09:01:22.060 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:19 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-30T09:01:22.454 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-30T09:01:22.684 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-30T09:01:22.848 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-30T09:01:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-30T09:01:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-30T09:01:23.424 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-30T09:01:23.563 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-30T09:01:23.600 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:01:23.603 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================= 2023-09-30T09:01:23.603 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:01:23.603 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================= 2023-09-30T09:01:23.603 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: ceph x86_64 2:17.2.6-1161.g191c1869.el8 ceph 6.4 k 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mds x86_64 2:17.2.6-1161.g191c1869.el8 ceph 2.3 M 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr x86_64 2:17.2.6-1161.g191c1869.el8 ceph 1.6 M 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 238 k 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mon x86_64 2:17.2.6-1161.g191c1869.el8 ceph 3.9 M 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-osd x86_64 2:17.2.6-1161.g191c1869.el8 ceph 17 M 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-30T09:01:23.604 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-30T09:01:23.605 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:23.606 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:01:23.607 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================= 2023-09-30T09:01:23.607 INFO:teuthology.orchestra.run.smithi019.stdout:Install 32 Packages 2023-09-30T09:01:23.607 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:23.607 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 28 M 2023-09-30T09:01:23.607 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 110 M 2023-09-30T09:01:23.607 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:01:23.752 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-30T09:01:23.873 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-30T09:01:23.888 INFO:teuthology.orchestra.run.smithi019.stdout:(1/32): ceph-17.2.6-1161.g191c1869.el8.x86_64.r 26 kB/s | 6.4 kB 00:00 2023-09-30T09:01:24.026 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-30T09:01:24.047 INFO:teuthology.orchestra.run.smithi019.stdout:(2/32): ceph-mds-17.2.6-1161.g191c1869.el8.x86_ 5.7 MB/s | 2.3 MB 00:00 2023-09-30T09:01:24.161 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-30T09:01:24.397 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-30T09:01:24.456 INFO:teuthology.orchestra.run.smithi019.stdout:(3/32): ceph-mon-17.2.6-1161.g191c1869.el8.x86_ 6.9 MB/s | 3.9 MB 00:00 2023-09-30T09:01:24.547 INFO:teuthology.orchestra.run.smithi019.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1161.g191c 2.5 MB/s | 238 kB 00:00 2023-09-30T09:01:24.608 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-30T09:01:24.723 INFO:teuthology.orchestra.run.smithi019.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 249 kB/s | 44 kB 00:00 2023-09-30T09:01:24.778 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-30T09:01:24.973 INFO:teuthology.orchestra.run.smithi019.stdout:(6/32): ceph-osd-17.2.6-1161.g191c1869.el8.x86_ 18 MB/s | 17 MB 00:00 2023-09-30T09:01:24.998 INFO:teuthology.orchestra.run.smithi019.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 673 kB/s | 185 kB 00:00 2023-09-30T09:01:25.036 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-30T09:01:25.123 INFO:teuthology.orchestra.run.smithi019.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 3.0 MB/s | 384 kB 00:00 2023-09-30T09:01:25.182 INFO:teuthology.orchestra.run.smithi019.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 190 kB/s | 11 kB 00:00 2023-09-30T09:01:25.207 INFO:teuthology.orchestra.run.smithi019.stdout:(10/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 738 kB/s | 173 kB 00:00 2023-09-30T09:01:25.241 INFO:teuthology.orchestra.run.smithi019.stdout:(11/32): ceph-mgr-17.2.6-1161.g191c1869.el8.x86 1.0 MB/s | 1.6 MB 00:01 2023-09-30T09:01:25.267 INFO:teuthology.orchestra.run.smithi019.stdout:(12/32): python3-jaraco-functools-2.0-4.el8.noa 210 kB/s | 18 kB 00:00 2023-09-30T09:01:25.292 INFO:teuthology.orchestra.run.smithi019.stdout:(13/32): python3-logutils-0.3.5-11.el8.noarch.r 581 kB/s | 49 kB 00:00 2023-09-30T09:01:25.350 INFO:teuthology.orchestra.run.smithi019.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 268 kB/s | 16 kB 00:00 2023-09-30T09:01:25.375 INFO:teuthology.orchestra.run.smithi019.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.5 MB/s | 283 kB 00:00 2023-09-30T09:01:25.409 INFO:teuthology.orchestra.run.smithi019.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 320 kB/s | 19 kB 00:00 2023-09-30T09:01:25.434 INFO:teuthology.orchestra.run.smithi019.stdout:(17/32): python3-more-itertools-7.2.0-3.el8.noa 305 kB/s | 59 kB 00:00 2023-09-30T09:01:25.459 INFO:teuthology.orchestra.run.smithi019.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 285 kB/s | 24 kB 00:00 2023-09-30T09:01:25.485 INFO:teuthology.orchestra.run.smithi019.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 388 kB/s | 29 kB 00:00 2023-09-30T09:01:25.518 INFO:teuthology.orchestra.run.smithi019.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 471 kB/s | 27 kB 00:00 2023-09-30T09:01:25.560 INFO:teuthology.orchestra.run.smithi019.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 3.3 MB/s | 251 kB 00:00 2023-09-30T09:01:25.611 INFO:teuthology.orchestra.run.smithi019.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 930 kB/s | 85 kB 00:00 2023-09-30T09:01:25.636 INFO:teuthology.orchestra.run.smithi019.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 305 kB/s | 23 kB 00:00 2023-09-30T09:01:25.661 INFO:teuthology.orchestra.run.smithi019.stdout:(24/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.2 MB/s | 255 kB 00:00 2023-09-30T09:01:25.712 INFO:teuthology.orchestra.run.smithi019.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 1.1 MB/s | 109 kB 00:00 2023-09-30T09:01:25.753 INFO:teuthology.orchestra.run.smithi019.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 4.7 MB/s | 559 kB 00:00 2023-09-30T09:01:25.778 INFO:teuthology.orchestra.run.smithi019.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 926 kB/s | 54 kB 00:00 2023-09-30T09:01:25.804 INFO:teuthology.orchestra.run.smithi019.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 770 kB/s | 39 kB 00:00 2023-09-30T09:01:25.829 INFO:teuthology.orchestra.run.smithi019.stdout:(29/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.5 MB/s | 238 kB 00:00 2023-09-30T09:01:25.855 INFO:teuthology.orchestra.run.smithi019.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-09-30T09:01:25.880 INFO:teuthology.orchestra.run.smithi019.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-09-30T09:01:25.905 INFO:teuthology.orchestra.run.smithi019.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.4 MB/s | 457 kB 00:00 2023-09-30T09:01:25.906 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:01:25.907 INFO:teuthology.orchestra.run.smithi019.stdout:Total 12 MB/s | 28 MB 00:02 2023-09-30T09:01:25.913 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-30T09:01:26.000 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:01:26.040 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:01:26.040 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:01:26.092 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-30T09:01:26.258 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:01:26.262 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:01:26.398 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-30T09:01:26.686 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-30T09:01:27.001 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1161.g191c1869.el8. 27/32 2023-09-30T09:01:27.154 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 28/32 2023-09-30T09:01:27.437 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:01:27.695 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-30T09:01:28.001 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-30T09:01:28.232 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-30T09:01:28.476 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-30T09:01:28.653 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 28/32 2023-09-30T09:01:28.687 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-30T09:01:28.845 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-osd-2:17.2.6-1161.g191c1869.el8.x86_64 29/32 2023-09-30T09:01:28.892 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-30T09:01:29.126 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-30T09:01:29.252 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-osd-2:17.2.6-1161.g191c1869.el8.x86_64 29/32 2023-09-30T09:01:29.381 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mon-2:17.2.6-1161.g191c1869.el8.x86_64 30/32 2023-09-30T09:01:29.500 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-30T09:01:29.598 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mon-2:17.2.6-1161.g191c1869.el8.x86_64 30/32 2023-09-30T09:01:29.731 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mds-2:17.2.6-1161.g191c1869.el8.x86_64 31/32 2023-09-30T09:01:29.749 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mds-2:17.2.6-1161.g191c1869.el8.x86_64 31/32 2023-09-30T09:01:29.765 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-30T09:01:29.862 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-2:17.2.6-1161.g191c1869.el8.x86_64 32/32 2023-09-30T09:01:30.044 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-30T09:01:30.222 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-2:17.2.6-1161.g191c1869.el8.x86_64 32/32 2023-09-30T09:01:30.222 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-2:17.2.6-1161.g191c1869.el8.x86_64 1/32 2023-09-30T09:01:30.222 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mds-2:17.2.6-1161.g191c1869.el8.x86_64 2/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 3/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mon-2:17.2.6-1161.g191c1869.el8.x86_64 4/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-osd-2:17.2.6-1161.g191c1869.el8.x86_64 5/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1161.g191c1869.el8. 6/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-30T09:01:30.223 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-30T09:01:30.224 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-30T09:01:30.225 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-30T09:01:30.225 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-30T09:01:30.225 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-30T09:01:30.225 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-30T09:01:30.225 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-30T09:01:30.288 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-30T09:01:30.495 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-30T09:01:30.714 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-30T09:01:30.884 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-30T09:01:31.048 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-30T09:01:31.207 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-30T09:01:31.388 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-30T09:01:31.525 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-30T09:01:31.752 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-30T09:01:31.805 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-30T09:01:31.805 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:30 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:32.549 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-30T09:01:32.766 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-30T09:01:33.046 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-30T09:01:33.310 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-30T09:01:33.532 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-30T09:01:33.853 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-30T09:01:34.139 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:01:34.216 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-30T09:01:34.325 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:34.325 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mds-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-modules-core-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mon-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-osd-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-30T09:01:34.326 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-portend-2.6-1.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-30T09:01:34.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:34.328 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:01:34.487 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-base 2023-09-30T09:01:34.653 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1161.g191c1869.el8. 27/32 2023-09-30T09:01:34.833 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:34.837 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 28/32 2023-09-30T09:01:35.654 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:34 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 28/32 2023-09-30T09:01:36.663 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-osd-2:17.2.6-1161.g191c1869.el8.x86_64 29/32 2023-09-30T09:01:36.917 INFO:teuthology.orchestra.run.smithi167.stdout:Package ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:01:37.026 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:01:37.027 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:01:37.027 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:01:37.094 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-osd-2:17.2.6-1161.g191c1869.el8.x86_64 29/32 2023-09-30T09:01:37.285 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-mon-2:17.2.6-1161.g191c1869.el8.x86_64 30/32 2023-09-30T09:01:37.514 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-mon-2:17.2.6-1161.g191c1869.el8.x86_64 30/32 2023-09-30T09:01:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-mds-2:17.2.6-1161.g191c1869.el8.x86_64 31/32 2023-09-30T09:01:37.655 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install cephadm 2023-09-30T09:01:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-mds-2:17.2.6-1161.g191c1869.el8.x86_64 31/32 2023-09-30T09:01:37.783 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-2:17.2.6-1161.g191c1869.el8.x86_64 32/32 2023-09-30T09:01:38.000 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-2:17.2.6-1161.g191c1869.el8.x86_64 32/32 2023-09-30T09:01:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-2:17.2.6-1161.g191c1869.el8.x86_64 1/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-mds-2:17.2.6-1161.g191c1869.el8.x86_64 2/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 3/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-mon-2:17.2.6-1161.g191c1869.el8.x86_64 4/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-osd-2:17.2.6-1161.g191c1869.el8.x86_64 5/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1161.g191c1869.el8. 6/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-30T09:01:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-30T09:01:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-30T09:01:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-30T09:01:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-30T09:01:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-30T09:01:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-30T09:01:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-30T09:01:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-30T09:01:38.906 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:37 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:39.741 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-30T09:01:39.742 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:36 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:40.275 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:01:40.276 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:01:40.276 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:01:40.276 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:01:40.277 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:01:40.277 INFO:teuthology.orchestra.run.smithi167.stdout: cephadm noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 80 k 2023-09-30T09:01:40.277 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:40.277 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:01:40.277 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:01:40.278 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2023-09-30T09:01:40.278 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:40.278 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 80 k 2023-09-30T09:01:40.278 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 363 k 2023-09-30T09:01:40.278 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:01:40.437 INFO:teuthology.orchestra.run.smithi167.stdout:cephadm-17.2.6-1161.g191c1869.el8.noarch.rpm 500 kB/s | 80 kB 00:00 2023-09-30T09:01:40.438 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:01:40.438 INFO:teuthology.orchestra.run.smithi167.stdout:Total 496 kB/s | 80 kB 00:00 2023-09-30T09:01:40.438 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:01:40.446 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:01:40.446 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:01:40.448 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:01:40.451 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:01:40.493 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:01:42.164 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/1 2023-09-30T09:01:42.221 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:01:42.311 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/1 2023-09-30T09:01:42.428 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:42.428 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:01:42.428 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mds-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-modules-core-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mon-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-osd-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-30T09:01:42.429 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-portend-2.6-1.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-30T09:01:42.430 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:42.431 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:01:42.478 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/1 2023-09-30T09:01:42.582 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-base 2023-09-30T09:01:42.951 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:43.576 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/1 2023-09-30T09:01:43.577 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:42 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:43.839 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:40 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:45.190 INFO:teuthology.orchestra.run.smithi019.stdout:Package ceph-base-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:01:45.310 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:01:45.311 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:01:45.311 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:01:45.413 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install cephadm 2023-09-30T09:01:45.779 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:46.306 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:01:46.490 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:46.490 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:01:46.490 INFO:teuthology.orchestra.run.smithi167.stdout: cephadm-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:01:46.490 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:46.490 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:01:46.624 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-immutable-object-cache 2023-09-30T09:01:46.712 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:43 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:46.982 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:47.792 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:46 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:48.242 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:01:48.243 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:01:48.243 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:01:48.243 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:01:48.243 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:01:48.243 INFO:teuthology.orchestra.run.smithi019.stdout: cephadm noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 80 k 2023-09-30T09:01:48.243 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:48.243 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:01:48.244 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:01:48.244 INFO:teuthology.orchestra.run.smithi019.stdout:Install 1 Package 2023-09-30T09:01:48.244 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:48.244 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 80 k 2023-09-30T09:01:48.244 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 363 k 2023-09-30T09:01:48.245 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:01:48.514 INFO:teuthology.orchestra.run.smithi019.stdout:cephadm-17.2.6-1161.g191c1869.el8.noarch.rpm 297 kB/s | 80 kB 00:00 2023-09-30T09:01:48.514 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:01:48.514 INFO:teuthology.orchestra.run.smithi019.stdout:Total 296 kB/s | 80 kB 00:00 2023-09-30T09:01:48.515 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:01:48.524 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:01:48.524 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:01:48.526 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:01:48.530 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:01:48.575 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:01:49.151 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:01:49.151 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:01:49.151 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repo Size 2023-09-30T09:01:49.151 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:01:49.152 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:01:49.152 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1161.g191c1869.el8 ceph 162 k 2023-09-30T09:01:49.152 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:49.152 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:01:49.152 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:01:49.152 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2023-09-30T09:01:49.152 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:49.153 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 162 k 2023-09-30T09:01:49.153 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 490 k 2023-09-30T09:01:49.153 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:01:49.368 INFO:teuthology.orchestra.run.smithi167.stdout:ceph-immutable-object-cache-17.2.6-1161.g191c18 752 kB/s | 162 kB 00:00 2023-09-30T09:01:49.368 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:01:49.368 INFO:teuthology.orchestra.run.smithi167.stdout:Total 748 kB/s | 162 kB 00:00 2023-09-30T09:01:49.369 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:01:49.379 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:01:49.379 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:01:49.431 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:01:49.435 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:01:49.561 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:01:49.736 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1161.g191c1869. 1/1 2023-09-30T09:01:50.092 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1161.g191c1869. 1/1 2023-09-30T09:01:51.087 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/1 2023-09-30T09:01:51.095 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1161.g191c1869. 1/1 2023-09-30T09:01:51.095 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:50 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:51.247 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/1 2023-09-30T09:01:51.424 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/1 2023-09-30T09:01:52.510 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/1 2023-09-30T09:01:52.511 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:49 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:55.798 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:01:55.980 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:55.996 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:01:55.996 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-immutable-object-cache-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:01:55.996 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:01:55.996 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:01:56.111 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr 2023-09-30T09:01:56.460 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:57.216 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:01:57.412 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:57.412 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:01:57.413 INFO:teuthology.orchestra.run.smithi019.stdout: cephadm-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:01:57.413 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:01:57.413 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:01:57.515 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:56 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:01:57.540 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-immutable-object-cache 2023-09-30T09:01:57.920 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:01:58.805 INFO:teuthology.orchestra.run.smithi167.stdout:Package ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:01:58.899 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:55 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:01:58.914 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:01:58.916 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:01:58.916 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:01:59.033 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr-dashboard 2023-09-30T09:01:59.373 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:02:00.281 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:59 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:02:00.421 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:02:00.421 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:02:00.421 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repo Size 2023-09-30T09:02:00.421 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:02:00.421 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:02:00.422 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1161.g191c1869.el8 ceph 162 k 2023-09-30T09:02:00.422 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:00.422 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:02:00.422 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:02:00.422 INFO:teuthology.orchestra.run.smithi019.stdout:Install 1 Package 2023-09-30T09:02:00.422 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:00.423 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 162 k 2023-09-30T09:02:00.423 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 490 k 2023-09-30T09:02:00.423 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:02:00.646 INFO:teuthology.orchestra.run.smithi019.stdout:ceph-immutable-object-cache-17.2.6-1161.g191c18 723 kB/s | 162 kB 00:00 2023-09-30T09:02:00.647 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:02:00.647 INFO:teuthology.orchestra.run.smithi019.stdout:Total 719 kB/s | 162 kB 00:00 2023-09-30T09:02:00.647 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:02:00.659 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:02:00.659 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:02:00.714 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:02:00.718 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:02:00.837 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:02:01.006 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1161.g191c1869. 1/1 2023-09-30T09:02:01.379 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1161.g191c1869. 1/1 2023-09-30T09:02:01.724 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:02:01.725 INFO:teuthology.orchestra.run.smithi167.stdout:=================================================================================================== 2023-09-30T09:02:01.725 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:02:01.725 INFO:teuthology.orchestra.run.smithi167.stdout:=================================================================================================== 2023-09-30T09:02:01.725 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:02:01.725 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 1.7 M 2023-09-30T09:02:01.725 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T09:02:01.725 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 23 k 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 14 k 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout:=================================================================================================== 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout:Install 6 Packages 2023-09-30T09:02:01.726 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:01.727 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 2.0 M 2023-09-30T09:02:01.727 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 8.0 M 2023-09-30T09:02:01.727 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:02:02.299 INFO:teuthology.orchestra.run.smithi167.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1161.g191c 91 kB/s | 14 kB 00:00 2023-09-30T09:02:02.416 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1161.g191c1869. 1/1 2023-09-30T09:02:02.416 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:01:59 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:02:02.416 INFO:teuthology.orchestra.run.smithi167.stdout:(2/6): ceph-grafana-dashboards-17.2.6-1161.g191 83 kB/s | 23 kB 00:00 2023-09-30T09:02:02.584 INFO:teuthology.orchestra.run.smithi167.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 119 kB/s | 34 kB 00:00 2023-09-30T09:02:02.633 INFO:teuthology.orchestra.run.smithi167.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 854 kB/s | 43 kB 00:00 2023-09-30T09:02:02.717 INFO:teuthology.orchestra.run.smithi167.stdout:(5/6): ceph-mgr-dashboard-17.2.6-1161.g191c1869 2.9 MB/s | 1.7 MB 00:00 2023-09-30T09:02:02.750 INFO:teuthology.orchestra.run.smithi167.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 595 kB/s | 196 kB 00:00 2023-09-30T09:02:02.754 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:02:02.754 INFO:teuthology.orchestra.run.smithi167.stdout:Total 1.9 MB/s | 2.0 MB 00:01 2023-09-30T09:02:02.766 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:02:02.776 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:02:02.776 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:02:02.836 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:02:02.840 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:02:03.114 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:02:03.268 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-30T09:02:03.465 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-30T09:02:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-30T09:02:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1161.g191c1869.el8.n 4/6 2023-09-30T09:02:04.078 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1161.g191c1869.el8. 5/6 2023-09-30T09:02:04.276 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1161.g191c1869.el8.noarc 6/6 2023-09-30T09:02:04.525 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1161.g191c1869.el8.noarc 6/6 2023-09-30T09:02:04.525 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1161.g191c1869.el8. 1/6 2023-09-30T09:02:04.526 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1161.g191c1869.el8.noarc 2/6 2023-09-30T09:02:04.526 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1161.g191c1869.el8.n 3/6 2023-09-30T09:02:04.526 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-30T09:02:04.526 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-30T09:02:05.107 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:02:05.304 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:05.304 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:02:05.304 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-immutable-object-cache-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:02:05.304 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:05.305 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:02:05.428 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-mgr 2023-09-30T09:02:05.580 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-30T09:02:05.580 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:02:04 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:02:05.796 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:02:06.890 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:03 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:02:08.269 INFO:teuthology.orchestra.run.smithi019.stdout:Package ceph-mgr-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:02:08.388 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:02:08.389 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:02:08.389 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:02:08.492 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-mgr-dashboard 2023-09-30T09:02:08.854 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:02:10.391 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:07 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:02:10.917 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:02:11.099 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:11.099 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:02:11.100 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-grafana-dashboards-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:11.100 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-dashboard-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:11.100 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-prometheus-alerts-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:11.100 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-30T09:02:11.100 INFO:teuthology.orchestra.run.smithi167.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-30T09:02:11.100 INFO:teuthology.orchestra.run.smithi167.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-30T09:02:11.100 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:11.100 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:02:11.233 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-30T09:02:11.583 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:02:11.908 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:02:11.909 INFO:teuthology.orchestra.run.smithi019.stdout:=================================================================================================== 2023-09-30T09:02:11.909 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:02:11.909 INFO:teuthology.orchestra.run.smithi019.stdout:=================================================================================================== 2023-09-30T09:02:11.909 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 1.7 M 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 23 k 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 14 k 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout:=================================================================================================== 2023-09-30T09:02:11.910 INFO:teuthology.orchestra.run.smithi019.stdout:Install 6 Packages 2023-09-30T09:02:11.911 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:11.911 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 2.0 M 2023-09-30T09:02:11.911 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 8.0 M 2023-09-30T09:02:11.911 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:02:12.476 INFO:teuthology.orchestra.run.smithi019.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1161.g191c 100 kB/s | 14 kB 00:00 2023-09-30T09:02:12.568 INFO:teuthology.orchestra.run.smithi019.stdout:(2/6): ceph-grafana-dashboards-17.2.6-1161.g191 97 kB/s | 23 kB 00:00 2023-09-30T09:02:12.910 INFO:teuthology.orchestra.run.smithi019.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 78 kB/s | 34 kB 00:00 2023-09-30T09:02:13.077 INFO:teuthology.orchestra.run.smithi019.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 255 kB/s | 43 kB 00:00 2023-09-30T09:02:13.145 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:02:12 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:02:13.194 INFO:teuthology.orchestra.run.smithi019.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 316 kB/s | 196 kB 00:00 2023-09-30T09:02:13.411 INFO:teuthology.orchestra.run.smithi019.stdout:(6/6): ceph-mgr-dashboard-17.2.6-1161.g191c1869 1.6 MB/s | 1.7 MB 00:01 2023-09-30T09:02:13.414 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:02:13.414 INFO:teuthology.orchestra.run.smithi019.stdout:Total 1.3 MB/s | 2.0 MB 00:01 2023-09-30T09:02:13.428 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:02:13.439 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:02:13.439 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:02:13.504 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:02:13.508 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:02:13.802 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:02:13.964 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-30T09:02:14.179 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-30T09:02:14.365 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-30T09:02:14.495 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1161.g191c1869.el8.n 4/6 2023-09-30T09:02:14.515 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================================= 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================================= 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 7.4 M 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-30T09:02:14.517 INFO:teuthology.orchestra.run.smithi167.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-30T09:02:14.518 INFO:teuthology.orchestra.run.smithi167.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-30T09:02:14.518 INFO:teuthology.orchestra.run.smithi167.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-30T09:02:14.518 INFO:teuthology.orchestra.run.smithi167.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-30T09:02:14.518 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:14.518 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:02:14.518 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================================= 2023-09-30T09:02:14.518 INFO:teuthology.orchestra.run.smithi167.stdout:Install 8 Packages 2023-09-30T09:02:14.518 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:14.519 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 35 M 2023-09-30T09:02:14.519 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 210 M 2023-09-30T09:02:14.519 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:02:14.697 INFO:teuthology.orchestra.run.smithi167.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 3.5 MB/s | 644 kB 00:00 2023-09-30T09:02:14.722 INFO:teuthology.orchestra.run.smithi167.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 845 kB/s | 171 kB 00:00 2023-09-30T09:02:14.829 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1161.g191c1869.el8. 5/6 2023-09-30T09:02:14.948 INFO:teuthology.orchestra.run.smithi167.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 17 MB/s | 3.7 MB 00:00 2023-09-30T09:02:15.012 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1161.g191c1869.el8.noarc 6/6 2023-09-30T09:02:15.148 INFO:teuthology.orchestra.run.smithi167.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 30 MB/s | 14 MB 00:00 2023-09-30T09:02:15.173 INFO:teuthology.orchestra.run.smithi167.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 997 kB/s | 225 kB 00:00 2023-09-30T09:02:15.261 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1161.g191c1869.el8.noarc 6/6 2023-09-30T09:02:15.261 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1161.g191c1869.el8. 1/6 2023-09-30T09:02:15.261 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1161.g191c1869.el8.noarc 2/6 2023-09-30T09:02:15.261 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1161.g191c1869.el8.n 3/6 2023-09-30T09:02:15.261 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-30T09:02:15.261 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-30T09:02:15.357 INFO:teuthology.orchestra.run.smithi167.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 23 MB/s | 4.9 MB 00:00 2023-09-30T09:02:15.440 INFO:teuthology.orchestra.run.smithi167.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.7 MB 00:00 2023-09-30T09:02:16.335 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-30T09:02:16.335 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:13 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:02:16.824 INFO:teuthology.orchestra.run.smithi167.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-116 3.2 MB/s | 7.4 MB 00:02 2023-09-30T09:02:16.825 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:02:16.826 INFO:teuthology.orchestra.run.smithi167.stdout:Total 15 MB/s | 35 MB 00:02 2023-09-30T09:02:17.070 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:02:17.089 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:02:17.089 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:02:17.383 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:02:17.387 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:02:17.962 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:02:18.115 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-30T09:02:18.243 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-30T09:02:18.365 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-30T09:02:18.967 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-30T09:02:19.107 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-30T09:02:19.692 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-30T09:02:19.923 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-30T09:02:20.602 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-30T09:02:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-30T09:02:21.079 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:02:21.115 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-30T09:02:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:21.276 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:02:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-grafana-dashboards-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-dashboard-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-prometheus-alerts-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-30T09:02:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-30T09:02:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-30T09:02:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:21.277 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:02:21.401 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-30T09:02:21.769 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:02:22.869 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-30T09:02:23.341 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:20 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:02:24.106 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-30T09:02:24.264 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1161.g191c186 8/8 2023-09-30T09:02:24.508 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1161.g191c186 8/8 2023-09-30T09:02:24.508 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1161.g191c186 1/8 2023-09-30T09:02:24.509 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-30T09:02:24.509 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-30T09:02:24.509 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-30T09:02:24.509 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-30T09:02:24.509 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-30T09:02:24.509 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-30T09:02:24.878 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:02:24.879 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================================= 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================================= 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 7.4 M 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-30T09:02:24.880 INFO:teuthology.orchestra.run.smithi019.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-30T09:02:24.881 INFO:teuthology.orchestra.run.smithi019.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-30T09:02:24.881 INFO:teuthology.orchestra.run.smithi019.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-30T09:02:24.881 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:24.881 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:02:24.881 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================================= 2023-09-30T09:02:24.881 INFO:teuthology.orchestra.run.smithi019.stdout:Install 8 Packages 2023-09-30T09:02:24.881 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:24.882 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 35 M 2023-09-30T09:02:24.882 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 210 M 2023-09-30T09:02:24.882 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:02:25.141 INFO:teuthology.orchestra.run.smithi019.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.4 MB/s | 644 kB 00:00 2023-09-30T09:02:25.158 INFO:teuthology.orchestra.run.smithi019.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 621 kB/s | 171 kB 00:00 2023-09-30T09:02:25.400 INFO:teuthology.orchestra.run.smithi019.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 15 MB/s | 3.7 MB 00:00 2023-09-30T09:02:25.590 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-30T09:02:25.591 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:02:24 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:02:25.634 INFO:teuthology.orchestra.run.smithi019.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 965 kB/s | 225 kB 00:00 2023-09-30T09:02:25.834 INFO:teuthology.orchestra.run.smithi019.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 20 MB/s | 14 MB 00:00 2023-09-30T09:02:25.951 INFO:teuthology.orchestra.run.smithi019.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 15 MB/s | 4.9 MB 00:00 2023-09-30T09:02:26.218 INFO:teuthology.orchestra.run.smithi019.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 12 MB/s | 4.7 MB 00:00 2023-09-30T09:02:27.818 INFO:teuthology.orchestra.run.smithi019.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-116 2.5 MB/s | 7.4 MB 00:02 2023-09-30T09:02:27.819 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:02:27.819 INFO:teuthology.orchestra.run.smithi019.stdout:Total 12 MB/s | 35 MB 00:02 2023-09-30T09:02:28.070 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:02:28.090 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:02:28.090 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:02:28.401 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:02:28.405 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:02:28.957 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:02:29.094 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-30T09:02:29.240 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-30T09:02:29.385 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-30T09:02:30.057 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-30T09:02:30.252 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-30T09:02:30.783 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:02:30.887 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-30T09:02:30.968 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:30.968 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:02:30.969 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:30.969 INFO:teuthology.orchestra.run.smithi167.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-30T09:02:30.969 INFO:teuthology.orchestra.run.smithi167.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-30T09:02:30.969 INFO:teuthology.orchestra.run.smithi167.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-30T09:02:30.969 INFO:teuthology.orchestra.run.smithi167.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-30T09:02:30.969 INFO:teuthology.orchestra.run.smithi167.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-30T09:02:30.969 INFO:teuthology.orchestra.run.smithi167.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-30T09:02:30.969 INFO:teuthology.orchestra.run.smithi167.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-30T09:02:30.970 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:30.970 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:02:31.077 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-30T09:02:31.121 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr-rook 2023-09-30T09:02:31.471 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:02:31.810 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-30T09:02:32.135 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-30T09:02:32.319 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-30T09:02:33.096 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:02:32 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:02:34.202 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-30T09:02:34.475 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:02:34.476 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================== 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================== 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-rook noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 47 k 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-30T09:02:34.477 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:02:34.478 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================================== 2023-09-30T09:02:34.479 INFO:teuthology.orchestra.run.smithi167.stdout:Install 13 Packages 2023-09-30T09:02:34.479 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:34.479 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 1.9 M 2023-09-30T09:02:34.479 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 25 M 2023-09-30T09:02:34.479 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:02:34.671 INFO:teuthology.orchestra.run.smithi167.stdout:(1/13): ceph-mgr-rook-17.2.6-1161.g191c1869.el8 297 kB/s | 47 kB 00:00 2023-09-30T09:02:34.855 INFO:teuthology.orchestra.run.smithi167.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 47 kB/s | 16 kB 00:00 2023-09-30T09:02:34.888 INFO:teuthology.orchestra.run.smithi167.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 88 kB/s | 33 kB 00:00 2023-09-30T09:02:35.114 INFO:teuthology.orchestra.run.smithi167.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 285 kB/s | 64 kB 00:00 2023-09-30T09:02:35.206 INFO:teuthology.orchestra.run.smithi167.stdout:(5/13): python3-google-auth-1.1.1-10.el8.noarch 152 kB/s | 81 kB 00:00 2023-09-30T09:02:35.264 INFO:teuthology.orchestra.run.smithi167.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 408 kB/s | 61 kB 00:00 2023-09-30T09:02:35.365 INFO:teuthology.orchestra.run.smithi167.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 269 kB/s | 43 kB 00:00 2023-09-30T09:02:35.381 INFO:teuthology.orchestra.run.smithi167.stdout:(8/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.3 MB/s | 155 kB 00:00 2023-09-30T09:02:35.465 INFO:teuthology.orchestra.run.smithi167.stdout:(9/13): python3-pyasn1-modules-0.3.7-6.el8.noar 1.1 MB/s | 110 kB 00:00 2023-09-30T09:02:35.480 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-30T09:02:35.490 INFO:teuthology.orchestra.run.smithi167.stdout:(10/13): python3-jsonpatch-1.21-2.el8.noarch.rp 244 kB/s | 27 kB 00:00 2023-09-30T09:02:35.524 INFO:teuthology.orchestra.run.smithi167.stdout:(11/13): python3-kubernetes-11.0.0-6.el8.noarch 1.7 MB/s | 1.2 MB 00:00 2023-09-30T09:02:35.566 INFO:teuthology.orchestra.run.smithi167.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.2 MB/s | 126 kB 00:00 2023-09-30T09:02:35.633 INFO:teuthology.orchestra.run.smithi167.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 141 kB/s | 20 kB 00:00 2023-09-30T09:02:35.635 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:02:35.635 INFO:teuthology.orchestra.run.smithi167.stdout:Total 1.7 MB/s | 1.9 MB 00:01 2023-09-30T09:02:35.660 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1161.g191c186 8/8 2023-09-30T09:02:35.696 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:02:35.711 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:02:35.711 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:02:35.782 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:02:35.786 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:02:35.927 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1161.g191c186 8/8 2023-09-30T09:02:35.927 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1161.g191c186 1/8 2023-09-30T09:02:35.927 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-30T09:02:35.927 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-30T09:02:35.927 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-30T09:02:35.927 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-30T09:02:35.927 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-30T09:02:35.927 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-30T09:02:36.240 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:02:36.437 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-30T09:02:36.649 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-30T09:02:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-30T09:02:36.969 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-30T09:02:37.030 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-30T09:02:37.030 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:34 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:02:37.190 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-30T09:02:37.584 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-30T09:02:37.782 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-30T09:02:38.011 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-30T09:02:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-30T09:02:38.342 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-30T09:02:39.448 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-30T09:02:39.704 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-30T09:02:39.829 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-rook-2:17.2.6-1161.g191c1869.el8.noarch 13/13 2023-09-30T09:02:40.073 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1161.g191c1869.el8.noarch 13/13 2023-09-30T09:02:40.073 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1161.g191c1869.el8.noarch 1/13 2023-09-30T09:02:40.073 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-30T09:02:40.073 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-30T09:02:40.073 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-30T09:02:40.073 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-30T09:02:40.073 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-30T09:02:40.074 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-30T09:02:40.074 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-30T09:02:40.074 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-30T09:02:40.074 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-30T09:02:40.074 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-30T09:02:40.074 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-30T09:02:41.213 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-30T09:02:41.213 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:02:40 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:02:42.960 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:02:43.168 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:43.168 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:02:43.168 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:43.168 INFO:teuthology.orchestra.run.smithi019.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-30T09:02:43.169 INFO:teuthology.orchestra.run.smithi019.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-30T09:02:43.169 INFO:teuthology.orchestra.run.smithi019.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-30T09:02:43.169 INFO:teuthology.orchestra.run.smithi019.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-30T09:02:43.169 INFO:teuthology.orchestra.run.smithi019.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-30T09:02:43.169 INFO:teuthology.orchestra.run.smithi019.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-30T09:02:43.169 INFO:teuthology.orchestra.run.smithi019.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-30T09:02:43.169 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:43.169 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:02:43.353 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-mgr-rook 2023-09-30T09:02:43.722 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:02:45.355 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:42 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:02:46.862 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:02:46.871 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:02:46.873 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================== 2023-09-30T09:02:46.873 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:02:46.873 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================== 2023-09-30T09:02:46.873 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:02:46.873 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-rook noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 47 k 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-30T09:02:46.874 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-30T09:02:46.875 INFO:teuthology.orchestra.run.smithi019.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-30T09:02:46.875 INFO:teuthology.orchestra.run.smithi019.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-30T09:02:46.875 INFO:teuthology.orchestra.run.smithi019.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-30T09:02:46.875 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:46.875 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:02:46.875 INFO:teuthology.orchestra.run.smithi019.stdout:======================================================================================================== 2023-09-30T09:02:46.875 INFO:teuthology.orchestra.run.smithi019.stdout:Install 13 Packages 2023-09-30T09:02:46.875 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:46.876 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 1.9 M 2023-09-30T09:02:46.876 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 25 M 2023-09-30T09:02:46.876 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:02:47.047 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:47.047 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:02:47.047 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-rook-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:47.047 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-30T09:02:47.048 INFO:teuthology.orchestra.run.smithi167.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-30T09:02:47.049 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:47.049 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:02:47.074 INFO:teuthology.orchestra.run.smithi019.stdout:(1/13): ceph-mgr-rook-17.2.6-1161.g191c1869.el8 281 kB/s | 47 kB 00:00 2023-09-30T09:02:47.174 INFO:teuthology.orchestra.run.smithi019.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 61 kB/s | 16 kB 00:00 2023-09-30T09:02:47.189 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr-cephadm 2023-09-30T09:02:47.199 INFO:teuthology.orchestra.run.smithi019.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 112 kB/s | 33 kB 00:00 2023-09-30T09:02:47.367 INFO:teuthology.orchestra.run.smithi019.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 386 kB/s | 64 kB 00:00 2023-09-30T09:02:47.417 INFO:teuthology.orchestra.run.smithi019.stdout:(5/13): python3-google-auth-1.1.1-10.el8.noarch 237 kB/s | 81 kB 00:00 2023-09-30T09:02:47.476 INFO:teuthology.orchestra.run.smithi019.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 564 kB/s | 61 kB 00:00 2023-09-30T09:02:47.543 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:02:47.584 INFO:teuthology.orchestra.run.smithi019.stdout:(7/13): python3-kubernetes-11.0.0-6.el8.noarch. 2.8 MB/s | 1.2 MB 00:00 2023-09-30T09:02:47.610 INFO:teuthology.orchestra.run.smithi019.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 222 kB/s | 43 kB 00:00 2023-09-30T09:02:47.651 INFO:teuthology.orchestra.run.smithi019.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 883 kB/s | 155 kB 00:00 2023-09-30T09:02:47.727 INFO:teuthology.orchestra.run.smithi019.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 774 kB/s | 110 kB 00:00 2023-09-30T09:02:47.752 INFO:teuthology.orchestra.run.smithi019.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 198 kB/s | 27 kB 00:00 2023-09-30T09:02:47.777 INFO:teuthology.orchestra.run.smithi019.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.0 MB/s | 126 kB 00:00 2023-09-30T09:02:47.844 INFO:teuthology.orchestra.run.smithi019.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 171 kB/s | 20 kB 00:00 2023-09-30T09:02:47.847 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:02:47.847 INFO:teuthology.orchestra.run.smithi019.stdout:Total 2.0 MB/s | 1.9 MB 00:00 2023-09-30T09:02:47.917 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:02:47.934 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:02:47.934 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:02:48.013 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:02:48.017 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:02:48.609 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:02:48.835 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-30T09:02:49.061 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:02:48 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:02:49.072 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-30T09:02:49.207 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-30T09:02:49.340 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-30T09:02:49.583 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-30T09:02:49.781 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-30T09:02:49.979 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-30T09:02:50.199 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-30T09:02:50.366 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-30T09:02:50.431 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:02:50.432 INFO:teuthology.orchestra.run.smithi167.stdout:=============================================================================================== 2023-09-30T09:02:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:02:50.432 INFO:teuthology.orchestra.run.smithi167.stdout:=============================================================================================== 2023-09-30T09:02:50.432 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:02:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 121 k 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout:=============================================================================================== 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout:Install 5 Packages 2023-09-30T09:02:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:02:50.434 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 6.0 M 2023-09-30T09:02:50.434 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 26 M 2023-09-30T09:02:50.434 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:02:50.568 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-30T09:02:50.857 INFO:teuthology.orchestra.run.smithi167.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 156 kB/s | 60 kB 00:00 2023-09-30T09:02:50.882 INFO:teuthology.orchestra.run.smithi167.stdout:(2/5): ceph-mgr-cephadm-17.2.6-1161.g191c1869.e 295 kB/s | 121 kB 00:00 2023-09-30T09:02:51.074 INFO:teuthology.orchestra.run.smithi167.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.4 MB/s | 538 kB 00:00 2023-09-30T09:02:51.170 INFO:teuthology.orchestra.run.smithi167.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 17 MB/s | 4.8 MB 00:00 2023-09-30T09:02:51.199 INFO:teuthology.orchestra.run.smithi167.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 654 kB/s | 476 kB 00:00 2023-09-30T09:02:51.200 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:02:51.200 INFO:teuthology.orchestra.run.smithi167.stdout:Total 7.8 MB/s | 6.0 MB 00:00 2023-09-30T09:02:51.254 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:02:51.265 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:02:51.265 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:02:51.343 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:02:51.347 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:02:51.758 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-30T09:02:52.043 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-30T09:02:52.191 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-mgr-rook-2:17.2.6-1161.g191c1869.el8.noarch 13/13 2023-09-30T09:02:52.288 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:02:52.443 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1161.g191c1869.el8.noarch 13/13 2023-09-30T09:02:52.443 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1161.g191c1869.el8.noarch 1/13 2023-09-30T09:02:52.443 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-30T09:02:52.443 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-30T09:02:52.444 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-30T09:02:52.622 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-30T09:02:52.845 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-30T09:02:53.166 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-30T09:02:53.562 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-30T09:02:53.562 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:50 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:02:54.609 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-30T09:02:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1161.g191c1869.el8.noarch 5/5 2023-09-30T09:02:55.624 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1161.g191c1869.el8.noarch 5/5 2023-09-30T09:02:55.624 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/5 2023-09-30T09:02:55.624 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-30T09:02:55.624 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-30T09:02:55.624 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-30T09:02:56.665 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-30T09:02:56.665 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:02:55 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:02:58.861 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:02:59.061 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:59.061 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:02:59.061 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-rook-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:02:59.061 INFO:teuthology.orchestra.run.smithi019.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-30T09:02:59.061 INFO:teuthology.orchestra.run.smithi019.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-30T09:02:59.061 INFO:teuthology.orchestra.run.smithi019.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:02:59.062 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:02:59.185 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-mgr-cephadm 2023-09-30T09:02:59.561 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:00.660 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:03:00.843 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:00.843 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:03:00.843 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-cephadm-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:03:00.844 INFO:teuthology.orchestra.run.smithi167.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-30T09:03:00.844 INFO:teuthology.orchestra.run.smithi167.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-30T09:03:00.844 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-30T09:03:00.844 INFO:teuthology.orchestra.run.smithi167.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-30T09:03:00.844 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:00.844 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:00.988 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:02:57 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:01.032 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-fuse 2023-09-30T09:03:01.387 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:02.517 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:03:02.518 INFO:teuthology.orchestra.run.smithi019.stdout:=============================================================================================== 2023-09-30T09:03:02.518 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:03:02.518 INFO:teuthology.orchestra.run.smithi019.stdout:=============================================================================================== 2023-09-30T09:03:02.518 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1161.g191c1869.el8 ceph-noarch 121 k 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout:=============================================================================================== 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout:Install 5 Packages 2023-09-30T09:03:02.519 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:02.520 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 6.0 M 2023-09-30T09:03:02.520 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 26 M 2023-09-30T09:03:02.520 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:03:02.866 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:01 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:03.160 INFO:teuthology.orchestra.run.smithi019.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1161.g191c1869.e 655 kB/s | 121 kB 00:00 2023-09-30T09:03:03.335 INFO:teuthology.orchestra.run.smithi019.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.0 MB/s | 538 kB 00:00 2023-09-30T09:03:03.361 INFO:teuthology.orchestra.run.smithi019.stdout:(3/5): python3-natsort-7.1.1-2.el8.noarch.rpm 156 kB/s | 60 kB 00:00 2023-09-30T09:03:03.652 INFO:teuthology.orchestra.run.smithi019.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 15 MB/s | 4.8 MB 00:00 2023-09-30T09:03:04.021 INFO:teuthology.orchestra.run.smithi019.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 455 kB/s | 476 kB 00:01 2023-09-30T09:03:04.022 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:03:04.022 INFO:teuthology.orchestra.run.smithi019.stdout:Total 4.0 MB/s | 6.0 MB 00:01 2023-09-30T09:03:04.081 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:03:04.094 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:03:04.094 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:03:04.197 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:03:04.201 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:03:04.231 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:04.232 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:04.232 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-09-30T09:03:04.232 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:04.232 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-fuse 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout: x86_64 2:17.2.6-1161.g191c1869.el8 ceph 874 k 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout:Install 2 Packages 2023-09-30T09:03:04.233 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:04.234 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 957 k 2023-09-30T09:03:04.234 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 3.0 M 2023-09-30T09:03:04.234 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:03:04.445 INFO:teuthology.orchestra.run.smithi167.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 393 kB/s | 83 kB 00:00 2023-09-30T09:03:04.536 INFO:teuthology.orchestra.run.smithi167.stdout:(2/2): ceph-fuse-17.2.6-1161.g191c1869.el8.x86_ 2.8 MB/s | 874 kB 00:00 2023-09-30T09:03:04.537 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:03:04.537 INFO:teuthology.orchestra.run.smithi167.stdout:Total 3.1 MB/s | 957 kB 00:00 2023-09-30T09:03:04.549 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:03:04.569 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:03:04.569 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:03:04.643 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:03:04.647 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:03:04.805 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:03:05.101 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-30T09:03:05.249 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 2/2 2023-09-30T09:03:05.582 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 2/2 2023-09-30T09:03:05.582 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 1/2 2023-09-30T09:03:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:03:06.670 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-30T09:03:06.670 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:05 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:06.684 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-30T09:03:06.967 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-30T09:03:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-30T09:03:07.584 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-30T09:03:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1161.g191c1869.el8.noarch 5/5 2023-09-30T09:03:08.019 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1161.g191c1869.el8.noarch 5/5 2023-09-30T09:03:08.019 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1161.g191c1869.el8.noarch 1/5 2023-09-30T09:03:08.019 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-30T09:03:08.019 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-30T09:03:08.020 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-30T09:03:09.110 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-30T09:03:09.111 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:06 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:12.796 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:03:12.976 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:12.976 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:03:12.976 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-30T09:03:12.976 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:12.976 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:13.098 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install librados-devel 2023-09-30T09:03:13.455 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:15.046 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:14 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:16.426 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:16.427 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repo Size 2023-09-30T09:03:16.427 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:16.427 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:03:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: librados-devel x86_64 2:17.2.6-1161.g191c1869.el8 ceph 130 k 2023-09-30T09:03:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:16.428 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:03:16.428 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:16.428 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2023-09-30T09:03:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:16.429 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 130 k 2023-09-30T09:03:16.429 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 464 k 2023-09-30T09:03:16.429 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:03:16.818 INFO:teuthology.orchestra.run.smithi167.stdout:librados-devel-17.2.6-1161.g191c1869.el8.x86_64 333 kB/s | 130 kB 00:00 2023-09-30T09:03:16.818 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:03:16.819 INFO:teuthology.orchestra.run.smithi167.stdout:Total 332 kB/s | 130 kB 00:00 2023-09-30T09:03:16.819 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:03:16.824 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:03:16.825 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:03:16.878 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:03:16.882 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:03:16.994 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:03:17.188 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : librados-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:17.409 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librados-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:17.620 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:03:17.820 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:17.821 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:03:17.821 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-mgr-cephadm-2:17.2.6-1161.g191c1869.el8.noarch 2023-09-30T09:03:17.821 INFO:teuthology.orchestra.run.smithi019.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-30T09:03:17.821 INFO:teuthology.orchestra.run.smithi019.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-30T09:03:17.821 INFO:teuthology.orchestra.run.smithi019.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-30T09:03:17.821 INFO:teuthology.orchestra.run.smithi019.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-30T09:03:17.821 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:17.821 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:03:17.985 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install ceph-fuse 2023-09-30T09:03:18.364 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:18.472 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librados-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:18.472 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:17 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:20.038 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:17 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:21.541 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:03:21.542 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:21.542 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2023-09-30T09:03:21.542 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:21.542 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:03:21.542 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-fuse 2023-09-30T09:03:21.543 INFO:teuthology.orchestra.run.smithi019.stdout: x86_64 2:17.2.6-1161.g191c1869.el8 ceph 874 k 2023-09-30T09:03:21.543 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2023-09-30T09:03:21.543 INFO:teuthology.orchestra.run.smithi019.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-30T09:03:21.543 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:21.543 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:03:21.543 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:21.543 INFO:teuthology.orchestra.run.smithi019.stdout:Install 2 Packages 2023-09-30T09:03:21.543 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:21.544 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 957 k 2023-09-30T09:03:21.544 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 3.0 M 2023-09-30T09:03:21.544 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:03:21.615 INFO:teuthology.orchestra.run.smithi019.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2023-09-30T09:03:21.840 INFO:teuthology.orchestra.run.smithi019.stdout:(2/2): ceph-fuse-17.2.6-1161.g191c1869.el8.x86_ 2.9 MB/s | 874 kB 00:00 2023-09-30T09:03:21.841 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:03:21.841 INFO:teuthology.orchestra.run.smithi019.stdout:Total 3.1 MB/s | 957 kB 00:00 2023-09-30T09:03:21.852 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:03:21.874 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:03:21.875 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:03:21.954 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:03:21.958 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:03:21.989 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:03:22.120 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:03:22.172 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:22.172 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:03:22.172 INFO:teuthology.orchestra.run.smithi167.stdout: librados-devel-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:03:22.172 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:22.173 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:22.308 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install libcephfs2 2023-09-30T09:03:22.451 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-30T09:03:22.611 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : ceph-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 2/2 2023-09-30T09:03:22.666 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 2/2 2023-09-30T09:03:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : ceph-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 1/2 2023-09-30T09:03:24.083 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-30T09:03:24.084 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:21 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:24.108 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:23 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:25.369 INFO:teuthology.orchestra.run.smithi167.stdout:Package libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:25.478 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:25.478 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:03:25.478 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:25.597 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install libcephfs-devel 2023-09-30T09:03:25.950 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:27.402 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:26 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:28.804 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:28.804 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:28.804 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repo Size 2023-09-30T09:03:28.804 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:28.805 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:03:28.805 INFO:teuthology.orchestra.run.smithi167.stdout: libcephfs-devel x86_64 2:17.2.6-1161.g191c1869.el8 ceph 25 k 2023-09-30T09:03:28.805 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:28.805 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:03:28.805 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:28.805 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2023-09-30T09:03:28.805 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:28.806 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 25 k 2023-09-30T09:03:28.806 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 109 k 2023-09-30T09:03:28.806 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:03:28.947 INFO:teuthology.orchestra.run.smithi167.stdout:libcephfs-devel-17.2.6-1161.g191c1869.el8.x86_6 175 kB/s | 25 kB 00:00 2023-09-30T09:03:28.948 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:03:28.948 INFO:teuthology.orchestra.run.smithi167.stdout:Total 174 kB/s | 25 kB 00:00 2023-09-30T09:03:28.948 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:03:28.952 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:03:28.952 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:03:28.953 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:03:28.957 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:03:29.006 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:03:29.153 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libcephfs-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:29.246 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:03:29.369 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:29.447 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:29.448 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:03:29.448 INFO:teuthology.orchestra.run.smithi019.stdout: ceph-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-30T09:03:29.448 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:29.448 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:03:29.570 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install librados-devel 2023-09-30T09:03:29.953 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:30.357 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libcephfs-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:30.357 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:29 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:30.875 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:27 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:32.334 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:03:32.334 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:32.334 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repo Size 2023-09-30T09:03:32.334 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:32.335 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:03:32.335 INFO:teuthology.orchestra.run.smithi019.stdout: librados-devel x86_64 2:17.2.6-1161.g191c1869.el8 ceph 130 k 2023-09-30T09:03:32.335 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:32.335 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:03:32.335 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:32.335 INFO:teuthology.orchestra.run.smithi019.stdout:Install 1 Package 2023-09-30T09:03:32.335 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:32.335 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 130 k 2023-09-30T09:03:32.336 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 464 k 2023-09-30T09:03:32.336 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:03:32.521 INFO:teuthology.orchestra.run.smithi019.stdout:librados-devel-17.2.6-1161.g191c1869.el8.x86_64 700 kB/s | 130 kB 00:00 2023-09-30T09:03:32.522 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:03:32.522 INFO:teuthology.orchestra.run.smithi019.stdout:Total 696 kB/s | 130 kB 00:00 2023-09-30T09:03:32.522 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:03:32.528 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:03:32.528 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:03:32.588 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:03:32.592 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:03:32.740 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:03:32.900 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : librados-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:33.144 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: librados-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:34.226 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : librados-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:34.226 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:31 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:34.643 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:03:34.830 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:34.830 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:03:34.830 INFO:teuthology.orchestra.run.smithi167.stdout: libcephfs-devel-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:03:34.830 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:34.830 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:34.963 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install librados2 2023-09-30T09:03:35.317 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:36.577 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:35 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:37.836 INFO:teuthology.orchestra.run.smithi167.stdout:Package librados2-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:37.944 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:37.945 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:03:37.945 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:38.058 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install librbd1 2023-09-30T09:03:38.414 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:39.385 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:03:39.476 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:38 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:39.581 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:39.581 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:03:39.582 INFO:teuthology.orchestra.run.smithi019.stdout: librados-devel-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:03:39.582 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:39.582 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:03:39.706 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install libcephfs2 2023-09-30T09:03:40.083 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:40.727 INFO:teuthology.orchestra.run.smithi167.stdout:Package librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:40.835 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:40.836 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:03:40.836 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:40.957 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install python3-rados 2023-09-30T09:03:41.315 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:41.616 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:38 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:43.009 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:42 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:43.011 INFO:teuthology.orchestra.run.smithi019.stdout:Package libcephfs2-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:43.132 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:03:43.133 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:03:43.134 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:03:43.238 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install libcephfs-devel 2023-09-30T09:03:43.614 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:44.256 INFO:teuthology.orchestra.run.smithi167.stdout:Package python3-rados-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:44.364 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:44.365 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:03:44.365 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:44.485 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install python3-rgw 2023-09-30T09:03:44.837 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:45.012 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:42 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:46.427 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:45 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:46.569 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repo Size 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout: libcephfs-devel x86_64 2:17.2.6-1161.g191c1869.el8 ceph 25 k 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:03:46.570 INFO:teuthology.orchestra.run.smithi019.stdout:Install 1 Package 2023-09-30T09:03:46.571 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:46.571 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 25 k 2023-09-30T09:03:46.571 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 109 k 2023-09-30T09:03:46.571 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:03:46.910 INFO:teuthology.orchestra.run.smithi019.stdout:libcephfs-devel-17.2.6-1161.g191c1869.el8.x86_6 73 kB/s | 25 kB 00:00 2023-09-30T09:03:46.910 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:03:46.910 INFO:teuthology.orchestra.run.smithi019.stdout:Total 73 kB/s | 25 kB 00:00 2023-09-30T09:03:46.911 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:03:46.915 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:03:46.915 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:03:46.917 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:03:46.921 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:03:46.971 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:03:47.140 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : libcephfs-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:47.351 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:47.743 INFO:teuthology.orchestra.run.smithi167.stdout:Package python3-rgw-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:47.853 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:47.854 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:03:47.854 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:47.961 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install python3-cephfs 2023-09-30T09:03:48.317 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:48.384 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : libcephfs-devel-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:48.384 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:45 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:49.824 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:48 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:51.168 INFO:teuthology.orchestra.run.smithi167.stdout:Package python3-cephfs-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:51.277 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:51.278 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:03:51.278 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:51.390 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install python3-rbd 2023-09-30T09:03:51.744 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:51.985 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:03:52.182 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:52.182 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:03:52.182 INFO:teuthology.orchestra.run.smithi019.stdout: libcephfs-devel-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:03:52.183 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:03:52.183 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:03:52.306 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install librados2 2023-09-30T09:03:52.607 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:51 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:52.685 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:53.555 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:50 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:53.922 INFO:teuthology.orchestra.run.smithi167.stdout:Package python3-rbd-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:54.029 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:54.031 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2023-09-30T09:03:54.031 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:03:54.150 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install rbd-fuse 2023-09-30T09:03:54.501 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:54.957 INFO:teuthology.orchestra.run.smithi019.stdout:Package librados2-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:55.075 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:03:55.076 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:03:55.076 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:03:55.182 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install librbd1 2023-09-30T09:03:55.314 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:54 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:55.560 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:56.456 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:53 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:03:56.716 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:03:56.716 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:56.716 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2023-09-30T09:03:56.717 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:56.717 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:03:56.717 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-fuse x86_64 2:17.2.6-1161.g191c1869.el8 ceph 90 k 2023-09-30T09:03:56.717 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:56.717 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:03:56.717 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:03:56.717 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2023-09-30T09:03:56.717 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:03:56.718 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 90 k 2023-09-30T09:03:56.718 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 264 k 2023-09-30T09:03:56.718 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:03:56.891 INFO:teuthology.orchestra.run.smithi167.stdout:rbd-fuse-17.2.6-1161.g191c1869.el8.x86_64.rpm 522 kB/s | 90 kB 00:00 2023-09-30T09:03:56.891 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:03:56.892 INFO:teuthology.orchestra.run.smithi167.stdout:Total 518 kB/s | 90 kB 00:00 2023-09-30T09:03:56.892 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:03:56.901 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:03:56.901 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:03:56.954 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:03:56.958 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:03:57.099 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:03:57.240 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : rbd-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:57.470 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:57.857 INFO:teuthology.orchestra.run.smithi019.stdout:Package librbd1-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:03:57.975 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:03:57.976 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:03:57.976 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:03:58.216 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install python3-rados 2023-09-30T09:03:58.533 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : rbd-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:03:58.534 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:03:57 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:03:58.592 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:03:59.516 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:56 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:00.890 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:04:00.906 INFO:teuthology.orchestra.run.smithi019.stdout:Package python3-rados-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:04:01.025 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:04:01.026 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:04:01.026 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:04:01.074 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:01.074 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:04:01.074 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:04:01.074 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:01.074 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:04:01.134 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install python3-rgw 2023-09-30T09:04:01.208 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install rbd-mirror 2023-09-30T09:04:01.510 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:04:01.563 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:04:02.550 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:04:01 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:04:02.569 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:03:59 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:03.971 INFO:teuthology.orchestra.run.smithi019.stdout:Package python3-rgw-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:04:03.983 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:04:03.984 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:04:03.984 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repo Size 2023-09-30T09:04:03.984 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:04:03.984 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:04:03.984 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-mirror x86_64 2:17.2.6-1161.g191c1869.el8 ceph 3.5 M 2023-09-30T09:04:03.984 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:03.984 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:04:03.984 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:04:03.985 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2023-09-30T09:04:03.985 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:03.985 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 3.5 M 2023-09-30T09:04:03.985 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 16 M 2023-09-30T09:04:03.986 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:04:04.092 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:04:04.093 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:04:04.093 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:04:04.200 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install python3-cephfs 2023-09-30T09:04:04.469 INFO:teuthology.orchestra.run.smithi167.stdout:rbd-mirror-17.2.6-1161.g191c1869.el8.x86_64.rpm 7.1 MB/s | 3.5 MB 00:00 2023-09-30T09:04:04.469 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:04:04.470 INFO:teuthology.orchestra.run.smithi167.stdout:Total 7.1 MB/s | 3.5 MB 00:00 2023-09-30T09:04:04.470 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:04:04.482 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:04:04.482 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:04:04.547 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:04:04.551 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:04:04.576 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:04:04.990 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:04:05.193 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : rbd-mirror-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:05.467 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:04:02 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:05.556 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:06.638 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : rbd-mirror-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:06.638 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:04:05 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:04:06.878 INFO:teuthology.orchestra.run.smithi019.stdout:Package python3-cephfs-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:04:06.998 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:04:06.999 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:04:06.999 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:04:07.100 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install python3-rbd 2023-09-30T09:04:07.476 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:04:09.069 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:04:06 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:10.058 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:04:10.239 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:10.240 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:04:10.240 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-mirror-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:04:10.240 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:10.240 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:04:10.373 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install rbd-nbd 2023-09-30T09:04:10.480 INFO:teuthology.orchestra.run.smithi019.stdout:Package python3-rbd-2:17.2.6-1161.g191c1869.el8.x86_64 is already installed. 2023-09-30T09:04:10.597 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:04:10.598 INFO:teuthology.orchestra.run.smithi019.stdout:Nothing to do. 2023-09-30T09:04:10.598 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:04:10.711 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install rbd-fuse 2023-09-30T09:04:10.731 INFO:teuthology.orchestra.run.smithi167.stdout:Updating Subscription Management repositories. 2023-09-30T09:04:11.086 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:04:11.665 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:04:10 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:04:11.960 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:04:08 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:13.046 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-09-30T09:04:13.046 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:04:13.046 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2023-09-30T09:04:13.047 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:04:13.047 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-09-30T09:04:13.047 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-nbd x86_64 2:17.2.6-1161.g191c1869.el8 ceph 173 k 2023-09-30T09:04:13.047 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:13.047 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-09-30T09:04:13.047 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-09-30T09:04:13.047 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2023-09-30T09:04:13.047 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:13.048 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 173 k 2023-09-30T09:04:13.048 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 552 k 2023-09-30T09:04:13.048 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-09-30T09:04:13.231 INFO:teuthology.orchestra.run.smithi167.stdout:rbd-nbd-17.2.6-1161.g191c1869.el8.x86_64.rpm 943 kB/s | 173 kB 00:00 2023-09-30T09:04:13.232 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:04:13.232 INFO:teuthology.orchestra.run.smithi167.stdout:Total 936 kB/s | 173 kB 00:00 2023-09-30T09:04:13.232 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-09-30T09:04:13.243 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-09-30T09:04:13.243 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-09-30T09:04:13.297 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-09-30T09:04:13.301 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-09-30T09:04:13.415 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-09-30T09:04:13.469 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:04:13.469 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:13.469 INFO:teuthology.orchestra.run.smithi019.stdout: Package Architecture Version Repository Size 2023-09-30T09:04:13.470 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:13.470 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:04:13.470 INFO:teuthology.orchestra.run.smithi019.stdout: rbd-fuse x86_64 2:17.2.6-1161.g191c1869.el8 ceph 90 k 2023-09-30T09:04:13.470 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:13.470 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:04:13.470 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:13.470 INFO:teuthology.orchestra.run.smithi019.stdout:Install 1 Package 2023-09-30T09:04:13.470 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:13.471 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 90 k 2023-09-30T09:04:13.471 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 264 k 2023-09-30T09:04:13.471 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:04:13.554 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : rbd-nbd-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:13.770 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:13.798 INFO:teuthology.orchestra.run.smithi019.stdout:rbd-fuse-17.2.6-1161.g191c1869.el8.x86_64.rpm 276 kB/s | 90 kB 00:00 2023-09-30T09:04:13.798 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:04:13.799 INFO:teuthology.orchestra.run.smithi019.stdout:Total 275 kB/s | 90 kB 00:00 2023-09-30T09:04:13.799 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:04:13.810 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:04:13.810 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:04:13.869 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:04:13.873 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:04:13.987 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:04:14.137 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : rbd-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:14.377 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:14.865 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : rbd-nbd-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:14.865 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:04:13 ago on Sat 30 Sep 2023 09:00:01 AM UTC. 2023-09-30T09:04:15.438 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : rbd-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:15.438 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:04:12 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:17.403 INFO:teuthology.orchestra.run.smithi167.stdout:Installed products updated. 2023-09-30T09:04:17.586 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:17.586 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-09-30T09:04:17.586 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-nbd-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:04:17.586 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:04:17.586 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-09-30T09:04:17.730 DEBUG:teuthology.parallel:result is None 2023-09-30T09:04:18.152 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:04:18.351 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:18.351 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:04:18.351 INFO:teuthology.orchestra.run.smithi019.stdout: rbd-fuse-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:04:18.351 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:18.351 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:04:18.475 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install rbd-mirror 2023-09-30T09:04:18.853 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:04:19.754 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:04:16 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:21.280 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:04:21.280 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repo Size 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: rbd-mirror x86_64 2:17.2.6-1161.g191c1869.el8 ceph 3.5 M 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout:Install 1 Package 2023-09-30T09:04:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:21.282 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 3.5 M 2023-09-30T09:04:21.282 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 16 M 2023-09-30T09:04:21.282 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:04:21.777 INFO:teuthology.orchestra.run.smithi019.stdout:rbd-mirror-17.2.6-1161.g191c1869.el8.x86_64.rpm 7.0 MB/s | 3.5 MB 00:00 2023-09-30T09:04:21.778 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:04:21.778 INFO:teuthology.orchestra.run.smithi019.stdout:Total 7.0 MB/s | 3.5 MB 00:00 2023-09-30T09:04:21.778 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:04:21.791 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:04:21.791 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:04:21.864 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:04:21.868 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:04:22.330 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:04:22.479 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : rbd-mirror-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:22.855 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:23.903 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : rbd-mirror-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:23.903 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:04:20 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:26.361 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:04:26.561 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:26.561 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:04:26.561 INFO:teuthology.orchestra.run.smithi019.stdout: rbd-mirror-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:04:26.562 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:26.562 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:04:26.689 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum -y install rbd-nbd 2023-09-30T09:04:27.065 INFO:teuthology.orchestra.run.smithi019.stdout:Updating Subscription Management repositories. 2023-09-30T09:04:27.921 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:04:24 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:29.442 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2023-09-30T09:04:29.442 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:29.442 INFO:teuthology.orchestra.run.smithi019.stdout: Package Architecture Version Repository Size 2023-09-30T09:04:29.443 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:29.443 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2023-09-30T09:04:29.443 INFO:teuthology.orchestra.run.smithi019.stdout: rbd-nbd x86_64 2:17.2.6-1161.g191c1869.el8 ceph 173 k 2023-09-30T09:04:29.443 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:29.443 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2023-09-30T09:04:29.443 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2023-09-30T09:04:29.443 INFO:teuthology.orchestra.run.smithi019.stdout:Install 1 Package 2023-09-30T09:04:29.443 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:29.444 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 173 k 2023-09-30T09:04:29.444 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 552 k 2023-09-30T09:04:29.444 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2023-09-30T09:04:29.622 INFO:teuthology.orchestra.run.smithi019.stdout:rbd-nbd-17.2.6-1161.g191c1869.el8.x86_64.rpm 971 kB/s | 173 kB 00:00 2023-09-30T09:04:29.623 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:04:29.624 INFO:teuthology.orchestra.run.smithi019.stdout:Total 965 kB/s | 173 kB 00:00 2023-09-30T09:04:29.624 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2023-09-30T09:04:29.635 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2023-09-30T09:04:29.635 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2023-09-30T09:04:29.693 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2023-09-30T09:04:29.697 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2023-09-30T09:04:29.818 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2023-09-30T09:04:29.976 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : rbd-nbd-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:30.209 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:31.287 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : rbd-nbd-2:17.2.6-1161.g191c1869.el8.x86_64 1/1 2023-09-30T09:04:31.287 INFO:teuthology.orchestra.run.smithi019.stdout:Last metadata expiration check: 0:04:28 ago on Sat 30 Sep 2023 09:00:03 AM UTC. 2023-09-30T09:04:33.729 INFO:teuthology.orchestra.run.smithi019.stdout:Installed products updated. 2023-09-30T09:04:33.930 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:33.931 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2023-09-30T09:04:33.931 INFO:teuthology.orchestra.run.smithi019.stdout: rbd-nbd-2:17.2.6-1161.g191c1869.el8.x86_64 2023-09-30T09:04:33.931 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:04:33.931 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2023-09-30T09:04:34.056 DEBUG:teuthology.parallel:result is None 2023-09-30T09:04:34.057 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-30T09:04:34.057 INFO:teuthology.packaging:ref: None 2023-09-30T09:04:34.057 INFO:teuthology.packaging:tag: None 2023-09-30T09:04:34.057 INFO:teuthology.packaging:branch: quincy 2023-09-30T09:04:34.057 INFO:teuthology.packaging:sha1: 1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:04:34.057 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-30T09:04:34.399 DEBUG:teuthology.orchestra.run.smithi019:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-30T09:04:34.422 INFO:teuthology.orchestra.run.smithi019.stdout:17.2.6-1161.g191c1869.el8 2023-09-30T09:04:34.422 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1161.g191c1869.el8 2023-09-30T09:04:34.422 INFO:teuthology.task.install:The correct ceph version 17.2.6-1161.g191c1869 is installed. 2023-09-30T09:04:34.423 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-30T09:04:34.423 INFO:teuthology.packaging:ref: None 2023-09-30T09:04:34.423 INFO:teuthology.packaging:tag: None 2023-09-30T09:04:34.423 INFO:teuthology.packaging:branch: quincy 2023-09-30T09:04:34.423 INFO:teuthology.packaging:sha1: 1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:04:34.423 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-30T09:04:34.778 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-30T09:04:34.807 INFO:teuthology.orchestra.run.smithi167.stdout:17.2.6-1161.g191c1869.el8 2023-09-30T09:04:34.807 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1161.g191c1869.el8 2023-09-30T09:04:34.807 INFO:teuthology.task.install:The correct ceph version 17.2.6-1161.g191c1869 is installed. 2023-09-30T09:04:34.808 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-30T09:04:34.808 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:04:34.808 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-30T09:04:34.839 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:04:34.839 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-30T09:04:34.880 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-30T09:04:34.881 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:04:34.881 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/usr/bin/daemon-helper 2023-09-30T09:04:34.910 DEBUG:teuthology.orchestra.run.smithi019:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-30T09:04:34.979 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:04:34.979 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/usr/bin/daemon-helper 2023-09-30T09:04:35.008 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-30T09:04:35.076 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-30T09:04:35.076 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:04:35.077 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-30T09:04:35.105 DEBUG:teuthology.orchestra.run.smithi019:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-30T09:04:35.173 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:04:35.173 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-30T09:04:35.201 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-30T09:04:35.269 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-30T09:04:35.269 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:04:35.269 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/usr/bin/stdin-killer 2023-09-30T09:04:35.297 DEBUG:teuthology.orchestra.run.smithi019:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-30T09:04:35.364 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:04:35.365 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/usr/bin/stdin-killer 2023-09-30T09:04:35.391 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-30T09:04:35.457 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-30T09:04:35.533 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '1d10b71792f3be8887a7631e69851ac2df3585af'} 2023-09-30T09:04:35.533 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-30T09:04:35.534 INFO:tasks.cephadm:Cluster fsid is 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:04:35.534 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-30T09:04:35.534 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.19', 'mon.c': '[v2:172.21.15.19:3301,v1:172.21.15.19:6790]', 'mon.b': '172.21.15.167'} 2023-09-30T09:04:35.534 INFO:tasks.cephadm:First mon is mon.a on smithi019 2023-09-30T09:04:35.534 INFO:tasks.cephadm:First mgr is y 2023-09-30T09:04:35.534 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-30T09:04:35.535 DEBUG:teuthology.orchestra.run.smithi019:> sudo hostname $(hostname -s) 2023-09-30T09:04:35.563 DEBUG:teuthology.orchestra.run.smithi167:> sudo hostname $(hostname -s) 2023-09-30T09:04:35.589 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-30T09:04:35.589 DEBUG:teuthology.orchestra.run.smithi019:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-30T09:04:35.835 INFO:teuthology.orchestra.run.smithi019.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 30 09:04 /home/ubuntu/cephtest/cephadm 2023-09-30T09:04:35.835 DEBUG:teuthology.orchestra.run.smithi167:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-30T09:04:36.026 INFO:teuthology.orchestra.run.smithi167.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 30 09:04 /home/ubuntu/cephtest/cephadm 2023-09-30T09:04:36.027 DEBUG:teuthology.orchestra.run.smithi019:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-30T09:04:36.045 DEBUG:teuthology.orchestra.run.smithi167:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-30T09:04:36.072 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-30T09:04:36.073 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-30T09:04:36.087 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-30T09:04:37.105 INFO:teuthology.orchestra.run.smithi167.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-30T09:04:37.239 INFO:teuthology.orchestra.run.smithi019.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-30T09:05:18.026 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:05:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph_version": "ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)", 2023-09-30T09:05:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "image_id": "8497a1260037b82e7a7d1f7b1252ba27ac4d4c1a1689efe0d7d603614c3dfeb9", 2023-09-30T09:05:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "repo_digests": [ 2023-09-30T09:05:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:59dfea07bd40b1827d644f258b611da4ee50ad9e03cd12567e2c0635c847dcac" 2023-09-30T09:05:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:05:18.027 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:05:20.458 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:05:20.459 INFO:teuthology.orchestra.run.smithi167.stdout: "ceph_version": "ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)", 2023-09-30T09:05:20.459 INFO:teuthology.orchestra.run.smithi167.stdout: "image_id": "8497a1260037b82e7a7d1f7b1252ba27ac4d4c1a1689efe0d7d603614c3dfeb9", 2023-09-30T09:05:20.459 INFO:teuthology.orchestra.run.smithi167.stdout: "repo_digests": [ 2023-09-30T09:05:20.459 INFO:teuthology.orchestra.run.smithi167.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:59dfea07bd40b1827d644f258b611da4ee50ad9e03cd12567e2c0635c847dcac" 2023-09-30T09:05:20.459 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:05:20.459 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:05:20.485 DEBUG:teuthology.orchestra.run.smithi019:> sudo mkdir -p /etc/ceph 2023-09-30T09:05:20.524 DEBUG:teuthology.orchestra.run.smithi167:> sudo mkdir -p /etc/ceph 2023-09-30T09:05:20.556 DEBUG:teuthology.orchestra.run.smithi019:> sudo chmod 777 /etc/ceph 2023-09-30T09:05:20.596 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 777 /etc/ceph 2023-09-30T09:05:20.624 INFO:tasks.cephadm:Writing seed config... 2023-09-30T09:05:20.625 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-30T09:05:20.625 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-30T09:05:20.625 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-30T09:05:20.626 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-30T09:05:20.626 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-30T09:05:20.626 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-09-30T09:05:20.626 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-30T09:05:20.626 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-30T09:05:20.626 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-30T09:05:20.626 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-30T09:05:20.626 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-30T09:05:20.627 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:05:20.627 DEBUG:teuthology.orchestra.run.smithi019:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-30T09:05:20.655 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 608495ac-5f70-11ee-8db4-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-30T09:05:20.655 DEBUG:teuthology.orchestra.run.smithi019:mon.a> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-30T09:05:20.697 DEBUG:teuthology.orchestra.run.smithi019:mgr.y> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-30T09:05:20.729 INFO:journalctl@ceph.mon.a.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:05:20.731 INFO:tasks.cephadm:Bootstrapping... 2023-09-30T09:05:20.731 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.19 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-30T09:05:20.760 INFO:journalctl@ceph.mgr.y.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:05:20.947 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2023-09-30T09:05:20.947 INFO:teuthology.orchestra.run.smithi019.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '608495ac-5f70-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.19', '--skip-admin-label'] 2023-09-30T09:05:21.009 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stdout 3.0.1 2023-09-30T09:05:21.009 INFO:teuthology.orchestra.run.smithi019.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-30T09:05:21.010 INFO:teuthology.orchestra.run.smithi019.stdout:Verifying podman|docker is present... 2023-09-30T09:05:21.071 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stdout 3.0.1 2023-09-30T09:05:21.071 INFO:teuthology.orchestra.run.smithi019.stdout:Verifying lvm2 is present... 2023-09-30T09:05:21.071 INFO:teuthology.orchestra.run.smithi019.stdout:Verifying time synchronization is in place... 2023-09-30T09:05:21.078 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-30T09:05:21.078 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-30T09:05:21.084 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-30T09:05:21.084 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout inactive 2023-09-30T09:05:21.090 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout enabled 2023-09-30T09:05:21.097 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout active 2023-09-30T09:05:21.097 INFO:teuthology.orchestra.run.smithi019.stdout:Unit chronyd.service is enabled and running 2023-09-30T09:05:21.097 INFO:teuthology.orchestra.run.smithi019.stdout:Repeating the final host check... 2023-09-30T09:05:21.160 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stdout 3.0.1 2023-09-30T09:05:21.160 INFO:teuthology.orchestra.run.smithi019.stdout:podman (/bin/podman) version 3.0.1 is present 2023-09-30T09:05:21.160 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl is present 2023-09-30T09:05:21.160 INFO:teuthology.orchestra.run.smithi019.stdout:lvcreate is present 2023-09-30T09:05:21.166 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-30T09:05:21.166 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-30T09:05:21.172 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-30T09:05:21.172 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout inactive 2023-09-30T09:05:21.178 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout enabled 2023-09-30T09:05:21.184 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout active 2023-09-30T09:05:21.184 INFO:teuthology.orchestra.run.smithi019.stdout:Unit chronyd.service is enabled and running 2023-09-30T09:05:21.184 INFO:teuthology.orchestra.run.smithi019.stdout:Host looks OK 2023-09-30T09:05:21.185 INFO:teuthology.orchestra.run.smithi019.stdout:Cluster fsid: 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:21.185 INFO:teuthology.orchestra.run.smithi019.stdout:Acquiring lock 139691619191272 on /run/cephadm/608495ac-5f70-11ee-8db4-212e2dc638e7.lock 2023-09-30T09:05:21.185 INFO:teuthology.orchestra.run.smithi019.stdout:Lock 139691619191272 acquired on /run/cephadm/608495ac-5f70-11ee-8db4-212e2dc638e7.lock 2023-09-30T09:05:21.185 INFO:teuthology.orchestra.run.smithi019.stdout:Verifying IP 172.21.15.19 port 3300 ... 2023-09-30T09:05:21.185 INFO:teuthology.orchestra.run.smithi019.stdout:Verifying IP 172.21.15.19 port 6789 ... 2023-09-30T09:05:21.186 INFO:teuthology.orchestra.run.smithi019.stdout:Base mon IP(s) is [172.21.15.19:3300, 172.21.15.19:6789], mon addrv is [v2:172.21.15.19:3300,v1:172.21.15.19:6789] 2023-09-30T09:05:21.189 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-09-30T09:05:21.189 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.19 metric 100 2023-09-30T09:05:21.192 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-30T09:05:21.192 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-09-30T09:05:21.192 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1793sec hoplimit 64 pref medium 2023-09-30T09:05:21.194 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-30T09:05:21.194 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-30T09:05:21.195 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-30T09:05:21.195 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout 3: ens1f0: mtu 1500 state UP qlen 1000 2023-09-30T09:05:21.195 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:161e/64 scope link 2023-09-30T09:05:21.195 INFO:teuthology.orchestra.run.smithi019.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-30T09:05:21.195 INFO:teuthology.orchestra.run.smithi019.stdout:Mon IP `172.21.15.19` is in CIDR network `172.21.0.0/20` 2023-09-30T09:05:21.195 INFO:teuthology.orchestra.run.smithi019.stdout:Mon IP `172.21.15.19` is in CIDR network `172.21.0.0/20` 2023-09-30T09:05:21.195 INFO:teuthology.orchestra.run.smithi019.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-30T09:05:21.196 INFO:teuthology.orchestra.run.smithi019.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-30T09:05:21.197 INFO:teuthology.orchestra.run.smithi019.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-30T09:05:21.580 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stdout 8497a1260037b82e7a7d1f7b1252ba27ac4d4c1a1689efe0d7d603614c3dfeb9 2023-09-30T09:05:21.580 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-30T09:05:21.580 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stderr Getting image source signatures 2023-09-30T09:05:21.580 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stderr Copying blob sha256:9ded06edb695fe18ba35c4fd73ee2e36b97f8c971e708e670c5fe24743e14d30 2023-09-30T09:05:21.580 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stderr Copying blob sha256:c16254b0810587cdd81d0647eccf1c3e4d200bf4eeba6596492e593ad063d4ee 2023-09-30T09:05:21.580 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stderr Copying config sha256:8497a1260037b82e7a7d1f7b1252ba27ac4d4c1a1689efe0d7d603614c3dfeb9 2023-09-30T09:05:21.580 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-30T09:05:21.580 INFO:teuthology.orchestra.run.smithi019.stdout:/bin/podman: stderr Storing signatures 2023-09-30T09:05:22.665 INFO:teuthology.orchestra.run.smithi019.stdout:ceph: stdout ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable) 2023-09-30T09:05:22.665 INFO:teuthology.orchestra.run.smithi019.stdout:Ceph version: ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable) 2023-09-30T09:05:22.665 INFO:teuthology.orchestra.run.smithi019.stdout:Extracting ceph user uid/gid from container image... 2023-09-30T09:05:23.642 INFO:teuthology.orchestra.run.smithi019.stdout:stat: stdout 167 167 2023-09-30T09:05:23.643 INFO:teuthology.orchestra.run.smithi019.stdout:Creating initial keys... 2023-09-30T09:05:24.593 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph-authtool: stdout AQDU5Bdl/cGqARAAgD3HKR0X4mFNzZs/f8kv6Q== 2023-09-30T09:05:25.574 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph-authtool: stdout AQDU5Bdlf5kIORAAK8GhpPGQy0wgs0u5FbaZIA== 2023-09-30T09:05:26.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph-authtool: stdout AQDV5BdlNey8OBAACsOFUrx2otVshw/DeBmiRA== 2023-09-30T09:05:26.472 INFO:teuthology.orchestra.run.smithi019.stdout:Creating initial monmap... 2023-09-30T09:05:27.399 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-30T09:05:27.400 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-30T09:05:27.400 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:27.400 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-30T09:05:27.401 INFO:teuthology.orchestra.run.smithi019.stdout:monmaptool for a [v2:172.21.15.19:3300,v1:172.21.15.19:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-30T09:05:27.401 INFO:teuthology.orchestra.run.smithi019.stdout:setting min_mon_release = octopus 2023-09-30T09:05:27.401 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/monmaptool: set fsid to 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:27.401 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-30T09:05:27.401 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:05:27.401 INFO:teuthology.orchestra.run.smithi019.stdout:Creating mon... 2023-09-30T09:05:28.398 INFO:teuthology.orchestra.run.smithi019.stdout:create mon.a on 2023-09-30T09:05:28.703 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-30T09:05:28.703 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-30T09:05:28.703 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7.target. 2023-09-30T09:05:28.703 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7.target. 2023-09-30T09:05:28.823 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a 2023-09-30T09:05:29.248 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Failed to reset failed state of unit ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-30T09:05:29.249 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7.target.wants/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@.service. 2023-09-30T09:05:29.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:28 smithi019 systemd[1]: Starting Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:05:29.521 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-30T09:05:29.566 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout disabled 2023-09-30T09:05:29.568 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-30T09:05:29.568 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout inactive 2023-09-30T09:05:29.568 INFO:teuthology.orchestra.run.smithi019.stdout:firewalld.service is not enabled 2023-09-30T09:05:29.568 INFO:teuthology.orchestra.run.smithi019.stdout:Not possible to enable service . firewalld.service is not available 2023-09-30T09:05:29.568 INFO:teuthology.orchestra.run.smithi019.stdout:Waiting for mon to start... 2023-09-30T09:05:29.568 INFO:teuthology.orchestra.run.smithi019.stdout:Waiting for mon... 2023-09-30T09:05:29.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:29 smithi019 bash[99194]: 9a353d7492ef6662c1a7aa05bb8064f5c414c6a2f0194bfba2fe6e20a9716783 2023-09-30T09:05:29.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:29 smithi019 systemd[1]: Started Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:05:29.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:29 smithi019 ceph-mon[99260]: mkfs 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:29.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:29 smithi019 ceph-mon[99260]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-30T09:05:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:30 smithi019 ceph-mon[99260]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-30T09:05:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:30 smithi019 ceph-mon[99260]: monmap e1: 1 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0]} removed_ranks: {} 2023-09-30T09:05:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:30 smithi019 ceph-mon[99260]: fsmap 2023-09-30T09:05:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:30 smithi019 ceph-mon[99260]: osdmap e1: 0 total, 0 up, 0 in 2023-09-30T09:05:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:30 smithi019 ceph-mon[99260]: mgrmap e1: no daemons active 2023-09-30T09:05:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:30 smithi019 ceph-mon[99260]: from='client.? 172.21.15.19:0/147426833' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-30T09:05:30.906 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout cluster: 2023-09-30T09:05:30.908 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout id: 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:30.908 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-30T09:05:30.908 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:30.908 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout services: 2023-09-30T09:05:30.908 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.587627s) 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout data: 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout pgs: 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:mon is available 2023-09-30T09:05:30.909 INFO:teuthology.orchestra.run.smithi019.stdout:Assimilating anything we can from ceph.conf... 2023-09-30T09:05:32.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:31 smithi019 ceph-mon[99260]: from='client.? 172.21.15.19:0/2697262349' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-30T09:05:32.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:31 smithi019 ceph-mon[99260]: from='client.? 172.21.15.19:0/2697262349' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-30T09:05:32.141 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:32.141 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout [global] 2023-09-30T09:05:32.141 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout fsid = 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:32.141 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.19:3300,v1:172.21.15.19:6789] 2023-09-30T09:05:32.141 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout [osd] 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-30T09:05:32.142 INFO:teuthology.orchestra.run.smithi019.stdout:Generating new minimal ceph.conf... 2023-09-30T09:05:33.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:32 smithi019 ceph-mon[99260]: from='client.? 172.21.15.19:0/1369460410' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:05:33.356 INFO:teuthology.orchestra.run.smithi019.stdout:Restarting the monitor... 2023-09-30T09:05:33.615 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:33 smithi019 systemd[1]: Stopping Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:05:33.888 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:33 smithi019 conmon[99236]: 2023-09-30T09:05:33.613+0000 7f70862a1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:05:33.889 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:33 smithi019 conmon[99236]: 2023-09-30T09:05:33.613+0000 7f70862a1700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-30T09:05:33.889 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:33 smithi019 bash[99818]: 9a353d7492ef6662c1a7aa05bb8064f5c414c6a2f0194bfba2fe6e20a9716783 2023-09-30T09:05:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-30T09:05:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 systemd[1]: Stopped Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:05:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 systemd[1]: Starting Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:05:34.783 INFO:teuthology.orchestra.run.smithi019.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-30T09:05:35.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: set uid:gid to 167:167 (ceph:ceph) 2023-09-30T09:05:35.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable), process ceph-mon, pid 7 2023-09-30T09:05:35.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: pidfile_write: ignore empty --pid-file 2023-09-30T09:05:35.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: load: jerasure load: lrc 2023-09-30T09:05:35.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: RocksDB version: 6.15.5 2023-09-30T09:05:35.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-30T09:05:35.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Compile date Sep 29 2023 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: DB SUMMARY 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: DB Session ID: GW0EDN8F5W7HAX0EXULK 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: CURRENT file: CURRENT 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: IDENTITY file: IDENTITY 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83585 ; 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.error_if_exists: 0 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.create_if_missing: 0 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.paranoid_checks: 1 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-30T09:05:35.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.env: 0x565131242a80 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.fs: Posix File System 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.info_log: 0x565132ea25a0 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_file_opening_threads: 16 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.statistics: (nil) 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.use_fsync: 0 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_log_file_size: 0 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.keep_log_file_num: 1000 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.recycle_log_file_num: 0 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.allow_fallocate: 1 2023-09-30T09:05:35.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.allow_mmap_reads: 0 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.allow_mmap_writes: 0 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.use_direct_reads: 0 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.create_missing_column_families: 0 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.db_log_dir: 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-30T09:05:35.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.advise_random_on_open: 1 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.db_write_buffer_size: 0 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.write_buffer_manager: 0x565133c14960 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.rate_limiter: (nil) 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.wal_recovery_mode: 2 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.enable_thread_tracking: 0 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.enable_pipelined_write: 0 2023-09-30T09:05:35.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.unordered_write: 0 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.row_cache: None 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.wal_filter: None 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.allow_ingest_behind: 0 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.preserve_deletes: 0 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.two_write_queues: 0 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.manual_wal_flush: 0 2023-09-30T09:05:35.027 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.atomic_flush: 0 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.log_readahead_size: 0 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.best_efforts_recovery: 0 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.allow_data_in_errors: 0 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.db_host_id: __hostname__ 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_background_jobs: 2 2023-09-30T09:05:35.028 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_background_compactions: -1 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_subcompactions: 1 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_total_wal_size: 0 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_open_files: -1 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bytes_per_sync: 0 2023-09-30T09:05:35.029 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_readahead_size: 0 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_background_flushes: -1 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Compression algorithms supported: 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: kZSTD supported: 0 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: kXpressCompression supported: 0 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: kLZ4HCCompression supported: 1 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: kZlibCompression supported: 1 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: kSnappyCompression supported: 1 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: kLZ4Compression supported: 1 2023-09-30T09:05:35.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: kBZip2Compression supported: 0 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.merge_operator: 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_filter: None 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_filter_factory: None 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.sst_partitioner_factory: None 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x565132ef4fb8) 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout: cache_index_and_filter_blocks: 1 2023-09-30T09:05:35.031 INFO:journalctl@ceph.mon.a.smithi019.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: pin_top_level_index_and_filter: 1 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_type: 0 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_block_index_type: 0 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_shortening: 1 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: hash_index_allow_collision: 1 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: checksum: 1 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: no_block_cache: 0 2023-09-30T09:05:35.032 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache: 0x565132f27090 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_name: BinnedLRUCache 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_options: 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: capacity : 536870912 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: num_shard_bits : 4 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: strict_capacity_limit : 0 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: high_pri_pool_ratio: 0.000 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_compressed: (nil) 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: persistent_cache: (nil) 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_size: 4096 2023-09-30T09:05:35.033 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_size_deviation: 10 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_restart_interval: 16 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_block_restart_interval: 1 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: metadata_block_size: 4096 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: partition_filters: 0 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: use_delta_encoding: 1 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: whole_key_filtering: 1 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: verify_compression: 0 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: read_amp_bytes_per_bit: 0 2023-09-30T09:05:35.034 INFO:journalctl@ceph.mon.a.smithi019.stdout: format_version: 4 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout: enable_index_compression: 1 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_align: 0 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.write_buffer_size: 33554432 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_write_buffer_number: 2 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compression: NoCompression 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bottommost_compression: Disabled 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.prefix_extractor: nullptr 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.num_levels: 7 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-30T09:05:35.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-30T09:05:35.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-30T09:05:35.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compression_opts.level: 32767 2023-09-30T09:05:35.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compression_opts.strategy: 0 2023-09-30T09:05:35.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-30T09:05:35.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compression_opts.enabled: false 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.target_file_size_base: 67108864 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-30T09:05:35.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.arena_block_size: 4194304 2023-09-30T09:05:35.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.disable_auto_compactions: 0 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-30T09:05:35.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.table_properties_collectors: 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.inplace_update_support: 0 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.bloom_locality: 0 2023-09-30T09:05:35.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.max_successive_merges: 0 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.paranoid_file_checks: 0 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.force_consistency_checks: 1 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.report_bg_io_stats: 0 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.ttl: 2592000 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.enable_blob_files: false 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.min_blob_size: 0 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.blob_file_size: 268435456 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-30T09:05:35.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064734761866, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064734762601, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80132, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78400, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72608, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696064734, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "93c200b6-f7d8-4377-8c33-cd7077544374", "db_session_id": "GW0EDN8F5W7HAX0EXULK"}} 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064734762944, "job": 1, "event": "recovery_finished"} 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-30T09:05:35.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x565132ebb180 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: DB pointer 0x565132f90000 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** DB Stats ** 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-30T09:05:35.044 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: L0 2/0 80.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: Sum 2/0 80.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:05:35.045 INFO:journalctl@ceph.mon.a.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative compaction: 0.00 GB write, 24.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval compaction: 0.00 GB write, 24.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:05:35.046 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: L0 2/0 80.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: Sum 2/0 80.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:05:35.047 INFO:journalctl@ceph.mon.a.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative compaction: 0.00 GB write, 24.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:05:35.048 INFO:journalctl@ceph.mon.a.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: starting mon.a rank 0 at public addrs [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] at bind addrs [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a@-1(???) e1 preinit fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a@-1(???).mds e1 new map 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a@-1(???).mds e1 print_map 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout: e1 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:05:35.049 INFO:journalctl@ceph.mon.a.smithi019.stdout: legacy client fscid: -1 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout: No filesystems configured 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 bash[100149]: 91f2eb5c552094bf4653affa7296c1bd6a5e866412d2bc337bc706732be19a5a 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 systemd[1]: Started Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-30T09:05:35.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: monmap e1: 1 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0]} removed_ranks: {} 2023-09-30T09:05:35.051 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: fsmap 2023-09-30T09:05:35.051 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: osdmap e1: 0 total, 0 up, 0 in 2023-09-30T09:05:35.051 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:34 smithi019 ceph-mon[100214]: mgrmap e1: no daemons active 2023-09-30T09:05:36.098 INFO:teuthology.orchestra.run.smithi019.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-30T09:05:36.098 INFO:teuthology.orchestra.run.smithi019.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-30T09:05:36.098 INFO:teuthology.orchestra.run.smithi019.stdout:Creating mgr... 2023-09-30T09:05:36.098 INFO:teuthology.orchestra.run.smithi019.stdout:Verifying port 9283 ... 2023-09-30T09:05:36.235 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y 2023-09-30T09:05:36.235 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Failed to reset failed state of unit ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-30T09:05:36.331 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7.target.wants/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@.service. 2023-09-30T09:05:36.429 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:36 smithi019 systemd[1]: Starting Ceph mgr.y for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:05:36.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:36 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2712626141' entity='client.admin' 2023-09-30T09:05:37.002 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-30T09:05:37.002 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout disabled 2023-09-30T09:05:37.010 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-30T09:05:37.010 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout inactive 2023-09-30T09:05:37.010 INFO:teuthology.orchestra.run.smithi019.stdout:firewalld.service is not enabled 2023-09-30T09:05:37.010 INFO:teuthology.orchestra.run.smithi019.stdout:Not possible to enable service . firewalld.service is not available 2023-09-30T09:05:37.018 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-30T09:05:37.018 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout disabled 2023-09-30T09:05:37.027 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-30T09:05:37.027 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout inactive 2023-09-30T09:05:37.028 INFO:teuthology.orchestra.run.smithi019.stdout:firewalld.service is not enabled 2023-09-30T09:05:37.028 INFO:teuthology.orchestra.run.smithi019.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-30T09:05:37.028 INFO:teuthology.orchestra.run.smithi019.stdout:Waiting for mgr to start... 2023-09-30T09:05:37.028 INFO:teuthology.orchestra.run.smithi019.stdout:Waiting for mgr... 2023-09-30T09:05:37.248 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:36 smithi019 bash[100595]: 08339cd406da9636bc25fbc7ec0177b9ada7c27042f99ae70b336a271243e40a 2023-09-30T09:05:37.249 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:36 smithi019 systemd[1]: Started Ceph mgr.y for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:05:37.249 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:37 smithi019 conmon[100636]: 2023-09-30T09:05:37.124+0000 7f7a75db11c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:05:37.249 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:37 smithi019 conmon[100636]: 2023-09-30T09:05:37.219+0000 7f7a75db11c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:05:37.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:37 smithi019 conmon[100636]: 2023-09-30T09:05:37.332+0000 7f7a75db11c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:05:37.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:37 smithi019 conmon[100636]: 2023-09-30T09:05:37.442+0000 7f7a75db11c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:05:38.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:37 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1061224619' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-30T09:05:38.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:37 smithi019 conmon[100636]: 2023-09-30T09:05:37.648+0000 7f7a75db11c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:05:38.263 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:38.269 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout { 2023-09-30T09:05:38.269 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "fsid": "608495ac-5f70-11ee-8db4-212e2dc638e7", 2023-09-30T09:05:38.269 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "health": { 2023-09-30T09:05:38.269 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-30T09:05:38.269 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-30T09:05:38.269 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 0 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "a" 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-30T09:05:38.270 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:38.271 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-30T09:05:38.271 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:38.272 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:38.273 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "restful" 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-30T09:05:38.274 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:38.275 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-30T09:05:38.275 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:38.275 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "modified": "2023-09-30T09:05:29.547973+0000", 2023-09-30T09:05:38.275 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-30T09:05:38.275 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:38.275 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-30T09:05:38.275 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout } 2023-09-30T09:05:38.275 INFO:teuthology.orchestra.run.smithi019.stdout:mgr not available, waiting (1/15)... 2023-09-30T09:05:38.406 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:38 smithi019 conmon[100636]: 2023-09-30T09:05:38.149+0000 7f7a75db11c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:05:38.406 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:38 smithi019 conmon[100636]: 2023-09-30T09:05:38.256+0000 7f7a75db11c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:05:38.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:38 smithi019 conmon[100636]: 2023-09-30T09:05:38.405+0000 7f7a75db11c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:05:38.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:38 smithi019 conmon[100636]: 2023-09-30T09:05:38.518+0000 7f7a75db11c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:05:39.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:39 smithi019 conmon[100636]: 2023-09-30T09:05:39.025+0000 7f7a75db11c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:05:40.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:39 smithi019 conmon[100636]: 2023-09-30T09:05:39.714+0000 7f7a75db11c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:05:40.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:39 smithi019 conmon[100636]: 2023-09-30T09:05:39.904+0000 7f7a75db11c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:05:41.206 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:40 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3074937552' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-30T09:05:41.206 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:40 smithi019 conmon[100636]: 2023-09-30T09:05:40.876+0000 7f7a75db11c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:05:41.207 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:40 smithi019 conmon[100636]: 2023-09-30T09:05:40.989+0000 7f7a75db11c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:05:41.508 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:41.508 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout { 2023-09-30T09:05:41.508 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "fsid": "608495ac-5f70-11ee-8db4-212e2dc638e7", 2023-09-30T09:05:41.508 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "health": { 2023-09-30T09:05:41.508 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-30T09:05:41.509 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-30T09:05:41.509 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-30T09:05:41.509 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:41.509 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-30T09:05:41.509 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-30T09:05:41.509 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 0 2023-09-30T09:05:41.509 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:41.509 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "a" 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-30T09:05:41.510 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:41.511 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-30T09:05:41.511 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-30T09:05:41.511 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-30T09:05:41.511 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-30T09:05:41.511 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-30T09:05:41.511 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-30T09:05:41.514 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:41.514 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-30T09:05:41.514 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-30T09:05:41.514 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-30T09:05:41.515 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "restful" 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:41.516 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-30T09:05:41.517 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:41.517 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-30T09:05:41.517 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:41.517 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "modified": "2023-09-30T09:05:29.547973+0000", 2023-09-30T09:05:41.517 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-30T09:05:41.517 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:41.517 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-30T09:05:41.517 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout } 2023-09-30T09:05:41.518 INFO:teuthology.orchestra.run.smithi019.stdout:mgr not available, waiting (2/15)... 2023-09-30T09:05:41.518 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:41 smithi019 conmon[100636]: 2023-09-30T09:05:41.205+0000 7f7a75db11c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:05:41.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:41 smithi019 conmon[100636]: 2023-09-30T09:05:41.547+0000 7f7a75db11c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:05:41.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:41 smithi019 conmon[100636]: 2023-09-30T09:05:41.650+0000 7f7a75db11c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:05:42.306 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:41 smithi019 conmon[100636]: 2023-09-30T09:05:41.986+0000 7f7a75db11c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:05:43.559 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:42 smithi019 conmon[100636]: 2023-09-30T09:05:42.305+0000 7f7a75db11c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:05:43.560 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:43 smithi019 conmon[100636]: 2023-09-30T09:05:43.156+0000 7f7a75db11c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:05:43.560 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:43 smithi019 conmon[100636]: 2023-09-30T09:05:43.314+0000 7f7a75db11c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:05:43.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:43 smithi019 conmon[100636]: 2023-09-30T09:05:43.559+0000 7f7a75db11c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:05:43.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:43 smithi019 conmon[100636]: 2023-09-30T09:05:43.699+0000 7f7a75db11c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:05:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:44 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4263946916' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-30T09:05:44.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:43 smithi019 conmon[100636]: 2023-09-30T09:05:43.970+0000 7f7a75db11c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:05:44.720 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:44.720 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout { 2023-09-30T09:05:44.720 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "fsid": "608495ac-5f70-11ee-8db4-212e2dc638e7", 2023-09-30T09:05:44.720 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "health": { 2023-09-30T09:05:44.720 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-30T09:05:44.720 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-30T09:05:44.720 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 0 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "a" 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:44.721 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-30T09:05:44.722 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-30T09:05:44.723 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "restful" 2023-09-30T09:05:44.724 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:44.725 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-30T09:05:44.725 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:44.725 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-30T09:05:44.725 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:44.725 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "modified": "2023-09-30T09:05:29.547973+0000", 2023-09-30T09:05:44.725 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-30T09:05:44.726 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:44.726 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-30T09:05:44.726 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout } 2023-09-30T09:05:44.726 INFO:teuthology.orchestra.run.smithi019.stdout:mgr not available, waiting (3/15)... 2023-09-30T09:05:45.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:45 smithi019 conmon[100636]: 2023-09-30T09:05:45.014+0000 7f7a75db11c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:05:46.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: Activating manager daemon y 2023-09-30T09:05:46.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: mgrmap e2: y(active, starting, since 0.00292799s) 2023-09-30T09:05:46.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:05:46.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:05:46.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:05:46.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:05:46.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:05:46.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: Manager daemon y is now available 2023-09-30T09:05:46.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:05:46.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:05:46.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' 2023-09-30T09:05:46.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' 2023-09-30T09:05:46.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:45 smithi019 ceph-mon[100214]: from='mgr.14100 172.21.15.19:0/2894062944' entity='mgr.y' 2023-09-30T09:05:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:47 smithi019 ceph-mon[100214]: mgrmap e3: y(active, since 1.00886s) 2023-09-30T09:05:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:48 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3857581791' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-30T09:05:48.465 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:48.465 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout { 2023-09-30T09:05:48.465 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "fsid": "608495ac-5f70-11ee-8db4-212e2dc638e7", 2023-09-30T09:05:48.465 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "health": { 2023-09-30T09:05:48.465 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-30T09:05:48.465 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-30T09:05:48.465 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-30T09:05:48.465 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 0 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "a" 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "quorum_age": 13, 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-30T09:05:48.466 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-30T09:05:48.467 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-30T09:05:48.470 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:48.470 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-30T09:05:48.470 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-30T09:05:48.470 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-30T09:05:48.471 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "restful" 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ], 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-30T09:05:48.472 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-30T09:05:48.473 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "modified": "2023-09-30T09:05:29.547973+0000", 2023-09-30T09:05:48.473 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-30T09:05:48.473 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout }, 2023-09-30T09:05:48.473 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-30T09:05:48.473 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout } 2023-09-30T09:05:48.473 INFO:teuthology.orchestra.run.smithi019.stdout:mgr is available 2023-09-30T09:05:49.327 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:49 smithi019 ceph-mon[100214]: mgrmap e4: y(active, since 3s) 2023-09-30T09:05:49.871 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:49.871 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout [global] 2023-09-30T09:05:49.871 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout fsid = 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:05:49.871 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-30T09:05:49.871 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout [osd] 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-30T09:05:49.872 INFO:teuthology.orchestra.run.smithi019.stdout:Enabling cephadm module... 2023-09-30T09:05:50.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:50 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/83707821' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-30T09:05:51.323 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:51 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4293072348' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-30T09:05:51.323 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:51 smithi019 conmon[100636]: ignoring --setuser ceph since I am not root 2023-09-30T09:05:51.323 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:51 smithi019 conmon[100636]: ignoring --setgroup ceph since I am not root 2023-09-30T09:05:51.323 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:51 smithi019 conmon[100636]: 2023-09-30T09:05:51.227+0000 7fdb7f0041c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:05:51.323 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:51 smithi019 conmon[100636]: 2023-09-30T09:05:51.321+0000 7fdb7f0041c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:05:51.583 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:51 smithi019 conmon[100636]: 2023-09-30T09:05:51.428+0000 7fdb7f0041c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:05:51.584 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:51 smithi019 conmon[100636]: 2023-09-30T09:05:51.537+0000 7fdb7f0041c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:05:51.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:51 smithi019 conmon[100636]: 2023-09-30T09:05:51.751+0000 7fdb7f0041c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:05:52.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:52 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4293072348' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-30T09:05:52.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:52 smithi019 ceph-mon[100214]: mgrmap e5: y(active, since 6s) 2023-09-30T09:05:52.419 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:52 smithi019 conmon[100636]: 2023-09-30T09:05:52.161+0000 7fdb7f0041c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:05:52.419 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:52 smithi019 conmon[100636]: 2023-09-30T09:05:52.268+0000 7fdb7f0041c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:05:52.777 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:52 smithi019 conmon[100636]: 2023-09-30T09:05:52.417+0000 7fdb7f0041c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:05:52.817 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:52 smithi019 conmon[100636]: 2023-09-30T09:05:52.531+0000 7fdb7f0041c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:05:53.112 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout { 2023-09-30T09:05:53.112 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-30T09:05:53.112 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-30T09:05:53.112 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-30T09:05:53.112 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-30T09:05:53.112 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout } 2023-09-30T09:05:53.112 INFO:teuthology.orchestra.run.smithi019.stdout:Waiting for the mgr to restart... 2023-09-30T09:05:53.113 INFO:teuthology.orchestra.run.smithi019.stdout:Waiting for mgr epoch 5... 2023-09-30T09:05:53.319 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:53 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2742826036' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-30T09:05:53.320 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:53 smithi019 conmon[100636]: 2023-09-30T09:05:53.047+0000 7fdb7f0041c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:05:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:53 smithi019 conmon[100636]: 2023-09-30T09:05:53.754+0000 7fdb7f0041c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:05:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:53 smithi019 conmon[100636]: 2023-09-30T09:05:53.944+0000 7fdb7f0041c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:05:55.220 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:54 smithi019 conmon[100636]: 2023-09-30T09:05:54.896+0000 7fdb7f0041c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:05:55.220 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:55 smithi019 conmon[100636]: 2023-09-30T09:05:55.005+0000 7fdb7f0041c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:05:55.563 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:55 smithi019 conmon[100636]: 2023-09-30T09:05:55.218+0000 7fdb7f0041c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:05:55.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:55 smithi019 conmon[100636]: 2023-09-30T09:05:55.561+0000 7fdb7f0041c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:05:55.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:55 smithi019 conmon[100636]: 2023-09-30T09:05:55.665+0000 7fdb7f0041c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:05:56.323 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:56 smithi019 conmon[100636]: 2023-09-30T09:05:56.003+0000 7fdb7f0041c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:05:56.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:56 smithi019 conmon[100636]: 2023-09-30T09:05:56.322+0000 7fdb7f0041c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:05:57.537 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:57 smithi019 conmon[100636]: 2023-09-30T09:05:57.143+0000 7fdb7f0041c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:05:57.538 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:57 smithi019 conmon[100636]: 2023-09-30T09:05:57.299+0000 7fdb7f0041c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:05:57.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:57 smithi019 conmon[100636]: 2023-09-30T09:05:57.536+0000 7fdb7f0041c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:05:57.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:57 smithi019 conmon[100636]: 2023-09-30T09:05:57.671+0000 7fdb7f0041c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:05:58.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:57 smithi019 conmon[100636]: 2023-09-30T09:05:57.937+0000 7fdb7f0041c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:05:59.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:05:58 smithi019 conmon[100636]: 2023-09-30T09:05:58.966+0000 7fdb7f0041c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:05:59.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: Active manager daemon y restarted 2023-09-30T09:05:59.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: Activating manager daemon y 2023-09-30T09:05:59.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: osdmap e2: 0 total, 0 up, 0 in 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: mgrmap e6: y(active, starting, since 0.00591044s) 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: Manager daemon y is now available 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:05:59.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:05:59 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:00 smithi019 ceph-mon[100214]: Found migration_current of "None". Setting to last migration. 2023-09-30T09:06:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:00 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:00 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:00 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:00 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:06:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:00 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:06:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:00 smithi019 ceph-mon[100214]: mgrmap e7: y(active, since 1.00596s) 2023-09-30T09:06:00.611 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout { 2023-09-30T09:06:00.611 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-30T09:06:00.611 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-30T09:06:00.611 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout } 2023-09-30T09:06:00.612 INFO:teuthology.orchestra.run.smithi019.stdout:mgr epoch 5 is available 2023-09-30T09:06:00.612 INFO:teuthology.orchestra.run.smithi019.stdout:Setting orchestrator backend to cephadm... 2023-09-30T09:06:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:01 smithi019 ceph-mon[100214]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-30T09:06:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:01 smithi019 ceph-mon[100214]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-30T09:06:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:01 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:01 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:02.301 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:02 smithi019 ceph-mon[100214]: [30/Sep/2023:09:06:00] ENGINE Bus STARTING 2023-09-30T09:06:02.301 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:02 smithi019 ceph-mon[100214]: [30/Sep/2023:09:06:00] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:06:02.301 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:02 smithi019 ceph-mon[100214]: [30/Sep/2023:09:06:00] ENGINE Bus STARTED 2023-09-30T09:06:02.301 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:02 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:02.301 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:02 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:02.302 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:02 smithi019 ceph-mon[100214]: mgrmap e8: y(active, since 2s) 2023-09-30T09:06:03.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:03 smithi019 ceph-mon[100214]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:03.419 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-30T09:06:03.419 INFO:teuthology.orchestra.run.smithi019.stdout:Generating ssh key... 2023-09-30T09:06:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:04 smithi019 ceph-mon[100214]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: Generating public/private rsa key pair. 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: Your identification has been saved in /tmp/tmphfa7fk24/key. 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: Your public key has been saved in /tmp/tmphfa7fk24/key.pub. 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: The key fingerprint is: 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: SHA256:ssP/Uiry6X7rMj63t6XB8jIxj5I8rtrZCHrq7uhuFLI ceph-608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: The key's randomart image is: 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: +---[RSA 3072]----+ 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: | | 2023-09-30T09:06:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: | | 2023-09-30T09:06:04.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: | | 2023-09-30T09:06:04.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: |.. | 2023-09-30T09:06:04.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: |... . S | 2023-09-30T09:06:04.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: |E. . oo.. | 2023-09-30T09:06:04.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: |. . .+..Bo . | 2023-09-30T09:06:04.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: |.o.o.+X+Ooo+ | 2023-09-30T09:06:04.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: |@O..=OO@=O=. | 2023-09-30T09:06:04.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:04 smithi019 conmon[100636]: +----[SHA256]-----+ 2023-09-30T09:06:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:05 smithi019 ceph-mon[100214]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:05 smithi019 ceph-mon[100214]: Generating ssh key... 2023-09-30T09:06:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:05 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:05 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:06.518 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQazCJm+YBLaYNnqY4I4Y5LfjjUiGwQAUFL3DHD4Xeh7g3vH0zU19jT/htEOHbD+iHQwrY3Z9dwlHHojprMelWGrj7e4JUqJ9f0WO4/yBfMcF2IVSinJbpa22kwoaEVC/Y/C7OT1fbogpWdenpujrf9DfI1ker5sh16LxUYPZ/uLLXAcnJJzk4BDkIdU6RVY5TgbMx37fK6l9+5rpN0S4V6wH0BfEMuZUEDwyy3T9BB9HkJB45WHDr3O3nkll/xuuKp7zpd5pgl9wz3du7BLunucrmCp+gJ3LO4F6Y7siQb83lGYuEZZFsPGJQRSbO4YNQp+OIEsLGGUZtNSpKL0LR/oQcQPvJpIBOnqPeCoKQPt3x+vLh4VkzCwSjxj8geZ2yPdGTKZPIMXqVmNQ41NM83Ed7UJ98GsjchbbiLAqHJj3OJKK6aVz4uJ2e+eX+Yb/bXarVNKAcqzRKtAvmHTog+cqIi3CcDdo21lbgwg9QG+urQ1yGsCHuTr3WbFIS0v0= ceph-608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:06:06.519 INFO:teuthology.orchestra.run.smithi019.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-30T09:06:06.519 INFO:teuthology.orchestra.run.smithi019.stdout:Adding key to root@localhost authorized_keys... 2023-09-30T09:06:06.521 INFO:teuthology.orchestra.run.smithi019.stdout:Adding host smithi019... 2023-09-30T09:06:06.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:06 smithi019 ceph-mon[100214]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:08.218 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:08 smithi019 ceph-mon[100214]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi019", "addr": "172.21.15.19", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:08.219 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:08 smithi019 ceph-mon[100214]: Deploying cephadm binary to smithi019 2023-09-30T09:06:10.009 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout Added host 'smithi019' with addr '172.21.15.19' 2023-09-30T09:06:10.009 INFO:teuthology.orchestra.run.smithi019.stdout:Deploying unmanaged mon service... 2023-09-30T09:06:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:10 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:10 smithi019 ceph-mon[100214]: Added host smithi019 2023-09-30T09:06:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:10 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:11.956 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-30T09:06:11.956 INFO:teuthology.orchestra.run.smithi019.stdout:Deploying unmanaged mgr service... 2023-09-30T09:06:12.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:11 smithi019 ceph-mon[100214]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:12.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:11 smithi019 ceph-mon[100214]: Saving service mon spec with placement count:5 2023-09-30T09:06:12.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:11 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:13.241 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:13 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:13.241 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:13 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:13.241 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:13 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:13.742 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-30T09:06:14.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:14 smithi019 ceph-mon[100214]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:14.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:14 smithi019 ceph-mon[100214]: Saving service mgr spec with placement count:2 2023-09-30T09:06:14.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:14 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:15.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:15 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3109568338' entity='client.admin' 2023-09-30T09:06:15.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:15 smithi019 ceph-mon[100214]: from='mgr.14120 172.21.15.19:0/603071197' entity='mgr.y' 2023-09-30T09:06:16.817 INFO:teuthology.orchestra.run.smithi019.stdout:Enabling the dashboard module... 2023-09-30T09:06:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:17 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3970091678' entity='client.admin' 2023-09-30T09:06:18.396 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:18 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/45748355' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-30T09:06:18.396 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:18 smithi019 conmon[100636]: ignoring --setuser ceph since I am not root 2023-09-30T09:06:18.397 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:18 smithi019 conmon[100636]: ignoring --setgroup ceph since I am not root 2023-09-30T09:06:18.397 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:18 smithi019 conmon[100636]: 2023-09-30T09:06:18.298+0000 7fe755c5c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:06:18.397 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:18 smithi019 conmon[100636]: 2023-09-30T09:06:18.394+0000 7fe755c5c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:06:18.724 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:18 smithi019 conmon[100636]: 2023-09-30T09:06:18.500+0000 7fe755c5c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:06:18.724 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:18 smithi019 conmon[100636]: 2023-09-30T09:06:18.610+0000 7fe755c5c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:06:18.980 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:18 smithi019 conmon[100636]: 2023-09-30T09:06:18.822+0000 7fe755c5c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:06:19.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:19 smithi019 conmon[100636]: 2023-09-30T09:06:19.225+0000 7fe755c5c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:06:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:19 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/45748355' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-30T09:06:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:19 smithi019 ceph-mon[100214]: mgrmap e9: y(active, since 19s) 2023-09-30T09:06:19.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:19 smithi019 conmon[100636]: 2023-09-30T09:06:19.334+0000 7fe755c5c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:06:19.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:19 smithi019 conmon[100636]: 2023-09-30T09:06:19.483+0000 7fe755c5c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:06:19.932 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:19 smithi019 conmon[100636]: 2023-09-30T09:06:19.596+0000 7fe755c5c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:06:20.325 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout { 2023-09-30T09:06:20.325 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-30T09:06:20.325 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-30T09:06:20.325 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-30T09:06:20.325 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-30T09:06:20.325 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout } 2023-09-30T09:06:20.325 INFO:teuthology.orchestra.run.smithi019.stdout:Waiting for the mgr to restart... 2023-09-30T09:06:20.325 INFO:teuthology.orchestra.run.smithi019.stdout:Waiting for mgr epoch 9... 2023-09-30T09:06:20.330 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:20 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2072451676' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-30T09:06:20.331 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:20 smithi019 conmon[100636]: 2023-09-30T09:06:20.116+0000 7fe755c5c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:06:20.834 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:20 smithi019 conmon[100636]: 2023-09-30T09:06:20.813+0000 7fe755c5c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:06:21.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:21 smithi019 conmon[100636]: 2023-09-30T09:06:21.004+0000 7fe755c5c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:06:22.280 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:21 smithi019 conmon[100636]: 2023-09-30T09:06:21.956+0000 7fe755c5c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:06:22.281 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:22 smithi019 conmon[100636]: 2023-09-30T09:06:22.064+0000 7fe755c5c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:06:22.281 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:22 smithi019 conmon[100636]: 2023-09-30T09:06:22.279+0000 7fe755c5c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:06:23.057 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:22 smithi019 conmon[100636]: 2023-09-30T09:06:22.622+0000 7fe755c5c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:06:23.057 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:22 smithi019 conmon[100636]: 2023-09-30T09:06:22.725+0000 7fe755c5c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:06:23.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:23 smithi019 conmon[100636]: 2023-09-30T09:06:23.055+0000 7fe755c5c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:06:23.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:23 smithi019 conmon[100636]: 2023-09-30T09:06:23.377+0000 7fe755c5c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:06:24.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:24 smithi019 conmon[100636]: 2023-09-30T09:06:24.200+0000 7fe755c5c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:06:24.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:24 smithi019 conmon[100636]: 2023-09-30T09:06:24.356+0000 7fe755c5c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:06:25.001 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:24 smithi019 conmon[100636]: 2023-09-30T09:06:24.594+0000 7fe755c5c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:06:25.001 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:24 smithi019 conmon[100636]: 2023-09-30T09:06:24.730+0000 7fe755c5c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:06:26.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:25 smithi019 conmon[100636]: 2023-09-30T09:06:24.999+0000 7fe755c5c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:06:26.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:06:26 smithi019 conmon[100636]: 2023-09-30T09:06:26.031+0000 7fe755c5c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:06:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: Active manager daemon y restarted 2023-09-30T09:06:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: Activating manager daemon y 2023-09-30T09:06:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: osdmap e3: 0 total, 0 up, 0 in 2023-09-30T09:06:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: mgrmap e10: y(active, starting, since 0.00311798s) 2023-09-30T09:06:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:06:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:06:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:06:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:06:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:06:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: Manager daemon y is now available 2023-09-30T09:06:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:27 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:06:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:27 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:06:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:27 smithi019 ceph-mon[100214]: mgrmap e11: y(active, since 1.00569s) 2023-09-30T09:06:27.675 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout { 2023-09-30T09:06:27.675 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-30T09:06:27.675 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-30T09:06:27.676 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout } 2023-09-30T09:06:27.676 INFO:teuthology.orchestra.run.smithi019.stdout:mgr epoch 9 is available 2023-09-30T09:06:27.676 INFO:teuthology.orchestra.run.smithi019.stdout:Generating a dashboard self-signed certificate... 2023-09-30T09:06:28.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:28 smithi019 ceph-mon[100214]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-30T09:06:28.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:28 smithi019 ceph-mon[100214]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-30T09:06:28.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:28 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:28.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:28 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:29.372 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:29 smithi019 ceph-mon[100214]: [30/Sep/2023:09:06:27] ENGINE Bus STARTING 2023-09-30T09:06:29.372 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:29 smithi019 ceph-mon[100214]: [30/Sep/2023:09:06:27] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:06:29.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:29 smithi019 ceph-mon[100214]: [30/Sep/2023:09:06:27] ENGINE Bus STARTED 2023-09-30T09:06:29.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:29.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:29 smithi019 ceph-mon[100214]: mgrmap e12: y(active, since 2s) 2023-09-30T09:06:29.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:29.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:29.399 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-30T09:06:29.399 INFO:teuthology.orchestra.run.smithi019.stdout:Creating initial admin user... 2023-09-30T09:06:30.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:30 smithi019 ceph-mon[100214]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:31.340 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$Qw9n53vDYYhquzKOItBkY.grihUO.NOOFu3WQkNwYJKWoHY8iniVu", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696064790, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-30T09:06:31.340 INFO:teuthology.orchestra.run.smithi019.stdout:Fetching dashboard port number... 2023-09-30T09:06:32.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:31 smithi019 ceph-mon[100214]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:32.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:31 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:32.806 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stdout 8443 2023-09-30T09:06:32.818 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-30T09:06:32.818 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout disabled 2023-09-30T09:06:32.826 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-30T09:06:32.826 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stdout inactive 2023-09-30T09:06:32.826 INFO:teuthology.orchestra.run.smithi019.stdout:firewalld.service is not enabled 2023-09-30T09:06:32.826 INFO:teuthology.orchestra.run.smithi019.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-30T09:06:32.828 INFO:teuthology.orchestra.run.smithi019.stdout:Ceph Dashboard is now available at: 2023-09-30T09:06:32.828 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:32.828 INFO:teuthology.orchestra.run.smithi019.stdout: URL: https://smithi019:8443/ 2023-09-30T09:06:32.828 INFO:teuthology.orchestra.run.smithi019.stdout: User: admin 2023-09-30T09:06:32.828 INFO:teuthology.orchestra.run.smithi019.stdout: Password: z77xxzonag 2023-09-30T09:06:32.828 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:32.828 INFO:teuthology.orchestra.run.smithi019.stdout:Saving cluster configuration to /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config directory 2023-09-30T09:06:32.829 INFO:teuthology.orchestra.run.smithi019.stdout:Enabling autotune for osd_memory_target 2023-09-30T09:06:33.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:32 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/608528695' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-30T09:06:34.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:33 smithi019 ceph-mon[100214]: mgrmap e13: y(active, since 6s) 2023-09-30T09:06:35.881 INFO:teuthology.orchestra.run.smithi019.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-30T09:06:35.881 INFO:teuthology.orchestra.run.smithi019.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-30T09:06:35.881 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:35.881 INFO:teuthology.orchestra.run.smithi019.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-30T09:06:35.881 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:35.881 INFO:teuthology.orchestra.run.smithi019.stdout:Or, if you are only running a single cluster on this host: 2023-09-30T09:06:35.881 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout: ceph telemetry on 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout:For more information see: 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:35.882 INFO:teuthology.orchestra.run.smithi019.stdout:Bootstrap complete. 2023-09-30T09:06:35.918 INFO:tasks.cephadm:Fetching config... 2023-09-30T09:06:35.918 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:06:35.918 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-30T09:06:35.935 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-30T09:06:35.935 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:06:35.935 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-30T09:06:35.993 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-30T09:06:35.994 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:06:35.994 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-30T09:06:36.064 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-30T09:06:36.064 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:06:36.064 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-30T09:06:36.122 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-30T09:06:36.122 DEBUG:teuthology.orchestra.run.smithi019:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQazCJm+YBLaYNnqY4I4Y5LfjjUiGwQAUFL3DHD4Xeh7g3vH0zU19jT/htEOHbD+iHQwrY3Z9dwlHHojprMelWGrj7e4JUqJ9f0WO4/yBfMcF2IVSinJbpa22kwoaEVC/Y/C7OT1fbogpWdenpujrf9DfI1ker5sh16LxUYPZ/uLLXAcnJJzk4BDkIdU6RVY5TgbMx37fK6l9+5rpN0S4V6wH0BfEMuZUEDwyy3T9BB9HkJB45WHDr3O3nkll/xuuKp7zpd5pgl9wz3du7BLunucrmCp+gJ3LO4F6Y7siQb83lGYuEZZFsPGJQRSbO4YNQp+OIEsLGGUZtNSpKL0LR/oQcQPvJpIBOnqPeCoKQPt3x+vLh4VkzCwSjxj8geZ2yPdGTKZPIMXqVmNQ41NM83Ed7UJ98GsjchbbiLAqHJj3OJKK6aVz4uJ2e+eX+Yb/bXarVNKAcqzRKtAvmHTog+cqIi3CcDdo21lbgwg9QG+urQ1yGsCHuTr3WbFIS0v0= ceph-608495ac-5f70-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-30T09:06:36.203 INFO:teuthology.orchestra.run.smithi019.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQazCJm+YBLaYNnqY4I4Y5LfjjUiGwQAUFL3DHD4Xeh7g3vH0zU19jT/htEOHbD+iHQwrY3Z9dwlHHojprMelWGrj7e4JUqJ9f0WO4/yBfMcF2IVSinJbpa22kwoaEVC/Y/C7OT1fbogpWdenpujrf9DfI1ker5sh16LxUYPZ/uLLXAcnJJzk4BDkIdU6RVY5TgbMx37fK6l9+5rpN0S4V6wH0BfEMuZUEDwyy3T9BB9HkJB45WHDr3O3nkll/xuuKp7zpd5pgl9wz3du7BLunucrmCp+gJ3LO4F6Y7siQb83lGYuEZZFsPGJQRSbO4YNQp+OIEsLGGUZtNSpKL0LR/oQcQPvJpIBOnqPeCoKQPt3x+vLh4VkzCwSjxj8geZ2yPdGTKZPIMXqVmNQ41NM83Ed7UJ98GsjchbbiLAqHJj3OJKK6aVz4uJ2e+eX+Yb/bXarVNKAcqzRKtAvmHTog+cqIi3CcDdo21lbgwg9QG+urQ1yGsCHuTr3WbFIS0v0= ceph-608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:06:36.217 DEBUG:teuthology.orchestra.run.smithi167:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQazCJm+YBLaYNnqY4I4Y5LfjjUiGwQAUFL3DHD4Xeh7g3vH0zU19jT/htEOHbD+iHQwrY3Z9dwlHHojprMelWGrj7e4JUqJ9f0WO4/yBfMcF2IVSinJbpa22kwoaEVC/Y/C7OT1fbogpWdenpujrf9DfI1ker5sh16LxUYPZ/uLLXAcnJJzk4BDkIdU6RVY5TgbMx37fK6l9+5rpN0S4V6wH0BfEMuZUEDwyy3T9BB9HkJB45WHDr3O3nkll/xuuKp7zpd5pgl9wz3du7BLunucrmCp+gJ3LO4F6Y7siQb83lGYuEZZFsPGJQRSbO4YNQp+OIEsLGGUZtNSpKL0LR/oQcQPvJpIBOnqPeCoKQPt3x+vLh4VkzCwSjxj8geZ2yPdGTKZPIMXqVmNQ41NM83Ed7UJ98GsjchbbiLAqHJj3OJKK6aVz4uJ2e+eX+Yb/bXarVNKAcqzRKtAvmHTog+cqIi3CcDdo21lbgwg9QG+urQ1yGsCHuTr3WbFIS0v0= ceph-608495ac-5f70-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-30T09:06:36.261 INFO:teuthology.orchestra.run.smithi167.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQazCJm+YBLaYNnqY4I4Y5LfjjUiGwQAUFL3DHD4Xeh7g3vH0zU19jT/htEOHbD+iHQwrY3Z9dwlHHojprMelWGrj7e4JUqJ9f0WO4/yBfMcF2IVSinJbpa22kwoaEVC/Y/C7OT1fbogpWdenpujrf9DfI1ker5sh16LxUYPZ/uLLXAcnJJzk4BDkIdU6RVY5TgbMx37fK6l9+5rpN0S4V6wH0BfEMuZUEDwyy3T9BB9HkJB45WHDr3O3nkll/xuuKp7zpd5pgl9wz3du7BLunucrmCp+gJ3LO4F6Y7siQb83lGYuEZZFsPGJQRSbO4YNQp+OIEsLGGUZtNSpKL0LR/oQcQPvJpIBOnqPeCoKQPt3x+vLh4VkzCwSjxj8geZ2yPdGTKZPIMXqVmNQ41NM83Ed7UJ98GsjchbbiLAqHJj3OJKK6aVz4uJ2e+eX+Yb/bXarVNKAcqzRKtAvmHTog+cqIi3CcDdo21lbgwg9QG+urQ1yGsCHuTr3WbFIS0v0= ceph-608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:06:36.274 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-30T09:06:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:36 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/66166354' entity='client.admin' 2023-09-30T09:06:36.631 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:06:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi019", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:06:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3057617418' entity='client.admin' 2023-09-30T09:06:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:39.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:39.491 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-30T09:06:39.491 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-30T09:06:40.259 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:06:42.796 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi167 2023-09-30T09:06:42.796 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:06:42.797 DEBUG:teuthology.orchestra.run.smithi167:> dd of=/etc/ceph/ceph.conf 2023-09-30T09:06:42.817 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:06:42.817 DEBUG:teuthology.orchestra.run.smithi167:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:06:42.876 INFO:tasks.cephadm:Adding host smithi167 to orchestrator... 2023-09-30T09:06:42.877 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi167 2023-09-30T09:06:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:43 smithi019 ceph-mon[100214]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:43 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:43 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:43 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:06:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:43 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:06:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:43 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:06:43.246 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:06:44.530 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:44 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:06:44.530 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:44 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:06:44.530 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:44 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:06:44.531 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:44.531 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:44.531 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:46.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:45 smithi019 ceph-mon[100214]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi167", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:46.949 INFO:teuthology.orchestra.run.smithi019.stdout:Added host 'smithi167' with addr '172.21.15.167' 2023-09-30T09:06:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:47 smithi019 ceph-mon[100214]: Deploying cephadm binary to smithi167 2023-09-30T09:06:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:47 smithi019 ceph-mon[100214]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:47 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:47 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:47.544 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-30T09:06:47.892 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:06:48.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:48 smithi019 ceph-mon[100214]: Added host smithi167 2023-09-30T09:06:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:48 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:48 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:49.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:49 smithi019 ceph-mon[100214]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:49.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:49 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:49.591 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:06:49.591 INFO:teuthology.orchestra.run.smithi019.stdout:[{"addr": "172.21.15.19", "hostname": "smithi019", "labels": [], "status": ""}, {"addr": "172.21.15.167", "hostname": "smithi167", "labels": [], "status": ""}] 2023-09-30T09:06:50.229 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-30T09:06:50.229 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-30T09:06:50.581 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:06:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:50 smithi019 ceph-mon[100214]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:06:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:50 smithi019 ceph-mon[100214]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:52.770 INFO:teuthology.orchestra.run.smithi019.stderr:adjusted tunables profile to default 2023-09-30T09:06:53.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:53.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:53.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:53.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:53.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi167", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:06:53.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:06:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:06:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:06:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:06:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1734286763' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-30T09:06:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:52 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:53.368 INFO:tasks.cephadm:Adding mon.a on smithi019 2023-09-30T09:06:53.368 INFO:tasks.cephadm:Adding mon.c on smithi019 2023-09-30T09:06:53.368 INFO:tasks.cephadm:Adding mon.b on smithi167 2023-09-30T09:06:53.368 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi167:172.21.15.167=b' 2023-09-30T09:06:53.749 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:06:54.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:53 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:06:54.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:53 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:06:54.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:53 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1734286763' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-30T09:06:54.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:53 smithi019 ceph-mon[100214]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:06:55.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:54 smithi019 ceph-mon[100214]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:55.336 INFO:teuthology.orchestra.run.smithi167.stdout:Scheduled mon update... 2023-09-30T09:06:55.952 DEBUG:teuthology.orchestra.run.smithi019:mon.c> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-30T09:06:55.955 DEBUG:teuthology.orchestra.run.smithi167:mon.b> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-30T09:06:55.958 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-30T09:06:55.958 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-30T09:06:55.988 INFO:journalctl@ceph.mon.b.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:06:55.992 INFO:journalctl@ceph.mon.c.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:06:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi167:172.21.15.167=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: Saving service mon spec with placement smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi167:172.21.15.167=b;count:3 2023-09-30T09:06:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:06:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:06:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:06:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:06:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: Deploying daemon mon.b on smithi167 2023-09-30T09:06:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:56 smithi019 ceph-mon[100214]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:56.930 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:06:58.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:58 smithi167 systemd[1]: Starting Ceph mon.b for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:06:59.245 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:06:59.245 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","modified":"2023-09-30T09:05:26.832956Z","created":"2023-09-30T09:05:26.832956Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3300","nonce":0},{"type":"v1","addr":"172.21.15.19:6789","nonce":0}]},"addr":"172.21.15.19:6789/0","public_addr":"172.21.15.19:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-30T09:06:59.247 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-09-30T09:06:59.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: set uid:gid to 167:167 (ceph:ceph) 2023-09-30T09:06:59.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable), process ceph-mon, pid 7 2023-09-30T09:06:59.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: pidfile_write: ignore empty --pid-file 2023-09-30T09:06:59.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: load: jerasure load: lrc 2023-09-30T09:06:59.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: RocksDB version: 6.15.5 2023-09-30T09:06:59.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-30T09:06:59.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Compile date Sep 29 2023 2023-09-30T09:06:59.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: DB SUMMARY 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: DB Session ID: B89JKHLL7988JR1Y64QP 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: CURRENT file: CURRENT 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: IDENTITY file: IDENTITY 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.error_if_exists: 0 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 bash[104319]: b5fffa651c9c3c7a6ab65360244bd148fd7e32bda77558c4813f2c0d212aa919 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.create_if_missing: 0 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.paranoid_checks: 1 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-30T09:06:59.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.env: 0x55d53d66aa80 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.fs: Posix File System 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.info_log: 0x55d53fe5e5a0 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_file_opening_threads: 16 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.statistics: (nil) 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.use_fsync: 0 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_log_file_size: 0 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.keep_log_file_num: 1000 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.recycle_log_file_num: 0 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.allow_fallocate: 1 2023-09-30T09:06:59.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.allow_mmap_reads: 0 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.allow_mmap_writes: 0 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.use_direct_reads: 0 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.create_missing_column_families: 0 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.db_log_dir: 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-30T09:06:59.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.advise_random_on_open: 1 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.db_write_buffer_size: 0 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.write_buffer_manager: 0x55d540bd0960 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.rate_limiter: (nil) 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.wal_recovery_mode: 2 2023-09-30T09:06:59.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.enable_thread_tracking: 0 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.enable_pipelined_write: 0 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.unordered_write: 0 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.row_cache: None 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.wal_filter: None 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.allow_ingest_behind: 0 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.preserve_deletes: 0 2023-09-30T09:06:59.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.two_write_queues: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.manual_wal_flush: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.atomic_flush: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.log_readahead_size: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.best_efforts_recovery: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.allow_data_in_errors: 0 2023-09-30T09:06:59.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.db_host_id: __hostname__ 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_background_jobs: 2 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_background_compactions: -1 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_subcompactions: 1 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_total_wal_size: 0 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-30T09:06:59.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_open_files: -1 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bytes_per_sync: 0 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_readahead_size: 0 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_background_flushes: -1 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Compression algorithms supported: 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: kZSTD supported: 0 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: kXpressCompression supported: 0 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: kLZ4HCCompression supported: 1 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: kZlibCompression supported: 1 2023-09-30T09:06:59.531 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: kSnappyCompression supported: 1 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: kLZ4Compression supported: 1 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: kBZip2Compression supported: 0 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.merge_operator: 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_filter: None 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_filter_factory: None 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.sst_partitioner_factory: None 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-30T09:06:59.532 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d53feb0fb8) 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: cache_index_and_filter_blocks: 1 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: pin_top_level_index_and_filter: 1 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: index_type: 0 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: data_block_index_type: 0 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: index_shortening: 1 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: hash_index_allow_collision: 1 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: checksum: 1 2023-09-30T09:06:59.533 INFO:journalctl@ceph.mon.b.smithi167.stdout: no_block_cache: 0 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_cache: 0x55d53fee3090 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_cache_name: BinnedLRUCache 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_cache_options: 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: capacity : 536870912 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: num_shard_bits : 4 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: strict_capacity_limit : 0 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: high_pri_pool_ratio: 0.000 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_cache_compressed: (nil) 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: persistent_cache: (nil) 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_size: 4096 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_size_deviation: 10 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_restart_interval: 16 2023-09-30T09:06:59.534 INFO:journalctl@ceph.mon.b.smithi167.stdout: index_block_restart_interval: 1 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: metadata_block_size: 4096 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: partition_filters: 0 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: use_delta_encoding: 1 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: whole_key_filtering: 1 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: verify_compression: 0 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: read_amp_bytes_per_bit: 0 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: format_version: 4 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: enable_index_compression: 1 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_align: 0 2023-09-30T09:06:59.535 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.write_buffer_size: 33554432 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_write_buffer_number: 2 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compression: NoCompression 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bottommost_compression: Disabled 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.prefix_extractor: nullptr 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.num_levels: 7 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-30T09:06:59.536 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-30T09:06:59.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:06:59.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-30T09:06:59.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-30T09:06:59.538 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-30T09:06:59.538 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compression_opts.level: 32767 2023-09-30T09:06:59.538 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compression_opts.strategy: 0 2023-09-30T09:06:59.538 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-30T09:06:59.538 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:06:59.538 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-30T09:06:59.538 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compression_opts.enabled: false 2023-09-30T09:06:59.538 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.target_file_size_base: 67108864 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-30T09:06:59.539 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.arena_block_size: 4194304 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.disable_auto_compactions: 0 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-30T09:06:59.540 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.table_properties_collectors: 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.inplace_update_support: 0 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-30T09:06:59.541 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.bloom_locality: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.max_successive_merges: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.paranoid_file_checks: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.force_consistency_checks: 1 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.report_bg_io_stats: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.ttl: 2592000 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.enable_blob_files: false 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.min_blob_size: 0 2023-09-30T09:06:59.542 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.blob_file_size: 268435456 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064819260839, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064819261212, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696064819, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "04629313-cfe7-4259-b5a8-54bfc3bd1673", "db_session_id": "B89JKHLL7988JR1Y64QP"}} 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-30T09:06:59.543 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064819261640, "job": 1, "event": "recovery_finished"} 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 systemd[1]: Started Ceph mon.b for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55d53fe77180 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: DB pointer 0x55d53ff4c000 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** DB Stats ** 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:06:59.544 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** Compaction Stats [default] ** 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** Compaction Stats [default] ** 2023-09-30T09:06:59.545 INFO:journalctl@ceph.mon.b.smithi167.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:06:59.546 INFO:journalctl@ceph.mon.b.smithi167.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** Compaction Stats [default] ** 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** Compaction Stats [default] ** 2023-09-30T09:06:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:06:59.548 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: using public_addr v2:172.21.15.167:0/0 -> [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: starting mon.b rank -1 at public addrs [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] at bind addrs [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(???) e0 preinit fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).mds e1 new map 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).mds e1 print_map 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout: e1 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout: legacy client fscid: -1 2023-09-30T09:06:59.549 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout: No filesystems configured 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:06:59.550 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi167", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:06:59.551 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1734286763' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1734286763' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi167:172.21.15.167=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:06:59.552 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: Saving service mon spec with placement smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi167:172.21.15.167=b;count:3 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: Deploying daemon mon.b on smithi167 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-30T09:06:59.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:06:59 smithi167 ceph-mon[104403]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-09-30T09:06:59.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:06:59 smithi019 ceph-mon[100214]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:01.001 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-30T09:07:01.001 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-30T09:07:01.347 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:07:04.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: mon.a calling monitor election 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: mon.b calling monitor election 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:04.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: monmap e2: 2 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} removed_ranks: {} 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: fsmap 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: mgrmap e13: y(active, since 38s) 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: overall HEALTH_OK 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:04.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:04.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:07:04.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: mon.a calling monitor election 2023-09-30T09:07:04.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:04.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: mon.b calling monitor election 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: monmap e2: 2 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} removed_ranks: {} 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: fsmap 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:07:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: mgrmap e13: y(active, since 38s) 2023-09-30T09:07:04.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: overall HEALTH_OK 2023-09-30T09:07:04.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:04.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:04.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:05 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:05 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:05 smithi167 ceph-mon[104403]: Deploying daemon mon.c on smithi019 2023-09-30T09:07:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:05 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:05.823 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:05 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:05.824 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:05 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:05.824 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:05 smithi019 ceph-mon[100214]: Deploying daemon mon.c on smithi019 2023-09-30T09:07:05.824 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:05 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:06.008 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:07:06.008 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":2,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","modified":"2023-09-30T09:06:59.285753Z","created":"2023-09-30T09:05:26.832956Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3300","nonce":0},{"type":"v1","addr":"172.21.15.19:6789","nonce":0}]},"addr":"172.21.15.19:6789/0","public_addr":"172.21.15.19:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:3300","nonce":0},{"type":"v1","addr":"172.21.15.167:6789","nonce":0}]},"addr":"172.21.15.167:6789/0","public_addr":"172.21.15.167:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-30T09:07:06.010 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 2 2023-09-30T09:07:06.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:06 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/2784779418' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:07:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:06 smithi167 ceph-mon[104403]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:06.824 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:06 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/2784779418' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:07:06.824 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:06 smithi019 ceph-mon[100214]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 systemd[1]: Starting Ceph mon.c for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:07:07.558 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-30T09:07:07.558 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-30T09:07:07.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: set uid:gid to 167:167 (ceph:ceph) 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable), process ceph-mon, pid 7 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pidfile_write: ignore empty --pid-file 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: load: jerasure load: lrc 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: RocksDB version: 6.15.5 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Compile date Sep 29 2023 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: DB SUMMARY 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: DB Session ID: XGE4ZQCCX1GR034HHO11 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: CURRENT file: CURRENT 2023-09-30T09:07:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: IDENTITY file: IDENTITY 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.error_if_exists: 0 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.create_if_missing: 0 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.paranoid_checks: 1 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.env: 0x5638c7453a80 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.fs: Posix File System 2023-09-30T09:07:07.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.info_log: 0x5638c80105a0 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_file_opening_threads: 16 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.statistics: (nil) 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.use_fsync: 0 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_log_file_size: 0 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.keep_log_file_num: 1000 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.recycle_log_file_num: 0 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.allow_fallocate: 1 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.allow_mmap_reads: 0 2023-09-30T09:07:07.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.allow_mmap_writes: 0 2023-09-30T09:07:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.use_direct_reads: 0 2023-09-30T09:07:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-30T09:07:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.create_missing_column_families: 0 2023-09-30T09:07:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.db_log_dir: 2023-09-30T09:07:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-30T09:07:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-30T09:07:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-30T09:07:07.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.advise_random_on_open: 1 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.db_write_buffer_size: 0 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.write_buffer_manager: 0x5638c8d82930 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-30T09:07:07.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.rate_limiter: (nil) 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.wal_recovery_mode: 2 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.enable_thread_tracking: 0 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.enable_pipelined_write: 0 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.unordered_write: 0 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.row_cache: None 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.wal_filter: None 2023-09-30T09:07:07.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.allow_ingest_behind: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.preserve_deletes: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.two_write_queues: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.manual_wal_flush: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.atomic_flush: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.log_readahead_size: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.best_efforts_recovery: 0 2023-09-30T09:07:07.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.allow_data_in_errors: 0 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.db_host_id: __hostname__ 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_background_jobs: 2 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_background_compactions: -1 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_subcompactions: 1 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_total_wal_size: 0 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-30T09:07:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_open_files: -1 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bytes_per_sync: 0 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_readahead_size: 0 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_background_flushes: -1 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Compression algorithms supported: 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: kZSTD supported: 0 2023-09-30T09:07:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: kXpressCompression supported: 0 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: kLZ4HCCompression supported: 1 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: kZlibCompression supported: 1 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: kSnappyCompression supported: 1 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: kLZ4Compression supported: 1 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: kBZip2Compression supported: 0 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.merge_operator: 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_filter: None 2023-09-30T09:07:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_filter_factory: None 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.sst_partitioner_factory: None 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5638c8062fb8) 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: cache_index_and_filter_blocks: 1 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: pin_top_level_index_and_filter: 1 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_type: 0 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_block_index_type: 0 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_shortening: 1 2023-09-30T09:07:07.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: hash_index_allow_collision: 1 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: checksum: 1 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: no_block_cache: 0 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache: 0x5638c8095090 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_name: BinnedLRUCache 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_options: 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: capacity : 536870912 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: num_shard_bits : 4 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: strict_capacity_limit : 0 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: high_pri_pool_ratio: 0.000 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_compressed: (nil) 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: persistent_cache: (nil) 2023-09-30T09:07:07.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_size: 4096 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_size_deviation: 10 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_restart_interval: 16 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_block_restart_interval: 1 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: metadata_block_size: 4096 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: partition_filters: 0 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: use_delta_encoding: 1 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: whole_key_filtering: 1 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: verify_compression: 0 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: read_amp_bytes_per_bit: 0 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: format_version: 4 2023-09-30T09:07:07.849 INFO:journalctl@ceph.mon.c.smithi019.stdout: enable_index_compression: 1 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_align: 0 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.write_buffer_size: 33554432 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_write_buffer_number: 2 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compression: NoCompression 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bottommost_compression: Disabled 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.prefix_extractor: nullptr 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.num_levels: 7 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-30T09:07:07.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compression_opts.level: 32767 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compression_opts.strategy: 0 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-30T09:07:07.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compression_opts.enabled: false 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.target_file_size_base: 67108864 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-30T09:07:07.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.arena_block_size: 4194304 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-30T09:07:07.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.disable_auto_compactions: 0 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-30T09:07:07.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.table_properties_collectors: 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.inplace_update_support: 0 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.bloom_locality: 0 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.max_successive_merges: 0 2023-09-30T09:07:07.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.paranoid_file_checks: 0 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.force_consistency_checks: 1 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.report_bg_io_stats: 0 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.ttl: 2592000 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.enable_blob_files: false 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.min_blob_size: 0 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.blob_file_size: 268435456 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-30T09:07:07.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064827619342, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064827619640, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696064827, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "abb82521-27c9-4c96-88f3-aef79f7e762d", "db_session_id": "XGE4ZQCCX1GR034HHO11"}} 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696064827620028, "job": 1, "event": "recovery_finished"} 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5638c8029180 2023-09-30T09:07:07.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: DB pointer 0x5638c80fe000 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** DB Stats ** 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:07:07.858 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.7 0.00 0.00 1 0.000 0 0 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.7 0.00 0.00 1 0.000 0 0 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.7 0.00 0.00 1 0.000 0 0 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.7 0.00 0.00 1 0.000 0 0 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:07:07.859 INFO:journalctl@ceph.mon.c.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:07:07.860 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.7 0.00 0.00 1 0.000 0 0 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.7 0.00 0.00 1 0.000 0 0 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.7 0.00 0.00 1 0.000 0 0 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-30T09:07:07.861 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: using public_addrv [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] 2023-09-30T09:07:07.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: starting mon.c rank -1 at public addrs [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] at bind addrs [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(???) e0 preinit fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).mds e1 new map 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: e1 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: legacy client fscid: -1 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: No filesystems configured 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-30T09:07:07.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:07:07.864 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi167", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1734286763' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1734286763' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi167:172.21.15.167=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: Saving service mon spec with placement smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi167:172.21.15.167=b;count:3 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: Deploying daemon mon.b on smithi167 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.a calling monitor election 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.867 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.b calling monitor election 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: monmap e2: 2 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} removed_ranks: {} 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: fsmap 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mgrmap e13: y(active, since 38s) 2023-09-30T09:07:07.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: overall HEALTH_OK 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: Deploying daemon mon.c on smithi019 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/2784779418' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 ceph-mon[118983]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-30T09:07:07.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 bash[118917]: 88df4d08fbd44b1cd0de427cc3e180e257a54a2a4418417b0652ce3153b6bf82 2023-09-30T09:07:07.870 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:07 smithi019 systemd[1]: Started Ceph mon.c for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:07:07.905 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:07:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:07:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: mon.a calling monitor election 2023-09-30T09:07:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: mon.b calling monitor election 2023-09-30T09:07:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: mon.c calling monitor election 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:07:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: fsmap 2023-09-30T09:07:13.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:07:13.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: mgrmap e13: y(active, since 46s) 2023-09-30T09:07:13.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: overall HEALTH_OK 2023-09-30T09:07:13.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: mon.a calling monitor election 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: mon.b calling monitor election 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: mon.c calling monitor election 2023-09-30T09:07:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: fsmap 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: mgrmap e13: y(active, since 46s) 2023-09-30T09:07:13.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: overall HEALTH_OK 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: mon.a calling monitor election 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: mon.b calling monitor election 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:07:13.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: mon.c calling monitor election 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:07:13.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: fsmap 2023-09-30T09:07:13.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: osdmap e4: 0 total, 0 up, 0 in 2023-09-30T09:07:13.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: mgrmap e13: y(active, since 46s) 2023-09-30T09:07:13.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: overall HEALTH_OK 2023-09-30T09:07:13.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:13.118 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-09-30T09:07:13.119 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":3,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","modified":"2023-09-30T09:07:07.641353Z","created":"2023-09-30T09:05:26.832956Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3300","nonce":0},{"type":"v1","addr":"172.21.15.19:6789","nonce":0}]},"addr":"172.21.15.19:6789/0","public_addr":"172.21.15.19:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:3300","nonce":0},{"type":"v1","addr":"172.21.15.167:6789","nonce":0}]},"addr":"172.21.15.167:6789/0","public_addr":"172.21.15.167:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3301","nonce":0},{"type":"v1","addr":"172.21.15.19:6790","nonce":0}]},"addr":"172.21.15.19:6790/0","public_addr":"172.21.15.19:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-09-30T09:07:13.121 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 3 2023-09-30T09:07:13.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:13.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:13 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/983997984' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:07:13.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:13.991 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-30T09:07:13.992 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-30T09:07:14.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:14.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:13 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/983997984' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:07:14.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:14.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:14.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:13 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/983997984' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:07:14.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:07:14.374 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:07:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:07:15.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:15.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:15.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:14 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:15.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:15.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:15.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:15.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:14 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:15 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:07:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:15 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:07:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:15 smithi167 ceph-mon[104403]: Reconfiguring mon.a (unknown last config time)... 2023-09-30T09:07:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:15 smithi167 ceph-mon[104403]: Reconfiguring daemon mon.a on smithi019 2023-09-30T09:07:16.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:15 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:07:16.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:15 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:07:16.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:15 smithi019 ceph-mon[100214]: Reconfiguring mon.a (unknown last config time)... 2023-09-30T09:07:16.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:15 smithi019 ceph-mon[100214]: Reconfiguring daemon mon.a on smithi019 2023-09-30T09:07:16.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:15 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:07:16.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:15 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:07:16.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:15 smithi019 ceph-mon[118983]: Reconfiguring mon.a (unknown last config time)... 2023-09-30T09:07:16.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:15 smithi019 ceph-mon[118983]: Reconfiguring daemon mon.a on smithi019 2023-09-30T09:07:16.440 INFO:teuthology.orchestra.run.smithi019.stdout:# minimal ceph.conf for 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:07:16.440 INFO:teuthology.orchestra.run.smithi019.stdout:[global] 2023-09-30T09:07:16.440 INFO:teuthology.orchestra.run.smithi019.stdout: fsid = 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:07:16.440 INFO:teuthology.orchestra.run.smithi019.stdout: mon_host = [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] 2023-09-30T09:07:17.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:17 smithi019 ceph-mon[118983]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:17.333 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:17 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:17.333 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:17 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2248445659' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:17.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:17 smithi019 ceph-mon[100214]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:17.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:17 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:17.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:17 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2248445659' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:17.349 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-30T09:07:17.350 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:07:17.350 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/ceph/ceph.conf 2023-09-30T09:07:17.419 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:07:17.419 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:07:17.490 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:07:17.490 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/ceph/ceph.conf 2023-09-30T09:07:17.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:17 smithi167 ceph-mon[104403]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:17.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:17 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:17 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2248445659' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:17.527 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:07:17.527 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:07:17.598 INFO:tasks.cephadm:Adding mgr.y on smithi019 2023-09-30T09:07:17.598 INFO:tasks.cephadm:Adding mgr.x on smithi167 2023-09-30T09:07:17.599 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi019=y;smithi167=x' 2023-09-30T09:07:17.972 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[100214]: Reconfiguring mon.c (monmap changed)... 2023-09-30T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[100214]: Reconfiguring daemon mon.c on smithi019 2023-09-30T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[100214]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[118983]: Reconfiguring mon.c (monmap changed)... 2023-09-30T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[118983]: Reconfiguring daemon mon.c on smithi019 2023-09-30T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:18 smithi019 ceph-mon[118983]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:18 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:18 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:18 smithi167 ceph-mon[104403]: Reconfiguring mon.c (monmap changed)... 2023-09-30T09:07:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:18 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:18 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:18 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:18 smithi167 ceph-mon[104403]: Reconfiguring daemon mon.c on smithi019 2023-09-30T09:07:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:18 smithi167 ceph-mon[104403]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:19.685 INFO:teuthology.orchestra.run.smithi167.stdout:Scheduled mgr update... 2023-09-30T09:07:19.987 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:19.987 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:19.987 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:19 smithi167 ceph-mon[104403]: Reconfiguring mon.b (monmap changed)... 2023-09-30T09:07:19.987 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:19.987 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:19.988 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:19.988 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:19 smithi167 ceph-mon[104403]: Reconfiguring daemon mon.b on smithi167 2023-09-30T09:07:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[100214]: Reconfiguring mon.b (monmap changed)... 2023-09-30T09:07:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[100214]: Reconfiguring daemon mon.b on smithi167 2023-09-30T09:07:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[118983]: Reconfiguring mon.b (monmap changed)... 2023-09-30T09:07:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:07:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:07:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:19 smithi019 ceph-mon[118983]: Reconfiguring daemon mon.b on smithi167 2023-09-30T09:07:20.606 DEBUG:teuthology.orchestra.run.smithi167:mgr.x> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-30T09:07:20.608 INFO:tasks.cephadm:Deploying OSDs... 2023-09-30T09:07:20.608 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:07:20.608 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/scratch_devs of=/dev/stdout 2023-09-30T09:07:20.630 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-30T09:07:20.630 DEBUG:teuthology.orchestra.run.smithi019:> stat /dev/vg_nvme/lv_1 2023-09-30T09:07:20.639 INFO:journalctl@ceph.mgr.x.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout:Device: 6h/6d Inode: 383869 Links: 1 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout:Context: system_u:object_r:device_t:s0 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout:Access: 2023-09-30 09:07:07.013227681 +0000 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout:Modify: 2023-09-30 08:58:27.757327715 +0000 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout:Change: 2023-09-30 08:58:27.757327715 +0000 2023-09-30T09:07:20.693 INFO:teuthology.orchestra.run.smithi019.stdout: Birth: - 2023-09-30T09:07:20.694 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-30T09:07:20.764 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records in 2023-09-30T09:07:20.764 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records out 2023-09-30T09:07:20.764 INFO:teuthology.orchestra.run.smithi019.stderr:512 bytes copied, 0.000245091 s, 2.1 MB/s 2023-09-30T09:07:20.765 DEBUG:teuthology.orchestra.run.smithi019:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-30T09:07:20.826 DEBUG:teuthology.orchestra.run.smithi019:> stat /dev/vg_nvme/lv_2 2023-09-30T09:07:20.831 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi019=y;smithi167=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:20.831 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: Saving service mgr spec with placement smithi019=y;smithi167=x;count:2 2023-09-30T09:07:20.831 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:20.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:20 smithi167 ceph-mon[104403]: Deploying daemon mgr.x on smithi167 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout:Device: 6h/6d Inode: 384634 Links: 1 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout:Context: system_u:object_r:device_t:s0 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout:Access: 2023-09-30 09:07:07.013227681 +0000 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout:Modify: 2023-09-30 08:58:28.279323503 +0000 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout:Change: 2023-09-30 08:58:28.279323503 +0000 2023-09-30T09:07:20.886 INFO:teuthology.orchestra.run.smithi019.stdout: Birth: - 2023-09-30T09:07:20.887 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-30T09:07:20.953 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi019=y;smithi167=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: Saving service mgr spec with placement smithi019=y;smithi167=x;count:2 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.954 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[118983]: Deploying daemon mgr.x on smithi167 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi019=y;smithi167=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: Saving service mgr spec with placement smithi019=y;smithi167=x;count:2 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.955 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:20.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:20 smithi019 ceph-mon[100214]: Deploying daemon mgr.x on smithi167 2023-09-30T09:07:20.956 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records in 2023-09-30T09:07:20.957 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records out 2023-09-30T09:07:20.957 INFO:teuthology.orchestra.run.smithi019.stderr:512 bytes copied, 0.000225991 s, 2.3 MB/s 2023-09-30T09:07:20.958 DEBUG:teuthology.orchestra.run.smithi019:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-30T09:07:21.018 DEBUG:teuthology.orchestra.run.smithi019:> stat /dev/vg_nvme/lv_3 2023-09-30T09:07:21.078 INFO:teuthology.orchestra.run.smithi019.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-30T09:07:21.078 INFO:teuthology.orchestra.run.smithi019.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-30T09:07:21.078 INFO:teuthology.orchestra.run.smithi019.stdout:Device: 6h/6d Inode: 392214 Links: 1 2023-09-30T09:07:21.078 INFO:teuthology.orchestra.run.smithi019.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-30T09:07:21.079 INFO:teuthology.orchestra.run.smithi019.stdout:Context: system_u:object_r:device_t:s0 2023-09-30T09:07:21.079 INFO:teuthology.orchestra.run.smithi019.stdout:Access: 2023-09-30 09:07:07.014227672 +0000 2023-09-30T09:07:21.079 INFO:teuthology.orchestra.run.smithi019.stdout:Modify: 2023-09-30 08:58:28.797319322 +0000 2023-09-30T09:07:21.079 INFO:teuthology.orchestra.run.smithi019.stdout:Change: 2023-09-30 08:58:28.797319322 +0000 2023-09-30T09:07:21.079 INFO:teuthology.orchestra.run.smithi019.stdout: Birth: - 2023-09-30T09:07:21.079 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-30T09:07:21.146 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records in 2023-09-30T09:07:21.146 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records out 2023-09-30T09:07:21.146 INFO:teuthology.orchestra.run.smithi019.stderr:512 bytes copied, 0.000248803 s, 2.1 MB/s 2023-09-30T09:07:21.148 DEBUG:teuthology.orchestra.run.smithi019:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-30T09:07:21.207 DEBUG:teuthology.orchestra.run.smithi019:> stat /dev/vg_nvme/lv_4 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout:Device: 6h/6d Inode: 391377 Links: 1 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout:Context: system_u:object_r:device_t:s0 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout:Access: 2023-09-30 09:07:07.014227672 +0000 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout:Modify: 2023-09-30 08:58:29.303315238 +0000 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout:Change: 2023-09-30 08:58:29.303315238 +0000 2023-09-30T09:07:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: Birth: - 2023-09-30T09:07:21.268 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-30T09:07:21.335 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records in 2023-09-30T09:07:21.335 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records out 2023-09-30T09:07:21.335 INFO:teuthology.orchestra.run.smithi019.stderr:512 bytes copied, 0.000196487 s, 2.6 MB/s 2023-09-30T09:07:21.337 DEBUG:teuthology.orchestra.run.smithi019:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-30T09:07:21.396 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:07:21.396 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/scratch_devs of=/dev/stdout 2023-09-30T09:07:21.413 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-30T09:07:21.413 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_1 2023-09-30T09:07:21.470 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-30T09:07:21.471 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-30T09:07:21.471 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 6h/6d Inode: 387683 Links: 1 2023-09-30T09:07:21.471 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-30T09:07:21.471 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2023-09-30T09:07:21.471 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2023-09-30 09:06:58.486460359 +0000 2023-09-30T09:07:21.471 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2023-09-30 08:58:06.036514857 +0000 2023-09-30T09:07:21.471 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2023-09-30 08:58:06.036514857 +0000 2023-09-30T09:07:21.471 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: - 2023-09-30T09:07:21.471 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-30T09:07:21.603 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2023-09-30T09:07:21.604 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2023-09-30T09:07:21.604 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000250845 s, 2.0 MB/s 2023-09-30T09:07:21.605 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-30T09:07:21.622 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_2 2023-09-30T09:07:21.680 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-30T09:07:21.680 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-30T09:07:21.680 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 6h/6d Inode: 392219 Links: 1 2023-09-30T09:07:21.680 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-30T09:07:21.681 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2023-09-30T09:07:21.681 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2023-09-30 09:07:21.553444805 +0000 2023-09-30T09:07:21.681 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2023-09-30 08:58:06.558515098 +0000 2023-09-30T09:07:21.681 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2023-09-30 08:58:06.558515098 +0000 2023-09-30T09:07:21.681 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: - 2023-09-30T09:07:21.681 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-30T09:07:21.748 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2023-09-30T09:07:21.748 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2023-09-30T09:07:21.748 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000209457 s, 2.4 MB/s 2023-09-30T09:07:21.749 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-30T09:07:21.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:21 smithi167 systemd[1]: Starting Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:07:21.789 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_3 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 6h/6d Inode: 392269 Links: 1 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2023-09-30 09:07:21.678444720 +0000 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2023-09-30 08:58:07.062515331 +0000 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2023-09-30 08:58:07.062515331 +0000 2023-09-30T09:07:21.847 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: - 2023-09-30T09:07:21.848 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-30T09:07:21.914 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2023-09-30T09:07:21.914 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2023-09-30T09:07:21.914 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000178658 s, 2.9 MB/s 2023-09-30T09:07:21.915 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-30T09:07:21.974 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_4 2023-09-30T09:07:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-30T09:07:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-30T09:07:22.032 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 6h/6d Inode: 393362 Links: 1 2023-09-30T09:07:22.032 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-30T09:07:22.032 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2023-09-30T09:07:22.032 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2023-09-30 09:07:21.678444720 +0000 2023-09-30T09:07:22.032 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2023-09-30 08:58:07.582515570 +0000 2023-09-30T09:07:22.032 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2023-09-30 08:58:07.582515570 +0000 2023-09-30T09:07:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: - 2023-09-30T09:07:22.032 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-30T09:07:22.097 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2023-09-30T09:07:22.097 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2023-09-30T09:07:22.097 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000224727 s, 2.3 MB/s 2023-09-30T09:07:22.098 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-30T09:07:22.156 INFO:tasks.cephadm:Deploying osd.0 on smithi019 with /dev/vg_nvme/lv_4... 2023-09-30T09:07:22.157 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-30T09:07:22.519 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:07:22.593 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:22 smithi167 ceph-mgr[107656]: -- 172.21.15.167:0/455122704 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556de0ba09c0 con 0x556de0b78c00 2023-09-30T09:07:22.593 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:22 smithi167 bash[107506]: 6176c234bba154419f88145510827da36ad02476c4b9d5d8326302f81baf7d5c 2023-09-30T09:07:22.593 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:22 smithi167 systemd[1]: Started Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:07:22.593 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:22 smithi167 conmon[107623]: 2023-09-30T09:07:22.422+0000 7fc48211d1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:07:22.593 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:22 smithi167 conmon[107623]: 2023-09-30T09:07:22.499+0000 7fc48211d1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:07:22.593 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:22 smithi167 conmon[107623]: 2023-09-30T09:07:22.591+0000 7fc48211d1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:07:22.874 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:22 smithi167 conmon[107623]: 2023-09-30T09:07:22.690+0000 7fc48211d1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:07:23.220 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:22 smithi167 conmon[107623]: 2023-09-30T09:07:22.871+0000 7fc48211d1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:07:23.220 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:23 smithi167 ceph-mon[104403]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:23.220 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.221 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.221 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.221 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.221 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[118983]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:23.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[100214]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:23.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:23.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:23.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:23 smithi167 conmon[107623]: 2023-09-30T09:07:23.218+0000 7fc48211d1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:07:23.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:23 smithi167 conmon[107623]: 2023-09-30T09:07:23.310+0000 7fc48211d1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:07:23.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:23 smithi167 conmon[107623]: 2023-09-30T09:07:23.448+0000 7fc48211d1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:07:23.970 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:23 smithi167 conmon[107623]: 2023-09-30T09:07:23.550+0000 7fc48211d1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:07:24.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:23 smithi167 conmon[107623]: 2023-09-30T09:07:23.968+0000 7fc48211d1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:07:24.403 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-30T09:07:24.403 INFO:teuthology.orchestra.run.smithi019.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-30T09:07:24.403 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10+0 records in 2023-09-30T09:07:24.403 INFO:teuthology.orchestra.run.smithi019.stderr:10+0 records out 2023-09-30T09:07:24.403 INFO:teuthology.orchestra.run.smithi019.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0126326 s, 830 MB/s 2023-09-30T09:07:24.403 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping successful for: 2023-09-30T09:07:24.775 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:24.775 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:24.775 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:24.775 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:24.775 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:24.775 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: Reconfiguring mgr.y (unknown last config time)... 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: Reconfiguring daemon mgr.y on smithi019 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[100214]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:24.776 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:24.777 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: Reconfiguring mgr.y (unknown last config time)... 2023-09-30T09:07:24.777 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:07:24.777 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:07:24.777 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:24.777 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: Reconfiguring daemon mgr.y on smithi019 2023-09-30T09:07:24.777 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:24 smithi019 ceph-mon[118983]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:25.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:24 smithi167 conmon[107623]: 2023-09-30T09:07:24.536+0000 7fc48211d1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:07:25.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:24 smithi167 conmon[107623]: 2023-09-30T09:07:24.704+0000 7fc48211d1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:07:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: Reconfiguring mgr.y (unknown last config time)... 2023-09-30T09:07:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:07:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:07:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: Reconfiguring daemon mgr.y on smithi019 2023-09-30T09:07:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:24 smithi167 ceph-mon[104403]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:25.426 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi019:vg_nvme/lv_4 2023-09-30T09:07:25.710 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:25 smithi167 conmon[107623]: 2023-09-30T09:07:25.455+0000 7fc48211d1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:07:25.710 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:25 smithi167 conmon[107623]: 2023-09-30T09:07:25.542+0000 7fc48211d1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:07:25.793 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:07:25.985 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:25 smithi167 conmon[107623]: 2023-09-30T09:07:25.708+0000 7fc48211d1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:07:26.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:25 smithi167 conmon[107623]: 2023-09-30T09:07:25.983+0000 7fc48211d1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:07:26.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:26 smithi167 conmon[107623]: 2023-09-30T09:07:26.068+0000 7fc48211d1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:07:26.610 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:26 smithi167 conmon[107623]: 2023-09-30T09:07:26.355+0000 7fc48211d1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:07:27.021 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:26 smithi167 conmon[107623]: 2023-09-30T09:07:26.608+0000 7fc48211d1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:07:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:26 smithi167 ceph-mon[104403]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[118983]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[100214]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:27.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:27.620 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:27 smithi167 conmon[107623]: 2023-09-30T09:07:27.307+0000 7fc48211d1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:07:27.620 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:27 smithi167 conmon[107623]: 2023-09-30T09:07:27.435+0000 7fc48211d1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:07:27.942 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:27 smithi167 conmon[107623]: 2023-09-30T09:07:27.618+0000 7fc48211d1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:07:27.942 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:27 smithi167 conmon[107623]: 2023-09-30T09:07:27.730+0000 7fc48211d1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:07:27.942 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:27 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:07:27.942 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:27 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:07:27.943 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:27 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:28.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:27 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:07:28.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:27 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:07:28.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:27 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:28.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:27 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:07:28.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:27 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:07:28.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:27 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:28.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:27 smithi167 conmon[107623]: 2023-09-30T09:07:27.940+0000 7fc48211d1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:07:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:28 smithi167 ceph-mon[104403]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:28 smithi167 ceph-mon[104403]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:29.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:07:28 smithi167 conmon[107623]: 2023-09-30T09:07:28.784+0000 7fc48211d1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:07:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:28 smithi019 ceph-mon[100214]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:28 smithi019 ceph-mon[100214]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:28 smithi019 ceph-mon[118983]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:28 smithi019 ceph-mon[118983]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:29 smithi167 ceph-mon[104403]: Standby manager daemon x started 2023-09-30T09:07:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:29 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:07:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:29 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:07:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:29 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:07:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:29 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:07:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[100214]: Standby manager daemon x started 2023-09-30T09:07:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:07:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:07:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:07:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:07:30.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[118983]: Standby manager daemon x started 2023-09-30T09:07:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:07:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:07:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:07:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:29 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1323210978' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:07:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:30 smithi167 ceph-mon[104403]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-30T09:07:31.034 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:30 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:07:31.035 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:30 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/4286437062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b26362-fc1a-4b53-a717-f8d48720a017"}]: dispatch 2023-09-30T09:07:31.035 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:30 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/4286437062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91b26362-fc1a-4b53-a717-f8d48720a017"}]': finished 2023-09-30T09:07:31.035 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:30 smithi167 ceph-mon[104403]: osdmap e5: 1 total, 0 up, 1 in 2023-09-30T09:07:31.035 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:30 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:31.035 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:30 smithi167 ceph-mon[104403]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:31.035 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:30 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3606757570' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:07:31.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[100214]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-30T09:07:31.140 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:07:31.140 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4286437062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b26362-fc1a-4b53-a717-f8d48720a017"}]: dispatch 2023-09-30T09:07:31.140 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4286437062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91b26362-fc1a-4b53-a717-f8d48720a017"}]': finished 2023-09-30T09:07:31.140 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[100214]: osdmap e5: 1 total, 0 up, 1 in 2023-09-30T09:07:31.140 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:31.140 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[100214]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:31.140 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3606757570' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:07:31.141 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[118983]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-30T09:07:31.141 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:07:31.141 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/4286437062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b26362-fc1a-4b53-a717-f8d48720a017"}]: dispatch 2023-09-30T09:07:31.141 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/4286437062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91b26362-fc1a-4b53-a717-f8d48720a017"}]': finished 2023-09-30T09:07:31.141 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[118983]: osdmap e5: 1 total, 0 up, 1 in 2023-09-30T09:07:31.141 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:31.141 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[118983]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:31.141 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:30 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3606757570' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:07:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:33 smithi019 ceph-mon[100214]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:33 smithi019 ceph-mon[118983]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:33 smithi167 ceph-mon[104403]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:35.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:35 smithi019 ceph-mon[100214]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:35.299 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:35 smithi019 ceph-mon[118983]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:35.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:35 smithi167 ceph-mon[104403]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:37.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:37 smithi019 ceph-mon[100214]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:37.197 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:07:37.197 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:37.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:37 smithi019 ceph-mon[118983]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:37.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:07:37.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:37 smithi167 ceph-mon[104403]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:07:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:38.290 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:38 smithi019 ceph-mon[100214]: Deploying daemon osd.0 on smithi019 2023-09-30T09:07:38.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:38 smithi019 ceph-mon[118983]: Deploying daemon osd.0 on smithi019 2023-09-30T09:07:38.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:38 smithi167 ceph-mon[104403]: Deploying daemon osd.0 on smithi019 2023-09-30T09:07:39.283 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:39 smithi019 ceph-mon[100214]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:39.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:39 smithi019 ceph-mon[118983]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:39.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:39 smithi167 ceph-mon[104403]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:40 smithi019 ceph-mon[100214]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:40 smithi019 ceph-mon[118983]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:40 smithi167 ceph-mon[104403]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:41 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:41 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:41 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:41 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:41 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:41 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:41 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:41 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:41 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:42 smithi019 ceph-mon[100214]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:42 smithi019 ceph-mon[118983]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:42 smithi167 ceph-mon[104403]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:44.533 INFO:teuthology.orchestra.run.smithi019.stdout:Created osd(s) 0 on host 'smithi019' 2023-09-30T09:07:44.729 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.729 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.730 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:44.730 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:44.730 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.730 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[118983]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:44.730 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.730 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.730 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:44.730 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:44.731 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.731 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:44 smithi019 ceph-mon[100214]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:44 smithi167 ceph-mon[104403]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:45.127 DEBUG:teuthology.orchestra.run.smithi019:osd.0> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-30T09:07:45.129 INFO:tasks.cephadm:Deploying osd.1 on smithi019 with /dev/vg_nvme/lv_3... 2023-09-30T09:07:45.130 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-30T09:07:45.161 INFO:journalctl@ceph.osd.0.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:07:45.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[100214]: from='osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:07:45.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[100214]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:07:45.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:45.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:45.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:45.405 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[118983]: from='osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:07:45.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[118983]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:07:45.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:45.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:45.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:45 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:45 smithi167 ceph-mon[104403]: from='osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:07:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:45 smithi167 ceph-mon[104403]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:07:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:45 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:45 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:07:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:45 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:46.182 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:07:46.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[100214]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:07:46.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[100214]: osdmap e6: 1 total, 0 up, 1 in 2023-09-30T09:07:46.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:46.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[100214]: from='osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:07:46.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[100214]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:07:46.444 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[100214]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:46.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[118983]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:07:46.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[118983]: osdmap e6: 1 total, 0 up, 1 in 2023-09-30T09:07:46.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:46.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[118983]: from='osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:07:46.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[118983]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:07:46.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:46 smithi019 ceph-mon[118983]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:46.445 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:07:46 smithi019 conmon[125233]: 2023-09-30T09:07:46.346+0000 7fbd4ba14700 -1 osd.0 0 waiting for initial osdmap 2023-09-30T09:07:46.445 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:07:46 smithi019 conmon[125233]: 2023-09-30T09:07:46.348+0000 7fbd4600e700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:07:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:46 smithi167 ceph-mon[104403]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:07:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:46 smithi167 ceph-mon[104403]: osdmap e6: 1 total, 0 up, 1 in 2023-09-30T09:07:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:46 smithi167 ceph-mon[104403]: from='osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:07:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:46 smithi167 ceph-mon[104403]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:07:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:46 smithi167 ceph-mon[104403]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:47.680 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:47 smithi019 ceph-mon[100214]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:07:47.680 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:47 smithi019 ceph-mon[100214]: osdmap e7: 1 total, 0 up, 1 in 2023-09-30T09:07:47.680 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:47 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:47.680 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:47 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:47.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:47 smithi019 ceph-mon[118983]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:07:47.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:47 smithi019 ceph-mon[118983]: osdmap e7: 1 total, 0 up, 1 in 2023-09-30T09:07:47.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:47 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:47.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:47 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:47 smithi167 ceph-mon[104403]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:07:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:47 smithi167 ceph-mon[104403]: osdmap e7: 1 total, 0 up, 1 in 2023-09-30T09:07:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:47 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:47 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:48 smithi167 ceph-mon[104403]: purged_snaps scrub starts 2023-09-30T09:07:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:48 smithi167 ceph-mon[104403]: purged_snaps scrub ok 2023-09-30T09:07:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:48 smithi167 ceph-mon[104403]: osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465] boot 2023-09-30T09:07:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:48 smithi167 ceph-mon[104403]: osdmap e8: 1 total, 1 up, 1 in 2023-09-30T09:07:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:48 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:48 smithi167 ceph-mon[104403]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[118983]: purged_snaps scrub starts 2023-09-30T09:07:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[118983]: purged_snaps scrub ok 2023-09-30T09:07:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[118983]: osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465] boot 2023-09-30T09:07:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[118983]: osdmap e8: 1 total, 1 up, 1 in 2023-09-30T09:07:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[118983]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[100214]: purged_snaps scrub starts 2023-09-30T09:07:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[100214]: purged_snaps scrub ok 2023-09-30T09:07:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[100214]: osd.0 [v2:172.21.15.19:6802/1200588465,v1:172.21.15.19:6803/1200588465] boot 2023-09-30T09:07:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[100214]: osdmap e8: 1 total, 1 up, 1 in 2023-09-30T09:07:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:07:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:48 smithi019 ceph-mon[100214]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-30T09:07:48.882 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-30T09:07:48.883 INFO:teuthology.orchestra.run.smithi019.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-30T09:07:48.883 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10+0 records in 2023-09-30T09:07:48.883 INFO:teuthology.orchestra.run.smithi019.stderr:10+0 records out 2023-09-30T09:07:48.883 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111303 s, 942 MB/s 2023-09-30T09:07:48.883 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping successful for: 2023-09-30T09:07:49.554 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi019:vg_nvme/lv_3 2023-09-30T09:07:49.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: osdmap e9: 1 total, 1 up, 1 in 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: Detected new or changed devices on smithi019 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi019 to 16149M 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:49 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: osdmap e9: 1 total, 1 up, 1 in 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: Detected new or changed devices on smithi019 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi019 to 16149M 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: osdmap e9: 1 total, 1 up, 1 in 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: Detected new or changed devices on smithi019 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi019 to 16149M 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:07:49.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:49 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:07:49.909 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:07:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:50 smithi019 ceph-mon[100214]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:51.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:50 smithi019 ceph-mon[118983]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:51.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:50 smithi167 ceph-mon[104403]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:52.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:51 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:07:52.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:51 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:07:52.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:51 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:52.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:51 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:07:52.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:51 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:07:52.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:51 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:52.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:51 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:07:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:51 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:07:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:51 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:07:53.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:52 smithi019 ceph-mon[118983]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:53.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:52 smithi019 ceph-mon[118983]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:53.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:52 smithi019 ceph-mon[100214]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:52 smithi019 ceph-mon[100214]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:52 smithi167 ceph-mon[104403]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:07:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:52 smithi167 ceph-mon[104403]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:55 smithi167 ceph-mon[104403]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:55 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3882021127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]: dispatch 2023-09-30T09:07:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:55 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]: dispatch 2023-09-30T09:07:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:55 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]': finished 2023-09-30T09:07:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:55 smithi167 ceph-mon[104403]: osdmap e10: 2 total, 1 up, 2 in 2023-09-30T09:07:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:55 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:07:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:55 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1522188336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:07:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[100214]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3882021127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]: dispatch 2023-09-30T09:07:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]: dispatch 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]': finished 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[100214]: osdmap e10: 2 total, 1 up, 2 in 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1522188336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[118983]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3882021127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]: dispatch 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]: dispatch 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64e9dc26-b495-4d5f-83f7-a38a9466d999"}]': finished 2023-09-30T09:07:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[118983]: osdmap e10: 2 total, 1 up, 2 in 2023-09-30T09:07:55.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:07:55.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:55 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1522188336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:07:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:57 smithi167 ceph-mon[104403]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:57 smithi019 ceph-mon[100214]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:57 smithi019 ceph-mon[118983]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:07:59 smithi167 ceph-mon[104403]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:59.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:07:59 smithi019 ceph-mon[100214]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:07:59.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:07:59 smithi019 ceph-mon[118983]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:00 smithi019 ceph-mon[118983]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:00 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:08:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:00 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:00.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:00 smithi019 ceph-mon[100214]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:00.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:00 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:08:00.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:00 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:00 smithi167 ceph-mon[104403]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:00 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:08:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:00 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:01.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:01 smithi019 ceph-mon[100214]: Deploying daemon osd.1 on smithi019 2023-09-30T09:08:01.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:01 smithi019 ceph-mon[118983]: Deploying daemon osd.1 on smithi019 2023-09-30T09:08:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:01 smithi167 ceph-mon[104403]: Deploying daemon osd.1 on smithi019 2023-09-30T09:08:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:02 smithi019 ceph-mon[100214]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:03.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:02 smithi019 ceph-mon[118983]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:02 smithi167 ceph-mon[104403]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:05 smithi167 ceph-mon[104403]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:05.984 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:05 smithi019 ceph-mon[100214]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:05 smithi019 ceph-mon[118983]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:06 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:06 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:06 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:06 smithi167 ceph-mon[104403]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:06 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:06 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:06 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:06 smithi019 ceph-mon[100214]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:06 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:06 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:06 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:06 smithi019 ceph-mon[118983]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:08.943 INFO:teuthology.orchestra.run.smithi019.stdout:Created osd(s) 1 on host 'smithi019' 2023-09-30T09:08:09.143 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:09.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:09.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[100214]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:09.605 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.605 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.606 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:09.606 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:09.606 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.606 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:08 smithi019 ceph-mon[118983]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:09.606 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.606 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.606 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:09.607 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:09.607 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:09.607 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:08 smithi167 ceph-mon[104403]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:09.608 DEBUG:teuthology.orchestra.run.smithi019:osd.1> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-30T09:08:09.611 INFO:tasks.cephadm:Deploying osd.2 on smithi019 with /dev/vg_nvme/lv_2... 2023-09-30T09:08:09.611 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-30T09:08:09.641 INFO:journalctl@ceph.osd.1.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:08:10.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:10.310 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:10.310 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:10.310 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[118983]: from='osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:08:10.310 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[118983]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:08:10.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:10.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:10.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:10.311 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[100214]: from='osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:08:10.311 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:09 smithi019 ceph-mon[100214]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:08:10.311 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:09 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:10.311 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:09 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:10.311 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:09 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:10.311 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:09 smithi167 ceph-mon[104403]: from='osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:08:10.311 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:09 smithi167 ceph-mon[104403]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:08:10.732 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:08:10.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[100214]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:08:10.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[100214]: osdmap e11: 2 total, 1 up, 2 in 2023-09-30T09:08:10.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:10.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[100214]: from='osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:10.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[100214]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:10.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[100214]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:10.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[118983]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:08:10.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[118983]: osdmap e11: 2 total, 1 up, 2 in 2023-09-30T09:08:10.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:10.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[118983]: from='osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:10.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[118983]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:10.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:10 smithi019 ceph-mon[118983]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:10.991 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:10 smithi019 conmon[134349]: 2023-09-30T09:08:10.919+0000 7f3173f6e700 -1 osd.1 0 waiting for initial osdmap 2023-09-30T09:08:10.991 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:10 smithi019 conmon[134349]: 2023-09-30T09:08:10.922+0000 7f316d566700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:08:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:10 smithi167 ceph-mon[104403]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:08:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:10 smithi167 ceph-mon[104403]: osdmap e11: 2 total, 1 up, 2 in 2023-09-30T09:08:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:10 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:10 smithi167 ceph-mon[104403]: from='osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:10 smithi167 ceph-mon[104403]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:10 smithi167 ceph-mon[104403]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-30T09:08:12.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:11 smithi019 ceph-mon[118983]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:08:12.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:11 smithi019 ceph-mon[118983]: osdmap e12: 2 total, 1 up, 2 in 2023-09-30T09:08:12.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:11 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:12.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:11 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:12.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:11 smithi019 ceph-mon[100214]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:08:12.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:11 smithi019 ceph-mon[100214]: osdmap e12: 2 total, 1 up, 2 in 2023-09-30T09:08:12.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:11 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:12.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:11 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:11 smithi167 ceph-mon[104403]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:08:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:11 smithi167 ceph-mon[104403]: osdmap e12: 2 total, 1 up, 2 in 2023-09-30T09:08:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:11 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:11 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:12.968 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-30T09:08:12.969 INFO:teuthology.orchestra.run.smithi019.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-30T09:08:12.969 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10+0 records in 2023-09-30T09:08:12.969 INFO:teuthology.orchestra.run.smithi019.stderr:10+0 records out 2023-09-30T09:08:12.969 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110056 s, 953 MB/s 2023-09-30T09:08:12.969 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping successful for: 2023-09-30T09:08:13.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:12 smithi167 ceph-mon[104403]: purged_snaps scrub starts 2023-09-30T09:08:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:12 smithi167 ceph-mon[104403]: purged_snaps scrub ok 2023-09-30T09:08:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:12 smithi167 ceph-mon[104403]: osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342] boot 2023-09-30T09:08:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:12 smithi167 ceph-mon[104403]: osdmap e13: 2 total, 2 up, 2 in 2023-09-30T09:08:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:12 smithi167 ceph-mon[104403]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[100214]: purged_snaps scrub starts 2023-09-30T09:08:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[100214]: purged_snaps scrub ok 2023-09-30T09:08:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[100214]: osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342] boot 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[100214]: osdmap e13: 2 total, 2 up, 2 in 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[100214]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[118983]: purged_snaps scrub starts 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[118983]: purged_snaps scrub ok 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[118983]: osd.1 [v2:172.21.15.19:6810/1005512342,v1:172.21.15.19:6811/1005512342] boot 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[118983]: osdmap e13: 2 total, 2 up, 2 in 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:08:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:12 smithi019 ceph-mon[118983]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:13.757 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi019:vg_nvme/lv_2 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: osdmap e14: 2 total, 2 up, 2 in 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: osdmap e14: 2 total, 2 up, 2 in 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:14.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:13 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.114 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:08:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: osdmap e14: 2 total, 2 up, 2 in 2023-09-30T09:08:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:14.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:14.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:14.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:13 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:14 smithi167 ceph-mon[104403]: Detected new or changed devices on smithi019 2023-09-30T09:08:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:14 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi019 to 8074M 2023-09-30T09:08:15.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:14 smithi167 ceph-mon[104403]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:15.328 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:14 smithi019 ceph-mon[100214]: Detected new or changed devices on smithi019 2023-09-30T09:08:15.328 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:14 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi019 to 8074M 2023-09-30T09:08:15.328 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:14 smithi019 ceph-mon[100214]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:15.328 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:14 smithi019 ceph-mon[118983]: Detected new or changed devices on smithi019 2023-09-30T09:08:15.328 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:14 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi019 to 8074M 2023-09-30T09:08:15.328 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:14 smithi019 ceph-mon[118983]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:15 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:08:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:15 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:08:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:15 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:15 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:08:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:15 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:08:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:15 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:15 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:08:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:15 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:08:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:15 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:16 smithi019 ceph-mon[100214]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:08:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:16 smithi019 ceph-mon[100214]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:16 smithi019 ceph-mon[118983]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:08:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:16 smithi019 ceph-mon[118983]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:17.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:16 smithi167 ceph-mon[104403]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:08:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:16 smithi167 ceph-mon[104403]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:19 smithi167 ceph-mon[104403]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:19 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1098994081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5256ec9-304c-4028-8256-970c6bd03bb2"}]: dispatch 2023-09-30T09:08:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:19 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1098994081' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5256ec9-304c-4028-8256-970c6bd03bb2"}]': finished 2023-09-30T09:08:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:19 smithi167 ceph-mon[104403]: osdmap e15: 3 total, 2 up, 3 in 2023-09-30T09:08:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:19 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1297926105' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:08:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[100214]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1098994081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5256ec9-304c-4028-8256-970c6bd03bb2"}]: dispatch 2023-09-30T09:08:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1098994081' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5256ec9-304c-4028-8256-970c6bd03bb2"}]': finished 2023-09-30T09:08:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[100214]: osdmap e15: 3 total, 2 up, 3 in 2023-09-30T09:08:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1297926105' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:08:19.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[118983]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:19.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1098994081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5256ec9-304c-4028-8256-970c6bd03bb2"}]: dispatch 2023-09-30T09:08:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1098994081' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5256ec9-304c-4028-8256-970c6bd03bb2"}]': finished 2023-09-30T09:08:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[118983]: osdmap e15: 3 total, 2 up, 3 in 2023-09-30T09:08:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:19 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1297926105' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:08:21.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:20 smithi019 ceph-mon[100214]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:20 smithi019 ceph-mon[118983]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:20 smithi167 ceph-mon[104403]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:23.418 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:23 smithi019 ceph-mon[100214]: pgmap v63: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:23.418 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:23 smithi019 ceph-mon[118983]: pgmap v63: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:23.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:23 smithi167 ceph-mon[104403]: pgmap v63: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:25.498 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:25 smithi019 ceph-mon[100214]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:25.498 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:25 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:08:25.499 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:25 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:25.499 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:25 smithi019 ceph-mon[118983]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:25.499 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:25 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:08:25.499 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:25 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:25.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:25 smithi167 ceph-mon[104403]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:25 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:08:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:25 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:26.130 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:26 smithi019 ceph-mon[100214]: Deploying daemon osd.2 on smithi019 2023-09-30T09:08:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:26 smithi167 ceph-mon[104403]: Deploying daemon osd.2 on smithi019 2023-09-30T09:08:26.580 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:26 smithi019 ceph-mon[118983]: Deploying daemon osd.2 on smithi019 2023-09-30T09:08:27.428 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:27 smithi019 ceph-mon[100214]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:27.428 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:27 smithi019 ceph-mon[118983]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:27 smithi167 ceph-mon[104403]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:29 smithi167 ceph-mon[104403]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:29.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:29 smithi019 ceph-mon[100214]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:29.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:29 smithi019 ceph-mon[118983]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:30.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:30 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:31.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:30 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:31.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:30 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:31.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:30 smithi019 ceph-mon[100214]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:31.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:30 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:31.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:30 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:31.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:30 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:31.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:30 smithi019 ceph-mon[118983]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:31.186 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:30 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:31.186 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:30 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:31.186 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:30 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:31.186 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:30 smithi167 ceph-mon[104403]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:32.918 INFO:teuthology.orchestra.run.smithi019.stdout:Created osd(s) 2 on host 'smithi019' 2023-09-30T09:08:33.139 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: pgmap v68: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:08:33.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[118983]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: pgmap v68: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:08:33.141 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:32 smithi019 ceph-mon[100214]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: pgmap v68: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:33.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:08:33.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:32 smithi167 ceph-mon[104403]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:08:33.569 DEBUG:teuthology.orchestra.run.smithi019:osd.2> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-30T09:08:33.571 INFO:tasks.cephadm:Deploying osd.3 on smithi019 with /dev/vg_nvme/lv_1... 2023-09-30T09:08:33.571 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-30T09:08:33.603 INFO:journalctl@ceph.osd.2.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:08:34.695 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:08:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[118983]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-30T09:08:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[118983]: osdmap e16: 3 total, 2 up, 3 in 2023-09-30T09:08:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[118983]: from='osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[118983]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[118983]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:34.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[100214]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-30T09:08:34.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[100214]: osdmap e16: 3 total, 2 up, 3 in 2023-09-30T09:08:34.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:34.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[100214]: from='osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:34.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[100214]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:34.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:34 smithi019 ceph-mon[100214]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:34.962 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:34 smithi019 conmon[143980]: 2023-09-30T09:08:34.916+0000 7efdc92fb700 -1 osd.2 0 waiting for initial osdmap 2023-09-30T09:08:34.962 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:34 smithi019 conmon[143980]: 2023-09-30T09:08:34.920+0000 7efdc50f8700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:08:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:34 smithi167 ceph-mon[104403]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-30T09:08:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:34 smithi167 ceph-mon[104403]: osdmap e16: 3 total, 2 up, 3 in 2023-09-30T09:08:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:34 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:34 smithi167 ceph-mon[104403]: from='osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:34 smithi167 ceph-mon[104403]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:08:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:34 smithi167 ceph-mon[104403]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:36.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:35 smithi019 ceph-mon[100214]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:08:36.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:35 smithi019 ceph-mon[100214]: osdmap e17: 3 total, 2 up, 3 in 2023-09-30T09:08:36.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:35 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:36.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:35 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:36.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:35 smithi019 ceph-mon[118983]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:08:36.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:35 smithi019 ceph-mon[118983]: osdmap e17: 3 total, 2 up, 3 in 2023-09-30T09:08:36.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:35 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:36.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:35 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:35 smithi167 ceph-mon[104403]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:08:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:35 smithi167 ceph-mon[104403]: osdmap e17: 3 total, 2 up, 3 in 2023-09-30T09:08:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:35 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:35 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:36.684 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-30T09:08:36.684 INFO:teuthology.orchestra.run.smithi019.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-30T09:08:36.684 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10+0 records in 2023-09-30T09:08:36.684 INFO:teuthology.orchestra.run.smithi019.stderr:10+0 records out 2023-09-30T09:08:36.684 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122324 s, 857 MB/s 2023-09-30T09:08:36.684 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping successful for: 2023-09-30T09:08:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[100214]: purged_snaps scrub starts 2023-09-30T09:08:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[100214]: purged_snaps scrub ok 2023-09-30T09:08:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[100214]: osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021] boot 2023-09-30T09:08:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[100214]: osdmap e18: 3 total, 3 up, 3 in 2023-09-30T09:08:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[100214]: pgmap v73: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-30T09:08:37.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[118983]: purged_snaps scrub starts 2023-09-30T09:08:37.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[118983]: purged_snaps scrub ok 2023-09-30T09:08:37.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:37.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[118983]: osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021] boot 2023-09-30T09:08:37.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[118983]: osdmap e18: 3 total, 3 up, 3 in 2023-09-30T09:08:37.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:37.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[118983]: pgmap v73: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:37.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:36 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-30T09:08:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:36 smithi167 ceph-mon[104403]: purged_snaps scrub starts 2023-09-30T09:08:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:36 smithi167 ceph-mon[104403]: purged_snaps scrub ok 2023-09-30T09:08:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:36 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:36 smithi167 ceph-mon[104403]: osd.2 [v2:172.21.15.19:6818/2893453021,v1:172.21.15.19:6819/2893453021] boot 2023-09-30T09:08:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:36 smithi167 ceph-mon[104403]: osdmap e18: 3 total, 3 up, 3 in 2023-09-30T09:08:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:36 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:08:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:36 smithi167 ceph-mon[104403]: pgmap v73: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-30T09:08:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:36 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-30T09:08:37.731 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi019:vg_nvme/lv_1 2023-09-30T09:08:38.106 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:08:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-30T09:08:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: osdmap e19: 3 total, 3 up, 3 in 2023-09-30T09:08:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:08:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:38.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:38.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:37 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-30T09:08:38.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: osdmap e19: 3 total, 3 up, 3 in 2023-09-30T09:08:38.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:08:38.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:38.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:38.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.307 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-30T09:08:38.307 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: osdmap e19: 3 total, 3 up, 3 in 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:37 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:38.942 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[118983]: Detected new or changed devices on smithi019 2023-09-30T09:08:38.942 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi019 to 5383M 2023-09-30T09:08:38.942 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:08:38.942 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[118983]: osdmap e20: 3 total, 3 up, 3 in 2023-09-30T09:08:38.942 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[118983]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:38 smithi167 ceph-mon[104403]: Detected new or changed devices on smithi019 2023-09-30T09:08:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:38 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi019 to 5383M 2023-09-30T09:08:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:38 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:08:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:38 smithi167 ceph-mon[104403]: osdmap e20: 3 total, 3 up, 3 in 2023-09-30T09:08:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:38 smithi167 ceph-mon[104403]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[100214]: Detected new or changed devices on smithi019 2023-09-30T09:08:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi019 to 5383M 2023-09-30T09:08:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:08:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[100214]: osdmap e20: 3 total, 3 up, 3 in 2023-09-30T09:08:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:38 smithi019 ceph-mon[100214]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:39.835 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:08:39 smithi019 sudo[150742]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-30T09:08:39.835 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:08:39 smithi019 sudo[150742]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:39.836 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:08:39 smithi019 sudo[150742]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:39.836 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:08:39 smithi019 sudo[150742]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:40.335 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:08:39 smithi019 sudo[150819]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-30T09:08:40.335 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:08:39 smithi019 sudo[150819]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:40.335 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:08:39 smithi019 sudo[150819]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:40.335 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:08:39 smithi019 sudo[150819]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:40.684 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:40 smithi019 sudo[151001]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-30T09:08:40.684 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:40 smithi019 sudo[151001]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:40.684 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:40 smithi019 sudo[151001]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:40.684 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:40 smithi019 sudo[151001]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:41.085 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:40 smithi019 sudo[151051]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-30T09:08:41.085 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:40 smithi019 sudo[151051]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:41.085 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:40 smithi019 sudo[151051]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:41.085 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:08:40 smithi019 sudo[151051]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[100214]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[100214]: osdmap e21: 3 total, 3 up, 3 in 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[100214]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[118983]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:08:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:08:41.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:41.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[118983]: osdmap e21: 3 total, 3 up, 3 in 2023-09-30T09:08:41.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:41.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:40 smithi019 ceph-mon[118983]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:40 smithi167 ceph-mon[104403]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:08:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:40 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:08:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:40 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:08:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:40 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:40 smithi167 ceph-mon[104403]: osdmap e21: 3 total, 3 up, 3 in 2023-09-30T09:08:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:40 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:40 smithi167 ceph-mon[104403]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:41.585 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:41 smithi019 sudo[151080]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-30T09:08:41.585 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:41 smithi019 sudo[151080]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:41.585 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:41 smithi019 sudo[151080]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:41.585 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:41 smithi019 sudo[151080]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:42.085 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:41 smithi019 sudo[151159]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-30T09:08:42.085 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:41 smithi019 sudo[151159]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:42.086 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:41 smithi019 sudo[151159]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:42.086 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:08:41 smithi019 sudo[151159]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 sudo[151180]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-30T09:08:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 sudo[151180]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 sudo[151180]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 sudo[151180]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:42.936 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:42 smithi167 ceph-mon[104403]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-30T09:08:42.936 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:42 smithi167 ceph-mon[104403]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:42.936 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:42 smithi167 sudo[108528]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-30T09:08:42.937 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:42 smithi167 sudo[108528]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:42.937 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:42 smithi167 sudo[108528]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:42.937 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:42 smithi167 sudo[108528]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:42 smithi019 ceph-mon[118983]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-30T09:08:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:42 smithi019 ceph-mon[118983]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 sudo[151208]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-30T09:08:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 sudo[151208]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 sudo[151208]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 sudo[151208]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 ceph-mon[100214]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-30T09:08:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:42 smithi019 ceph-mon[100214]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:43 smithi167 sudo[108531]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-30T09:08:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:43 smithi167 sudo[108531]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:43 smithi167 sudo[108531]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:43 smithi167 sudo[108531]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:44.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:43 smithi019 sudo[151246]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-30T09:08:44.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:43 smithi019 sudo[151246]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:44.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:43 smithi019 sudo[151246]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:44.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:43 smithi019 sudo[151246]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1892966322' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]: dispatch 2023-09-30T09:08:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]: dispatch 2023-09-30T09:08:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]': finished 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: osdmap e22: 4 total, 3 up, 4 in 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1892966322' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]: dispatch 2023-09-30T09:08:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]: dispatch 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]': finished 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: osdmap e22: 4 total, 3 up, 4 in 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:44.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 sudo[151249]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 sudo[151249]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 sudo[151249]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 sudo[151249]: pam_unix(sudo:session): session closed for user root 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1892966322' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]: dispatch 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1f9fc59d-faab-4c30-938c-f244c7f972d9"}]': finished 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: osdmap e22: 4 total, 3 up, 4 in 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:44.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:44.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:08:44.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:08:44.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:08:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:45 smithi167 ceph-mon[104403]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:45 smithi167 ceph-mon[104403]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:45 smithi167 ceph-mon[104403]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:45 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2317933327' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:08:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:45 smithi019 ceph-mon[100214]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:45 smithi019 ceph-mon[100214]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:45 smithi019 ceph-mon[100214]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:45 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2317933327' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:08:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:45 smithi019 ceph-mon[118983]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-30T09:08:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:45 smithi019 ceph-mon[118983]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:45.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:45 smithi019 ceph-mon[118983]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-30T09:08:45.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:45 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2317933327' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:08:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:47 smithi019 ceph-mon[100214]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:47.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:47 smithi019 ceph-mon[118983]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:47 smithi167 ceph-mon[104403]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:08:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:49 smithi167 ceph-mon[104403]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-30T09:08:49.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:49 smithi019 ceph-mon[100214]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-30T09:08:49.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:49 smithi019 ceph-mon[118983]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-30T09:08:50.367 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:50 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:08:50.368 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:50 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:50.368 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:50 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:08:50.368 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:50 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:50 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:08:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:50 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:51.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:51 smithi019 ceph-mon[100214]: Deploying daemon osd.3 on smithi019 2023-09-30T09:08:51.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:51 smithi019 ceph-mon[100214]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-30T09:08:51.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:51 smithi019 ceph-mon[118983]: Deploying daemon osd.3 on smithi019 2023-09-30T09:08:51.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:51 smithi019 ceph-mon[118983]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-30T09:08:51.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:51 smithi167 ceph-mon[104403]: Deploying daemon osd.3 on smithi019 2023-09-30T09:08:51.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:51 smithi167 ceph-mon[104403]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-30T09:08:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:53 smithi167 ceph-mon[104403]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-30T09:08:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:53 smithi019 ceph-mon[100214]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-30T09:08:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:53 smithi019 ceph-mon[118983]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-30T09:08:55.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:55 smithi019 ceph-mon[100214]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2023-09-30T09:08:55.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:55 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:55.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:55 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:55.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:55 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:55 smithi167 ceph-mon[104403]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2023-09-30T09:08:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:55 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:55 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:55 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:55 smithi019 ceph-mon[118983]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2023-09-30T09:08:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:55 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:55 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:55 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:57.441 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:57 smithi019 ceph-mon[100214]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-30T09:08:57.442 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:57 smithi019 ceph-mon[118983]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-30T09:08:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:57 smithi167 ceph-mon[104403]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-30T09:08:58.026 INFO:teuthology.orchestra.run.smithi019.stdout:Created osd(s) 3 on host 'smithi019' 2023-09-30T09:08:58.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-30T09:08:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:08:58.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[100214]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:08:58.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:08:58.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:08:58 smithi019 ceph-mon[118983]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:08:58.665 DEBUG:teuthology.orchestra.run.smithi019:osd.3> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-30T09:08:58.667 INFO:tasks.cephadm:Deploying osd.4 on smithi167 with /dev/vg_nvme/lv_4... 2023-09-30T09:08:58.667 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-30T09:08:58.699 INFO:journalctl@ceph.osd.3.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:08:58.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:08:58.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:08:58.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:08:58.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-30T09:08:58.701 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:08:58.701 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:08:58.701 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:08:58 smithi167 ceph-mon[104403]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:08:59.010 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:09:00.173 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[100214]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-30T09:09:00.173 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[100214]: osdmap e23: 4 total, 3 up, 4 in 2023-09-30T09:09:00.173 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:00.173 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[100214]: from='osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:09:00.173 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[100214]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:09:00.173 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[118983]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-30T09:09:00.174 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[118983]: osdmap e23: 4 total, 3 up, 4 in 2023-09-30T09:09:00.174 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:00.174 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[118983]: from='osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:09:00.174 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:00 smithi019 ceph-mon[118983]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:09:00.174 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:09:00 smithi019 conmon[153794]: 2023-09-30T09:09:00.037+0000 7f545fb12700 -1 osd.3 0 waiting for initial osdmap 2023-09-30T09:09:00.174 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:09:00 smithi019 conmon[153794]: 2023-09-30T09:09:00.041+0000 7f545c911700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:09:00.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:00 smithi167 ceph-mon[104403]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-30T09:09:00.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:00 smithi167 ceph-mon[104403]: osdmap e23: 4 total, 3 up, 4 in 2023-09-30T09:09:00.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:00 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:00.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:00 smithi167 ceph-mon[104403]: from='osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:09:00.181 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:00 smithi167 ceph-mon[104403]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:09:00.612 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-30T09:09:00.612 INFO:teuthology.orchestra.run.smithi167.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-30T09:09:00.612 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10+0 records in 2023-09-30T09:09:00.612 INFO:teuthology.orchestra.run.smithi167.stderr:10+0 records out 2023-09-30T09:09:00.612 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111215 s, 943 MB/s 2023-09-30T09:09:00.612 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping successful for: 2023-09-30T09:09:01.158 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi167:vg_nvme/lv_4 2023-09-30T09:09:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[118983]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:09:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[118983]: osdmap e24: 4 total, 3 up, 4 in 2023-09-30T09:09:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[118983]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:09:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[100214]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:09:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[100214]: osdmap e24: 4 total, 3 up, 4 in 2023-09-30T09:09:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[100214]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:09:01.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:01 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:01 smithi167 ceph-mon[104403]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2023-09-30T09:09:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:01 smithi167 ceph-mon[104403]: osdmap e24: 4 total, 3 up, 4 in 2023-09-30T09:09:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:01 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:01 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:01 smithi167 ceph-mon[104403]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-30T09:09:01.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:01 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:01.532 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:09:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: purged_snaps scrub starts 2023-09-30T09:09:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: purged_snaps scrub ok 2023-09-30T09:09:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925] boot 2023-09-30T09:09:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: osdmap e25: 4 total, 4 up, 4 in 2023-09-30T09:09:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: purged_snaps scrub starts 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: purged_snaps scrub ok 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925] boot 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: osdmap e25: 4 total, 4 up, 4 in 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:02.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:02.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:02 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: purged_snaps scrub starts 2023-09-30T09:09:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: purged_snaps scrub ok 2023-09-30T09:09:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: osd.3 [v2:172.21.15.19:6826/2039937925,v1:172.21.15.19:6827/2039937925] boot 2023-09-30T09:09:02.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: osdmap e25: 4 total, 4 up, 4 in 2023-09-30T09:09:02.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:09:02.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:02.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:02.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:02.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:02.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:02 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:03 smithi019 ceph-mon[100214]: Detected new or changed devices on smithi019 2023-09-30T09:09:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:03 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi019 to 4037M 2023-09-30T09:09:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:03 smithi019 ceph-mon[100214]: osdmap e26: 4 total, 4 up, 4 in 2023-09-30T09:09:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:03 smithi019 ceph-mon[100214]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:03 smithi019 ceph-mon[118983]: Detected new or changed devices on smithi019 2023-09-30T09:09:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:03 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi019 to 4037M 2023-09-30T09:09:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:03 smithi019 ceph-mon[118983]: osdmap e26: 4 total, 4 up, 4 in 2023-09-30T09:09:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:03 smithi019 ceph-mon[118983]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:03.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:03 smithi167 ceph-mon[104403]: Detected new or changed devices on smithi019 2023-09-30T09:09:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:03 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi019 to 4037M 2023-09-30T09:09:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:03 smithi167 ceph-mon[104403]: osdmap e26: 4 total, 4 up, 4 in 2023-09-30T09:09:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:03 smithi167 ceph-mon[104403]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:04 smithi019 ceph-mon[100214]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:04 smithi019 ceph-mon[118983]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:04.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:04 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:04.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:04 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:04.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:04 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:04.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:04 smithi167 ceph-mon[104403]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:04.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:04.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:04.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:05 smithi019 ceph-mon[100214]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:05 smithi019 ceph-mon[118983]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:05 smithi167 ceph-mon[104403]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/2781406509' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]: dispatch 2023-09-30T09:09:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]: dispatch 2023-09-30T09:09:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]': finished 2023-09-30T09:09:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[100214]: osdmap e27: 5 total, 4 up, 5 in 2023-09-30T09:09:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/1264222109' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/2781406509' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]: dispatch 2023-09-30T09:09:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]: dispatch 2023-09-30T09:09:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]': finished 2023-09-30T09:09:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[118983]: osdmap e27: 5 total, 4 up, 5 in 2023-09-30T09:09:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:06 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/1264222109' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:06 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/2781406509' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]: dispatch 2023-09-30T09:09:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:06 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]: dispatch 2023-09-30T09:09:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:06 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb4cb89f-3933-48cd-a588-95a2f2bf8d13"}]': finished 2023-09-30T09:09:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:06 smithi167 ceph-mon[104403]: osdmap e27: 5 total, 4 up, 5 in 2023-09-30T09:09:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:06 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:06.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:06 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/1264222109' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:07 smithi019 ceph-mon[100214]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:07.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:07 smithi019 ceph-mon[118983]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:07.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:07 smithi167 ceph-mon[104403]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:09 smithi167 ceph-mon[104403]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:09 smithi019 ceph-mon[100214]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:09.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:09 smithi019 ceph-mon[118983]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:10 smithi019 ceph-mon[100214]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:11.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:10 smithi019 ceph-mon[118983]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:11.142 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:10 smithi167 ceph-mon[104403]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:12.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:11 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-30T09:09:12.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:11 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:12.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:11 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-30T09:09:12.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:11 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:12.099 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:11 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-30T09:09:12.099 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:11 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:13.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:12 smithi019 ceph-mon[100214]: Deploying daemon osd.4 on smithi167 2023-09-30T09:09:13.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:12 smithi019 ceph-mon[100214]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:13.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:12 smithi019 ceph-mon[118983]: Deploying daemon osd.4 on smithi167 2023-09-30T09:09:13.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:12 smithi019 ceph-mon[118983]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:13.228 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:12 smithi167 ceph-mon[104403]: Deploying daemon osd.4 on smithi167 2023-09-30T09:09:13.228 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:12 smithi167 ceph-mon[104403]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:15 smithi167 ceph-mon[104403]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:15.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:15 smithi019 ceph-mon[100214]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:15.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:15 smithi019 ceph-mon[118983]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:17 smithi167 ceph-mon[104403]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:17 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:17 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:17 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:17 smithi019 ceph-mon[100214]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:17 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:17 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:17 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:17.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:17 smithi019 ceph-mon[118983]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:17 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:17 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:17 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:19.257 INFO:teuthology.orchestra.run.smithi167.stdout:Created osd(s) 4 on host 'smithi167' 2023-09-30T09:09:19.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:19 smithi167 ceph-mon[104403]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:19.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:19.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:19.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:19 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[118983]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:19.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[100214]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:19 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:19.779 DEBUG:teuthology.orchestra.run.smithi167:osd.4> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-30T09:09:19.780 INFO:tasks.cephadm:Deploying osd.5 on smithi167 with /dev/vg_nvme/lv_3... 2023-09-30T09:09:19.781 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-30T09:09:19.810 INFO:journalctl@ceph.osd.4.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:09:20.373 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:20.374 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:20.374 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:20 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:20.374 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:20 smithi167 ceph-mon[104403]: from='osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-30T09:09:20.374 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:20 smithi167 ceph-mon[104403]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-30T09:09:20.374 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:20 smithi167 ceph-mon[104403]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[100214]: from='osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-30T09:09:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[100214]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-30T09:09:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[100214]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[118983]: from='osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-30T09:09:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[118983]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-30T09:09:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:20 smithi019 ceph-mon[118983]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:20.761 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:09:21.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:21 smithi167 ceph-mon[104403]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-30T09:09:21.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:21 smithi167 ceph-mon[104403]: osdmap e28: 5 total, 4 up, 5 in 2023-09-30T09:09:21.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:21 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:21.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:21 smithi167 ceph-mon[104403]: from='osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:21.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:21 smithi167 ceph-mon[104403]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:21.400 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:09:21 smithi167 conmon[112464]: 2023-09-30T09:09:21.235+0000 7f7bd9eed700 -1 osd.4 0 waiting for initial osdmap 2023-09-30T09:09:21.400 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:09:21 smithi167 conmon[112464]: 2023-09-30T09:09:21.239+0000 7f7bd54e9700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:09:21.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[100214]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-30T09:09:21.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[100214]: osdmap e28: 5 total, 4 up, 5 in 2023-09-30T09:09:21.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:21.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[100214]: from='osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:21.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[100214]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:21.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[118983]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-30T09:09:21.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[118983]: osdmap e28: 5 total, 4 up, 5 in 2023-09-30T09:09:21.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:21.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[118983]: from='osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:21.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:21 smithi019 ceph-mon[118983]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:22.416 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:22 smithi167 ceph-mon[104403]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:09:22.416 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:22 smithi167 ceph-mon[104403]: osdmap e29: 5 total, 4 up, 5 in 2023-09-30T09:09:22.416 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:22 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:22.416 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:22 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:22.416 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:22 smithi167 ceph-mon[104403]: pgmap v107: 1 pgs: 1 peering; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[100214]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:09:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[100214]: osdmap e29: 5 total, 4 up, 5 in 2023-09-30T09:09:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[100214]: pgmap v107: 1 pgs: 1 peering; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:22.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[118983]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:09:22.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[118983]: osdmap e29: 5 total, 4 up, 5 in 2023-09-30T09:09:22.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:22.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:22.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:22 smithi019 ceph-mon[118983]: pgmap v107: 1 pgs: 1 peering; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-30T09:09:22.693 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-30T09:09:22.694 INFO:teuthology.orchestra.run.smithi167.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-30T09:09:22.694 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10+0 records in 2023-09-30T09:09:22.694 INFO:teuthology.orchestra.run.smithi167.stderr:10+0 records out 2023-09-30T09:09:22.694 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011096 s, 945 MB/s 2023-09-30T09:09:22.694 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping successful for: 2023-09-30T09:09:23.288 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi167:vg_nvme/lv_3 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: purged_snaps scrub starts 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: purged_snaps scrub ok 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573] boot 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: osdmap e30: 5 total, 5 up, 5 in 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: Detected new or changed devices on smithi167 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi167 to 17168M 2023-09-30T09:09:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: purged_snaps scrub starts 2023-09-30T09:09:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: purged_snaps scrub ok 2023-09-30T09:09:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573] boot 2023-09-30T09:09:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: osdmap e30: 5 total, 5 up, 5 in 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: Detected new or changed devices on smithi167 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi167 to 17168M 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: purged_snaps scrub starts 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: purged_snaps scrub ok 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: osd.4 [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573] boot 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: osdmap e30: 5 total, 5 up, 5 in 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: Detected new or changed devices on smithi167 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi167 to 17168M 2023-09-30T09:09:23.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:23.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:23.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:23.619 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:09:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:24 smithi167 ceph-mon[104403]: osdmap e31: 5 total, 5 up, 5 in 2023-09-30T09:09:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:24 smithi167 ceph-mon[104403]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:24 smithi019 ceph-mon[100214]: osdmap e31: 5 total, 5 up, 5 in 2023-09-30T09:09:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:24 smithi019 ceph-mon[100214]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:24.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:24 smithi019 ceph-mon[118983]: osdmap e31: 5 total, 5 up, 5 in 2023-09-30T09:09:24.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:24 smithi019 ceph-mon[118983]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:25 smithi167 ceph-mon[104403]: osdmap e32: 5 total, 5 up, 5 in 2023-09-30T09:09:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:25 smithi167 ceph-mon[104403]: osdmap e33: 5 total, 5 up, 5 in 2023-09-30T09:09:25.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:25 smithi019 ceph-mon[100214]: osdmap e32: 5 total, 5 up, 5 in 2023-09-30T09:09:25.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:25 smithi019 ceph-mon[100214]: osdmap e33: 5 total, 5 up, 5 in 2023-09-30T09:09:25.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:25 smithi019 ceph-mon[118983]: osdmap e32: 5 total, 5 up, 5 in 2023-09-30T09:09:25.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:25 smithi019 ceph-mon[118983]: osdmap e33: 5 total, 5 up, 5 in 2023-09-30T09:09:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:26 smithi167 ceph-mon[104403]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:26 smithi167 ceph-mon[104403]: osdmap e34: 5 total, 5 up, 5 in 2023-09-30T09:09:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:26 smithi167 ceph-mon[104403]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[100214]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[100214]: osdmap e34: 5 total, 5 up, 5 in 2023-09-30T09:09:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[100214]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[118983]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[118983]: osdmap e34: 5 total, 5 up, 5 in 2023-09-30T09:09:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:26 smithi019 ceph-mon[118983]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:28.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/3742766741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]: dispatch 2023-09-30T09:09:28.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]: dispatch 2023-09-30T09:09:28.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]': finished 2023-09-30T09:09:28.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[100214]: osdmap e35: 6 total, 5 up, 6 in 2023-09-30T09:09:28.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:28.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/3742766741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]: dispatch 2023-09-30T09:09:28.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]: dispatch 2023-09-30T09:09:28.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]': finished 2023-09-30T09:09:28.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[118983]: osdmap e35: 6 total, 5 up, 6 in 2023-09-30T09:09:28.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:27 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:27 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/3742766741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]: dispatch 2023-09-30T09:09:28.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:27 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]: dispatch 2023-09-30T09:09:28.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:27 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1bc3ed47-225a-429a-a825-36b3357cf271"}]': finished 2023-09-30T09:09:28.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:27 smithi167 ceph-mon[104403]: osdmap e35: 6 total, 5 up, 6 in 2023-09-30T09:09:28.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:27 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:28 smithi019 ceph-mon[100214]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-30T09:09:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:28 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/1490491435' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:28 smithi019 ceph-mon[118983]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-30T09:09:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:28 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/1490491435' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:28 smithi167 ceph-mon[104403]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-30T09:09:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:28 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/1490491435' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:31.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:30 smithi019 ceph-mon[100214]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2023-09-30T09:09:31.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:30 smithi019 ceph-mon[118983]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2023-09-30T09:09:31.168 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:30 smithi167 ceph-mon[104403]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2023-09-30T09:09:33.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:33 smithi167 ceph-mon[104403]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:09:33.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:33 smithi019 ceph-mon[100214]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:09:33.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:33 smithi019 ceph-mon[118983]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:09:34.458 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:34 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-30T09:09:34.458 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:34 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:34 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-30T09:09:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:34 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:34 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-30T09:09:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:34 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:35 smithi167 ceph-mon[104403]: Deploying daemon osd.5 on smithi167 2023-09-30T09:09:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:35 smithi167 ceph-mon[104403]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-30T09:09:35.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:35 smithi019 ceph-mon[100214]: Deploying daemon osd.5 on smithi167 2023-09-30T09:09:35.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:35 smithi019 ceph-mon[100214]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-30T09:09:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:35 smithi019 ceph-mon[118983]: Deploying daemon osd.5 on smithi167 2023-09-30T09:09:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:35 smithi019 ceph-mon[118983]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-30T09:09:37.481 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:37 smithi167 ceph-mon[104403]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:09:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:37 smithi019 ceph-mon[100214]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:09:37.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:37 smithi019 ceph-mon[118983]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:09:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:39 smithi167 ceph-mon[104403]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:39 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:39 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:39 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:39 smithi019 ceph-mon[100214]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:39 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:39 smithi019 ceph-mon[118983]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:39 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:39 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:39 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:41.280 INFO:teuthology.orchestra.run.smithi167.stdout:Created osd(s) 5 on host 'smithi167' 2023-09-30T09:09:41.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:41 smithi167 ceph-mon[104403]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:41.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:41 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:41 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:41 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:41.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:41 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:41.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:41 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[118983]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[100214]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:41 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:41.809 DEBUG:teuthology.orchestra.run.smithi167:osd.5> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-30T09:09:41.811 INFO:tasks.cephadm:Deploying osd.6 on smithi167 with /dev/vg_nvme/lv_2... 2023-09-30T09:09:41.811 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-30T09:09:41.840 INFO:journalctl@ceph.osd.5.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:09:42.420 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:42 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:42.420 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:42 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:42.420 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:42 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:42.420 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:42 smithi167 ceph-mon[104403]: from='osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:09:42.421 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:42 smithi167 ceph-mon[104403]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:09:42.421 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:42 smithi167 ceph-mon[104403]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[118983]: from='osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[118983]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[118983]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[100214]: from='osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[100214]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:09:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:42 smithi019 ceph-mon[100214]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:42.824 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:09:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:43 smithi167 ceph-mon[104403]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-30T09:09:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:43 smithi167 ceph-mon[104403]: osdmap e36: 6 total, 5 up, 6 in 2023-09-30T09:09:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:43 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:43 smithi167 ceph-mon[104403]: from='osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:43 smithi167 ceph-mon[104403]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:43.523 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:09:43 smithi167 conmon[121532]: 2023-09-30T09:09:43.246+0000 7ff9a9d0a700 -1 osd.5 0 waiting for initial osdmap 2023-09-30T09:09:43.523 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:09:43 smithi167 conmon[121532]: 2023-09-30T09:09:43.251+0000 7ff9a4304700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:09:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[118983]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-30T09:09:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[118983]: osdmap e36: 6 total, 5 up, 6 in 2023-09-30T09:09:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[118983]: from='osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[118983]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:43.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[100214]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-30T09:09:43.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[100214]: osdmap e36: 6 total, 5 up, 6 in 2023-09-30T09:09:43.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:43.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[100214]: from='osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:43.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:43 smithi019 ceph-mon[100214]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:09:44.513 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:44 smithi167 ceph-mon[104403]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:09:44.514 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:44 smithi167 ceph-mon[104403]: osdmap e37: 6 total, 5 up, 6 in 2023-09-30T09:09:44.514 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:44.514 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:44.514 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:44 smithi167 ceph-mon[104403]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[100214]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:09:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[100214]: osdmap e37: 6 total, 5 up, 6 in 2023-09-30T09:09:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[100214]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[118983]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:09:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[118983]: osdmap e37: 6 total, 5 up, 6 in 2023-09-30T09:09:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:44 smithi019 ceph-mon[118983]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-30T09:09:44.826 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-30T09:09:44.826 INFO:teuthology.orchestra.run.smithi167.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-30T09:09:44.826 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10+0 records in 2023-09-30T09:09:44.826 INFO:teuthology.orchestra.run.smithi167.stderr:10+0 records out 2023-09-30T09:09:44.826 INFO:teuthology.orchestra.run.smithi167.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0130139 s, 806 MB/s 2023-09-30T09:09:44.826 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping successful for: 2023-09-30T09:09:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:45 smithi167 ceph-mon[104403]: purged_snaps scrub starts 2023-09-30T09:09:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:45 smithi167 ceph-mon[104403]: purged_snaps scrub ok 2023-09-30T09:09:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:45 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:45 smithi167 ceph-mon[104403]: osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136] boot 2023-09-30T09:09:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:45 smithi167 ceph-mon[104403]: osdmap e38: 6 total, 6 up, 6 in 2023-09-30T09:09:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:45 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:45 smithi167 ceph-mon[104403]: osdmap e39: 6 total, 6 up, 6 in 2023-09-30T09:09:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[118983]: purged_snaps scrub starts 2023-09-30T09:09:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[118983]: purged_snaps scrub ok 2023-09-30T09:09:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[118983]: osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136] boot 2023-09-30T09:09:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[118983]: osdmap e38: 6 total, 6 up, 6 in 2023-09-30T09:09:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[118983]: osdmap e39: 6 total, 6 up, 6 in 2023-09-30T09:09:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[100214]: purged_snaps scrub starts 2023-09-30T09:09:45.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[100214]: purged_snaps scrub ok 2023-09-30T09:09:45.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:45.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[100214]: osd.5 [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136] boot 2023-09-30T09:09:45.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[100214]: osdmap e38: 6 total, 6 up, 6 in 2023-09-30T09:09:45.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:09:45.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:45 smithi019 ceph-mon[100214]: osdmap e39: 6 total, 6 up, 6 in 2023-09-30T09:09:45.588 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi167:vg_nvme/lv_2 2023-09-30T09:09:45.930 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: Detected new or changed devices on smithi167 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi167 to 8584M 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:46.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: osdmap e40: 6 total, 6 up, 6 in 2023-09-30T09:09:46.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:46 smithi167 ceph-mon[104403]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: Detected new or changed devices on smithi167 2023-09-30T09:09:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi167 to 8584M 2023-09-30T09:09:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: osdmap e40: 6 total, 6 up, 6 in 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[100214]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: Detected new or changed devices on smithi167 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:09:46.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi167 to 8584M 2023-09-30T09:09:46.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:46.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:09:46.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:09:46.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: osdmap e40: 6 total, 6 up, 6 in 2023-09-30T09:09:46.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:46 smithi019 ceph-mon[118983]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:47 smithi167 ceph-mon[104403]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:09:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:47 smithi167 ceph-mon[104403]: osdmap e41: 6 total, 6 up, 6 in 2023-09-30T09:09:47.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:47 smithi019 ceph-mon[100214]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:09:47.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:47 smithi019 ceph-mon[100214]: osdmap e41: 6 total, 6 up, 6 in 2023-09-30T09:09:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:47 smithi019 ceph-mon[118983]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:09:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:47 smithi019 ceph-mon[118983]: osdmap e41: 6 total, 6 up, 6 in 2023-09-30T09:09:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:48 smithi167 ceph-mon[104403]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:48 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:48 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:48 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:48 smithi167 ceph-mon[104403]: osdmap e42: 6 total, 6 up, 6 in 2023-09-30T09:09:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:48 smithi167 ceph-mon[104403]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[100214]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[100214]: osdmap e42: 6 total, 6 up, 6 in 2023-09-30T09:09:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[100214]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[118983]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:09:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:09:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:09:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[118983]: osdmap e42: 6 total, 6 up, 6 in 2023-09-30T09:09:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:48 smithi019 ceph-mon[118983]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/3910371457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]: dispatch 2023-09-30T09:09:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]: dispatch 2023-09-30T09:09:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]': finished 2023-09-30T09:09:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[100214]: osdmap e43: 7 total, 6 up, 7 in 2023-09-30T09:09:51.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:09:51.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[100214]: pgmap v135: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:51.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/2135881868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/3910371457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]: dispatch 2023-09-30T09:09:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]: dispatch 2023-09-30T09:09:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]': finished 2023-09-30T09:09:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[118983]: osdmap e43: 7 total, 6 up, 7 in 2023-09-30T09:09:51.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:09:51.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[118983]: pgmap v135: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:51.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:50 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/2135881868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:50 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/3910371457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]: dispatch 2023-09-30T09:09:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:50 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]: dispatch 2023-09-30T09:09:51.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:50 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3a4720ae-d64a-4bda-b7d3-478ec2742819"}]': finished 2023-09-30T09:09:51.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:50 smithi167 ceph-mon[104403]: osdmap e43: 7 total, 6 up, 7 in 2023-09-30T09:09:51.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:50 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:09:51.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:50 smithi167 ceph-mon[104403]: pgmap v135: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:09:51.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:50 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/2135881868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:09:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:52 smithi167 ceph-mon[104403]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:09:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:52 smithi167 ceph-mon[104403]: Cluster is now healthy 2023-09-30T09:09:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:52 smithi019 ceph-mon[100214]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:09:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:52 smithi019 ceph-mon[100214]: Cluster is now healthy 2023-09-30T09:09:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:52 smithi019 ceph-mon[118983]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:09:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:52 smithi019 ceph-mon[118983]: Cluster is now healthy 2023-09-30T09:09:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:53 smithi167 ceph-mon[104403]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-30T09:09:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:53 smithi019 ceph-mon[100214]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-30T09:09:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:53 smithi019 ceph-mon[118983]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-30T09:09:55.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:55 smithi167 ceph-mon[104403]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:09:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:55 smithi019 ceph-mon[100214]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:09:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:55 smithi019 ceph-mon[118983]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:09:56.466 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:56 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-30T09:09:56.466 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:56 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-30T09:09:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:56.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:56 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-30T09:09:56.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:56 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:09:57.461 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:57 smithi167 ceph-mon[104403]: Deploying daemon osd.6 on smithi167 2023-09-30T09:09:57.461 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:57 smithi167 ceph-mon[104403]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-30T09:09:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:57 smithi019 ceph-mon[100214]: Deploying daemon osd.6 on smithi167 2023-09-30T09:09:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:57 smithi019 ceph-mon[100214]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-30T09:09:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:57 smithi019 ceph-mon[118983]: Deploying daemon osd.6 on smithi167 2023-09-30T09:09:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:57 smithi019 ceph-mon[118983]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-30T09:09:58.484 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:09:58 smithi167 ceph-mon[104403]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:09:58.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:09:58 smithi019 ceph-mon[100214]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:09:58.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:09:58 smithi019 ceph-mon[118983]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:10:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[100214]: overall HEALTH_OK 2023-09-30T09:10:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[100214]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-30T09:10:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[118983]: overall HEALTH_OK 2023-09-30T09:10:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[118983]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-30T09:10:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:00 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:01.251 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:00 smithi167 ceph-mon[104403]: overall HEALTH_OK 2023-09-30T09:10:01.251 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:00 smithi167 ceph-mon[104403]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-30T09:10:01.251 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:00 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:01.251 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:00 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:01.251 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:00 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:03.371 INFO:teuthology.orchestra.run.smithi167.stdout:Created osd(s) 6 on host 'smithi167' 2023-09-30T09:10:03.396 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:03 smithi167 ceph-mon[104403]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-30T09:10:03.396 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:03 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.397 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:03 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.397 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:03 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:03.397 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:03 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:03.397 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:03 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[100214]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-30T09:10:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[118983]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-30T09:10:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:03 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:03.906 DEBUG:teuthology.orchestra.run.smithi167:osd.6> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-30T09:10:03.909 INFO:tasks.cephadm:Deploying osd.7 on smithi167 with /dev/vg_nvme/lv_1... 2023-09-30T09:10:03.909 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-30T09:10:03.938 INFO:journalctl@ceph.osd.6.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:10:04.427 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:10:04.500 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:04.500 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:04.500 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:04 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:04.501 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:04 smithi167 ceph-mon[104403]: from='osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-30T09:10:04.501 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:04 smithi167 ceph-mon[104403]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-30T09:10:04.501 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:04 smithi167 ceph-mon[104403]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:10:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[100214]: from='osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-30T09:10:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[100214]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-30T09:10:04.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[100214]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:10:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:04.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[118983]: from='osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-30T09:10:04.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[118983]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-30T09:10:04.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:04 smithi019 ceph-mon[118983]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:10:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:05 smithi167 ceph-mon[104403]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-30T09:10:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:05 smithi167 ceph-mon[104403]: osdmap e44: 7 total, 6 up, 7 in 2023-09-30T09:10:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:05 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:05 smithi167 ceph-mon[104403]: from='osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:05 smithi167 ceph-mon[104403]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:05.523 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:10:05 smithi167 conmon[130376]: 2023-09-30T09:10:05.342+0000 7fac2995f700 -1 osd.6 0 waiting for initial osdmap 2023-09-30T09:10:05.523 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:10:05 smithi167 conmon[130376]: 2023-09-30T09:10:05.349+0000 7fac2575c700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:10:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[100214]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-30T09:10:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[100214]: osdmap e44: 7 total, 6 up, 7 in 2023-09-30T09:10:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[100214]: from='osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[100214]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:05.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[118983]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-30T09:10:05.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[118983]: osdmap e44: 7 total, 6 up, 7 in 2023-09-30T09:10:05.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:05.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[118983]: from='osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:05.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:05 smithi019 ceph-mon[118983]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:06 smithi167 ceph-mon[104403]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:10:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:06 smithi167 ceph-mon[104403]: osdmap e45: 7 total, 6 up, 7 in 2023-09-30T09:10:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:06 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:06 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:06 smithi167 ceph-mon[104403]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:10:06.667 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-30T09:10:06.667 INFO:teuthology.orchestra.run.smithi167.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-30T09:10:06.667 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10+0 records in 2023-09-30T09:10:06.667 INFO:teuthology.orchestra.run.smithi167.stderr:10+0 records out 2023-09-30T09:10:06.667 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112042 s, 936 MB/s 2023-09-30T09:10:06.668 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping successful for: 2023-09-30T09:10:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[100214]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:10:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[100214]: osdmap e45: 7 total, 6 up, 7 in 2023-09-30T09:10:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[100214]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:10:06.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[118983]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:10:06.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[118983]: osdmap e45: 7 total, 6 up, 7 in 2023-09-30T09:10:06.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:06.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:06.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:06 smithi019 ceph-mon[118983]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-30T09:10:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:07 smithi167 ceph-mon[104403]: purged_snaps scrub starts 2023-09-30T09:10:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:07 smithi167 ceph-mon[104403]: purged_snaps scrub ok 2023-09-30T09:10:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:07 smithi167 ceph-mon[104403]: osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455] boot 2023-09-30T09:10:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:07 smithi167 ceph-mon[104403]: osdmap e46: 7 total, 7 up, 7 in 2023-09-30T09:10:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:07 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:07.565 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi167:vg_nvme/lv_1 2023-09-30T09:10:07.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[100214]: purged_snaps scrub starts 2023-09-30T09:10:07.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[100214]: purged_snaps scrub ok 2023-09-30T09:10:07.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[100214]: osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455] boot 2023-09-30T09:10:07.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[100214]: osdmap e46: 7 total, 7 up, 7 in 2023-09-30T09:10:07.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[118983]: purged_snaps scrub starts 2023-09-30T09:10:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[118983]: purged_snaps scrub ok 2023-09-30T09:10:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[118983]: osd.6 [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455] boot 2023-09-30T09:10:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[118983]: osdmap e46: 7 total, 7 up, 7 in 2023-09-30T09:10:07.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:07 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:10:07.904 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:10:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: osdmap e47: 7 total, 7 up, 7 in 2023-09-30T09:10:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: Detected new or changed devices on smithi167 2023-09-30T09:10:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi167 to 5722M 2023-09-30T09:10:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:08 smithi167 ceph-mon[104403]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: osdmap e47: 7 total, 7 up, 7 in 2023-09-30T09:10:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: Detected new or changed devices on smithi167 2023-09-30T09:10:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi167 to 5722M 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[100214]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: osdmap e47: 7 total, 7 up, 7 in 2023-09-30T09:10:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: Detected new or changed devices on smithi167 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi167 to 5722M 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:08 smithi019 ceph-mon[118983]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:09 smithi167 ceph-mon[104403]: osdmap e48: 7 total, 7 up, 7 in 2023-09-30T09:10:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:09 smithi019 ceph-mon[100214]: osdmap e48: 7 total, 7 up, 7 in 2023-09-30T09:10:09.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:09 smithi019 ceph-mon[118983]: osdmap e48: 7 total, 7 up, 7 in 2023-09-30T09:10:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:10 smithi167 ceph-mon[104403]: osdmap e49: 7 total, 7 up, 7 in 2023-09-30T09:10:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:10 smithi167 ceph-mon[104403]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:10 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:10:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:10 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:10:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:10 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:10 smithi167 ceph-mon[104403]: pgmap v151: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-30T09:10:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[100214]: osdmap e49: 7 total, 7 up, 7 in 2023-09-30T09:10:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[100214]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:10:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:10:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[100214]: pgmap v151: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-30T09:10:10.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[118983]: osdmap e49: 7 total, 7 up, 7 in 2023-09-30T09:10:10.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[118983]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:10.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-30T09:10:10.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-30T09:10:10.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:10.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:10 smithi019 ceph-mon[118983]: pgmap v151: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-30T09:10:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:11 smithi167 ceph-mon[104403]: osdmap e50: 7 total, 7 up, 7 in 2023-09-30T09:10:12.382 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:11 smithi167 ceph-mon[104403]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:10:12.383 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:11 smithi019 ceph-mon[100214]: osdmap e50: 7 total, 7 up, 7 in 2023-09-30T09:10:12.383 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:11 smithi019 ceph-mon[100214]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:10:12.383 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:11 smithi019 ceph-mon[118983]: osdmap e50: 7 total, 7 up, 7 in 2023-09-30T09:10:12.383 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:11 smithi019 ceph-mon[118983]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:10:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:12 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/1921019391' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]: dispatch 2023-09-30T09:10:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:12 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]: dispatch 2023-09-30T09:10:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:12 smithi167 ceph-mon[104403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]': finished 2023-09-30T09:10:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:12 smithi167 ceph-mon[104403]: osdmap e51: 8 total, 7 up, 8 in 2023-09-30T09:10:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:12 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:12 smithi167 ceph-mon[104403]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-30T09:10:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/1921019391' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]: dispatch 2023-09-30T09:10:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]: dispatch 2023-09-30T09:10:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[100214]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]': finished 2023-09-30T09:10:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[100214]: osdmap e51: 8 total, 7 up, 8 in 2023-09-30T09:10:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[100214]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-30T09:10:12.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/1921019391' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]: dispatch 2023-09-30T09:10:12.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]: dispatch 2023-09-30T09:10:12.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[118983]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "88b4cb7b-99e4-46e9-a9aa-24f886b0174b"}]': finished 2023-09-30T09:10:12.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[118983]: osdmap e51: 8 total, 7 up, 8 in 2023-09-30T09:10:12.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:12.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:12 smithi019 ceph-mon[118983]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-30T09:10:13.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:13 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/1715695078' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:10:13.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:13 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/1715695078' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:10:13.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:13 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/1715695078' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-30T09:10:14.768 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:14 smithi167 ceph-mon[104403]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-30T09:10:14.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:14 smithi019 ceph-mon[118983]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-30T09:10:14.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:14 smithi019 ceph-mon[100214]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-30T09:10:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:15 smithi167 ceph-mon[104403]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-30T09:10:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:15 smithi167 ceph-mon[104403]: Cluster is now healthy 2023-09-30T09:10:15.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:15 smithi019 ceph-mon[100214]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-30T09:10:15.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:15 smithi019 ceph-mon[100214]: Cluster is now healthy 2023-09-30T09:10:15.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:15 smithi019 ceph-mon[118983]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-30T09:10:15.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:15 smithi019 ceph-mon[118983]: Cluster is now healthy 2023-09-30T09:10:16.720 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:16 smithi167 ceph-mon[104403]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-30T09:10:16.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:16 smithi019 ceph-mon[100214]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-30T09:10:16.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:16 smithi019 ceph-mon[118983]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-30T09:10:18.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:17 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-30T09:10:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:17 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:17 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-30T09:10:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:17 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:18.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:17 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-30T09:10:18.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:17 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:19.169 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:18 smithi167 ceph-mon[104403]: Deploying daemon osd.7 on smithi167 2023-09-30T09:10:19.169 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:18 smithi167 ceph-mon[104403]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-30T09:10:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:18 smithi019 ceph-mon[100214]: Deploying daemon osd.7 on smithi167 2023-09-30T09:10:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:18 smithi019 ceph-mon[100214]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-30T09:10:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:18 smithi019 ceph-mon[118983]: Deploying daemon osd.7 on smithi167 2023-09-30T09:10:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:18 smithi019 ceph-mon[118983]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-30T09:10:21.019 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:20 smithi167 ceph-mon[104403]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-30T09:10:21.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:20 smithi019 ceph-mon[100214]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-30T09:10:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:20 smithi019 ceph-mon[118983]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-30T09:10:23.506 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:23 smithi167 ceph-mon[104403]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-30T09:10:23.506 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:23.506 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:23.507 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:23 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:23 smithi019 ceph-mon[100214]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-30T09:10:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:23 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:23.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:23 smithi019 ceph-mon[118983]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-30T09:10:23.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:23.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:23.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:23 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:25.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:25 smithi167 ceph-mon[104403]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-30T09:10:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:25 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:25 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:25.580 INFO:teuthology.orchestra.run.smithi167.stdout:Created osd(s) 7 on host 'smithi167' 2023-09-30T09:10:25.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:25 smithi019 ceph-mon[118983]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-30T09:10:25.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:25 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:25.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:25 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:25.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:25 smithi019 ceph-mon[100214]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-30T09:10:25.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:25 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:25.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:25 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.145 DEBUG:teuthology.orchestra.run.smithi167:osd.7> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-30T09:10:26.147 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-30T09:10:26.147 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-30T09:10:26.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:26.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:26.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:26.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:26 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:26 smithi167 ceph-mon[104403]: from='osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-30T09:10:26.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:26 smithi167 ceph-mon[104403]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-30T09:10:26.177 INFO:journalctl@ceph.osd.7.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:10:26.517 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:10:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[100214]: from='osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-30T09:10:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[100214]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-30T09:10:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:26.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:26.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:26.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:26.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[118983]: from='osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-30T09:10:26.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:26 smithi019 ceph-mon[118983]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-30T09:10:27.432 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:27 smithi167 ceph-mon[104403]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:27.432 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:27 smithi167 ceph-mon[104403]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-30T09:10:27.432 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:27 smithi167 ceph-mon[104403]: osdmap e52: 8 total, 7 up, 8 in 2023-09-30T09:10:27.432 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:27 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:27.433 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:27 smithi167 ceph-mon[104403]: from='osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:27.433 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:27 smithi167 ceph-mon[104403]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[100214]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[100214]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-30T09:10:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[100214]: osdmap e52: 8 total, 7 up, 8 in 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[100214]: from='osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[100214]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[118983]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[118983]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[118983]: osdmap e52: 8 total, 7 up, 8 in 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[118983]: from='osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:27.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:27 smithi019 ceph-mon[118983]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:10:27.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:10:27 smithi167 conmon[139912]: 2023-09-30T09:10:27.579+0000 7f24361d1700 -1 osd.7 0 waiting for initial osdmap 2023-09-30T09:10:27.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:10:27 smithi167 conmon[139912]: 2023-09-30T09:10:27.587+0000 7f24307cb700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:10:28.397 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:10:28.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:28 smithi167 ceph-mon[104403]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:10:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:28 smithi167 ceph-mon[104403]: osdmap e53: 8 total, 7 up, 8 in 2023-09-30T09:10:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:28 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:28 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:28 smithi167 ceph-mon[104403]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:28 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3993650598' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-30T09:10:28.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[100214]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:10:28.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[100214]: osdmap e53: 8 total, 7 up, 8 in 2023-09-30T09:10:28.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:28.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:28.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[100214]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:28.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3993650598' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-30T09:10:28.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[118983]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-09-30T09:10:28.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[118983]: osdmap e53: 8 total, 7 up, 8 in 2023-09-30T09:10:28.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:28.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:28.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[118983]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-30T09:10:28.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:28 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3993650598' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-30T09:10:29.183 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1696065006,"num_in_osds":8,"osd_in_since":1696065011,"num_remapped_pgs":0} 2023-09-30T09:10:29.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: purged_snaps scrub starts 2023-09-30T09:10:29.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: purged_snaps scrub ok 2023-09-30T09:10:29.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506] boot 2023-09-30T09:10:29.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: osdmap e54: 8 total, 8 up, 8 in 2023-09-30T09:10:29.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:29.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: Detected new or changed devices on smithi167 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi167 to 4292M 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: purged_snaps scrub starts 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: purged_snaps scrub ok 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506] boot 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: osdmap e54: 8 total, 8 up, 8 in 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: Detected new or changed devices on smithi167 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:29.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:29.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi167 to 4292M 2023-09-30T09:10:29.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:29.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:29.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:29.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:29 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: purged_snaps scrub starts 2023-09-30T09:10:30.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: purged_snaps scrub ok 2023-09-30T09:10:30.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: osd.7 [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506] boot 2023-09-30T09:10:30.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: osdmap e54: 8 total, 8 up, 8 in 2023-09-30T09:10:30.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:10:30.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: Detected new or changed devices on smithi167 2023-09-30T09:10:30.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:30.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi167 to 4292M 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:30.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:29 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:30.186 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-30T09:10:30.554 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:10:30.928 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:30 smithi019 ceph-mon[100214]: osdmap e55: 8 total, 8 up, 8 in 2023-09-30T09:10:30.929 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:30 smithi019 ceph-mon[100214]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:30.929 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:30 smithi019 ceph-mon[118983]: osdmap e55: 8 total, 8 up, 8 in 2023-09-30T09:10:30.929 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:30 smithi019 ceph-mon[118983]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:30 smithi167 ceph-mon[104403]: osdmap e55: 8 total, 8 up, 8 in 2023-09-30T09:10:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:30 smithi167 ceph-mon[104403]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:31.950 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:31 smithi019 ceph-mon[100214]: osdmap e56: 8 total, 8 up, 8 in 2023-09-30T09:10:31.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:31 smithi019 ceph-mon[118983]: osdmap e56: 8 total, 8 up, 8 in 2023-09-30T09:10:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:31 smithi167 ceph-mon[104403]: osdmap e56: 8 total, 8 up, 8 in 2023-09-30T09:10:32.349 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:10:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:32 smithi019 ceph-mon[100214]: osdmap e57: 8 total, 8 up, 8 in 2023-09-30T09:10:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:32 smithi019 ceph-mon[100214]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:32 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3521620549' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-30T09:10:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:32 smithi019 ceph-mon[118983]: osdmap e57: 8 total, 8 up, 8 in 2023-09-30T09:10:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:32 smithi019 ceph-mon[118983]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:32 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3521620549' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-30T09:10:33.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:32 smithi167 ceph-mon[104403]: osdmap e57: 8 total, 8 up, 8 in 2023-09-30T09:10:33.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:32 smithi167 ceph-mon[104403]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:33.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:32 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3521620549' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-30T09:10:33.132 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1696065028,"num_in_osds":8,"osd_in_since":1696065011,"num_remapped_pgs":0} 2023-09-30T09:10:33.133 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:10:33.493 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:10:33.884 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:33 smithi019 ceph-mon[100214]: osdmap e58: 8 total, 8 up, 8 in 2023-09-30T09:10:33.884 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:33 smithi019 ceph-mon[118983]: osdmap e58: 8 total, 8 up, 8 in 2023-09-30T09:10:34.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:33 smithi167 ceph-mon[104403]: osdmap e58: 8 total, 8 up, 8 in 2023-09-30T09:10:34.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:34 smithi019 ceph-mon[100214]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2023-09-30T09:10:34.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:34 smithi019 ceph-mon[118983]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2023-09-30T09:10:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:34 smithi167 ceph-mon[104403]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2023-09-30T09:10:35.229 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:10:35.229 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":58,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:10:32.597386+0000","last_up_change":"2023-09-30T09:10:28.570615+0000","last_in_change":"2023-09-30T09:10:11.880926+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6803","nonce":1200588465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6805","nonce":1200588465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6809","nonce":1200588465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6807","nonce":1200588465}]},"public_addr":"172.21.15.19:6803/1200588465","cluster_addr":"172.21.15.19:6805/1200588465","heartbeat_back_addr":"172.21.15.19:6809/1200588465","heartbeat_front_addr":"172.21.15.19:6807/1200588465","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6811","nonce":1005512342}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6813","nonce":1005512342}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6817","nonce":1005512342}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6815","nonce":1005512342}]},"public_addr":"172.21.15.19:6811/1005512342","cluster_addr":"172.21.15.19:6813/1005512342","heartbeat_back_addr":"172.21.15.19:6817/1005512342","heartbeat_front_addr":"172.21.15.19:6815/1005512342","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6819","nonce":2893453021}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6821","nonce":2893453021}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6825","nonce":2893453021}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6823","nonce":2893453021}]},"public_addr":"172.21.15.19:6819/2893453021","cluster_addr":"172.21.15.19:6821/2893453021","heartbeat_back_addr":"172.21.15.19:6825/2893453021","heartbeat_front_addr":"172.21.15.19:6823/2893453021","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6827","nonce":2039937925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6829","nonce":2039937925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6833","nonce":2039937925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6831","nonce":2039937925}]},"public_addr":"172.21.15.19:6827/2039937925","cluster_addr":"172.21.15.19:6829/2039937925","heartbeat_back_addr":"172.21.15.19:6833/2039937925","heartbeat_front_addr":"172.21.15.19:6831/2039937925","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6809","nonce":2834758136}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6811","nonce":2834758136}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6815","nonce":2834758136}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6813","nonce":2834758136}]},"public_addr":"172.21.15.167:6809/2834758136","cluster_addr":"172.21.15.167:6811/2834758136","heartbeat_back_addr":"172.21.15.167:6815/2834758136","heartbeat_front_addr":"172.21.15.167:6813/2834758136","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:10:35.990 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-30T09:08:36.585315+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-30T09:10:36.193 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-30T09:10:36.194 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:35 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/295670870' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:10:36.194 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:35 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/295670870' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:10:36.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:35 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/295670870' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:10:36.558 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:10:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:36 smithi019 ceph-mon[100214]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-30T09:10:37.816 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:36 smithi019 ceph-mon[118983]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-30T09:10:37.816 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:36 smithi167 ceph-mon[104403]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-30T09:10:38.276 INFO:teuthology.orchestra.run.smithi019.stdout:pg_num: 1 2023-09-30T09:10:39.022 INFO:tasks.cephadm:Adding prometheus.a on smithi167 2023-09-30T09:10:39.023 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi167=a' 2023-09-30T09:10:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:39 smithi019 ceph-mon[100214]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2023-09-30T09:10:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:39 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3077913370' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-30T09:10:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:39 smithi019 ceph-mon[118983]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2023-09-30T09:10:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:39 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3077913370' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-30T09:10:39.380 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:10:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:39 smithi167 ceph-mon[104403]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2023-09-30T09:10:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:39 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3077913370' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-30T09:10:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:40 smithi167 ceph-mon[104403]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2023-09-30T09:10:41.060 INFO:teuthology.orchestra.run.smithi167.stdout:Scheduled prometheus update... 2023-09-30T09:10:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:40 smithi019 ceph-mon[100214]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2023-09-30T09:10:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:40 smithi019 ceph-mon[118983]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2023-09-30T09:10:41.658 DEBUG:teuthology.orchestra.run.smithi167:prometheus.a> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-30T09:10:41.660 INFO:tasks.cephadm:Adding node-exporter.a on smithi019 2023-09-30T09:10:41.661 INFO:tasks.cephadm:Adding node-exporter.b on smithi167 2023-09-30T09:10:41.661 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi019=a;smithi167=b' 2023-09-30T09:10:41.693 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:10:42.027 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:10:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[118983]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi167=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[118983]: Saving service prometheus spec with placement smithi167=a;count:1 2023-09-30T09:10:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[118983]: Deploying daemon prometheus.a on smithi167 2023-09-30T09:10:42.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[100214]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi167=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:42.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[100214]: Saving service prometheus spec with placement smithi167=a;count:1 2023-09-30T09:10:42.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:42.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:42.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:42.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:42.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:42.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:42 smithi019 ceph-mon[100214]: Deploying daemon prometheus.a on smithi167 2023-09-30T09:10:42.390 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:42 smithi167 ceph-mon[104403]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi167=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:42.391 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:42 smithi167 ceph-mon[104403]: Saving service prometheus spec with placement smithi167=a;count:1 2023-09-30T09:10:42.391 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:42 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:42.391 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:42 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:10:42.391 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:42 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:10:42.391 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:42 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:10:42.391 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:42 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:42.391 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:42 smithi167 ceph-mon[104403]: Deploying daemon prometheus.a on smithi167 2023-09-30T09:10:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:43 smithi019 ceph-mon[118983]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:10:43.437 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:43 smithi019 ceph-mon[100214]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:10:43.438 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:43 smithi167 ceph-mon[104403]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-30T09:10:43.886 INFO:teuthology.orchestra.run.smithi167.stdout:Scheduled node-exporter update... 2023-09-30T09:10:45.101 DEBUG:teuthology.orchestra.run.smithi019:node-exporter.a> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-30T09:10:45.104 DEBUG:teuthology.orchestra.run.smithi167:node-exporter.b> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-30T09:10:45.106 INFO:tasks.cephadm:Adding alertmanager.a on smithi019 2023-09-30T09:10:45.106 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi019=a' 2023-09-30T09:10:45.136 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:10:45.141 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:10:45.158 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:44 smithi167 ceph-mon[104403]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi019=a;smithi167=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:45.158 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:44 smithi167 ceph-mon[104403]: Saving service node-exporter spec with placement smithi019=a;smithi167=b;count:2 2023-09-30T09:10:45.159 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:44 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:45.159 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:44 smithi167 ceph-mon[104403]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-30T09:10:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:44 smithi019 ceph-mon[100214]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi019=a;smithi167=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:44 smithi019 ceph-mon[100214]: Saving service node-exporter spec with placement smithi019=a;smithi167=b;count:2 2023-09-30T09:10:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:44 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:44 smithi019 ceph-mon[100214]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-30T09:10:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:44 smithi019 ceph-mon[118983]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi019=a;smithi167=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:44 smithi019 ceph-mon[118983]: Saving service node-exporter spec with placement smithi019=a;smithi167=b;count:2 2023-09-30T09:10:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:44 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:44 smithi019 ceph-mon[118983]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-30T09:10:45.472 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:10:47.213 INFO:teuthology.orchestra.run.smithi167.stdout:Scheduled alertmanager update... 2023-09-30T09:10:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:47 smithi167 ceph-mon[104403]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:47.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:47 smithi019 ceph-mon[100214]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:47.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:47 smithi019 ceph-mon[118983]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:48.275 DEBUG:teuthology.orchestra.run.smithi019:alertmanager.a> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-30T09:10:48.277 INFO:tasks.cephadm:Adding grafana.a on smithi167 2023-09-30T09:10:48.278 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi167=a' 2023-09-30T09:10:48.311 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:10:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:48 smithi167 ceph-mon[104403]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi019=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:48 smithi167 ceph-mon[104403]: Saving service alertmanager spec with placement smithi019=a;count:1 2023-09-30T09:10:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:48 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:48 smithi167 ceph-mon[104403]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:48 smithi019 ceph-mon[100214]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi019=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:48 smithi019 ceph-mon[100214]: Saving service alertmanager spec with placement smithi019=a;count:1 2023-09-30T09:10:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:48 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:48 smithi019 ceph-mon[100214]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:48 smithi019 ceph-mon[118983]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi019=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:48 smithi019 ceph-mon[118983]: Saving service alertmanager spec with placement smithi019=a;count:1 2023-09-30T09:10:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:48 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:48 smithi019 ceph-mon[118983]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:50.734 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:10:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:50 smithi019 ceph-mon[100214]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:51.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:50 smithi019 ceph-mon[118983]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:51.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:50 smithi167 ceph-mon[104403]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:53 smithi167 ceph-mon[104403]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:53 smithi019 ceph-mon[100214]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:53 smithi019 ceph-mon[118983]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:55 smithi167 ceph-mon[104403]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:55 smithi019 ceph-mon[100214]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:55 smithi019 ceph-mon[118983]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:55.729 INFO:teuthology.orchestra.run.smithi167.stdout:Scheduled grafana update... 2023-09-30T09:10:56.721 DEBUG:teuthology.orchestra.run.smithi167:grafana.a> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-30T09:10:56.723 INFO:tasks.cephadm:Setting up client nodes... 2023-09-30T09:10:56.724 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-30T09:10:56.782 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:56 smithi167 ceph-mon[104403]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi167=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:56.874 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:56 smithi167 ceph-mon[104403]: Saving service grafana spec with placement smithi167=a;count:1 2023-09-30T09:10:56.874 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:56 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:56.874 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:56 smithi167 ceph-mon[104403]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:56.875 INFO:journalctl@ceph.grafana.a.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:10:57.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:56 smithi019 ceph-mon[100214]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi167=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:57.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:56 smithi019 ceph-mon[100214]: Saving service grafana spec with placement smithi167=a;count:1 2023-09-30T09:10:57.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:56 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:57.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:56 smithi019 ceph-mon[100214]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:57.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:56 smithi019 ceph-mon[118983]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi167=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:10:57.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:56 smithi019 ceph-mon[118983]: Saving service grafana spec with placement smithi167=a;count:1 2023-09-30T09:10:57.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:56 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:57.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:56 smithi019 ceph-mon[118983]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:57.089 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:10:57.216 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:56 smithi167 systemd[1]: Starting Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:10:58.882 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:58.891 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:58.891 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:58.891 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-30T09:10:58.891 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[100214]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:58.891 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.380Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-30T09:10:58.891 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.380Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-30T09:10:58.892 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.380Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi167 (none))" 2023-09-30T09:10:58.892 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.380Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-30T09:10:58.892 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.380Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-30T09:10:58.892 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.381Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-30T09:10:58.892 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.381Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-30T09:10:59.035 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.382Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-30T09:10:59.035 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.382Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-30T09:10:59.035 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 bash[148236]: 4089b72911965cc7b8b68bd2b10302ea20be883869a2005b417acf36367dddbc 2023-09-30T09:10:59.035 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.386Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-30T09:10:59.035 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.386Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.047µs 2023-09-30T09:10:59.201 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.386Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-30T09:10:59.201 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.386Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-30T09:10:59.202 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.386Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.226µs wal_replay_duration=238.986µs wbl_replay_duration=87ns total_replay_duration=305.531µs 2023-09-30T09:10:59.202 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.387Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-30T09:10:59.202 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.387Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-30T09:10:59.202 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.387Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-30T09:10:59.202 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 systemd[1]: Started Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:10:59.203 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.402Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.40547ms db_storage=688ns remote_storage=1.059µs web_handler=371ns query_engine=609ns scrape=262.469µs scrape_sd=17.703µs notify=866ns notify_sd=1.91µs rules=14.947769ms tracing=6.191µs 2023-09-30T09:10:59.203 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.402Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-30T09:10:59.203 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:10:57 smithi167 conmon[148277]: ts=2023-09-30T09:10:57.402Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-30T09:10:59.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:59.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:59.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:59.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-30T09:10:59.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mon[118983]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:59.205 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:58 smithi167 conmon[107623]: ignoring --setuser ceph since I am not root 2023-09-30T09:10:59.208 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:58 smithi167 conmon[107623]: ignoring --setgroup ceph since I am not root 2023-09-30T09:10:59.208 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:58 smithi167 ceph-mgr[107656]: -- 172.21.15.167:0/2633731666 <== mon.0 v2:172.21.15.19:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b6ea7ea9c0 con 0x55b6ea7c2800 2023-09-30T09:10:59.209 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:58 smithi167 conmon[107623]: 2023-09-30T09:10:58.635+0000 7f4d1556c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:10:59.209 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:58 smithi167 conmon[107623]: 2023-09-30T09:10:58.712+0000 7f4d1556c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:10:59.209 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:58 smithi019 conmon[100636]: ignoring --setuser ceph since I am not root 2023-09-30T09:10:59.209 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:58 smithi019 conmon[100636]: ignoring --setgroup ceph since I am not root 2023-09-30T09:10:59.210 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:58 smithi019 ceph-mgr[100660]: -- 172.21.15.19:0/3339503882 <== mon.0 v2:172.21.15.19:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dd34e6e9c0 con 0x55dd34e47000 2023-09-30T09:10:59.210 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:58 smithi019 conmon[100636]: 2023-09-30T09:10:58.666+0000 7f3aa8d2c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:10:59.210 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:59.210 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:59.210 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' 2023-09-30T09:10:59.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:58 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-30T09:10:59.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:58 smithi167 ceph-mon[104403]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:10:59.211 INFO:teuthology.orchestra.run.smithi019.stdout:[client.0] 2023-09-30T09:10:59.211 INFO:teuthology.orchestra.run.smithi019.stdout: key = AQAi5hdlO4C6MRAAvF6HzVSSshuR2/i5WV/TuA== 2023-09-30T09:10:59.212 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:58 smithi167 conmon[107623]: 2023-09-30T09:10:58.800+0000 7f4d1556c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:10:59.212 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:58 smithi167 conmon[107623]: 2023-09-30T09:10:58.891+0000 7f4d1556c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:10:59.355 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:58 smithi019 conmon[100636]: 2023-09-30T09:10:58.760+0000 7f3aa8d2c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:10:59.356 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:58 smithi019 conmon[100636]: 2023-09-30T09:10:58.869+0000 7f3aa8d2c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:10:59.357 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:58 smithi019 conmon[100636]: 2023-09-30T09:10:58.985+0000 7f3aa8d2c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:10:59.668 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:59 smithi019 conmon[100636]: 2023-09-30T09:10:59.192+0000 7f3aa8d2c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:10:59.669 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:59 smithi167 conmon[107623]: 2023-09-30T09:10:59.056+0000 7f4d1556c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:10:59.670 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:10:59.670 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-30T09:10:59.670 DEBUG:teuthology.orchestra.run.smithi019:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-30T09:10:59.675 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:59 smithi167 conmon[107623]: 2023-09-30T09:10:59.362+0000 7f4d1556c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:11:01.958 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:59 smithi167 conmon[107623]: 2023-09-30T09:10:59.449+0000 7f4d1556c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:11:01.958 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:59 smithi167 conmon[107623]: 2023-09-30T09:10:59.582+0000 7f4d1556c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:11:01.958 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:59 smithi167 ceph-mon[104403]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-30T09:11:01.958 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:59 smithi167 ceph-mon[104403]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-30T09:11:01.958 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:59 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/763286672' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:01.959 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:10:59 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/763286672' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-30T09:11:01.959 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:10:59 smithi167 conmon[107623]: 2023-09-30T09:10:59.674+0000 7f4d1556c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:11:01.959 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:00 smithi167 conmon[107623]: 2023-09-30T09:11:00.095+0000 7f4d1556c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:11:01.959 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:00 smithi167 conmon[107623]: 2023-09-30T09:11:00.657+0000 7f4d1556c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:11:01.959 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:00 smithi167 conmon[107623]: 2023-09-30T09:11:00.825+0000 7f4d1556c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:11:01.960 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:01 smithi167 conmon[107623]: 2023-09-30T09:11:01.583+0000 7f4d1556c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:11:01.998 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:01 smithi167 conmon[107623]: 2023-09-30T09:11:01.673+0000 7f4d1556c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:11:01.999 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:59 smithi019 conmon[100636]: 2023-09-30T09:10:59.593+0000 7f3aa8d2c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:11:01.999 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:59 smithi019 conmon[100636]: 2023-09-30T09:10:59.701+0000 7f3aa8d2c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:11:01.999 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:59 smithi019 ceph-mon[100214]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-30T09:11:01.999 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:59 smithi019 ceph-mon[100214]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-30T09:11:02.003 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:59 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/763286672' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:02.003 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:10:59 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/763286672' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-30T09:11:02.004 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:59 smithi019 ceph-mon[118983]: from='mgr.14152 172.21.15.19:0/1653529200' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-30T09:11:02.004 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:59 smithi019 ceph-mon[118983]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-30T09:11:02.004 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:59 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/763286672' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:02.004 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:10:59 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/763286672' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-30T09:11:02.005 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-30T09:11:02.006 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:59 smithi019 conmon[100636]: 2023-09-30T09:10:59.854+0000 7f3aa8d2c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:11:02.006 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:10:59 smithi019 conmon[100636]: 2023-09-30T09:10:59.967+0000 7f3aa8d2c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:11:02.006 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:00 smithi019 conmon[100636]: 2023-09-30T09:11:00.473+0000 7f3aa8d2c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:11:02.006 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:01 smithi019 conmon[100636]: 2023-09-30T09:11:01.152+0000 7f3aa8d2c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:11:02.006 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:01 smithi019 conmon[100636]: 2023-09-30T09:11:01.342+0000 7f3aa8d2c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:11:02.125 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:01 smithi167 conmon[107623]: 2023-09-30T09:11:01.841+0000 7f4d1556c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:11:02.363 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:11:02.521 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:02 smithi167 conmon[107623]: 2023-09-30T09:11:02.124+0000 7f4d1556c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:11:02.630 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:02 smithi167 conmon[107623]: 2023-09-30T09:11:02.219+0000 7f4d1556c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:11:02.634 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:02 smithi019 conmon[100636]: 2023-09-30T09:11:02.300+0000 7f3aa8d2c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:11:02.634 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:02 smithi019 conmon[100636]: 2023-09-30T09:11:02.410+0000 7f3aa8d2c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:11:02.807 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:02 smithi167 conmon[107623]: 2023-09-30T09:11:02.534+0000 7f4d1556c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:11:02.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:02 smithi019 conmon[100636]: 2023-09-30T09:11:02.626+0000 7f3aa8d2c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:11:03.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:02 smithi167 conmon[107623]: 2023-09-30T09:11:02.810+0000 7f4d1556c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:11:03.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:02 smithi019 conmon[100636]: 2023-09-30T09:11:02.971+0000 7f3aa8d2c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:11:03.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:03 smithi019 conmon[100636]: 2023-09-30T09:11:03.076+0000 7f3aa8d2c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:11:03.736 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:03 smithi167 conmon[107623]: 2023-09-30T09:11:03.549+0000 7f4d1556c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:11:03.736 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:03 smithi167 conmon[107623]: 2023-09-30T09:11:03.687+0000 7f4d1556c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:11:03.736 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:03 smithi019 conmon[100636]: 2023-09-30T09:11:03.410+0000 7f3aa8d2c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:11:04.021 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:03 smithi167 conmon[107623]: 2023-09-30T09:11:03.892+0000 7f4d1556c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:11:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:03 smithi019 conmon[100636]: 2023-09-30T09:11:03.736+0000 7f3aa8d2c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:11:04.217 INFO:teuthology.orchestra.run.smithi167.stdout:[client.1] 2023-09-30T09:11:04.217 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQAo5hdl4Pe7DBAA7QRRJS8u2pAKdHyG2/hCmA== 2023-09-30T09:11:04.442 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:04 smithi167 conmon[107623]: 2023-09-30T09:11:04.021+0000 7f4d1556c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:11:04.442 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:04 smithi167 conmon[107623]: 2023-09-30T09:11:04.254+0000 7f4d1556c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:11:04.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:04 smithi167 ceph-mon[104403]: from='client.? 172.21.15.167:0/2573012889' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:04.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:04 smithi167 ceph-mon[104403]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:04.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:04 smithi167 ceph-mon[104403]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-30T09:11:04.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:04 smithi019 ceph-mon[118983]: from='client.? 172.21.15.167:0/2573012889' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:04.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:04 smithi019 ceph-mon[118983]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:04.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:04 smithi019 ceph-mon[118983]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-30T09:11:04.566 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:04 smithi019 ceph-mon[100214]: from='client.? 172.21.15.167:0/2573012889' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:04.566 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:04 smithi019 ceph-mon[100214]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-30T09:11:04.566 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:04 smithi019 ceph-mon[100214]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-30T09:11:04.829 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-09-30T09:11:04.829 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-30T09:11:04.829 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-30T09:11:04.834 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:04 smithi019 conmon[100636]: 2023-09-30T09:11:04.564+0000 7f3aa8d2c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:11:04.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:04 smithi019 conmon[100636]: 2023-09-30T09:11:04.723+0000 7f3aa8d2c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:11:04.874 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-30T09:11:04.874 DEBUG:teuthology.orchestra.run.smithi019:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-30T09:11:05.197 INFO:teuthology.misc.health.smithi019.stdout: 2023-09-30T09:11:05.197 INFO:teuthology.misc.health.smithi019.stdout:{"epoch":58,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:10:32.597386+0000","last_up_change":"2023-09-30T09:10:28.570615+0000","last_in_change":"2023-09-30T09:10:11.880926+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6803","nonce":1200588465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6805","nonce":1200588465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6809","nonce":1200588465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6807","nonce":1200588465}]},"public_addr":"172.21.15.19:6803/1200588465","cluster_addr":"172.21.15.19:6805/1200588465","heartbeat_back_addr":"172.21.15.19:6809/1200588465","heartbeat_front_addr":"172.21.15.19:6807/1200588465","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6811","nonce":1005512342}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6813","nonce":1005512342}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6817","nonce":1005512342}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6815","nonce":1005512342}]},"public_addr":"172.21.15.19:6811/1005512342","cluster_addr":"172.21.15.19:6813/1005512342","heartbeat_back_addr":"172.21.15.19:6817/1005512342","heartbeat_front_addr":"172.21.15.19:6815/1005512342","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6819","nonce":2893453021}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6821","nonce":2893453021}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6825","nonce":2893453021}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6823","nonce":2893453021}]},"public_addr":"172.21.15.19:6819/2893453021","cluster_addr":"172.21.15.19:6821/2893453021","heartbeat_back_addr":"172.21.15.19:6825/2893453021","heartbeat_front_addr":"172.21.15.19:6823/2893453021","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6827","nonce":2039937925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6829","nonce":2039937925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6833","nonce":2039937925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6831","nonce":2039937925}]},"public_addr":"172.21.15.19:6827/2039937925","cluster_addr":"172.21.15.19:6829/2039937925","heartbeat_back_addr":"172.21.15.19:6833/2039937925","heartbeat_front_addr":"172.21.15.19:6831/2039937925","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6809","nonce":2834758136}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6811","nonce":2834758136}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6815","nonce":2834758136}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6813","nonce":2834758136}]},"public_addr":"172.21.15.167:6809/2834758136","cluster_addr":"172.21.15.167:6811/2834758136","heartbeat_back_addr":"172.21.15.167:6815/2834758136","heartbeat_front_addr":"172.21.15.167:6813/2834758136","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:11:05.210 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-30T09:11:05.210 INFO:tasks.cephadm:Creating RBD pool 2023-09-30T09:11:05.210 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-30T09:11:05.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:04 smithi019 conmon[100636]: 2023-09-30T09:11:04.969+0000 7f3aa8d2c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:11:05.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:05 smithi019 conmon[100636]: 2023-09-30T09:11:05.112+0000 7f3aa8d2c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:11:05.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:05 smithi167 conmon[107623]: 2023-09-30T09:11:05.143+0000 7f4d1556c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:11:05.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:05 smithi167 conmon[107623]: [30/Sep/2023:09:11:05] ENGINE Bus STARTING 2023-09-30T09:11:05.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:05 smithi167 conmon[107623]: CherryPy Checker: 2023-09-30T09:11:05.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:05 smithi167 conmon[107623]: The Application mounted at '' has an empty config. 2023-09-30T09:11:05.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:05 smithi167 conmon[107623]: 2023-09-30T09:11:05.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:05 smithi167 conmon[107623]: [30/Sep/2023:09:11:05] ENGINE Serving on http://:::9283 2023-09-30T09:11:05.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:05 smithi167 conmon[107623]: [30/Sep/2023:09:11:05] ENGINE Bus STARTED 2023-09-30T09:11:05.575 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:05.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:05 smithi019 conmon[100636]: 2023-09-30T09:11:05.390+0000 7f3aa8d2c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:11:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[100214]: Standby manager daemon x restarted 2023-09-30T09:11:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[100214]: Standby manager daemon x started 2023-09-30T09:11:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:11:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:11:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:11:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:11:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/637494426' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[118983]: Standby manager daemon x restarted 2023-09-30T09:11:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[118983]: Standby manager daemon x started 2023-09-30T09:11:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:11:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:11:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:11:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:11:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:05 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/637494426' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:06.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:05 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:11:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:11:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:05 smithi167 ceph-mon[104403]: Standby manager daemon x restarted 2023-09-30T09:11:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:05 smithi167 ceph-mon[104403]: Standby manager daemon x started 2023-09-30T09:11:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:05 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:11:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:05 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:11:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:05 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:11:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:05 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/87553673' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:11:06.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:05 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/637494426' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:06.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:06 smithi019 conmon[100636]: 2023-09-30T09:11:06.484+0000 7f3aa8d2c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:11:06.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:06 smithi019 conmon[100636]: [30/Sep/2023:09:11:06] ENGINE Bus STARTING 2023-09-30T09:11:06.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:06 smithi019 conmon[100636]: CherryPy Checker: 2023-09-30T09:11:06.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:06 smithi019 conmon[100636]: The Application mounted at '' has an empty config. 2023-09-30T09:11:06.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:06 smithi019 conmon[100636]: 2023-09-30T09:11:06.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:06 smithi019 conmon[100636]: [30/Sep/2023:09:11:06] ENGINE Serving on http://:::9283 2023-09-30T09:11:06.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:06 smithi019 conmon[100636]: [30/Sep/2023:09:11:06] ENGINE Bus STARTED 2023-09-30T09:11:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-30T09:11:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: Active manager daemon y restarted 2023-09-30T09:11:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: Activating manager daemon y 2023-09-30T09:11:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: osdmap e59: 8 total, 8 up, 8 in 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: mgrmap e18: y(active, starting, since 0.00825957s), standbys: x 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:11:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: Manager daemon y is now available 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:11:07.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:11:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-30T09:11:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: Active manager daemon y restarted 2023-09-30T09:11:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: Activating manager daemon y 2023-09-30T09:11:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: osdmap e59: 8 total, 8 up, 8 in 2023-09-30T09:11:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: mgrmap e18: y(active, starting, since 0.00825957s), standbys: x 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:11:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: Manager daemon y is now available 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:11:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: Active manager daemon y restarted 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: Activating manager daemon y 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: osdmap e59: 8 total, 8 up, 8 in 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: mgrmap e18: y(active, starting, since 0.00825957s), standbys: x 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:11:07.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:11:07.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: Manager daemon y is now available 2023-09-30T09:11:07.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:07.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:07.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:07.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:07.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:11:07.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:11:07.906 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'rbd' created 2023-09-30T09:11:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:07 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/399887722' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-30T09:11:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:07 smithi167 ceph-mon[104403]: mgrmap e19: y(active, since 1.36125s), standbys: x 2023-09-30T09:11:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:07 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/399887722' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-30T09:11:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:07 smithi019 ceph-mon[100214]: mgrmap e19: y(active, since 1.36125s), standbys: x 2023-09-30T09:11:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:07 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/399887722' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-30T09:11:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:07 smithi019 ceph-mon[118983]: mgrmap e19: y(active, since 1.36125s), standbys: x 2023-09-30T09:11:08.768 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-30T09:11:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: [30/Sep/2023:09:11:07] ENGINE Bus STARTING 2023-09-30T09:11:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/399887722' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-30T09:11:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: osdmap e60: 8 total, 8 up, 8 in 2023-09-30T09:11:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: [30/Sep/2023:09:11:07] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:11:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: [30/Sep/2023:09:11:07] ENGINE Bus STARTED 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: [30/Sep/2023:09:11:07] ENGINE Bus STARTING 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/399887722' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: osdmap e60: 8 total, 8 up, 8 in 2023-09-30T09:11:09.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: [30/Sep/2023:09:11:07] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:11:09.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: [30/Sep/2023:09:11:07] ENGINE Bus STARTED 2023-09-30T09:11:09.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:09.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:09.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:09.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:09.219 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: [30/Sep/2023:09:11:07] ENGINE Bus STARTING 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/399887722' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: osdmap e60: 8 total, 8 up, 8 in 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: [30/Sep/2023:09:11:07] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: [30/Sep/2023:09:11:07] ENGINE Bus STARTED 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: osdmap e61: 8 total, 8 up, 8 in 2023-09-30T09:11:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: mgrmap e20: y(active, since 2s), standbys: x 2023-09-30T09:11:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi167 to 4036M 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: osdmap e61: 8 total, 8 up, 8 in 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: mgrmap e20: y(active, since 2s), standbys: x 2023-09-30T09:11:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi167 to 4036M 2023-09-30T09:11:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:09 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: osdmap e61: 8 total, 8 up, 8 in 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: mgrmap e20: y(active, since 2s), standbys: x 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:11:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi167 to 4036M 2023-09-30T09:11:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:09 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: osdmap e62: 8 total, 8 up, 8 in 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:11:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:10 smithi167 ceph-mon[104403]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: osdmap e62: 8 total, 8 up, 8 in 2023-09-30T09:11:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:11:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:11:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:11:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:11:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[118983]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: osdmap e62: 8 total, 8 up, 8 in 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:11:11.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:11:11.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:11:11.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:10 smithi019 ceph-mon[100214]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:11.932 INFO:teuthology.orchestra.run.smithi019.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3926622225' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:11 smithi167 ceph-mon[104403]: Deploying daemon alertmanager.a on smithi019 2023-09-30T09:11:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:11:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:11:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:11:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:11:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3926622225' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:11:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[100214]: Deploying daemon alertmanager.a on smithi019 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:11:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3926622225' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:12.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:11 smithi019 ceph-mon[118983]: Deploying daemon alertmanager.a on smithi019 2023-09-30T09:11:12.902 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-30T09:11:12.902 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-30T09:11:12.902 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-30T09:11:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:12 smithi167 ceph-mon[104403]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:11:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:12 smithi167 ceph-mon[104403]: osdmap e63: 8 total, 8 up, 8 in 2023-09-30T09:11:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:12 smithi167 ceph-mon[104403]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:13.275 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:12 smithi019 ceph-mon[100214]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:11:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:12 smithi019 ceph-mon[100214]: osdmap e63: 8 total, 8 up, 8 in 2023-09-30T09:11:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:12 smithi019 ceph-mon[100214]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:12 smithi019 ceph-mon[118983]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:11:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:12 smithi019 ceph-mon[118983]: osdmap e63: 8 total, 8 up, 8 in 2023-09-30T09:11:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:12 smithi019 ceph-mon[118983]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:14 smithi167 ceph-mon[104403]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:15.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:14 smithi019 ceph-mon[100214]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:15.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:14 smithi019 ceph-mon[118983]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:16.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:15 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:11:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:11:16.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:16 smithi019 ceph-mon[100214]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:16 smithi019 ceph-mon[118983]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:16 smithi167 ceph-mon[104403]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:17.610 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:11:17.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:17 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3605792889' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:11:17.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:17 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3605792889' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:11:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:17 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3605792889' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:11:18.718 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":20,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6800","nonce":2650629233},{"type":"v1","addr":"172.21.15.19:6801","nonce":2650629233}]},"active_addr":"172.21.15.19:6801/2650629233","active_change":"2023-09-30T09:11:06.498433+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24361,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":956155}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":3931489393}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":238111816}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":1638946842}]}]} 2023-09-30T09:11:18.723 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-30T09:11:18.723 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-30T09:11:18.723 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:11:18.939 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:18 smithi019 ceph-mon[118983]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:18.939 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:18 smithi019 ceph-mon[100214]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:18 smithi167 ceph-mon[104403]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:19.872 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:20.153 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:20 smithi019 systemd[1]: Starting Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:11:21.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:20 smithi019 ceph-mon[100214]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:21.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:20 smithi019 ceph-mon[118983]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:21.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:20 smithi167 ceph-mon[104403]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:21.828 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[100636]: [30/Sep/2023:09:11:21] ENGINE Bus STOPPING 2023-09-30T09:11:21.828 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[166358]: ts=2023-09-30T09:11:21.631Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[166358]: ts=2023-09-30T09:11:21.631Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[166358]: ts=2023-09-30T09:11:21.633Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.19 port=9094 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[166358]: ts=2023-09-30T09:11:21.634Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 bash[166272]: bbb8160f99ae4abfa52a08949679012b8752d3be589b7f0c26f0b52511794ea4 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 systemd[1]: Started Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[166358]: ts=2023-09-30T09:11:21.668Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[166358]: ts=2023-09-30T09:11:21.668Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[166358]: ts=2023-09-30T09:11:21.671Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-30T09:11:21.829 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[166358]: ts=2023-09-30T09:11:21.671Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-30T09:11:22.065 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:11:22.065 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":63,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:11:11.919326+0000","last_up_change":"2023-09-30T09:10:28.570615+0000","last_in_change":"2023-09-30T09:10:11.880926+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-30T09:11:07.846610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6803","nonce":1200588465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6805","nonce":1200588465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6809","nonce":1200588465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6807","nonce":1200588465}]},"public_addr":"172.21.15.19:6803/1200588465","cluster_addr":"172.21.15.19:6805/1200588465","heartbeat_back_addr":"172.21.15.19:6809/1200588465","heartbeat_front_addr":"172.21.15.19:6807/1200588465","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6811","nonce":1005512342}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6813","nonce":1005512342}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6817","nonce":1005512342}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6815","nonce":1005512342}]},"public_addr":"172.21.15.19:6811/1005512342","cluster_addr":"172.21.15.19:6813/1005512342","heartbeat_back_addr":"172.21.15.19:6817/1005512342","heartbeat_front_addr":"172.21.15.19:6815/1005512342","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6819","nonce":2893453021}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6821","nonce":2893453021}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6825","nonce":2893453021}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6823","nonce":2893453021}]},"public_addr":"172.21.15.19:6819/2893453021","cluster_addr":"172.21.15.19:6821/2893453021","heartbeat_back_addr":"172.21.15.19:6825/2893453021","heartbeat_front_addr":"172.21.15.19:6823/2893453021","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6827","nonce":2039937925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6829","nonce":2039937925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6833","nonce":2039937925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6831","nonce":2039937925}]},"public_addr":"172.21.15.19:6827/2039937925","cluster_addr":"172.21.15.19:6829/2039937925","heartbeat_back_addr":"172.21.15.19:6833/2039937925","heartbeat_front_addr":"172.21.15.19:6831/2039937925","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6809","nonce":2834758136}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6811","nonce":2834758136}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6815","nonce":2834758136}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6813","nonce":2834758136}]},"public_addr":"172.21.15.167:6809/2834758136","cluster_addr":"172.21.15.167:6811/2834758136","heartbeat_back_addr":"172.21.15.167:6815/2834758136","heartbeat_front_addr":"172.21.15.167:6813/2834758136","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:6801/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3152350528":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/1679148187":"2023-10-01T09:11:06.498249+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/3324367156":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6800/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/936416645":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:11:22.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[100636]: [30/Sep/2023:09:11:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:11:22.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[100636]: [30/Sep/2023:09:11:21] ENGINE Bus STOPPED 2023-09-30T09:11:22.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[100636]: [30/Sep/2023:09:11:21] ENGINE Bus STARTING 2023-09-30T09:11:22.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[100636]: [30/Sep/2023:09:11:21] ENGINE Serving on http://:::9283 2023-09-30T09:11:22.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:11:21 smithi019 conmon[100636]: [30/Sep/2023:09:11:21] ENGINE Bus STARTED 2023-09-30T09:11:22.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: Deploying daemon grafana.a on smithi167 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3997679148' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[100214]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-30T09:11:22.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-30T09:11:22.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:22.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: Deploying daemon grafana.a on smithi167 2023-09-30T09:11:22.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3997679148' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:22.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:22 smithi019 ceph-mon[118983]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:22.885 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-30T09:11:22.885 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:11:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-30T09:11:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-30T09:11:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: Deploying daemon grafana.a on smithi167 2023-09-30T09:11:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3997679148' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:22 smithi167 ceph-mon[104403]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:23.246 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:23.834 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:23 smithi019 conmon[166358]: ts=2023-09-30T09:11:23.634Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00015232s 2023-09-30T09:11:24.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:24 smithi019 ceph-mon[100214]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:24 smithi019 ceph-mon[118983]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:24.948 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:11:24.948 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":63,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:11:11.919326+0000","last_up_change":"2023-09-30T09:10:28.570615+0000","last_in_change":"2023-09-30T09:10:11.880926+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-30T09:11:07.846610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6803","nonce":1200588465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6805","nonce":1200588465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6809","nonce":1200588465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6807","nonce":1200588465}]},"public_addr":"172.21.15.19:6803/1200588465","cluster_addr":"172.21.15.19:6805/1200588465","heartbeat_back_addr":"172.21.15.19:6809/1200588465","heartbeat_front_addr":"172.21.15.19:6807/1200588465","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6811","nonce":1005512342}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6813","nonce":1005512342}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6817","nonce":1005512342}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6815","nonce":1005512342}]},"public_addr":"172.21.15.19:6811/1005512342","cluster_addr":"172.21.15.19:6813/1005512342","heartbeat_back_addr":"172.21.15.19:6817/1005512342","heartbeat_front_addr":"172.21.15.19:6815/1005512342","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6819","nonce":2893453021}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6821","nonce":2893453021}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6825","nonce":2893453021}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6823","nonce":2893453021}]},"public_addr":"172.21.15.19:6819/2893453021","cluster_addr":"172.21.15.19:6821/2893453021","heartbeat_back_addr":"172.21.15.19:6825/2893453021","heartbeat_front_addr":"172.21.15.19:6823/2893453021","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6827","nonce":2039937925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6829","nonce":2039937925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6833","nonce":2039937925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6831","nonce":2039937925}]},"public_addr":"172.21.15.19:6827/2039937925","cluster_addr":"172.21.15.19:6829/2039937925","heartbeat_back_addr":"172.21.15.19:6833/2039937925","heartbeat_front_addr":"172.21.15.19:6831/2039937925","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6809","nonce":2834758136}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6811","nonce":2834758136}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6815","nonce":2834758136}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6813","nonce":2834758136}]},"public_addr":"172.21.15.167:6809/2834758136","cluster_addr":"172.21.15.167:6811/2834758136","heartbeat_back_addr":"172.21.15.167:6815/2834758136","heartbeat_front_addr":"172.21.15.167:6813/2834758136","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:6801/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3152350528":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/1679148187":"2023-10-01T09:11:06.498249+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/3324367156":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6800/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/936416645":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:11:25.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:24 smithi167 ceph-mon[104403]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:25.590 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-30T09:11:25.590 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-30T09:11:25.591 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-30T09:11:25.591 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-30T09:11:25.592 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-30T09:11:25.592 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-30T09:11:25.592 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-30T09:11:25.593 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-30T09:11:26.261 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:26.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:25 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:11:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:11:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:25 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3728215732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:26.310 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:25 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3728215732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:25 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3728215732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:11:26.581 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:26.942 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:27.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:26 smithi019 ceph-mon[100214]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:27.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:26 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:27.175 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:26 smithi019 ceph-mon[118983]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:27.175 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:26 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:27.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:26 smithi167 ceph-mon[104403]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:26 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:28.089 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:28.346 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:28.511 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:28.783 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:28 smithi019 ceph-mon[100214]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:28.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:28 smithi019 ceph-mon[118983]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:29.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:28 smithi167 ceph-mon[104403]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:31.013 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:30 smithi019 ceph-mon[100214]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:31.013 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:30 smithi019 ceph-mon[118983]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:30 smithi167 ceph-mon[104403]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:31.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:11:31 smithi019 conmon[166358]: ts=2023-09-30T09:11:31.636Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001924267s 2023-09-30T09:11:32.723 INFO:teuthology.orchestra.run.smithi019.stdout:128849018907 2023-09-30T09:11:32.723 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-30T09:11:32.946 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:32 smithi019 ceph-mon[100214]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:32.947 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:32 smithi019 ceph-mon[118983]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:32.970 INFO:teuthology.orchestra.run.smithi019.stdout:197568495634 2023-09-30T09:11:32.970 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-30T09:11:33.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:32 smithi167 ceph-mon[104403]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:34.036 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:34.712 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:34 smithi167 ceph-mon[104403]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:34.798 INFO:teuthology.orchestra.run.smithi019.stdout:107374182433 2023-09-30T09:11:34.799 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-30T09:11:34.866 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:34 smithi019 ceph-mon[100214]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:34.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:34 smithi019 ceph-mon[118983]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:35.555 INFO:teuthology.orchestra.run.smithi019.stdout:77309411365 2023-09-30T09:11:35.555 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-30T09:11:35.603 INFO:teuthology.orchestra.run.smithi019.stdout:231928233998 2023-09-30T09:11:35.604 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-30T09:11:36.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:35 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:11:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:11:36.788 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:36 smithi019 ceph-mon[100214]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:36.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:36 smithi019 ceph-mon[118983]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:36.856 INFO:teuthology.orchestra.run.smithi019.stdout:163208757272 2023-09-30T09:11:36.856 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-30T09:11:36.926 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:36 smithi167 ceph-mon[104403]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:37.326 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:37.327 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:37.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:37 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:37.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:37 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:38.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:37 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:38.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:38 smithi019 ceph-mon[100214]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:38.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:38 smithi019 ceph-mon[118983]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:38 smithi167 ceph-mon[104403]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:39.609 INFO:teuthology.orchestra.run.smithi019.stdout:55834574890 2023-09-30T09:11:39.610 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-30T09:11:39.735 INFO:teuthology.orchestra.run.smithi019.stdout:34359738416 2023-09-30T09:11:39.735 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-30T09:11:39.843 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:40.643 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:40.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:40 smithi019 ceph-mon[100214]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:40.890 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:40 smithi019 ceph-mon[118983]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:40.892 INFO:teuthology.orchestra.run.smithi019.stdout:128849018909 2023-09-30T09:11:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:40 smithi167 ceph-mon[104403]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:41.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:41 smithi167 systemd[1]: Starting Ceph grafana.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:11:42.020 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:41 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3177966909' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:11:42.020 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:41 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3177966909' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:11:42.148 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:41 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3177966909' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:11:42.431 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:42.461 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018909 for osd.4 2023-09-30T09:11:42.461 DEBUG:teuthology.parallel:result is None 2023-09-30T09:11:42.570 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 bash[151225]: 1e414220bf374cfcc91ff68c39306554766b69236339d14cf7a9f05f0c76d33f 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 systemd[1]: Started Ceph grafana.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444413617Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444586878Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444604115Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444610582Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444616504Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444622972Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444628655Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444634277Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444640313Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-30T09:11:42.571 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444645813Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444651002Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444656057Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444661502Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=settings t=2023-09-30T09:11:42.444667826Z level=info msg="App mode production" 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=sqlstore t=2023-09-30T09:11:42.444734303Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=sqlstore t=2023-09-30T09:11:42.444750212Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.445573529Z level=info msg="Starting DB migrations" 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.445647399Z level=info msg="Executing migration" id="create migration_log table" 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.461297316Z level=info msg="Executing migration" id="create user table" 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.478278541Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.511656656Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-30T09:11:42.572 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.570277917Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-30T09:11:42.823 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.612594212Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-30T09:11:42.823 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.629667346Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-30T09:11:42.823 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.663012242Z level=info msg="Executing migration" id="create user table v2" 2023-09-30T09:11:42.823 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.679935887Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-30T09:11:42.823 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.696902898Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-30T09:11:42.824 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.738723087Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-30T09:11:42.824 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.755547865Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-30T09:11:42.824 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.772430654Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-30T09:11:42.824 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.789406889Z level=info msg="Executing migration" id="Update user table charset" 2023-09-30T09:11:42.824 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.806208978Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-30T09:11:42.824 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.823091146Z level=info msg="Executing migration" id="Add missing user data" 2023-09-30T09:11:42.981 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:43.092 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.865079324Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.906964062Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.923774362Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.940836143Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.957739683Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.974673908Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:42 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:42.991612717Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.008620621Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.025349484Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.04229672Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-30T09:11:43.093 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.059098723Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-30T09:11:43.094 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.092726214Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-30T09:11:43.127 INFO:teuthology.orchestra.run.smithi019.stdout:231928233999 2023-09-30T09:11:43.361 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.159773492Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-30T09:11:43.361 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.176492922Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-30T09:11:43.362 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.193335972Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-30T09:11:43.362 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.210276633Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-30T09:11:43.362 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.227058427Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-30T09:11:43.362 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.252332857Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-30T09:11:43.362 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.26932787Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-30T09:11:43.362 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.294471627Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-30T09:11:43.362 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.319606631Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-30T09:11:43.362 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.361417329Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-30T09:11:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[100214]: Deploying daemon node-exporter.a on smithi019 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[100214]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1456029067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[118983]: Deploying daemon node-exporter.a on smithi019 2023-09-30T09:11:43.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[118983]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:43.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:43 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1456029067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:11:43.630 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:43 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.630 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:43 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.630 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:43 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.630 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:43 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:43.630 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:43 smithi167 ceph-mon[104403]: Deploying daemon node-exporter.a on smithi019 2023-09-30T09:11:43.630 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:43 smithi167 ceph-mon[104403]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:43.630 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:43 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1456029067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:11:43.631 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.403532958Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-30T09:11:43.631 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.445474505Z level=info msg="Executing migration" id="create star table" 2023-09-30T09:11:43.631 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.487327265Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-30T09:11:43.631 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.512546174Z level=info msg="Executing migration" id="create org table v1" 2023-09-30T09:11:43.631 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.537695372Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-30T09:11:43.631 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.562810808Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-30T09:11:43.631 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.588229766Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-30T09:11:43.631 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.629985917Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-30T09:11:43.898 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.672345369Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-30T09:11:43.899 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.714244151Z level=info msg="Executing migration" id="Update org table charset" 2023-09-30T09:11:43.899 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.789508734Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-30T09:11:43.899 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.831455881Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-30T09:11:43.899 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.873350567Z level=info msg="Executing migration" id="create dashboard table" 2023-09-30T09:11:43.899 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.89855273Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-30T09:11:44.025 INFO:teuthology.orchestra.run.smithi019.stdout:107374182434 2023-09-30T09:11:44.159 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.924084364Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-30T09:11:44.159 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.949061486Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-30T09:11:44.159 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.974346033Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-30T09:11:44.160 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:43 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:43.999488841Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-30T09:11:44.160 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.025040789Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-30T09:11:44.160 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.050246698Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-30T09:11:44.160 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.092124234Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-30T09:11:44.160 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.134160331Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-30T09:11:44.160 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.159192345Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-30T09:11:44.411 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.201229589Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-30T09:11:44.411 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.226472927Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-30T09:11:44.411 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.251636356Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-30T09:11:44.411 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.268406591Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-30T09:11:44.411 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.285203253Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-30T09:11:44.411 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.318771144Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-30T09:11:44.662 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:44 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2915751540' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:11:44.775 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.410750499Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-30T09:11:44.775 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.461302212Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-30T09:11:44.776 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.503062541Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-30T09:11:44.776 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.528449079Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-30T09:11:44.776 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.553475857Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-30T09:11:44.776 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.595407393Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-30T09:11:44.776 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.637416452Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-30T09:11:44.776 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.662592288Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-30T09:11:44.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:44 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2915751540' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:11:44.875 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:44 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2915751540' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:11:44.939 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.704699566Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-30T09:11:44.939 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.751149265Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-30T09:11:44.939 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.771733206Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-30T09:11:44.940 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.796948941Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-30T09:11:44.940 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.822079574Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-30T09:11:44.940 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.864044107Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-30T09:11:44.940 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.939355084Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-30T09:11:45.225 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:44 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:44.981592453Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-30T09:11:45.226 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.00664144Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-30T09:11:45.226 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.031813841Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-30T09:11:45.226 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.07372494Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-30T09:11:45.226 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.098845943Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-30T09:11:45.226 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.12424389Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-30T09:11:45.226 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.149251607Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-30T09:11:45.226 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.174543249Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-30T09:11:45.226 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.225163209Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-30T09:11:45.229 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233998 got 231928233999 for osd.7 2023-09-30T09:11:45.229 DEBUG:teuthology.parallel:result is None 2023-09-30T09:11:45.266 INFO:teuthology.orchestra.run.smithi019.stdout:77309411367 2023-09-30T09:11:45.291 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182433 got 107374182434 for osd.3 2023-09-30T09:11:45.291 DEBUG:teuthology.parallel:result is None 2023-09-30T09:11:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:45 smithi167 ceph-mon[104403]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:45 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1031920290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:11:45.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.275624827Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-30T09:11:45.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.300641485Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-30T09:11:45.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.325845662Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-30T09:11:45.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.350900837Z level=info msg="Executing migration" id="create data_source table" 2023-09-30T09:11:45.523 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.393040943Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-30T09:11:45.523 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.418212099Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-30T09:11:45.523 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.460103767Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-30T09:11:45.523 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:45 smithi019 ceph-mon[100214]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:45.523 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:45 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1031920290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:11:45.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:45 smithi019 ceph-mon[118983]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:45.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:45 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1031920290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.535337022Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.560509244Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.585835878Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.627687841Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.652901101Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.694698882Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.719815331Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.761736217Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-30T09:11:45.787 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.786727808Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-30T09:11:46.047 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:11:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:11:46.047 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.837282079Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-30T09:11:46.047 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.879149388Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-30T09:11:46.047 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.904362704Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-30T09:11:46.047 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.929618888Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-30T09:11:46.048 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.954825613Z level=info msg="Executing migration" id="Add uid column" 2023-09-30T09:11:46.048 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:45 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:45.979998677Z level=info msg="Executing migration" id="Update uid value" 2023-09-30T09:11:46.048 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.021852264Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-30T09:11:46.048 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.047015859Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-30T09:11:46.112 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:46.234 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411367 for osd.2 2023-09-30T09:11:46.235 DEBUG:teuthology.parallel:result is None 2023-09-30T09:11:46.307 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.09760981Z level=info msg="Executing migration" id="create api_key table" 2023-09-30T09:11:46.308 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.181238874Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-30T09:11:46.308 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.206416796Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-30T09:11:46.308 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.23154155Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-30T09:11:46.308 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.256796783Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-30T09:11:46.308 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.281837902Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-30T09:11:46.308 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.307091079Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-30T09:11:46.560 INFO:teuthology.orchestra.run.smithi019.stdout:197568495637 2023-09-30T09:11:46.590 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.332536862Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-30T09:11:46.590 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.374405205Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-30T09:11:46.591 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.399666132Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-30T09:11:46.591 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.424702106Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-30T09:11:46.591 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.475264157Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-30T09:11:46.591 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.50045337Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-30T09:11:46.591 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.525567591Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-30T09:11:46.591 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.550795338Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-30T09:11:46.712 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:46 smithi019 ceph-mon[100214]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:46.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:46 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4034328306' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:11:46.713 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:46 smithi019 ceph-mon[118983]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:46.713 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:46 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/4034328306' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:11:46.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:46 smithi167 ceph-mon[104403]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:46.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:46 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/4034328306' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:11:46.861 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.592553916Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-30T09:11:46.861 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.643072805Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-30T09:11:46.862 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.685037131Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-30T09:11:46.862 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.710400152Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-30T09:11:46.862 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.735564436Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-30T09:11:46.862 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.819007056Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-30T09:11:46.862 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.860985487Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.89481724Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.919897953Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.945021623Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:46 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:46.970362524Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.012193827Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.037269993Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.062597705Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.104543693Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-30T09:11:47.130 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.129699776Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-30T09:11:47.398 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.171895121Z level=info msg="Executing migration" id="create quota table v1" 2023-09-30T09:11:47.399 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.196968192Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-30T09:11:47.399 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.222327936Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-30T09:11:47.399 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.247534696Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-30T09:11:47.399 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.272547285Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-30T09:11:47.399 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.297818606Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-30T09:11:47.399 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.364816667Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-30T09:11:47.399 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.398366283Z level=info msg="Executing migration" id="create session table" 2023-09-30T09:11:47.581 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:47 smithi019 bash[173101]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-30T09:11:47.676 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:47 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:47.676 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.415259465Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-30T09:11:47.676 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.448875035Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-30T09:11:47.676 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.465661157Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-30T09:11:47.676 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.482621731Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-30T09:11:47.676 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.51605008Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-30T09:11:47.676 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.532904296Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-30T09:11:47.676 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.55817284Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-30T09:11:47.677 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.583771158Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-30T09:11:47.677 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.625409259Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-30T09:11:47.677 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.675666015Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-30T09:11:47.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:47 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:47 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:47.969 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.726190415Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-30T09:11:47.969 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.751496153Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-30T09:11:47.969 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.776683916Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-30T09:11:47.970 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.801821315Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-30T09:11:47.970 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.843714849Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-30T09:11:47.970 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.885521412Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-30T09:11:47.970 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:47 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:47.969238538Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-30T09:11:48.238 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.019716377Z level=info msg="Executing migration" id="create alert table v1" 2023-09-30T09:11:48.238 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.044890725Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-30T09:11:48.238 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.07003541Z level=info msg="Executing migration" id="add index alert state" 2023-09-30T09:11:48.238 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.095201762Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-30T09:11:48.238 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.137239697Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-30T09:11:48.238 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.162457603Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-30T09:11:48.238 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.18754136Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-30T09:11:48.238 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.212736848Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-30T09:11:48.239 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.237713325Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-30T09:11:48.279 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:47 smithi019 bash[173101]: Getting image source signatures 2023-09-30T09:11:48.279 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:48 smithi019 bash[173101]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-30T09:11:48.280 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:48 smithi019 bash[173101]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-30T09:11:48.425 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495634 got 197568495637 for osd.6 2023-09-30T09:11:48.425 DEBUG:teuthology.parallel:result is None 2023-09-30T09:11:48.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.288458796Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-30T09:11:48.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.330191567Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-30T09:11:48.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.355435734Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-30T09:11:48.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.397374862Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-30T09:11:48.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.422489875Z level=info msg="Executing migration" id="Add column is_default" 2023-09-30T09:11:48.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.464364054Z level=info msg="Executing migration" id="Add column frequency" 2023-09-30T09:11:48.585 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:48 smithi019 bash[173101]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-30T09:11:48.712 INFO:teuthology.orchestra.run.smithi019.stdout:163208757275 2023-09-30T09:11:48.835 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:48 smithi167 ceph-mon[104403]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:48.836 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.548085161Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-30T09:11:48.836 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.590107231Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-30T09:11:48.836 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.6319969Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-30T09:11:48.836 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.674129724Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-30T09:11:48.836 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.699217649Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-30T09:11:48.836 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.741187629Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-30T09:11:48.836 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.766280403Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-30T09:11:48.836 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.791499472Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-30T09:11:48.837 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.835549118Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-30T09:11:48.880 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:48 smithi019 ceph-mon[100214]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:48.880 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:48 smithi019 ceph-mon[118983]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:48.880 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:48 smithi019 bash[173101]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-30T09:11:49.104 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.886115912Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-30T09:11:49.105 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.903101498Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-30T09:11:49.105 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.936600506Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-30T09:11:49.105 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.953476239Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-30T09:11:49.105 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.970509662Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-30T09:11:49.105 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:48 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:48.987107305Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-30T09:11:49.105 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.004171302Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-30T09:11:49.105 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.037650691Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-30T09:11:49.105 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.104582009Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-30T09:11:49.335 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:48 smithi019 bash[173101]: Writing manifest to image destination 2023-09-30T09:11:49.335 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:48 smithi019 bash[173101]: Storing signatures 2023-09-30T09:11:49.390 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.155096497Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-30T09:11:49.391 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.180390029Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-30T09:11:49.391 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.205447042Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-30T09:11:49.391 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.230775559Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-30T09:11:49.391 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.25587517Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-30T09:11:49.391 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.280998942Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-30T09:11:49.391 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.32280488Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-30T09:11:49.391 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.348315064Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-30T09:11:49.391 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.390196097Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-30T09:11:49.667 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:49 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2798962317' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:11:49.667 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.432057098Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-30T09:11:49.667 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.457428476Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-30T09:11:49.668 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.482548037Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-30T09:11:49.668 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.507715551Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-30T09:11:49.668 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.532816846Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-30T09:11:49.668 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.558250241Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-30T09:11:49.668 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.600000178Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-30T09:11:49.668 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.667011071Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-30T09:11:49.786 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:49 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2798962317' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:11:49.786 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:49 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2798962317' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:11:49.928 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.717447202Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-30T09:11:49.928 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.734358754Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-30T09:11:49.928 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.751203066Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-30T09:11:49.928 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.784762415Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-30T09:11:49.928 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.801736777Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-30T09:11:49.928 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.826900195Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-30T09:11:49.929 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.877473115Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-30T09:11:49.929 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.894278253Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-30T09:11:49.929 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.927834206Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-30T09:11:50.189 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.969930828Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-30T09:11:50.189 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:49 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:49.986911253Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.003703253Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.020681092Z level=info msg="Executing migration" id="Move region to single row" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.037479534Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.070857534Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.087764094Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.104697586Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.12168054Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.138477348Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-30T09:11:50.190 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.1554784Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-30T09:11:50.191 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.172279334Z level=info msg="Executing migration" id="create test_data table" 2023-09-30T09:11:50.191 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.189066253Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-30T09:11:50.457 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.272866007Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-30T09:11:50.458 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.314644814Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-30T09:11:50.458 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.33983277Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-30T09:11:50.458 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.365217839Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-30T09:11:50.458 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.407030251Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-30T09:11:50.458 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.432177225Z level=info msg="Executing migration" id="create team table" 2023-09-30T09:11:50.458 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.457497512Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-30T09:11:50.495 INFO:teuthology.orchestra.run.smithi019.stdout:55834574893 2023-09-30T09:11:50.517 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757272 got 163208757275 for osd.5 2023-09-30T09:11:50.517 DEBUG:teuthology.parallel:result is None 2023-09-30T09:11:50.750 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.49932664Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-30T09:11:50.750 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.532900782Z level=info msg="Executing migration" id="create team member table" 2023-09-30T09:11:50.751 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.566493769Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-30T09:11:50.751 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.599812317Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-30T09:11:50.751 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.650111911Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-30T09:11:50.751 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.700296678Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-30T09:11:50.751 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.750445671Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-30T09:11:51.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:50 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/77767440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:11:51.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:50 smithi167 ceph-mon[104403]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:51.003 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.775828635Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-30T09:11:51.003 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.800947452Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-30T09:11:51.003 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.83461721Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-30T09:11:51.003 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.909913893Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-30T09:11:51.003 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.934903135Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-30T09:11:51.003 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:50 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:50.976900667Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-30T09:11:51.003 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.002119519Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-30T09:11:51.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:50 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/77767440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:11:51.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:50 smithi019 ceph-mon[118983]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:50 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/77767440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:11:51.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:50 smithi019 ceph-mon[100214]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:51.262 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.052971684Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-30T09:11:51.263 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.094807262Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-30T09:11:51.263 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.136778765Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-30T09:11:51.263 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.161758299Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-30T09:11:51.263 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.186978389Z level=info msg="Executing migration" id="create tag table" 2023-09-30T09:11:51.263 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.212201Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-30T09:11:51.263 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.237281225Z level=info msg="Executing migration" id="create login attempt table" 2023-09-30T09:11:51.263 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.262513613Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-30T09:11:51.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.287916405Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-30T09:11:51.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.312877054Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-30T09:11:51.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.37987834Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-30T09:11:51.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.421774453Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-30T09:11:51.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.446967639Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-30T09:11:51.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.472155661Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-30T09:11:51.662 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574893 for osd.1 2023-09-30T09:11:51.662 DEBUG:teuthology.parallel:result is None 2023-09-30T09:11:51.799 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.547592121Z level=info msg="Executing migration" id="create user auth table" 2023-09-30T09:11:51.799 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.572622179Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-30T09:11:51.799 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.597786825Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-30T09:11:51.799 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.639818146Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-30T09:11:51.800 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.681650645Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-30T09:11:51.800 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.706889023Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-30T09:11:51.800 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.732033415Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-30T09:11:51.800 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.774154923Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-30T09:11:51.800 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.799169927Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-30T09:11:51.906 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:51 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:51.907 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:51 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:52.060 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:51 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:11:52.060 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.833133646Z level=info msg="Executing migration" id="create server_lock table" 2023-09-30T09:11:52.060 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.874892178Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-30T09:11:52.060 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.900026342Z level=info msg="Executing migration" id="create user auth token table" 2023-09-30T09:11:52.060 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.942087233Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-30T09:11:52.060 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.96718062Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-30T09:11:52.060 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:51 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:51.992412124Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-30T09:11:52.061 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.017752122Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-30T09:11:52.061 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.059608203Z level=info msg="Executing migration" id="create cache_data table" 2023-09-30T09:11:52.320 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.13513544Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-30T09:11:52.320 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.176826475Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-30T09:11:52.320 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.227402135Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-30T09:11:52.320 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.294565335Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-30T09:11:52.320 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.319668288Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-30T09:11:52.335 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.022Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-30T09:11:52.335 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.022Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.023Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.023Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.023Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.023Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=arp 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-30T09:11:52.336 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=edac 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-30T09:11:52.337 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=os 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-30T09:11:52.338 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=stat 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=time 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=uname 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-30T09:11:52.339 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-30T09:11:52.340 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-30T09:11:52.340 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-30T09:11:52.340 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 conmon[173766]: ts=2023-09-30T09:11:52.024Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-30T09:11:52.340 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 bash[173101]: 63714dd11e25f25601eb6135fdd8e97c1c42466033c264e1c99bb0bfed906da1 2023-09-30T09:11:52.340 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:11:52 smithi019 systemd[1]: Started Ceph node-exporter.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:11:52.475 INFO:teuthology.orchestra.run.smithi019.stdout:34359738419 2023-09-30T09:11:52.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.362030534Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-30T09:11:52.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.412245083Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-30T09:11:52.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.454206571Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-30T09:11:52.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.479373567Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-30T09:11:52.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.504481227Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-30T09:11:52.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.546448346Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-30T09:11:52.841 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:52 smithi167 systemd[1]: Starting Ceph node-exporter.b for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.588676299Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.630702376Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.69771081Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.723050015Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.756642253Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.773430863Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.790303833Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.824163699Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-30T09:11:52.842 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.840930173Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-30T09:11:53.117 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:53 smithi167 bash[151695]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-30T09:11:53.117 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.882995255Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-30T09:11:53.117 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.899839131Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-30T09:11:53.117 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.916743169Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-30T09:11:53.117 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.93354674Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-30T09:11:53.117 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:52 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:52.975552142Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-30T09:11:53.118 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.000661042Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-30T09:11:53.118 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.025829016Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-30T09:11:53.118 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.050999638Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-30T09:11:53.118 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.076151111Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-30T09:11:53.203 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738419 for osd.0 2023-09-30T09:11:53.204 DEBUG:teuthology.parallel:result is None 2023-09-30T09:11:53.204 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-30T09:11:53.204 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:11:53.396 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:53 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.396 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:53 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.396 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:53 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.396 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:53 smithi167 ceph-mon[104403]: Deploying daemon node-exporter.b on smithi167 2023-09-30T09:11:53.396 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:53 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/539206527' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:11:53.396 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:53 smithi167 ceph-mon[104403]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.118243197Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.160329351Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.17756647Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.194477593Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.261484095Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.28656254Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.311684625Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.32860413Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.345483043Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.362475431Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-30T09:11:53.397 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.395950689Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-30T09:11:53.568 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[100214]: Deploying daemon node-exporter.b on smithi167 2023-09-30T09:11:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/539206527' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:11:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[100214]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[118983]: Deploying daemon node-exporter.b on smithi167 2023-09-30T09:11:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/539206527' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:11:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:53 smithi019 ceph-mon[118983]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:53.665 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:53 smithi167 bash[151695]: Getting image source signatures 2023-09-30T09:11:53.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.438033407Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-30T09:11:53.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.463250014Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-30T09:11:53.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.488657681Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-30T09:11:53.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.513861127Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-30T09:11:53.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.555561345Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-30T09:11:53.666 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.580878113Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-30T09:11:53.666 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.622720211Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-30T09:11:53.666 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.664810429Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-30T09:11:53.976 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:53 smithi167 bash[151695]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-30T09:11:53.976 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:53 smithi167 bash[151695]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-30T09:11:53.976 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:53 smithi167 bash[151695]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-30T09:11:53.976 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.706937409Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-30T09:11:53.976 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.765420387Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-30T09:11:53.977 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.89373958Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-30T09:11:53.977 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:53 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:53.975401872Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-30T09:11:54.272 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.050289561Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-30T09:11:54.272 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.142428536Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-30T09:11:54.272 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.209436533Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-30T09:11:54.590 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:54 smithi167 bash[151695]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-30T09:11:54.590 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.301518449Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-30T09:11:54.590 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.406209227Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-30T09:11:54.590 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.49275283Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-30T09:11:54.881 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:54 smithi167 bash[151695]: Writing manifest to image destination 2023-09-30T09:11:54.882 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:54 smithi167 bash[151695]: Storing signatures 2023-09-30T09:11:54.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:54 smithi167 ceph-mon[104403]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:54.882 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.594306269Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-30T09:11:54.882 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.653530635Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-30T09:11:54.882 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.695509067Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-30T09:11:54.883 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.738151779Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-30T09:11:54.883 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.813780443Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-30T09:11:54.883 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.839384832Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-30T09:11:55.008 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:54 smithi019 ceph-mon[100214]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:55.009 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:54 smithi019 ceph-mon[118983]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:55.149 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.881537579Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-30T09:11:55.149 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:54 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:54.932512481Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-30T09:11:55.149 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.011931667Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-30T09:11:55.149 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.038695501Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-30T09:11:55.149 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.063849899Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-30T09:11:55.149 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.097664846Z level=info msg="Executing migration" id="create alert_image table" 2023-09-30T09:11:55.149 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.148634363Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-30T09:11:55.421 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.190874315Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-30T09:11:55.421 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.224584639Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-30T09:11:55.422 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.258205489Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-30T09:11:55.422 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.291788386Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-30T09:11:55.422 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.291932075Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-30T09:11:55.422 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.342281639Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-30T09:11:55.422 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.367672146Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-30T09:11:55.422 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.421309696Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-30T09:11:55.422 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.421483895Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-30T09:11:55.423 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:11:55.424 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:11:55.747 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.421658189Z level=info msg="alerts found to migrate" alerts=0 2023-09-30T09:11:55.747 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.494310214Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-30T09:11:55.747 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.603534917Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-30T09:11:55.747 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.670942705Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-30T09:11:56.017 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:11:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:11:56.017 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:55 smithi167 ceph-mon[104403]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:11:56.018 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.746793515Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-30T09:11:56.018 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.822850935Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-30T09:11:56.018 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.890008652Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-30T09:11:56.018 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.915447045Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-30T09:11:56.018 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:55 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:55.966470241Z level=info msg="Executing migration" id="create data_keys table" 2023-09-30T09:11:56.018 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.01704098Z level=info msg="Executing migration" id="create secrets table" 2023-09-30T09:11:56.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:55 smithi019 ceph-mon[118983]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:11:56.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:55 smithi019 ceph-mon[100214]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:11:56.198 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":31,"stamp":"2023-09-30T09:11:54.521723+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50352,"kb_used_data":6640,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910864,"statfs":{"total":767960285184,"available":767908724736,"internally_reserved":0,"allocated":6799360,"data_stored":3874917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002740"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:11.318321+0000","last_change":"2023-09-30T09:11:08.916599+0000","last_active":"2023-09-30T09:11:11.318321+0000","last_peered":"2023-09-30T09:11:11.318321+0000","last_clean":"2023-09-30T09:11:11.318321+0000","last_became_active":"2023-09-30T09:11:08.916301+0000","last_became_peered":"2023-09-30T09:11:08.916301+0000","last_unstale":"2023-09-30T09:11:11.318321+0000","last_undegraded":"2023-09-30T09:11:11.318321+0000","last_fullsized":"2023-09-30T09:11:11.318321+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:19.518596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921541+0000","last_change":"2023-09-30T09:11:09.118530+0000","last_active":"2023-09-30T09:11:09.921541+0000","last_peered":"2023-09-30T09:11:09.921541+0000","last_clean":"2023-09-30T09:11:09.921541+0000","last_became_active":"2023-09-30T09:11:09.118152+0000","last_became_peered":"2023-09-30T09:11:09.118152+0000","last_unstale":"2023-09-30T09:11:09.921541+0000","last_undegraded":"2023-09-30T09:11:09.921541+0000","last_fullsized":"2023-09-30T09:11:09.921541+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:29:57.045104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921809+0000","last_change":"2023-09-30T09:11:09.118453+0000","last_active":"2023-09-30T09:11:09.921809+0000","last_peered":"2023-09-30T09:11:09.921809+0000","last_clean":"2023-09-30T09:11:09.921809+0000","last_became_active":"2023-09-30T09:11:09.118151+0000","last_became_peered":"2023-09-30T09:11:09.118151+0000","last_unstale":"2023-09-30T09:11:09.921809+0000","last_undegraded":"2023-09-30T09:11:09.921809+0000","last_fullsized":"2023-09-30T09:11:09.921809+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:53:19.764905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921828+0000","last_change":"2023-09-30T09:11:08.918926+0000","last_active":"2023-09-30T09:11:09.921828+0000","last_peered":"2023-09-30T09:11:09.921828+0000","last_clean":"2023-09-30T09:11:09.921828+0000","last_became_active":"2023-09-30T09:11:08.918633+0000","last_became_peered":"2023-09-30T09:11:08.918633+0000","last_unstale":"2023-09-30T09:11:09.921828+0000","last_undegraded":"2023-09-30T09:11:09.921828+0000","last_fullsized":"2023-09-30T09:11:09.921828+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:10:07.071958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921791+0000","last_change":"2023-09-30T09:10:32.609108+0000","last_active":"2023-09-30T09:11:09.921791+0000","last_peered":"2023-09-30T09:11:09.921791+0000","last_clean":"2023-09-30T09:11:09.921791+0000","last_became_active":"2023-09-30T09:10:32.608566+0000","last_became_peered":"2023-09-30T09:10:32.608566+0000","last_unstale":"2023-09-30T09:11:09.921791+0000","last_undegraded":"2023-09-30T09:11:09.921791+0000","last_fullsized":"2023-09-30T09:11:09.921791+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:37:39.870350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:10.351468+0000","last_change":"2023-09-30T09:11:08.916682+0000","last_active":"2023-09-30T09:11:10.351468+0000","last_peered":"2023-09-30T09:11:10.351468+0000","last_clean":"2023-09-30T09:11:10.351468+0000","last_became_active":"2023-09-30T09:11:08.916369+0000","last_became_peered":"2023-09-30T09:11:08.916369+0000","last_unstale":"2023-09-30T09:11:10.351468+0000","last_undegraded":"2023-09-30T09:11:10.351468+0000","last_fullsized":"2023-09-30T09:11:10.351468+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:21.617802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:10.849070+0000","last_change":"2023-09-30T09:11:08.988754+0000","last_active":"2023-09-30T09:11:10.849070+0000","last_peered":"2023-09-30T09:11:10.849070+0000","last_clean":"2023-09-30T09:11:10.849070+0000","last_became_active":"2023-09-30T09:11:08.988424+0000","last_became_peered":"2023-09-30T09:11:08.988424+0000","last_unstale":"2023-09-30T09:11:10.849070+0000","last_undegraded":"2023-09-30T09:11:10.849070+0000","last_fullsized":"2023-09-30T09:11:10.849070+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:05:03.608949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:10.351468+0000","last_change":"2023-09-30T09:11:08.921010+0000","last_active":"2023-09-30T09:11:10.351468+0000","last_peered":"2023-09-30T09:11:10.351468+0000","last_clean":"2023-09-30T09:11:10.351468+0000","last_became_active":"2023-09-30T09:11:08.920716+0000","last_became_peered":"2023-09-30T09:11:08.920716+0000","last_unstale":"2023-09-30T09:11:10.351468+0000","last_undegraded":"2023-09-30T09:11:10.351468+0000","last_fullsized":"2023-09-30T09:11:10.351468+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:33:29.433934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921562+0000","last_change":"2023-09-30T09:11:08.919069+0000","last_active":"2023-09-30T09:11:09.921562+0000","last_peered":"2023-09-30T09:11:09.921562+0000","last_clean":"2023-09-30T09:11:09.921562+0000","last_became_active":"2023-09-30T09:11:08.918732+0000","last_became_peered":"2023-09-30T09:11:08.918732+0000","last_unstale":"2023-09-30T09:11:09.921562+0000","last_undegraded":"2023-09-30T09:11:09.921562+0000","last_fullsized":"2023-09-30T09:11:09.921562+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:45:47.599770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234002,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6324,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738828,"statfs":{"total":95995035648,"available":95988559872,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.69099999999999995}]},{"osd":1,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.53600000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.60599999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.496}]},{"osd":4,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.71199999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.73099999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.74199999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6380,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738772,"statfs":{"total":95995035648,"available":95988502528,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.435}]},{"osd":1,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.58899999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.58099999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.624}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6512,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738640,"statfs":{"total":95995035648,"available":95988367360,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:13 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.38600000000000001,"15min":0.374},"min":{"1min":0.28899999999999998,"5min":0.223,"15min":0.223},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.26900000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.78100000000000003,"5min":0.87,"15min":0.87},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.44500000000000001,"15min":0.43099999999999999},"min":{"1min":0.28100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.9769999999999999,"5min":2.9769999999999999,"15min":2.9769999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.46300000000000002,"15min":0.45700000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.6549999999999998,"5min":2.6549999999999998,"15min":2.6549999999999998},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:11:07 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.46300000000000002,"15min":0.46400000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.64600000000000002,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.38},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53000000000000003,"15min":0.52700000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.46200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:11:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.626,"15min":0.61699999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.726,"5min":2.726,"15min":2.726},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.65900000000000003,"15min":0.64600000000000002},"min":{"1min":0.49399999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":2.8570000000000002,"5min":2.8570000000000002,"15min":2.8570000000000002},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.64300000000000002,"15min":0.63600000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":2.8100000000000001,"5min":2.8100000000000001,"15min":2.8100000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.64300000000000002,"15min":0.63500000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":2.7469999999999999,"5min":2.7469999999999999,"15min":2.7469999999999999},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:11:07 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:11:29 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.8849999999999998,"5min":2.8849999999999998,"15min":2.8849999999999998},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":2.7709999999999999,"5min":2.7709999999999999,"15min":2.7709999999999999},"last":0.504}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7156,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737996,"statfs":{"total":95995035648,"available":95987707904,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:11:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.36699999999999999,"15min":0.35499999999999998},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36699999999999999,"15min":0.35399999999999998},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.39400000000000002,"15min":0.39800000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41999999999999998,"15min":0.42199999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.63400000000000001,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.50600000000000001,"15min":0.50900000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.64800000000000002,"15min":0.66000000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78900000000000003,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.64800000000000002,"15min":0.66000000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82299999999999995,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.621}]},{"osd":5,"last update":"Sat Sep 30 09:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.60599999999999998,"15min":0.61099999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.63500000000000001,"15min":0.64100000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.70799999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.58499999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6440,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738712,"statfs":{"total":95995035648,"available":95988441088,"internally_reserved":0,"allocated":622592,"data_stored":262693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.39100000000000001,"15min":0.39200000000000002},"min":{"1min":0.25700000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51900000000000002,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41499999999999998,"15min":0.41899999999999998},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61099999999999999,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41199999999999998,"15min":0.41799999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.55300000000000005,"5min":0.623,"15min":0.623},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42099999999999999,"15min":0.42199999999999999},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63900000000000001,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:11:03 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.499,"15min":0.501},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.60099999999999998,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.501,"15min":0.50700000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.63800000000000001,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:11:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.74299999999999999,"15min":0.76700000000000002},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.85599999999999998,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.495},{"interface":"front","average":{"1min":0.63,"5min":0.68600000000000005,"15min":0.69599999999999995},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.82399999999999995,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.70999999999999996,"15min":0.73399999999999999},"min":{"1min":0.41099999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84699999999999998,"5min":2.6419999999999999,"15min":2.6419999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.63800000000000001,"15min":0.65000000000000002},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.53600000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:11:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.52600000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.251},{"interface":"front","average":{"1min":0.433,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.61299999999999999,"5min":0.62,"15min":0.62},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.47599999999999998,"15min":0.48499999999999999},"min":{"1min":0.27600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.65500000000000003,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.32400000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.504,"15min":0.50600000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72099999999999997,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.47199999999999998,"15min":0.46899999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.61499999999999999,"15min":0.61099999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.627}]},{"osd":6,"last update":"Sat Sep 30 09:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.353,"5min":1.353,"15min":1.353},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":11.801,"5min":11.801,"15min":11.801},"last":0.64000000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5864,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739288,"statfs":{"total":95995035648,"available":95989030912,"internally_reserved":0,"allocated":622592,"data_stored":262693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.55100000000000005,"15min":0.55900000000000005},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.73399999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.626},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.5}]},{"osd":1,"last update":"Sat Sep 30 09:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.38,"5min":0.376,"15min":0.376},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56000000000000005,"15min":0.56499999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71599999999999997,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Sep 30 09:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.75,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.56699999999999995,"15min":0.56899999999999995},"min":{"1min":0.37,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.75700000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.441}]},{"osd":3,"last update":"Sat Sep 30 09:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.40799999999999997,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.57999999999999996,"15min":0.58999999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.70099999999999996,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48099999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.41099999999999998,"15min":0.39800000000000002},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.496,"5min":0.42599999999999999,"15min":0.41399999999999998},"min":{"1min":0.30499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70099999999999996,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:11:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.59599999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5804,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739348,"statfs":{"total":95995035648,"available":95989092352,"internally_reserved":0,"allocated":626688,"data_stored":262900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.57899999999999996,"15min":0.58899999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69099999999999995,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.60599999999999998,"15min":0.61599999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.879,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.751,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.443}]},{"osd":2,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.81200000000000006,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.38200000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.91600000000000004,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76900000000000002,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.505},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84399999999999997,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.38}]},{"osd":4,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.45000000000000001,"15min":0.443},"min":{"1min":0.30399999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.72899999999999998,"5min":0.875,"15min":0.875},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.432},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.53200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:11:56.201 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:11:56.271 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.134114289Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-30T09:11:56.272 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.201959218Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-30T09:11:56.272 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.235970738Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-30T09:11:56.272 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.261256077Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-30T09:11:56.569 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:56.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.322259441Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-30T09:11:56.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.387206949Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-30T09:11:56.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.420750064Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-30T09:11:56.589 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.462694489Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-30T09:11:56.590 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.504744989Z level=info msg="Executing migration" id="create permission table" 2023-09-30T09:11:56.590 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.546709592Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-30T09:11:56.590 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.588552841Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-30T09:11:56.591 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.355Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-30T09:11:56.591 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.355Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-30T09:11:56.591 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.356Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-30T09:11:56.591 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.356Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-30T09:11:56.591 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.356Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-30T09:11:56.591 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.356Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-30T09:11:56.592 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-30T09:11:56.592 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=arp 2023-09-30T09:11:56.592 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-30T09:11:56.592 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-30T09:11:56.592 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-30T09:11:56.592 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-30T09:11:56.593 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-30T09:11:56.593 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-30T09:11:56.593 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-30T09:11:56.593 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-30T09:11:56.593 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=edac 2023-09-30T09:11:56.593 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-30T09:11:56.593 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-30T09:11:56.594 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-30T09:11:56.594 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-30T09:11:56.594 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-30T09:11:56.594 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-30T09:11:56.594 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-30T09:11:56.594 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-30T09:11:56.595 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-30T09:11:56.595 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-30T09:11:56.595 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-30T09:11:56.595 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-30T09:11:56.595 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-30T09:11:56.595 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-30T09:11:56.596 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-30T09:11:56.596 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-30T09:11:56.596 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=os 2023-09-30T09:11:56.596 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-30T09:11:56.596 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-30T09:11:56.596 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-30T09:11:56.597 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-30T09:11:56.597 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-30T09:11:56.597 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-30T09:11:56.597 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-30T09:11:56.597 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=stat 2023-09-30T09:11:56.597 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-30T09:11:56.597 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-30T09:11:56.598 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-30T09:11:56.598 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=time 2023-09-30T09:11:56.598 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-30T09:11:56.598 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=uname 2023-09-30T09:11:56.598 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-30T09:11:56.598 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-30T09:11:56.599 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-30T09:11:56.599 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-30T09:11:56.599 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151770]: ts=2023-09-30T09:11:56.357Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-30T09:11:56.599 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 bash[151695]: 23727543383f3736c6549181ca9f0cdf1763b64852ada7cdd903a919dcb43a54 2023-09-30T09:11:56.599 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:11:56 smithi167 systemd[1]: Started Ceph node-exporter.b for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:11:56.849 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.666332369Z level=info msg="Executing migration" id="create role table" 2023-09-30T09:11:56.849 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.706531791Z level=info msg="Executing migration" id="add column display_name" 2023-09-30T09:11:56.849 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.773445648Z level=info msg="Executing migration" id="add column group_name" 2023-09-30T09:11:56.849 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.798632938Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-30T09:11:56.850 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.823774383Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-30T09:11:56.850 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.848967069Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-30T09:11:57.101 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.891031861Z level=info msg="Executing migration" id="create team role table" 2023-09-30T09:11:57.102 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.933034456Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-30T09:11:57.102 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.958312433Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-30T09:11:57.102 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:56 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:56.98356722Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-30T09:11:57.102 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.025482924Z level=info msg="Executing migration" id="create user role table" 2023-09-30T09:11:57.102 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.050733898Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-30T09:11:57.102 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.075857977Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-30T09:11:57.102 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.101059717Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-30T09:11:57.378 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.151605913Z level=info msg="Executing migration" id="create builtin role table" 2023-09-30T09:11:57.379 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.176823237Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-30T09:11:57.379 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.227310961Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-30T09:11:57.379 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.291312928Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-30T09:11:57.379 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.378439006Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-30T09:11:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[100214]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[118983]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:57 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:57.664 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.428888244Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-30T09:11:57.664 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.494428373Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-30T09:11:57.664 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.546483618Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-30T09:11:57.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.563527963Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-30T09:11:57.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.580341509Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-30T09:11:57.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.622246981Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-30T09:11:57.665 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.664246699Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-30T09:11:57.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:57 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:57 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:57 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:57 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:57 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:57 smithi167 ceph-mon[104403]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:57.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:57 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:11:57.667 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:57 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:11:57.934 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.71473608Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-30T09:11:57.935 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.739939999Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-30T09:11:57.935 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.790272623Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-30T09:11:57.935 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.815482506Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-30T09:11:57.935 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.857441715Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-30T09:11:57.935 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.932657118Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-30T09:11:58.184 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:57 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:57.983209312Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-30T09:11:58.185 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.025086577Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-30T09:11:58.185 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.091973651Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-30T09:11:58.185 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.167828601Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-30T09:11:58.185 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.18451365Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-30T09:11:58.439 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:11:58.441 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:11:58.462 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.226679557Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-30T09:11:58.462 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.277097279Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-30T09:11:58.462 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.335708721Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-30T09:11:58.462 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.413802895Z level=info msg="Executing migration" id="add correlation config column" 2023-09-30T09:11:58.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:58 smithi167 ceph-mon[104403]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:11:58.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:11:58 smithi167 ceph-mon[104403]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:58.748 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.46178971Z level=info msg="Executing migration" id="create entity_events table" 2023-09-30T09:11:58.748 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.529073126Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-30T09:11:58.748 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.554276642Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-30T09:11:58.748 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.554443702Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-30T09:11:58.748 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.596180935Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-30T09:11:58.749 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.596330941Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-30T09:11:58.749 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.671519137Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-30T09:11:58.749 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.747266645Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-30T09:11:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:58 smithi019 ceph-mon[100214]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:11:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:11:58 smithi019 ceph-mon[100214]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:58 smithi019 ceph-mon[118983]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:11:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:11:58 smithi019 ceph-mon[118983]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:11:59.022 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.789239912Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-30T09:11:59.022 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.875105428Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-30T09:11:59.022 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.898444336Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-30T09:11:59.022 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.92362041Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-30T09:11:59.022 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:58 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:58.982803582Z level=info msg="Executing migration" id="Drop public config table" 2023-09-30T09:11:59.110 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":32,"stamp":"2023-09-30T09:11:56.522153+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50352,"kb_used_data":6640,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910864,"statfs":{"total":767960285184,"available":767908724736,"internally_reserved":0,"allocated":6799360,"data_stored":3874917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002761"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:11.318321+0000","last_change":"2023-09-30T09:11:08.916599+0000","last_active":"2023-09-30T09:11:11.318321+0000","last_peered":"2023-09-30T09:11:11.318321+0000","last_clean":"2023-09-30T09:11:11.318321+0000","last_became_active":"2023-09-30T09:11:08.916301+0000","last_became_peered":"2023-09-30T09:11:08.916301+0000","last_unstale":"2023-09-30T09:11:11.318321+0000","last_undegraded":"2023-09-30T09:11:11.318321+0000","last_fullsized":"2023-09-30T09:11:11.318321+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:19.518596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921541+0000","last_change":"2023-09-30T09:11:09.118530+0000","last_active":"2023-09-30T09:11:09.921541+0000","last_peered":"2023-09-30T09:11:09.921541+0000","last_clean":"2023-09-30T09:11:09.921541+0000","last_became_active":"2023-09-30T09:11:09.118152+0000","last_became_peered":"2023-09-30T09:11:09.118152+0000","last_unstale":"2023-09-30T09:11:09.921541+0000","last_undegraded":"2023-09-30T09:11:09.921541+0000","last_fullsized":"2023-09-30T09:11:09.921541+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:29:57.045104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921809+0000","last_change":"2023-09-30T09:11:09.118453+0000","last_active":"2023-09-30T09:11:09.921809+0000","last_peered":"2023-09-30T09:11:09.921809+0000","last_clean":"2023-09-30T09:11:09.921809+0000","last_became_active":"2023-09-30T09:11:09.118151+0000","last_became_peered":"2023-09-30T09:11:09.118151+0000","last_unstale":"2023-09-30T09:11:09.921809+0000","last_undegraded":"2023-09-30T09:11:09.921809+0000","last_fullsized":"2023-09-30T09:11:09.921809+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:53:19.764905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921828+0000","last_change":"2023-09-30T09:11:08.918926+0000","last_active":"2023-09-30T09:11:09.921828+0000","last_peered":"2023-09-30T09:11:09.921828+0000","last_clean":"2023-09-30T09:11:09.921828+0000","last_became_active":"2023-09-30T09:11:08.918633+0000","last_became_peered":"2023-09-30T09:11:08.918633+0000","last_unstale":"2023-09-30T09:11:09.921828+0000","last_undegraded":"2023-09-30T09:11:09.921828+0000","last_fullsized":"2023-09-30T09:11:09.921828+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:10:07.071958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921791+0000","last_change":"2023-09-30T09:10:32.609108+0000","last_active":"2023-09-30T09:11:09.921791+0000","last_peered":"2023-09-30T09:11:09.921791+0000","last_clean":"2023-09-30T09:11:09.921791+0000","last_became_active":"2023-09-30T09:10:32.608566+0000","last_became_peered":"2023-09-30T09:10:32.608566+0000","last_unstale":"2023-09-30T09:11:09.921791+0000","last_undegraded":"2023-09-30T09:11:09.921791+0000","last_fullsized":"2023-09-30T09:11:09.921791+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:37:39.870350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:10.351468+0000","last_change":"2023-09-30T09:11:08.916682+0000","last_active":"2023-09-30T09:11:10.351468+0000","last_peered":"2023-09-30T09:11:10.351468+0000","last_clean":"2023-09-30T09:11:10.351468+0000","last_became_active":"2023-09-30T09:11:08.916369+0000","last_became_peered":"2023-09-30T09:11:08.916369+0000","last_unstale":"2023-09-30T09:11:10.351468+0000","last_undegraded":"2023-09-30T09:11:10.351468+0000","last_fullsized":"2023-09-30T09:11:10.351468+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:21.617802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:10.849070+0000","last_change":"2023-09-30T09:11:08.988754+0000","last_active":"2023-09-30T09:11:10.849070+0000","last_peered":"2023-09-30T09:11:10.849070+0000","last_clean":"2023-09-30T09:11:10.849070+0000","last_became_active":"2023-09-30T09:11:08.988424+0000","last_became_peered":"2023-09-30T09:11:08.988424+0000","last_unstale":"2023-09-30T09:11:10.849070+0000","last_undegraded":"2023-09-30T09:11:10.849070+0000","last_fullsized":"2023-09-30T09:11:10.849070+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:05:03.608949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:10.351468+0000","last_change":"2023-09-30T09:11:08.921010+0000","last_active":"2023-09-30T09:11:10.351468+0000","last_peered":"2023-09-30T09:11:10.351468+0000","last_clean":"2023-09-30T09:11:10.351468+0000","last_became_active":"2023-09-30T09:11:08.920716+0000","last_became_peered":"2023-09-30T09:11:08.920716+0000","last_unstale":"2023-09-30T09:11:10.351468+0000","last_undegraded":"2023-09-30T09:11:10.351468+0000","last_fullsized":"2023-09-30T09:11:10.351468+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:33:29.433934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-30T09:11:09.921562+0000","last_change":"2023-09-30T09:11:08.919069+0000","last_active":"2023-09-30T09:11:09.921562+0000","last_peered":"2023-09-30T09:11:09.921562+0000","last_clean":"2023-09-30T09:11:09.921562+0000","last_became_active":"2023-09-30T09:11:08.918732+0000","last_became_peered":"2023-09-30T09:11:08.918732+0000","last_unstale":"2023-09-30T09:11:09.921562+0000","last_undegraded":"2023-09-30T09:11:09.921562+0000","last_fullsized":"2023-09-30T09:11:09.921562+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:45:47.599770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234002,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6324,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738828,"statfs":{"total":95995035648,"available":95988559872,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.69099999999999995}]},{"osd":1,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.53600000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.60599999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.496}]},{"osd":4,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.71199999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.73099999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.74199999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6380,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738772,"statfs":{"total":95995035648,"available":95988502528,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.435}]},{"osd":1,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.58899999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.58099999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.624}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6512,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738640,"statfs":{"total":95995035648,"available":95988367360,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:13 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.38600000000000001,"15min":0.374},"min":{"1min":0.28899999999999998,"5min":0.223,"15min":0.223},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.26900000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.78100000000000003,"5min":0.87,"15min":0.87},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.44500000000000001,"15min":0.43099999999999999},"min":{"1min":0.28100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.9769999999999999,"5min":2.9769999999999999,"15min":2.9769999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.46300000000000002,"15min":0.45700000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.6549999999999998,"5min":2.6549999999999998,"15min":2.6549999999999998},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:11:07 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.46300000000000002,"15min":0.46400000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.64600000000000002,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.38},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53000000000000003,"15min":0.52700000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.46200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:11:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.626,"15min":0.61699999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.726,"5min":2.726,"15min":2.726},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.65900000000000003,"15min":0.64600000000000002},"min":{"1min":0.49399999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":2.8570000000000002,"5min":2.8570000000000002,"15min":2.8570000000000002},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.64300000000000002,"15min":0.63600000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":2.8100000000000001,"5min":2.8100000000000001,"15min":2.8100000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.64300000000000002,"15min":0.63500000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":2.7469999999999999,"5min":2.7469999999999999,"15min":2.7469999999999999},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:11:07 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:11:29 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.8849999999999998,"5min":2.8849999999999998,"15min":2.8849999999999998},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":2.7709999999999999,"5min":2.7709999999999999,"15min":2.7709999999999999},"last":0.504}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7156,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737996,"statfs":{"total":95995035648,"available":95987707904,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:11:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.36699999999999999,"15min":0.35499999999999998},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36699999999999999,"15min":0.35399999999999998},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.39400000000000002,"15min":0.39800000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41999999999999998,"15min":0.42199999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.63400000000000001,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.50600000000000001,"15min":0.50900000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.64800000000000002,"15min":0.66000000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78900000000000003,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.64800000000000002,"15min":0.66000000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82299999999999995,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.621}]},{"osd":5,"last update":"Sat Sep 30 09:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.60599999999999998,"15min":0.61099999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.63500000000000001,"15min":0.64100000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.70799999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.58499999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6440,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738712,"statfs":{"total":95995035648,"available":95988441088,"internally_reserved":0,"allocated":622592,"data_stored":262693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.39100000000000001,"15min":0.39200000000000002},"min":{"1min":0.25700000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51900000000000002,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41499999999999998,"15min":0.41899999999999998},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61099999999999999,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41199999999999998,"15min":0.41799999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.55300000000000005,"5min":0.623,"15min":0.623},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42099999999999999,"15min":0.42199999999999999},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63900000000000001,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:11:03 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.499,"15min":0.501},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.60099999999999998,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.501,"15min":0.50700000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.63800000000000001,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:11:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.74299999999999999,"15min":0.76700000000000002},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.85599999999999998,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.495},{"interface":"front","average":{"1min":0.63,"5min":0.68600000000000005,"15min":0.69599999999999995},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.82399999999999995,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.70999999999999996,"15min":0.73399999999999999},"min":{"1min":0.41099999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84699999999999998,"5min":2.6419999999999999,"15min":2.6419999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.63800000000000001,"15min":0.65000000000000002},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.53600000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:11:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.52600000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.251},{"interface":"front","average":{"1min":0.433,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.61299999999999999,"5min":0.62,"15min":0.62},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.47599999999999998,"15min":0.48499999999999999},"min":{"1min":0.27600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.65500000000000003,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.32400000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.504,"15min":0.50600000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72099999999999997,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.47199999999999998,"15min":0.46899999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.61499999999999999,"15min":0.61099999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.627}]},{"osd":6,"last update":"Sat Sep 30 09:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.353,"5min":1.353,"15min":1.353},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":11.801,"5min":11.801,"15min":11.801},"last":0.64000000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5864,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739288,"statfs":{"total":95995035648,"available":95989030912,"internally_reserved":0,"allocated":622592,"data_stored":262693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.55100000000000005,"15min":0.55900000000000005},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.73399999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.626},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.5}]},{"osd":1,"last update":"Sat Sep 30 09:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.38,"5min":0.376,"15min":0.376},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56000000000000005,"15min":0.56499999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71599999999999997,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Sep 30 09:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.75,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.56699999999999995,"15min":0.56899999999999995},"min":{"1min":0.37,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.75700000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.441}]},{"osd":3,"last update":"Sat Sep 30 09:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.40799999999999997,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.57999999999999996,"15min":0.58999999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.70099999999999996,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48099999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.41099999999999998,"15min":0.39800000000000002},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.496,"5min":0.42599999999999999,"15min":0.41399999999999998},"min":{"1min":0.30499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70099999999999996,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:11:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.59599999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5804,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739348,"statfs":{"total":95995035648,"available":95989092352,"internally_reserved":0,"allocated":626688,"data_stored":262900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.57899999999999996,"15min":0.58899999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69099999999999995,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.60599999999999998,"15min":0.61599999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.879,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.751,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.443}]},{"osd":2,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.81200000000000006,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.38200000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.91600000000000004,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76900000000000002,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.505},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84399999999999997,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.38}]},{"osd":4,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.45000000000000001,"15min":0.443},"min":{"1min":0.30399999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.72899999999999998,"5min":0.875,"15min":0.875},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.432},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.53200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:11:59.113 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-30T09:11:59.113 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-30T09:11:59.113 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-30T09:11:59.113 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-30T09:11:59.346 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.079213896Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-30T09:11:59.346 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.133247716Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-30T09:11:59.347 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.183435647Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-30T09:11:59.347 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.208674309Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-30T09:11:59.347 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.233903193Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-30T09:11:59.347 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.259103382Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-30T09:11:59.347 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.346370869Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-30T09:11:59.506 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:11:59.679 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.418447846Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-30T09:11:59.679 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.46883978Z level=info msg="Executing migration" id="add share column" 2023-09-30T09:11:59.679 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.52758455Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-30T09:11:59.679 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.577860016Z level=info msg="Executing migration" id="create file table" 2023-09-30T09:11:59.679 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.678783042Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-30T09:11:59.947 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.704028601Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-30T09:11:59.947 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.74585625Z level=info msg="Executing migration" id="create file_meta table" 2023-09-30T09:11:59.947 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.787936558Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-30T09:11:59.948 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.812965833Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-30T09:11:59.948 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.838233417Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-30T09:11:59.948 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.880029916Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-30T09:11:59.948 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.905197766Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-30T09:11:59.948 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.947191442Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.972776322Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:11:59 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:11:59.99782168Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.023035183Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.048186541Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.073514659Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.115355516Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.140506425Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.165815938Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-30T09:12:00.233 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.232752524Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-30T09:12:00.493 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.274838937Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-30T09:12:00.493 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.300091129Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-30T09:12:00.493 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.325305596Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-30T09:12:00.493 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.350538808Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-30T09:12:00.493 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.392341888Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-30T09:12:00.493 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.417546212Z level=info msg="Executing migration" id="create folder table" 2023-09-30T09:12:00.494 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.442752557Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-30T09:12:00.494 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.467818566Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-30T09:12:00.494 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.493074301Z level=info msg="Executing migration" id="Update folder title length" 2023-09-30T09:12:00.763 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.543631201Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-30T09:12:00.764 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=migrator t=2023-09-30T09:12:00.585516862Z level=info msg="migrations completed" performed=485 skipped=0 duration=18.139889196s 2023-09-30T09:12:00.764 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=sqlstore t=2023-09-30T09:12:00.586071706Z level=info msg="Created default organization" 2023-09-30T09:12:00.764 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=plugin.loader t=2023-09-30T09:12:00.67079967Z level=info msg="Plugin registered" pluginID=input 2023-09-30T09:12:00.764 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=plugin.loader t=2023-09-30T09:12:00.697631128Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-30T09:12:00.764 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=plugin.loader t=2023-09-30T09:12:00.697671954Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-30T09:12:00.764 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=secrets t=2023-09-30T09:12:00.697721332Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-30T09:12:00.764 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=query_data t=2023-09-30T09:12:00.699491183Z level=info msg="Query Service initialization" 2023-09-30T09:12:00.764 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:00 smithi167 conmon[151266]: logger=live.push_http t=2023-09-30T09:12:00.702145909Z level=info msg="Live Push Gateway initialization" 2023-09-30T09:12:01.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:00 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:00 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:00 smithi167 ceph-mon[104403]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:01.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:00 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:01.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:00 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:01.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:00 smithi019 ceph-mon[100214]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:01.067 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:00 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:01.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:00 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:01.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:00 smithi019 ceph-mon[118983]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:01.674 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:12:01.675 INFO:teuthology.orchestra.run.smithi019.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-30T09:12:01.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=infra.usagestats.collector t=2023-09-30T09:12:01.516165938Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-30T09:12:01.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=provisioning.datasources t=2023-09-30T09:12:01.516676831Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-30T09:12:01.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=provisioning.datasources t=2023-09-30T09:12:01.549911685Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-30T09:12:01.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=provisioning.datasources t=2023-09-30T09:12:01.630563822Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-30T09:12:01.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=provisioning.alerting t=2023-09-30T09:12:01.695948844Z level=info msg="starting to provision alerting" 2023-09-30T09:12:01.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=provisioning.alerting t=2023-09-30T09:12:01.695987329Z level=info msg="finished to provision alerting" 2023-09-30T09:12:01.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=ngalert.state.manager t=2023-09-30T09:12:01.696055164Z level=info msg="Warming state cache for startup" 2023-09-30T09:12:01.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=grafanaStorageLogger t=2023-09-30T09:12:01.696112482Z level=info msg="storage starting" 2023-09-30T09:12:01.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=http.server t=2023-09-30T09:12:01.697876098Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-30T09:12:01.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=sqlstore.transactions t=2023-09-30T09:12:01.707509528Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-30T09:12:01.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=ngalert.state.manager t=2023-09-30T09:12:01.739608096Z level=info msg="State cache has been initialized" states=0 duration=43.551713ms 2023-09-30T09:12:01.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=ticker t=2023-09-30T09:12:01.73967212Z level=info msg=starting first_tick=2023-09-30T09:12:10Z 2023-09-30T09:12:01.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:12:01 smithi167 conmon[151266]: logger=ngalert.multiorg.alertmanager t=2023-09-30T09:12:01.739670502Z level=info msg="starting MultiOrg Alertmanager" 2023-09-30T09:12:02.394 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-30T09:12:02.394 INFO:tasks.cephadm:Setup complete, yielding 2023-09-30T09:12:02.395 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-30T09:12:02.408 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi019.front.sepia.ceph.com 2023-09-30T09:12:02.408 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-30T09:12:02.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:02.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[118983]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[118983]: Reconfiguring daemon alertmanager.a on smithi019 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2184380457' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:02.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[100214]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-30T09:12:02.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[100214]: Reconfiguring daemon alertmanager.a on smithi019 2023-09-30T09:12:02.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:02 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2184380457' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:12:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:02 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:02 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:02 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:02 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:02 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:02 smithi167 ceph-mon[104403]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-30T09:12:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:02 smithi167 ceph-mon[104403]: Reconfiguring daemon alertmanager.a on smithi019 2023-09-30T09:12:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:02 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2184380457' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:12:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:03 smithi019 ceph-mon[100214]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:03.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:03 smithi019 ceph-mon[118983]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:03.659 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:03.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:03 smithi167 ceph-mon[104403]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:04 smithi167 ceph-mon[104403]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:05.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:04 smithi019 ceph-mon[100214]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:04 smithi019 ceph-mon[118983]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:05.826 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:05 smithi019 systemd[1]: Stopping Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:12:06.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:05 smithi019 conmon[166358]: ts=2023-09-30T09:12:05.824Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-30T09:12:06.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:06 smithi019 bash[176839]: bbb8160f99ae4abfa52a08949679012b8752d3be589b7f0c26f0b52511794ea4 2023-09-30T09:12:06.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:12:05 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:12:07.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:06 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-30T09:12:07.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:06 smithi019 systemd[1]: Stopped Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:12:07.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:06 smithi019 systemd[1]: Starting Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:12:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:07 smithi167 ceph-mon[104403]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:12:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:07 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-30T09:12:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:07 smithi167 ceph-mon[104403]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:07 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:07.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:07 smithi019 ceph-mon[118983]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:12:07.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:07 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-30T09:12:07.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:07 smithi019 ceph-mon[118983]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:07.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:07 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:07 smithi019 ceph-mon[100214]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:12:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:07 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-30T09:12:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:07 smithi019 ceph-mon[100214]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:07 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:07.836 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[177261]: ts=2023-09-30T09:12:07.515Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-30T09:12:07.836 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[177261]: ts=2023-09-30T09:12:07.515Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-30T09:12:07.836 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[177261]: ts=2023-09-30T09:12:07.517Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.19 port=9094 2023-09-30T09:12:07.837 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[177261]: ts=2023-09-30T09:12:07.518Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-30T09:12:07.837 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 bash[177220]: 51fb7ff237b45584ec3fe8f3ccb2742f25c78682c3b61c68f9c7a6c0ca637203 2023-09-30T09:12:07.837 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 systemd[1]: Started Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:12:07.837 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[177261]: ts=2023-09-30T09:12:07.551Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-30T09:12:07.837 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[177261]: ts=2023-09-30T09:12:07.551Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-30T09:12:07.837 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[177261]: ts=2023-09-30T09:12:07.554Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-30T09:12:07.837 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[177261]: ts=2023-09-30T09:12:07.554Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-30T09:12:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:07 smithi019 conmon[100190]: 2023-09-30T09:12:07.939+0000 7f51bd65e700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-30T09:12:08.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-30T09:12:08.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: osdmap e64: 8 total, 8 up, 8 in 2023-09-30T09:12:08.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-30T09:12:08.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-30T09:12:08.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: osdmap e65: 8 total, 8 up, 8 in 2023-09-30T09:12:08.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-30T09:12:08.509 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-30T09:12:08.509 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-30T09:12:08.509 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: osdmap e66: 8 total, 8 up, 8 in 2023-09-30T09:12:08.509 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-30T09:12:08.509 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: fsmap foo:0 2023-09-30T09:12:08.509 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:08 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-30T09:12:08.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: osdmap e64: 8 total, 8 up, 8 in 2023-09-30T09:12:08.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-30T09:12:08.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: osdmap e65: 8 total, 8 up, 8 in 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: osdmap e66: 8 total, 8 up, 8 in 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: fsmap foo:0 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.696 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: osdmap e64: 8 total, 8 up, 8 in 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: osdmap e65: 8 total, 8 up, 8 in 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: osdmap e66: 8 total, 8 up, 8 in 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-30T09:12:08.697 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: fsmap foo:0 2023-09-30T09:12:08.698 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:08 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:08.711 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-30T09:12:09.078 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[100214]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:12:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[100214]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:12:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[100214]: Saving service mds.foo spec with placement count:2 2023-09-30T09:12:09.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[100214]: pgmap v41: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:09.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[100214]: osdmap e67: 8 total, 8 up, 8 in 2023-09-30T09:12:09.586 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:09 smithi019 conmon[177261]: ts=2023-09-30T09:12:09.518Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000629603s 2023-09-30T09:12:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[118983]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:12:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[118983]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:12:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[118983]: Saving service mds.foo spec with placement count:2 2023-09-30T09:12:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[118983]: pgmap v41: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:09.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:09 smithi019 ceph-mon[118983]: osdmap e67: 8 total, 8 up, 8 in 2023-09-30T09:12:09.677 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:09 smithi167 ceph-mon[104403]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:12:09.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:09 smithi167 ceph-mon[104403]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:12:09.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:09 smithi167 ceph-mon[104403]: Saving service mds.foo spec with placement count:2 2023-09-30T09:12:09.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:09 smithi167 ceph-mon[104403]: pgmap v41: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:09.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:09 smithi167 ceph-mon[104403]: osdmap e67: 8 total, 8 up, 8 in 2023-09-30T09:12:10.721 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 systemd[1]: Stopping Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:12:11.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:10 smithi167 ceph-mon[104403]: osdmap e68: 8 total, 8 up, 8 in 2023-09-30T09:12:11.102 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:10 smithi167 ceph-mon[104403]: pgmap v44: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:11.103 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-30T09:12:11.103 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-30T09:12:11.103 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.720Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.721Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.721Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-30T09:12:11.104 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:10 smithi167 conmon[148277]: ts=2023-09-30T09:12:10.721Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-30T09:12:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:10 smithi019 ceph-mon[100214]: osdmap e68: 8 total, 8 up, 8 in 2023-09-30T09:12:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:10 smithi019 ceph-mon[100214]: pgmap v44: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:10 smithi019 ceph-mon[118983]: osdmap e68: 8 total, 8 up, 8 in 2023-09-30T09:12:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:10 smithi019 ceph-mon[118983]: pgmap v44: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:11.603 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-30T09:12:11.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:11 smithi167 bash[153310]: 4089b72911965cc7b8b68bd2b10302ea20be883869a2005b417acf36367dddbc 2023-09-30T09:12:11.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:11 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-30T09:12:11.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:11 smithi167 systemd[1]: Stopped Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:12:11.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:11 smithi167 systemd[1]: Starting Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:12:11.965 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:12.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.019Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-30T09:12:12.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.019Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-30T09:12:12.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.019Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi167 (none))" 2023-09-30T09:12:12.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.019Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-30T09:12:12.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.019Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-30T09:12:12.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.021Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-30T09:12:12.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.022Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-30T09:12:12.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 bash[153642]: 8187f7a0d1a5802ceb93b826b2500065857624ef8525235754e788a8c8a1c7eb 2023-09-30T09:12:12.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.022Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-30T09:12:12.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.022Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-30T09:12:12.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.024Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-30T09:12:12.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.024Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.025µs 2023-09-30T09:12:12.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.024Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-30T09:12:12.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.024Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-30T09:12:12.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.025Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-30T09:12:12.275 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.025Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.92µs wal_replay_duration=683.737µs wbl_replay_duration=131ns total_replay_duration=755.932µs 2023-09-30T09:12:12.275 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.026Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-30T09:12:12.275 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.026Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-30T09:12:12.275 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.026Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-30T09:12:12.275 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 systemd[1]: Started Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:12:12.275 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.040Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.611886ms db_storage=817ns remote_storage=1.65µs web_handler=547ns query_engine=715ns scrape=299.941µs scrape_sd=25.353µs notify=24.903µs notify_sd=7.088µs rules=14.026498ms tracing=7.323µs 2023-09-30T09:12:12.276 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.040Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-30T09:12:12.276 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:12:12 smithi167 conmon[153683]: ts=2023-09-30T09:12:12.040Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-30T09:12:12.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STOPPING 2023-09-30T09:12:12.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:12:12.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STOPPED 2023-09-30T09:12:12.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STARTING 2023-09-30T09:12:12.336 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Serving on http://:::9283 2023-09-30T09:12:12.336 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STARTED 2023-09-30T09:12:12.336 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STOPPING 2023-09-30T09:12:12.336 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:12:12.336 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STOPPED 2023-09-30T09:12:12.336 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STARTING 2023-09-30T09:12:12.336 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Serving on http://:::9283 2023-09-30T09:12:12.336 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STARTED 2023-09-30T09:12:12.337 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STOPPING 2023-09-30T09:12:12.337 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:12:12.337 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STOPPED 2023-09-30T09:12:12.337 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STARTING 2023-09-30T09:12:12.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Serving on http://:::9283 2023-09-30T09:12:12.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:12 smithi019 conmon[100636]: [30/Sep/2023:09:12:12] ENGINE Bus STARTED 2023-09-30T09:12:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:12:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:12:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019:9093"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019:9093"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi167.front.sepia.ceph.com:9095"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi167.front.sepia.ceph.com:9095"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi167.front.sepia.ceph.com:3000"}]: dispatch 2023-09-30T09:12:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi167.front.sepia.ceph.com:3000"}]: dispatch 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[118983]: pgmap v45: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019:9093"}]: dispatch 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019:9093"}]: dispatch 2023-09-30T09:12:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi167.front.sepia.ceph.com:9095"}]: dispatch 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi167.front.sepia.ceph.com:9095"}]: dispatch 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi167.front.sepia.ceph.com:3000"}]: dispatch 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi167.front.sepia.ceph.com:3000"}]: dispatch 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:13.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:13 smithi019 ceph-mon[100214]: pgmap v45: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:12:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:12:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019:9093"}]: dispatch 2023-09-30T09:12:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019:9093"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi167.front.sepia.ceph.com:9095"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi167.front.sepia.ceph.com:9095"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi167.front.sepia.ceph.com:3000"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi167.front.sepia.ceph.com:3000"}]: dispatch 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:13.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:13 smithi167 ceph-mon[104403]: pgmap v45: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:14.615 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-30T09:12:14.625 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-30T09:12:14.627 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-30T09:12:14.627 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-30T09:12:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:14 smithi167 ceph-mon[104403]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:15.035 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:15.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:14 smithi019 ceph-mon[100214]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:15.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:14 smithi019 ceph-mon[118983]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:15 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:15 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:15 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:15 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:15 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:15 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:16 smithi019 ceph-mon[100214]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:16 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:16 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:16 smithi019 ceph-mon[118983]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:16 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:17.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:16 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:12:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:16 smithi167 ceph-mon[104403]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:16 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:16 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:17.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:12:17 smithi019 conmon[177261]: ts=2023-09-30T09:12:17.521Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003225555s 2023-09-30T09:12:17.974 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":20,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6800","nonce":2650629233},{"type":"v1","addr":"172.21.15.19:6801","nonce":2650629233}]},"active_addr":"172.21.15.19:6801/2650629233","active_change":"2023-09-30T09:11:06.498433+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24361,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":956155}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":3931489393}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":238111816}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":1638946842}]}]} 2023-09-30T09:12:17.981 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-30T09:12:17.981 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-30T09:12:17.982 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:12:18.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:18.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:18.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:18.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:12:18.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-30T09:12:18.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:18.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[100214]: Deploying daemon mds.foo.smithi019.kcgktg on smithi019 2023-09-30T09:12:18.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3465126482' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:12:18.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:18.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:18.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:18.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:12:18.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-30T09:12:18.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:18.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[118983]: Deploying daemon mds.foo.smithi019.kcgktg on smithi019 2023-09-30T09:12:18.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:17 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3465126482' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:12:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:17 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:17 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:17 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:17 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:12:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:17 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-30T09:12:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:17 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:17 smithi167 ceph-mon[104403]: Deploying daemon mds.foo.smithi019.kcgktg on smithi019 2023-09-30T09:12:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:17 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3465126482' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:12:18.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:18 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:18] "GET /metrics HTTP/1.1" 200 203129 "" "Prometheus/2.43.0" 2023-09-30T09:12:19.043 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:19.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:18 smithi019 ceph-mon[100214]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:19.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:18 smithi019 ceph-mon[118983]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:18 smithi167 ceph-mon[104403]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[118983]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[100214]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-30T09:12:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:20 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:12:21.229 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":68,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:12:09.948503+0000","last_up_change":"2023-09-30T09:10:28.570615+0000","last_in_change":"2023-09-30T09:10:11.880926+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-30T09:11:07.846610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-30T09:12:06.877159+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-30T09:12:07.385950+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6803","nonce":1200588465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6805","nonce":1200588465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6809","nonce":1200588465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6807","nonce":1200588465}]},"public_addr":"172.21.15.19:6803/1200588465","cluster_addr":"172.21.15.19:6805/1200588465","heartbeat_back_addr":"172.21.15.19:6809/1200588465","heartbeat_front_addr":"172.21.15.19:6807/1200588465","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6811","nonce":1005512342}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6813","nonce":1005512342}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6817","nonce":1005512342}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6815","nonce":1005512342}]},"public_addr":"172.21.15.19:6811/1005512342","cluster_addr":"172.21.15.19:6813/1005512342","heartbeat_back_addr":"172.21.15.19:6817/1005512342","heartbeat_front_addr":"172.21.15.19:6815/1005512342","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6819","nonce":2893453021}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6821","nonce":2893453021}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6825","nonce":2893453021}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6823","nonce":2893453021}]},"public_addr":"172.21.15.19:6819/2893453021","cluster_addr":"172.21.15.19:6821/2893453021","heartbeat_back_addr":"172.21.15.19:6825/2893453021","heartbeat_front_addr":"172.21.15.19:6823/2893453021","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6827","nonce":2039937925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6829","nonce":2039937925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6833","nonce":2039937925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6831","nonce":2039937925}]},"public_addr":"172.21.15.19:6827/2039937925","cluster_addr":"172.21.15.19:6829/2039937925","heartbeat_back_addr":"172.21.15.19:6833/2039937925","heartbeat_front_addr":"172.21.15.19:6831/2039937925","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6809","nonce":2834758136}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6811","nonce":2834758136}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6815","nonce":2834758136}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6813","nonce":2834758136}]},"public_addr":"172.21.15.167:6809/2834758136","cluster_addr":"172.21.15.167:6811/2834758136","heartbeat_back_addr":"172.21.15.167:6815/2834758136","heartbeat_front_addr":"172.21.15.167:6813/2834758136","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:6801/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3152350528":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/1679148187":"2023-10-01T09:11:06.498249+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/3324367156":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6800/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/936416645":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:12:21.239 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:20 smithi167 ceph-mon[104403]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:21.239 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:20 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.239 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:20 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.239 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:20 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:21.239 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:20 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:12:21.239 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:20 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-30T09:12:21.239 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:20 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:21.950 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-30T09:12:21.951 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:12:22.179 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: Deploying daemon mds.foo.smithi167.bnuarp on smithi167 2023-09-30T09:12:22.179 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:boot 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: daemon mds.foo.smithi019.kcgktg assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: Cluster is now healthy 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: fsmap foo:0 1 up:standby 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:creating} 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: daemon mds.foo.smithi019.kcgktg is now active in filesystem foo as rank 0 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3480693833' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:12:22.180 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:21 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:22.306 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: Deploying daemon mds.foo.smithi167.bnuarp on smithi167 2023-09-30T09:12:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:boot 2023-09-30T09:12:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: daemon mds.foo.smithi019.kcgktg assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-30T09:12:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-30T09:12:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-30T09:12:22.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: Cluster is now healthy 2023-09-30T09:12:22.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: fsmap foo:0 1 up:standby 2023-09-30T09:12:22.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:12:22.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:creating} 2023-09-30T09:12:22.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: daemon mds.foo.smithi019.kcgktg is now active in filesystem foo as rank 0 2023-09-30T09:12:22.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3480693833' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:12:22.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:22.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: Deploying daemon mds.foo.smithi167.bnuarp on smithi167 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:boot 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: daemon mds.foo.smithi019.kcgktg assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: Cluster is now healthy 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: fsmap foo:0 1 up:standby 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:creating} 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: daemon mds.foo.smithi019.kcgktg is now active in filesystem foo as rank 0 2023-09-30T09:12:22.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3480693833' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:12:22.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:21 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:22 smithi167 ceph-mon[104403]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:active 2023-09-30T09:12:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:22 smithi167 ceph-mon[104403]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 2023-09-30T09:12:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:22 smithi167 ceph-mon[104403]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:23.333 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:22 smithi019 ceph-mon[118983]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:active 2023-09-30T09:12:23.333 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:22 smithi019 ceph-mon[118983]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 2023-09-30T09:12:23.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:22 smithi019 ceph-mon[118983]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:23.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:22 smithi019 ceph-mon[100214]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:active 2023-09-30T09:12:23.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:22 smithi019 ceph-mon[100214]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 2023-09-30T09:12:23.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:22 smithi019 ceph-mon[100214]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:23.929 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: mds.? [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] up:boot 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:23.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:12:24.058 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:12:24.058 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":68,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:12:09.948503+0000","last_up_change":"2023-09-30T09:10:28.570615+0000","last_in_change":"2023-09-30T09:10:11.880926+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-30T09:11:07.846610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-30T09:12:06.877159+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-30T09:12:07.385950+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6803","nonce":1200588465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6805","nonce":1200588465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6809","nonce":1200588465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":1200588465},{"type":"v1","addr":"172.21.15.19:6807","nonce":1200588465}]},"public_addr":"172.21.15.19:6803/1200588465","cluster_addr":"172.21.15.19:6805/1200588465","heartbeat_back_addr":"172.21.15.19:6809/1200588465","heartbeat_front_addr":"172.21.15.19:6807/1200588465","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6811","nonce":1005512342}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6813","nonce":1005512342}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6817","nonce":1005512342}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":1005512342},{"type":"v1","addr":"172.21.15.19:6815","nonce":1005512342}]},"public_addr":"172.21.15.19:6811/1005512342","cluster_addr":"172.21.15.19:6813/1005512342","heartbeat_back_addr":"172.21.15.19:6817/1005512342","heartbeat_front_addr":"172.21.15.19:6815/1005512342","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6819","nonce":2893453021}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6821","nonce":2893453021}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6825","nonce":2893453021}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2893453021},{"type":"v1","addr":"172.21.15.19:6823","nonce":2893453021}]},"public_addr":"172.21.15.19:6819/2893453021","cluster_addr":"172.21.15.19:6821/2893453021","heartbeat_back_addr":"172.21.15.19:6825/2893453021","heartbeat_front_addr":"172.21.15.19:6823/2893453021","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6827","nonce":2039937925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6829","nonce":2039937925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6833","nonce":2039937925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2039937925},{"type":"v1","addr":"172.21.15.19:6831","nonce":2039937925}]},"public_addr":"172.21.15.19:6827/2039937925","cluster_addr":"172.21.15.19:6829/2039937925","heartbeat_back_addr":"172.21.15.19:6833/2039937925","heartbeat_front_addr":"172.21.15.19:6831/2039937925","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6809","nonce":2834758136}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6811","nonce":2834758136}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6815","nonce":2834758136}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6813","nonce":2834758136}]},"public_addr":"172.21.15.167:6809/2834758136","cluster_addr":"172.21.15.167:6811/2834758136","heartbeat_back_addr":"172.21.15.167:6815/2834758136","heartbeat_front_addr":"172.21.15.167:6813/2834758136","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:6801/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3152350528":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/1679148187":"2023-10-01T09:11:06.498249+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/3324367156":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6800/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/936416645":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:12:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: mds.? [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] up:boot 2023-09-30T09:12:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:24.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:12:24.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:24.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:23 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: mds.? [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] up:boot 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:24.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:23 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:12:24.750 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-30T09:12:24.751 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-30T09:12:24.751 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-30T09:12:24.752 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-30T09:12:24.752 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-30T09:12:24.752 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-30T09:12:24.753 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-30T09:12:24.753 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-30T09:12:25.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:25 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/232698530' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:12:25.026 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:25 smithi019 ceph-mon[100214]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:25 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/232698530' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:12:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:25 smithi019 ceph-mon[118983]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:25.497 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:25.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:25 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/232698530' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:12:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:25 smithi167 ceph-mon[104403]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:25.745 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:26.048 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:26.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:12:25 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:12:26.793 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:26 smithi167 ceph-mon[104403]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:active 2023-09-30T09:12:26.793 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:26 smithi167 ceph-mon[104403]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:26.793 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:26 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:26.793 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:26 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:26.794 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:26 smithi019 ceph-mon[118983]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:active 2023-09-30T09:12:26.794 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:26 smithi019 ceph-mon[118983]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:26.794 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:26 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:26.794 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:26 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:26.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:26 smithi019 ceph-mon[100214]: mds.? [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] up:active 2023-09-30T09:12:26.795 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:26 smithi019 ceph-mon[100214]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:26.795 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:26 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:26.795 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:26 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:26.795 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:26.795 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:27.025 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:27.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:27 smithi019 ceph-mon[100214]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:27.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:27 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:27 smithi019 ceph-mon[118983]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:27 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:27 smithi167 ceph-mon[104403]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:27 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:28 smithi167 ceph-mon[104403]: mds.? [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] up:standby 2023-09-30T09:12:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:28 smithi167 ceph-mon[104403]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:28 smithi167 ceph-mon[104403]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:29.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:28 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:28] "GET /metrics HTTP/1.1" 200 238056 "" "Prometheus/2.43.0" 2023-09-30T09:12:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:28 smithi019 ceph-mon[100214]: mds.? [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] up:standby 2023-09-30T09:12:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:28 smithi019 ceph-mon[100214]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:29.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:28 smithi019 ceph-mon[100214]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:29.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:28 smithi019 ceph-mon[118983]: mds.? [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] up:standby 2023-09-30T09:12:29.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:28 smithi019 ceph-mon[118983]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:12:29.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:28 smithi019 ceph-mon[118983]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:29.191 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:29.926 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:31.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:31.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:31.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:31.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[100214]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:31.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:31.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:31.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:31.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:30 smithi019 ceph-mon[118983]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:31.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:30 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:30 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:30 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:30 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:30 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:30 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:12:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:30 smithi167 ceph-mon[104403]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:32.267 INFO:teuthology.orchestra.run.smithi019.stdout:55834574902 2023-09-30T09:12:32.268 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-30T09:12:32.941 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:32 smithi019 ceph-mon[100214]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:32.942 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:32 smithi019 ceph-mon[118983]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:33.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:32 smithi167 ceph-mon[104403]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:34.075 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:33 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:34.075 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:33 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:34.075 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:33 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:34.075 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:33 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:34.251 INFO:teuthology.orchestra.run.smithi019.stdout:34359738428 2023-09-30T09:12:34.253 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-30T09:12:34.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:33 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:33 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:34.570 INFO:teuthology.orchestra.run.smithi019.stdout:128849018921 2023-09-30T09:12:34.571 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-30T09:12:34.621 INFO:teuthology.orchestra.run.smithi019.stdout:77309411379 2023-09-30T09:12:34.621 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-30T09:12:35.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:34 smithi019 ceph-mon[100214]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:35.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:34 smithi019 ceph-mon[118983]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:35.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:34 smithi167 ceph-mon[104403]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-30T09:12:36.194 INFO:teuthology.orchestra.run.smithi019.stdout:197568495648 2023-09-30T09:12:36.194 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-30T09:12:36.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:12:35 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:12:36.279 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:36.715 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:36 smithi019 ceph-mon[100214]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-30T09:12:36.716 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:36 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:36.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:36 smithi019 ceph-mon[118983]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-30T09:12:36.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:36 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:36 smithi167 ceph-mon[104403]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-30T09:12:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:36 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:37.821 INFO:teuthology.orchestra.run.smithi019.stdout:107374182446 2023-09-30T09:12:37.821 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-30T09:12:37.859 INFO:teuthology.orchestra.run.smithi019.stdout:231928234011 2023-09-30T09:12:37.859 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-30T09:12:37.973 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:38.932 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:38 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:38] "GET /metrics HTTP/1.1" 200 238630 "" "Prometheus/2.43.0" 2023-09-30T09:12:38.932 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:38 smithi019 ceph-mon[118983]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-30T09:12:38.932 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:38 smithi019 ceph-mon[100214]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-30T09:12:38.953 INFO:teuthology.orchestra.run.smithi019.stdout:163208757285 2023-09-30T09:12:38.953 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-30T09:12:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:38 smithi167 ceph-mon[104403]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-30T09:12:39.286 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:39.443 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:40.117 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:40.120 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:40.517 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:40.633 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:40.688 INFO:teuthology.orchestra.run.smithi019.stdout:34359738430 2023-09-30T09:12:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:40 smithi019 ceph-mon[100214]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:40 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/593579154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:12:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:40 smithi019 ceph-mon[118983]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:40 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/593579154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:12:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:40 smithi167 ceph-mon[104403]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:40 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/593579154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:12:42.307 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738428 got 34359738430 for osd.0 2023-09-30T09:12:42.307 DEBUG:teuthology.parallel:result is None 2023-09-30T09:12:42.501 INFO:teuthology.orchestra.run.smithi019.stdout:128849018922 2023-09-30T09:12:42.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:42 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2154687072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:12:42.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:42 smithi019 ceph-mon[100214]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:42 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2154687072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:12:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:42 smithi019 ceph-mon[118983]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:42 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2154687072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:12:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:42 smithi167 ceph-mon[104403]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:44.564 INFO:teuthology.orchestra.run.smithi019.stdout:55834574904 2023-09-30T09:12:44.713 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:44 smithi019 ceph-mon[118983]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:44.713 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:44 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1187987620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:12:44.714 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:44 smithi019 ceph-mon[100214]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:44.714 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:44 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1187987620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:12:44.729 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018921 got 128849018922 for osd.4 2023-09-30T09:12:44.729 DEBUG:teuthology.parallel:result is None 2023-09-30T09:12:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:44 smithi167 ceph-mon[104403]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:44 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1187987620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:12:46.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:12:45 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:12:46.602 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574902 got 55834574904 for osd.1 2023-09-30T09:12:46.603 DEBUG:teuthology.parallel:result is None 2023-09-30T09:12:46.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:46 smithi019 ceph-mon[100214]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:46.909 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:46 smithi019 ceph-mon[118983]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:47.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:46 smithi167 ceph-mon[104403]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:47.515 INFO:teuthology.orchestra.run.smithi019.stdout:107374182448 2023-09-30T09:12:48.144 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:47 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1794978016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:12:48.144 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:47 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1794978016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:12:48.145 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:47 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1794978016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:12:48.145 INFO:teuthology.orchestra.run.smithi019.stdout:77309411381 2023-09-30T09:12:48.145 INFO:teuthology.orchestra.run.smithi019.stdout:197568495650 2023-09-30T09:12:48.611 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:48 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:48] "GET /metrics HTTP/1.1" 200 238630 "" "Prometheus/2.43.0" 2023-09-30T09:12:48.611 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:48 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3698837044' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:12:48.611 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:48 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/763081123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:12:48.611 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:48 smithi019 ceph-mon[100214]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:48.955 INFO:teuthology.orchestra.run.smithi019.stdout:163208757287 2023-09-30T09:12:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:48 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3698837044' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:12:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:48 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/763081123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:12:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:48 smithi167 ceph-mon[104403]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:48 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1733075970' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:12:49.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:48 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1733075970' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:12:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:48 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3698837044' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:12:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:48 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/763081123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:12:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:48 smithi019 ceph-mon[118983]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:48 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1733075970' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:12:49.202 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411379 got 77309411381 for osd.2 2023-09-30T09:12:49.203 DEBUG:teuthology.parallel:result is None 2023-09-30T09:12:49.664 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182446 got 107374182448 for osd.3 2023-09-30T09:12:49.664 DEBUG:teuthology.parallel:result is None 2023-09-30T09:12:50.153 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495648 got 197568495650 for osd.6 2023-09-30T09:12:50.153 DEBUG:teuthology.parallel:result is None 2023-09-30T09:12:50.282 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757285 got 163208757287 for osd.5 2023-09-30T09:12:50.282 DEBUG:teuthology.parallel:result is None 2023-09-30T09:12:50.519 INFO:teuthology.orchestra.run.smithi019.stdout:231928234014 2023-09-30T09:12:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:50 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1962287202' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:12:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:50 smithi019 ceph-mon[100214]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:51.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:50 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1962287202' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:12:51.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:50 smithi019 ceph-mon[118983]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:51.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:50 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1962287202' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:12:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:50 smithi167 ceph-mon[104403]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:51.408 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234011 got 231928234014 for osd.7 2023-09-30T09:12:51.409 DEBUG:teuthology.parallel:result is None 2023-09-30T09:12:51.409 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-30T09:12:51.409 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:12:51.952 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:52.155 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:51 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:52.156 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:51 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:52.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:51 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:12:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:52 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:52 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:52 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:52 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:52 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:52 smithi167 ceph-mon[104403]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:53.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:53.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:53.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[100214]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:53.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:12:53.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:12:53.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:12:53.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:52 smithi019 ceph-mon[118983]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:53.703 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:12:53.704 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:12:54.429 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":65,"stamp":"2023-09-30T09:12:52.535249+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52400,"kb_used_data":7216,"kb_used_omap":0,"kb_used_meta":45120,"kb_avail":749908816,"statfs":{"total":767960285184,"available":767906627584,"internally_reserved":0,"allocated":7389184,"data_stored":4172507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46202880},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003114"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.014962+0000","last_change":"2023-09-30T09:12:09.014492+0000","last_active":"2023-09-30T09:12:09.014962+0000","last_peered":"2023-09-30T09:12:09.014962+0000","last_clean":"2023-09-30T09:12:09.014962+0000","last_became_active":"2023-09-30T09:12:09.008964+0000","last_became_peered":"2023-09-30T09:12:09.008964+0000","last_unstale":"2023-09-30T09:12:09.014962+0000","last_undegraded":"2023-09-30T09:12:09.014962+0000","last_fullsized":"2023-09-30T09:12:09.014962+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:15:25.163331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.872158+0000","last_change":"2023-09-30T09:12:07.941891+0000","last_active":"2023-09-30T09:12:20.872158+0000","last_peered":"2023-09-30T09:12:20.872158+0000","last_clean":"2023-09-30T09:12:20.872158+0000","last_became_active":"2023-09-30T09:12:07.941062+0000","last_became_peered":"2023-09-30T09:12:07.941062+0000","last_unstale":"2023-09-30T09:12:20.872158+0000","last_undegraded":"2023-09-30T09:12:20.872158+0000","last_fullsized":"2023-09-30T09:12:20.872158+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:46:46.904341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.643561+0000","last_change":"2023-09-30T09:12:09.643163+0000","last_active":"2023-09-30T09:12:09.643561+0000","last_peered":"2023-09-30T09:12:09.643561+0000","last_clean":"2023-09-30T09:12:09.643561+0000","last_became_active":"2023-09-30T09:12:09.642880+0000","last_became_peered":"2023-09-30T09:12:09.642880+0000","last_unstale":"2023-09-30T09:12:09.643561+0000","last_undegraded":"2023-09-30T09:12:09.643561+0000","last_fullsized":"2023-09-30T09:12:09.643561+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:08.883041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.063379+0000","last_change":"2023-09-30T09:12:07.950886+0000","last_active":"2023-09-30T09:12:10.063379+0000","last_peered":"2023-09-30T09:12:10.063379+0000","last_clean":"2023-09-30T09:12:10.063379+0000","last_became_active":"2023-09-30T09:12:07.950425+0000","last_became_peered":"2023-09-30T09:12:07.950425+0000","last_unstale":"2023-09-30T09:12:10.063379+0000","last_undegraded":"2023-09-30T09:12:10.063379+0000","last_fullsized":"2023-09-30T09:12:10.063379+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:45:09.519541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.643772+0000","last_change":"2023-09-30T09:12:09.643522+0000","last_active":"2023-09-30T09:12:09.643772+0000","last_peered":"2023-09-30T09:12:09.643772+0000","last_clean":"2023-09-30T09:12:09.643772+0000","last_became_active":"2023-09-30T09:12:09.643195+0000","last_became_peered":"2023-09-30T09:12:09.643195+0000","last_unstale":"2023-09-30T09:12:09.643772+0000","last_undegraded":"2023-09-30T09:12:09.643772+0000","last_fullsized":"2023-09-30T09:12:09.643772+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:22:18.627117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.873279+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:12:20.873279+0000","last_peered":"2023-09-30T09:12:20.873279+0000","last_clean":"2023-09-30T09:12:20.873279+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:12:20.873279+0000","last_undegraded":"2023-09-30T09:12:20.873279+0000","last_fullsized":"2023-09-30T09:12:20.873279+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017454+0000","last_change":"2023-09-30T09:12:09.015680+0000","last_active":"2023-09-30T09:12:09.017454+0000","last_peered":"2023-09-30T09:12:09.017454+0000","last_clean":"2023-09-30T09:12:09.017454+0000","last_became_active":"2023-09-30T09:12:09.015128+0000","last_became_peered":"2023-09-30T09:12:09.015128+0000","last_unstale":"2023-09-30T09:12:09.017454+0000","last_undegraded":"2023-09-30T09:12:09.017454+0000","last_fullsized":"2023-09-30T09:12:09.017454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:14:26.966437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.871183+0000","last_change":"2023-09-30T09:12:07.947839+0000","last_active":"2023-09-30T09:12:20.871183+0000","last_peered":"2023-09-30T09:12:20.871183+0000","last_clean":"2023-09-30T09:12:20.871183+0000","last_became_active":"2023-09-30T09:12:07.947530+0000","last_became_peered":"2023-09-30T09:12:07.947530+0000","last_unstale":"2023-09-30T09:12:20.871183+0000","last_undegraded":"2023-09-30T09:12:20.871183+0000","last_fullsized":"2023-09-30T09:12:20.871183+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:38.822389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.534139+0000","last_change":"2023-09-30T09:12:08.094365+0000","last_active":"2023-09-30T09:12:09.534139+0000","last_peered":"2023-09-30T09:12:09.534139+0000","last_clean":"2023-09-30T09:12:09.534139+0000","last_became_active":"2023-09-30T09:12:08.094023+0000","last_became_peered":"2023-09-30T09:12:08.094023+0000","last_unstale":"2023-09-30T09:12:09.534139+0000","last_undegraded":"2023-09-30T09:12:09.534139+0000","last_fullsized":"2023-09-30T09:12:09.534139+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:07:53.843136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.063110+0000","last_change":"2023-09-30T09:12:07.945701+0000","last_active":"2023-09-30T09:12:10.063110+0000","last_peered":"2023-09-30T09:12:10.063110+0000","last_clean":"2023-09-30T09:12:10.063110+0000","last_became_active":"2023-09-30T09:12:07.945064+0000","last_became_peered":"2023-09-30T09:12:07.945064+0000","last_unstale":"2023-09-30T09:12:10.063110+0000","last_undegraded":"2023-09-30T09:12:10.063110+0000","last_fullsized":"2023-09-30T09:12:10.063110+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:55.000129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017430+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:12:09.017430+0000","last_peered":"2023-09-30T09:12:09.017430+0000","last_clean":"2023-09-30T09:12:09.017430+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:12:09.017430+0000","last_undegraded":"2023-09-30T09:12:09.017430+0000","last_fullsized":"2023-09-30T09:12:09.017430+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.063330+0000","last_change":"2023-09-30T09:12:07.942405+0000","last_active":"2023-09-30T09:12:10.063330+0000","last_peered":"2023-09-30T09:12:10.063330+0000","last_clean":"2023-09-30T09:12:10.063330+0000","last_became_active":"2023-09-30T09:12:07.942066+0000","last_became_peered":"2023-09-30T09:12:07.942066+0000","last_unstale":"2023-09-30T09:12:10.063330+0000","last_undegraded":"2023-09-30T09:12:10.063330+0000","last_fullsized":"2023-09-30T09:12:10.063330+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:19:40.632109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954027+0000","last_change":"2023-09-30T09:11:08.916599+0000","last_active":"2023-09-30T09:12:08.954027+0000","last_peered":"2023-09-30T09:12:08.954027+0000","last_clean":"2023-09-30T09:12:08.954027+0000","last_became_active":"2023-09-30T09:11:08.916301+0000","last_became_peered":"2023-09-30T09:11:08.916301+0000","last_unstale":"2023-09-30T09:12:08.954027+0000","last_undegraded":"2023-09-30T09:12:08.954027+0000","last_fullsized":"2023-09-30T09:12:08.954027+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:19.518596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.760245+0000","last_change":"2023-09-30T09:12:09.759528+0000","last_active":"2023-09-30T09:12:09.760245+0000","last_peered":"2023-09-30T09:12:09.760245+0000","last_clean":"2023-09-30T09:12:09.760245+0000","last_became_active":"2023-09-30T09:12:09.759199+0000","last_became_peered":"2023-09-30T09:12:09.759199+0000","last_unstale":"2023-09-30T09:12:09.760245+0000","last_undegraded":"2023-09-30T09:12:09.760245+0000","last_fullsized":"2023-09-30T09:12:09.760245+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:44.333971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954289+0000","last_change":"2023-09-30T09:11:09.118530+0000","last_active":"2023-09-30T09:12:08.954289+0000","last_peered":"2023-09-30T09:12:08.954289+0000","last_clean":"2023-09-30T09:12:08.954289+0000","last_became_active":"2023-09-30T09:11:09.118152+0000","last_became_peered":"2023-09-30T09:11:09.118152+0000","last_unstale":"2023-09-30T09:12:08.954289+0000","last_undegraded":"2023-09-30T09:12:08.954289+0000","last_fullsized":"2023-09-30T09:12:08.954289+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:29:57.045104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.870634+0000","last_change":"2023-09-30T09:12:07.952075+0000","last_active":"2023-09-30T09:12:20.870634+0000","last_peered":"2023-09-30T09:12:20.870634+0000","last_clean":"2023-09-30T09:12:20.870634+0000","last_became_active":"2023-09-30T09:12:07.951745+0000","last_became_peered":"2023-09-30T09:12:07.951745+0000","last_unstale":"2023-09-30T09:12:20.870634+0000","last_undegraded":"2023-09-30T09:12:20.870634+0000","last_fullsized":"2023-09-30T09:12:20.870634+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:16.750757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642182+0000","last_change":"2023-09-30T09:12:08.658689+0000","last_active":"2023-09-30T09:12:09.642182+0000","last_peered":"2023-09-30T09:12:09.642182+0000","last_clean":"2023-09-30T09:12:09.642182+0000","last_became_active":"2023-09-30T09:12:08.658407+0000","last_became_peered":"2023-09-30T09:12:08.658407+0000","last_unstale":"2023-09-30T09:12:09.642182+0000","last_undegraded":"2023-09-30T09:12:09.642182+0000","last_fullsized":"2023-09-30T09:12:09.642182+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:42.921517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954180+0000","last_change":"2023-09-30T09:12:07.948494+0000","last_active":"2023-09-30T09:12:08.954180+0000","last_peered":"2023-09-30T09:12:08.954180+0000","last_clean":"2023-09-30T09:12:08.954180+0000","last_became_active":"2023-09-30T09:12:07.948128+0000","last_became_peered":"2023-09-30T09:12:07.948128+0000","last_unstale":"2023-09-30T09:12:08.954180+0000","last_undegraded":"2023-09-30T09:12:08.954180+0000","last_fullsized":"2023-09-30T09:12:08.954180+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:25.537779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.758954+0000","last_change":"2023-09-30T09:11:09.118453+0000","last_active":"2023-09-30T09:12:09.758954+0000","last_peered":"2023-09-30T09:12:09.758954+0000","last_clean":"2023-09-30T09:12:09.758954+0000","last_became_active":"2023-09-30T09:11:09.118151+0000","last_became_peered":"2023-09-30T09:11:09.118151+0000","last_unstale":"2023-09-30T09:12:09.758954+0000","last_undegraded":"2023-09-30T09:12:09.758954+0000","last_fullsized":"2023-09-30T09:12:09.758954+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:53:19.764905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642328+0000","last_change":"2023-09-30T09:12:07.961450+0000","last_active":"2023-09-30T09:12:09.642328+0000","last_peered":"2023-09-30T09:12:09.642328+0000","last_clean":"2023-09-30T09:12:09.642328+0000","last_became_active":"2023-09-30T09:12:07.961110+0000","last_became_peered":"2023-09-30T09:12:07.961110+0000","last_unstale":"2023-09-30T09:12:09.642328+0000","last_undegraded":"2023-09-30T09:12:09.642328+0000","last_fullsized":"2023-09-30T09:12:09.642328+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:15.930823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642004+0000","last_change":"2023-09-30T09:12:07.946576+0000","last_active":"2023-09-30T09:12:09.642004+0000","last_peered":"2023-09-30T09:12:09.642004+0000","last_clean":"2023-09-30T09:12:09.642004+0000","last_became_active":"2023-09-30T09:12:07.946090+0000","last_became_peered":"2023-09-30T09:12:07.946090+0000","last_unstale":"2023-09-30T09:12:09.642004+0000","last_undegraded":"2023-09-30T09:12:09.642004+0000","last_fullsized":"2023-09-30T09:12:09.642004+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:59:12.592884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.758689+0000","last_change":"2023-09-30T09:11:08.918926+0000","last_active":"2023-09-30T09:12:09.758689+0000","last_peered":"2023-09-30T09:12:09.758689+0000","last_clean":"2023-09-30T09:12:09.758689+0000","last_became_active":"2023-09-30T09:11:08.918633+0000","last_became_peered":"2023-09-30T09:11:08.918633+0000","last_unstale":"2023-09-30T09:12:09.758689+0000","last_undegraded":"2023-09-30T09:12:09.758689+0000","last_fullsized":"2023-09-30T09:12:09.758689+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:10:07.071958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.761142+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:12:09.761142+0000","last_peered":"2023-09-30T09:12:09.761142+0000","last_clean":"2023-09-30T09:12:09.761142+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:12:09.761142+0000","last_undegraded":"2023-09-30T09:12:09.761142+0000","last_fullsized":"2023-09-30T09:12:09.761142+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953287+0000","last_change":"2023-09-30T09:12:07.953074+0000","last_active":"2023-09-30T09:12:08.953287+0000","last_peered":"2023-09-30T09:12:08.953287+0000","last_clean":"2023-09-30T09:12:08.953287+0000","last_became_active":"2023-09-30T09:12:07.951152+0000","last_became_peered":"2023-09-30T09:12:07.951152+0000","last_unstale":"2023-09-30T09:12:08.953287+0000","last_undegraded":"2023-09-30T09:12:08.953287+0000","last_fullsized":"2023-09-30T09:12:08.953287+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:45.636524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":276,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.758615+0000","last_change":"2023-09-30T09:10:32.609108+0000","last_active":"2023-09-30T09:12:09.758615+0000","last_peered":"2023-09-30T09:12:09.758615+0000","last_clean":"2023-09-30T09:12:09.758615+0000","last_became_active":"2023-09-30T09:10:32.608566+0000","last_became_peered":"2023-09-30T09:10:32.608566+0000","last_unstale":"2023-09-30T09:12:09.758615+0000","last_undegraded":"2023-09-30T09:12:09.758615+0000","last_fullsized":"2023-09-30T09:12:09.758615+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:37:39.870350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950423+0000","last_change":"2023-09-30T09:11:08.916682+0000","last_active":"2023-09-30T09:12:08.950423+0000","last_peered":"2023-09-30T09:12:08.950423+0000","last_clean":"2023-09-30T09:12:08.950423+0000","last_became_active":"2023-09-30T09:11:08.916369+0000","last_became_peered":"2023-09-30T09:11:08.916369+0000","last_unstale":"2023-09-30T09:12:08.950423+0000","last_undegraded":"2023-09-30T09:12:08.950423+0000","last_fullsized":"2023-09-30T09:12:08.950423+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:21.617802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.644302+0000","last_change":"2023-09-30T09:12:09.643472+0000","last_active":"2023-09-30T09:12:09.644302+0000","last_peered":"2023-09-30T09:12:09.644302+0000","last_clean":"2023-09-30T09:12:09.644302+0000","last_became_active":"2023-09-30T09:12:09.643121+0000","last_became_peered":"2023-09-30T09:12:09.643121+0000","last_unstale":"2023-09-30T09:12:09.644302+0000","last_undegraded":"2023-09-30T09:12:09.644302+0000","last_fullsized":"2023-09-30T09:12:09.644302+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:34.435180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953803+0000","last_change":"2023-09-30T09:12:07.945781+0000","last_active":"2023-09-30T09:12:08.953803+0000","last_peered":"2023-09-30T09:12:08.953803+0000","last_clean":"2023-09-30T09:12:08.953803+0000","last_became_active":"2023-09-30T09:12:07.945479+0000","last_became_peered":"2023-09-30T09:12:07.945479+0000","last_unstale":"2023-09-30T09:12:08.953803+0000","last_undegraded":"2023-09-30T09:12:08.953803+0000","last_fullsized":"2023-09-30T09:12:08.953803+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:42.046448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.533853+0000","last_change":"2023-09-30T09:11:08.988754+0000","last_active":"2023-09-30T09:12:09.533853+0000","last_peered":"2023-09-30T09:12:09.533853+0000","last_clean":"2023-09-30T09:12:09.533853+0000","last_became_active":"2023-09-30T09:11:08.988424+0000","last_became_peered":"2023-09-30T09:11:08.988424+0000","last_unstale":"2023-09-30T09:12:09.533853+0000","last_undegraded":"2023-09-30T09:12:09.533853+0000","last_fullsized":"2023-09-30T09:12:09.533853+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:05:03.608949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.535370+0000","last_change":"2023-09-30T09:12:09.534926+0000","last_active":"2023-09-30T09:12:09.535370+0000","last_peered":"2023-09-30T09:12:09.535370+0000","last_clean":"2023-09-30T09:12:09.535370+0000","last_became_active":"2023-09-30T09:12:09.534644+0000","last_became_peered":"2023-09-30T09:12:09.534644+0000","last_unstale":"2023-09-30T09:12:09.535370+0000","last_undegraded":"2023-09-30T09:12:09.535370+0000","last_fullsized":"2023-09-30T09:12:09.535370+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:28:06.343901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.008814+0000","last_change":"2023-09-30T09:12:07.947037+0000","last_active":"2023-09-30T09:12:09.008814+0000","last_peered":"2023-09-30T09:12:09.008814+0000","last_clean":"2023-09-30T09:12:09.008814+0000","last_became_active":"2023-09-30T09:12:07.946797+0000","last_became_peered":"2023-09-30T09:12:07.946797+0000","last_unstale":"2023-09-30T09:12:09.008814+0000","last_undegraded":"2023-09-30T09:12:09.008814+0000","last_fullsized":"2023-09-30T09:12:09.008814+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:56:34.851327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950295+0000","last_change":"2023-09-30T09:11:08.921010+0000","last_active":"2023-09-30T09:12:08.950295+0000","last_peered":"2023-09-30T09:12:08.950295+0000","last_clean":"2023-09-30T09:12:08.950295+0000","last_became_active":"2023-09-30T09:11:08.920716+0000","last_became_peered":"2023-09-30T09:11:08.920716+0000","last_unstale":"2023-09-30T09:12:08.950295+0000","last_undegraded":"2023-09-30T09:12:08.950295+0000","last_fullsized":"2023-09-30T09:12:08.950295+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:33:29.433934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.015746+0000","last_change":"2023-09-30T09:12:09.015260+0000","last_active":"2023-09-30T09:12:09.015746+0000","last_peered":"2023-09-30T09:12:09.015746+0000","last_clean":"2023-09-30T09:12:09.015746+0000","last_became_active":"2023-09-30T09:12:09.014920+0000","last_became_peered":"2023-09-30T09:12:09.014920+0000","last_unstale":"2023-09-30T09:12:09.015746+0000","last_undegraded":"2023-09-30T09:12:09.015746+0000","last_fullsized":"2023-09-30T09:12:09.015746+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:31:53.981691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950645+0000","last_change":"2023-09-30T09:12:07.952887+0000","last_active":"2023-09-30T09:12:08.950645+0000","last_peered":"2023-09-30T09:12:08.950645+0000","last_clean":"2023-09-30T09:12:08.950645+0000","last_became_active":"2023-09-30T09:12:07.952590+0000","last_became_peered":"2023-09-30T09:12:07.952590+0000","last_unstale":"2023-09-30T09:12:08.950645+0000","last_undegraded":"2023-09-30T09:12:08.950645+0000","last_fullsized":"2023-09-30T09:12:08.950645+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:31:44.696202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954067+0000","last_change":"2023-09-30T09:11:08.919069+0000","last_active":"2023-09-30T09:12:08.954067+0000","last_peered":"2023-09-30T09:12:08.954067+0000","last_clean":"2023-09-30T09:12:08.954067+0000","last_became_active":"2023-09-30T09:11:08.918732+0000","last_became_peered":"2023-09-30T09:11:08.918732+0000","last_unstale":"2023-09-30T09:12:08.954067+0000","last_undegraded":"2023-09-30T09:12:08.954067+0000","last_fullsized":"2023-09-30T09:12:08.954067+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:45:47.599770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.015656+0000","last_change":"2023-09-30T09:12:09.015232+0000","last_active":"2023-09-30T09:12:09.015656+0000","last_peered":"2023-09-30T09:12:09.015656+0000","last_clean":"2023-09-30T09:12:09.015656+0000","last_became_active":"2023-09-30T09:12:09.014949+0000","last_became_peered":"2023-09-30T09:12:09.014949+0000","last_unstale":"2023-09-30T09:12:09.015656+0000","last_undegraded":"2023-09-30T09:12:09.015656+0000","last_fullsized":"2023-09-30T09:12:09.015656+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:22:22.362529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953331+0000","last_change":"2023-09-30T09:12:07.952393+0000","last_active":"2023-09-30T09:12:08.953331+0000","last_peered":"2023-09-30T09:12:08.953331+0000","last_clean":"2023-09-30T09:12:08.953331+0000","last_became_active":"2023-09-30T09:12:07.952001+0000","last_became_peered":"2023-09-30T09:12:07.952001+0000","last_unstale":"2023-09-30T09:12:08.953331+0000","last_undegraded":"2023-09-30T09:12:08.953331+0000","last_fullsized":"2023-09-30T09:12:08.953331+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:31.068241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017441+0000","last_change":"2023-09-30T09:12:09.016837+0000","last_active":"2023-09-30T09:12:09.017441+0000","last_peered":"2023-09-30T09:12:09.017441+0000","last_clean":"2023-09-30T09:12:09.017441+0000","last_became_active":"2023-09-30T09:12:09.016418+0000","last_became_peered":"2023-09-30T09:12:09.016418+0000","last_unstale":"2023-09-30T09:12:09.017441+0000","last_undegraded":"2023-09-30T09:12:09.017441+0000","last_fullsized":"2023-09-30T09:12:09.017441+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:55:35.980668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.014757+0000","last_change":"2023-09-30T09:12:07.945521+0000","last_active":"2023-09-30T09:12:09.014757+0000","last_peered":"2023-09-30T09:12:09.014757+0000","last_clean":"2023-09-30T09:12:09.014757+0000","last_became_active":"2023-09-30T09:12:07.944823+0000","last_became_peered":"2023-09-30T09:12:07.944823+0000","last_unstale":"2023-09-30T09:12:09.014757+0000","last_undegraded":"2023-09-30T09:12:09.014757+0000","last_fullsized":"2023-09-30T09:12:09.014757+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:00:14.114953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.535296+0000","last_change":"2023-09-30T09:12:09.535095+0000","last_active":"2023-09-30T09:12:09.535296+0000","last_peered":"2023-09-30T09:12:09.535296+0000","last_clean":"2023-09-30T09:12:09.535296+0000","last_became_active":"2023-09-30T09:12:09.534785+0000","last_became_peered":"2023-09-30T09:12:09.534785+0000","last_unstale":"2023-09-30T09:12:09.535296+0000","last_undegraded":"2023-09-30T09:12:09.535296+0000","last_fullsized":"2023-09-30T09:12:09.535296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:38:46.809605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954701+0000","last_change":"2023-09-30T09:12:07.948029+0000","last_active":"2023-09-30T09:12:08.954701+0000","last_peered":"2023-09-30T09:12:08.954701+0000","last_clean":"2023-09-30T09:12:08.954701+0000","last_became_active":"2023-09-30T09:12:07.947750+0000","last_became_peered":"2023-09-30T09:12:07.947750+0000","last_unstale":"2023-09-30T09:12:08.954701+0000","last_undegraded":"2023-09-30T09:12:08.954701+0000","last_fullsized":"2023-09-30T09:12:08.954701+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:45.735654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.016499+0000","last_change":"2023-09-30T09:12:09.015235+0000","last_active":"2023-09-30T09:12:09.016499+0000","last_peered":"2023-09-30T09:12:09.016499+0000","last_clean":"2023-09-30T09:12:09.016499+0000","last_became_active":"2023-09-30T09:12:09.014835+0000","last_became_peered":"2023-09-30T09:12:09.014835+0000","last_unstale":"2023-09-30T09:12:09.016499+0000","last_undegraded":"2023-09-30T09:12:09.016499+0000","last_fullsized":"2023-09-30T09:12:09.016499+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:31.079441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.870976+0000","last_change":"2023-09-30T09:12:07.950514+0000","last_active":"2023-09-30T09:12:20.870976+0000","last_peered":"2023-09-30T09:12:20.870976+0000","last_clean":"2023-09-30T09:12:20.870976+0000","last_became_active":"2023-09-30T09:12:07.950178+0000","last_became_peered":"2023-09-30T09:12:07.950178+0000","last_unstale":"2023-09-30T09:12:20.870976+0000","last_undegraded":"2023-09-30T09:12:20.870976+0000","last_fullsized":"2023-09-30T09:12:20.870976+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:28:38.340356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642106+0000","last_change":"2023-09-30T09:12:07.962829+0000","last_active":"2023-09-30T09:12:09.642106+0000","last_peered":"2023-09-30T09:12:09.642106+0000","last_clean":"2023-09-30T09:12:09.642106+0000","last_became_active":"2023-09-30T09:12:07.962534+0000","last_became_peered":"2023-09-30T09:12:07.962534+0000","last_unstale":"2023-09-30T09:12:09.642106+0000","last_undegraded":"2023-09-30T09:12:09.642106+0000","last_fullsized":"2023-09-30T09:12:09.642106+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:32:46.328589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950450+0000","last_change":"2023-09-30T09:12:07.952897+0000","last_active":"2023-09-30T09:12:08.950450+0000","last_peered":"2023-09-30T09:12:08.950450+0000","last_clean":"2023-09-30T09:12:08.950450+0000","last_became_active":"2023-09-30T09:12:07.952622+0000","last_became_peered":"2023-09-30T09:12:07.952622+0000","last_unstale":"2023-09-30T09:12:08.950450+0000","last_undegraded":"2023-09-30T09:12:08.950450+0000","last_fullsized":"2023-09-30T09:12:08.950450+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:03.628387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.759666+0000","last_change":"2023-09-30T09:12:09.759390+0000","last_active":"2023-09-30T09:12:09.759666+0000","last_peered":"2023-09-30T09:12:09.759666+0000","last_clean":"2023-09-30T09:12:09.759666+0000","last_became_active":"2023-09-30T09:12:09.759143+0000","last_became_peered":"2023-09-30T09:12:09.759143+0000","last_unstale":"2023-09-30T09:12:09.759666+0000","last_undegraded":"2023-09-30T09:12:09.759666+0000","last_fullsized":"2023-09-30T09:12:09.759666+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:58:24.272665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.064222+0000","last_change":"2023-09-30T09:12:07.944640+0000","last_active":"2023-09-30T09:12:10.064222+0000","last_peered":"2023-09-30T09:12:10.064222+0000","last_clean":"2023-09-30T09:12:10.064222+0000","last_became_active":"2023-09-30T09:12:07.944362+0000","last_became_peered":"2023-09-30T09:12:07.944362+0000","last_unstale":"2023-09-30T09:12:10.064222+0000","last_undegraded":"2023-09-30T09:12:10.064222+0000","last_fullsized":"2023-09-30T09:12:10.064222+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:14.703475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017547+0000","last_change":"2023-09-30T09:12:09.017229+0000","last_active":"2023-09-30T09:12:09.017547+0000","last_peered":"2023-09-30T09:12:09.017547+0000","last_clean":"2023-09-30T09:12:09.017547+0000","last_became_active":"2023-09-30T09:12:09.016966+0000","last_became_peered":"2023-09-30T09:12:09.016966+0000","last_unstale":"2023-09-30T09:12:09.017547+0000","last_undegraded":"2023-09-30T09:12:09.017547+0000","last_fullsized":"2023-09-30T09:12:09.017547+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:27:48.033742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.872441+0000","last_change":"2023-09-30T09:12:07.947365+0000","last_active":"2023-09-30T09:12:20.872441+0000","last_peered":"2023-09-30T09:12:20.872441+0000","last_clean":"2023-09-30T09:12:20.872441+0000","last_became_active":"2023-09-30T09:12:07.947076+0000","last_became_peered":"2023-09-30T09:12:07.947076+0000","last_unstale":"2023-09-30T09:12:20.872441+0000","last_undegraded":"2023-09-30T09:12:20.872441+0000","last_fullsized":"2023-09-30T09:12:20.872441+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:10:12.657377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.761024+0000","last_change":"2023-09-30T09:12:09.760712+0000","last_active":"2023-09-30T09:12:09.761024+0000","last_peered":"2023-09-30T09:12:09.761024+0000","last_clean":"2023-09-30T09:12:09.761024+0000","last_became_active":"2023-09-30T09:12:09.760435+0000","last_became_peered":"2023-09-30T09:12:09.760435+0000","last_unstale":"2023-09-30T09:12:09.761024+0000","last_undegraded":"2023-09-30T09:12:09.761024+0000","last_fullsized":"2023-09-30T09:12:09.761024+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:03.896756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.064297+0000","last_change":"2023-09-30T09:12:07.947099+0000","last_active":"2023-09-30T09:12:10.064297+0000","last_peered":"2023-09-30T09:12:10.064297+0000","last_clean":"2023-09-30T09:12:10.064297+0000","last_became_active":"2023-09-30T09:12:07.946790+0000","last_became_peered":"2023-09-30T09:12:07.946790+0000","last_unstale":"2023-09-30T09:12:10.064297+0000","last_undegraded":"2023-09-30T09:12:10.064297+0000","last_fullsized":"2023-09-30T09:12:10.064297+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:19.132459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.760204+0000","last_change":"2023-09-30T09:12:09.759547+0000","last_active":"2023-09-30T09:12:09.760204+0000","last_peered":"2023-09-30T09:12:09.760204+0000","last_clean":"2023-09-30T09:12:09.760204+0000","last_became_active":"2023-09-30T09:12:09.759241+0000","last_became_peered":"2023-09-30T09:12:09.759241+0000","last_unstale":"2023-09-30T09:12:09.760204+0000","last_undegraded":"2023-09-30T09:12:09.760204+0000","last_fullsized":"2023-09-30T09:12:09.760204+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:22.119179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.873505+0000","last_change":"2023-09-30T09:12:07.940067+0000","last_active":"2023-09-30T09:12:20.873505+0000","last_peered":"2023-09-30T09:12:20.873505+0000","last_clean":"2023-09-30T09:12:20.873505+0000","last_became_active":"2023-09-30T09:12:07.939797+0000","last_became_peered":"2023-09-30T09:12:07.939797+0000","last_unstale":"2023-09-30T09:12:20.873505+0000","last_undegraded":"2023-09-30T09:12:20.873505+0000","last_fullsized":"2023-09-30T09:12:20.873505+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:28:44.481894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642239+0000","last_change":"2023-09-30T09:12:08.659348+0000","last_active":"2023-09-30T09:12:09.642239+0000","last_peered":"2023-09-30T09:12:09.642239+0000","last_clean":"2023-09-30T09:12:09.642239+0000","last_became_active":"2023-09-30T09:12:08.658925+0000","last_became_peered":"2023-09-30T09:12:08.658925+0000","last_unstale":"2023-09-30T09:12:09.642239+0000","last_undegraded":"2023-09-30T09:12:09.642239+0000","last_fullsized":"2023-09-30T09:12:09.642239+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:40.725283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.758620+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:12:09.758620+0000","last_peered":"2023-09-30T09:12:09.758620+0000","last_clean":"2023-09-30T09:12:09.758620+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:12:09.758620+0000","last_undegraded":"2023-09-30T09:12:09.758620+0000","last_fullsized":"2023-09-30T09:12:09.758620+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.761228+0000","last_change":"2023-09-30T09:12:09.760771+0000","last_active":"2023-09-30T09:12:09.761228+0000","last_peered":"2023-09-30T09:12:09.761228+0000","last_clean":"2023-09-30T09:12:09.761228+0000","last_became_active":"2023-09-30T09:12:09.760514+0000","last_became_peered":"2023-09-30T09:12:09.760514+0000","last_unstale":"2023-09-30T09:12:09.761228+0000","last_undegraded":"2023-09-30T09:12:09.761228+0000","last_fullsized":"2023-09-30T09:12:09.761228+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:19:15.004373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.641960+0000","last_change":"2023-09-30T09:12:07.945845+0000","last_active":"2023-09-30T09:12:09.641960+0000","last_peered":"2023-09-30T09:12:09.641960+0000","last_clean":"2023-09-30T09:12:09.641960+0000","last_became_active":"2023-09-30T09:12:07.945545+0000","last_became_peered":"2023-09-30T09:12:07.945545+0000","last_unstale":"2023-09-30T09:12:09.641960+0000","last_undegraded":"2023-09-30T09:12:09.641960+0000","last_fullsized":"2023-09-30T09:12:09.641960+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:47.395756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.760132+0000","last_change":"2023-09-30T09:12:09.759365+0000","last_active":"2023-09-30T09:12:09.760132+0000","last_peered":"2023-09-30T09:12:09.760132+0000","last_clean":"2023-09-30T09:12:09.760132+0000","last_became_active":"2023-09-30T09:12:09.758986+0000","last_became_peered":"2023-09-30T09:12:09.758986+0000","last_unstale":"2023-09-30T09:12:09.760132+0000","last_undegraded":"2023-09-30T09:12:09.760132+0000","last_fullsized":"2023-09-30T09:12:09.760132+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:19:44.539397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.872841+0000","last_change":"2023-09-30T09:12:07.949827+0000","last_active":"2023-09-30T09:12:20.872841+0000","last_peered":"2023-09-30T09:12:20.872841+0000","last_clean":"2023-09-30T09:12:20.872841+0000","last_became_active":"2023-09-30T09:12:07.949537+0000","last_became_peered":"2023-09-30T09:12:07.949537+0000","last_unstale":"2023-09-30T09:12:20.872841+0000","last_undegraded":"2023-09-30T09:12:20.872841+0000","last_fullsized":"2023-09-30T09:12:20.872841+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:56.783979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.016679+0000","last_change":"2023-09-30T09:12:09.015237+0000","last_active":"2023-09-30T09:12:09.016679+0000","last_peered":"2023-09-30T09:12:09.016679+0000","last_clean":"2023-09-30T09:12:09.016679+0000","last_became_active":"2023-09-30T09:12:09.014839+0000","last_became_peered":"2023-09-30T09:12:09.014839+0000","last_unstale":"2023-09-30T09:12:09.016679+0000","last_undegraded":"2023-09-30T09:12:09.016679+0000","last_fullsized":"2023-09-30T09:12:09.016679+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:03:23.891794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.873893+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:12:20.873893+0000","last_peered":"2023-09-30T09:12:20.873893+0000","last_clean":"2023-09-30T09:12:20.873893+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:12:20.873893+0000","last_undegraded":"2023-09-30T09:12:20.873893+0000","last_fullsized":"2023-09-30T09:12:20.873893+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.535701+0000","last_change":"2023-09-30T09:12:09.535404+0000","last_active":"2023-09-30T09:12:09.535701+0000","last_peered":"2023-09-30T09:12:09.535701+0000","last_clean":"2023-09-30T09:12:09.535701+0000","last_became_active":"2023-09-30T09:12:09.535079+0000","last_became_peered":"2023-09-30T09:12:09.535079+0000","last_unstale":"2023-09-30T09:12:09.535701+0000","last_undegraded":"2023-09-30T09:12:09.535701+0000","last_fullsized":"2023-09-30T09:12:09.535701+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:57:34.552136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.872829+0000","last_change":"2023-09-30T09:12:07.952351+0000","last_active":"2023-09-30T09:12:20.872829+0000","last_peered":"2023-09-30T09:12:20.872829+0000","last_clean":"2023-09-30T09:12:20.872829+0000","last_became_active":"2023-09-30T09:12:07.951985+0000","last_became_peered":"2023-09-30T09:12:07.951985+0000","last_unstale":"2023-09-30T09:12:20.872829+0000","last_undegraded":"2023-09-30T09:12:20.872829+0000","last_fullsized":"2023-09-30T09:12:20.872829+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:05:11.694255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.760107+0000","last_change":"2023-09-30T09:12:09.759379+0000","last_active":"2023-09-30T09:12:09.760107+0000","last_peered":"2023-09-30T09:12:09.760107+0000","last_clean":"2023-09-30T09:12:09.760107+0000","last_became_active":"2023-09-30T09:12:09.759063+0000","last_became_peered":"2023-09-30T09:12:09.759063+0000","last_unstale":"2023-09-30T09:12:09.760107+0000","last_undegraded":"2023-09-30T09:12:09.760107+0000","last_fullsized":"2023-09-30T09:12:09.760107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:12:39.717308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950535+0000","last_change":"2023-09-30T09:12:07.945831+0000","last_active":"2023-09-30T09:12:08.950535+0000","last_peered":"2023-09-30T09:12:08.950535+0000","last_clean":"2023-09-30T09:12:08.950535+0000","last_became_active":"2023-09-30T09:12:07.945557+0000","last_became_peered":"2023-09-30T09:12:07.945557+0000","last_unstale":"2023-09-30T09:12:08.950535+0000","last_undegraded":"2023-09-30T09:12:08.950535+0000","last_fullsized":"2023-09-30T09:12:08.950535+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:19:32.608191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.958975+0000","last_change":"2023-09-30T09:12:08.958737+0000","last_active":"2023-09-30T09:12:08.958975+0000","last_peered":"2023-09-30T09:12:08.958975+0000","last_clean":"2023-09-30T09:12:08.958975+0000","last_became_active":"2023-09-30T09:12:08.958451+0000","last_became_peered":"2023-09-30T09:12:08.958451+0000","last_unstale":"2023-09-30T09:12:08.958975+0000","last_undegraded":"2023-09-30T09:12:08.958975+0000","last_fullsized":"2023-09-30T09:12:08.958975+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:22:36.866388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642366+0000","last_change":"2023-09-30T09:12:07.949148+0000","last_active":"2023-09-30T09:12:09.642366+0000","last_peered":"2023-09-30T09:12:09.642366+0000","last_clean":"2023-09-30T09:12:09.642366+0000","last_became_active":"2023-09-30T09:12:07.948862+0000","last_became_peered":"2023-09-30T09:12:07.948862+0000","last_unstale":"2023-09-30T09:12:09.642366+0000","last_undegraded":"2023-09-30T09:12:09.642366+0000","last_fullsized":"2023-09-30T09:12:09.642366+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:20:35.484179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.957252+0000","last_change":"2023-09-30T09:12:08.957017+0000","last_active":"2023-09-30T09:12:08.957252+0000","last_peered":"2023-09-30T09:12:08.957252+0000","last_clean":"2023-09-30T09:12:08.957252+0000","last_became_active":"2023-09-30T09:12:08.956769+0000","last_became_peered":"2023-09-30T09:12:08.956769+0000","last_unstale":"2023-09-30T09:12:08.957252+0000","last_undegraded":"2023-09-30T09:12:08.957252+0000","last_fullsized":"2023-09-30T09:12:08.957252+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:58.002090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953417+0000","last_change":"2023-09-30T09:12:07.948020+0000","last_active":"2023-09-30T09:12:08.953417+0000","last_peered":"2023-09-30T09:12:08.953417+0000","last_clean":"2023-09-30T09:12:08.953417+0000","last_became_active":"2023-09-30T09:12:07.947546+0000","last_became_peered":"2023-09-30T09:12:07.947546+0000","last_unstale":"2023-09-30T09:12:08.953417+0000","last_undegraded":"2023-09-30T09:12:08.953417+0000","last_fullsized":"2023-09-30T09:12:08.953417+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:39.392899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642242+0000","last_change":"2023-09-30T09:12:08.659724+0000","last_active":"2023-09-30T09:12:09.642242+0000","last_peered":"2023-09-30T09:12:09.642242+0000","last_clean":"2023-09-30T09:12:09.642242+0000","last_became_active":"2023-09-30T09:12:08.659386+0000","last_became_peered":"2023-09-30T09:12:08.659386+0000","last_unstale":"2023-09-30T09:12:09.642242+0000","last_undegraded":"2023-09-30T09:12:09.642242+0000","last_fullsized":"2023-09-30T09:12:09.642242+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:00.336588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953943+0000","last_change":"2023-09-30T09:12:07.952014+0000","last_active":"2023-09-30T09:12:08.953943+0000","last_peered":"2023-09-30T09:12:08.953943+0000","last_clean":"2023-09-30T09:12:08.953943+0000","last_became_active":"2023-09-30T09:12:07.951700+0000","last_became_peered":"2023-09-30T09:12:07.951700+0000","last_unstale":"2023-09-30T09:12:08.953943+0000","last_undegraded":"2023-09-30T09:12:08.953943+0000","last_fullsized":"2023-09-30T09:12:08.953943+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:53.078385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017429+0000","last_change":"2023-09-30T09:12:09.017177+0000","last_active":"2023-09-30T09:12:09.017429+0000","last_peered":"2023-09-30T09:12:09.017429+0000","last_clean":"2023-09-30T09:12:09.017429+0000","last_became_active":"2023-09-30T09:12:09.016900+0000","last_became_peered":"2023-09-30T09:12:09.016900+0000","last_unstale":"2023-09-30T09:12:09.017429+0000","last_undegraded":"2023-09-30T09:12:09.017429+0000","last_fullsized":"2023-09-30T09:12:09.017429+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:12:51.092516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.063538+0000","last_change":"2023-09-30T09:12:07.945009+0000","last_active":"2023-09-30T09:12:10.063538+0000","last_peered":"2023-09-30T09:12:10.063538+0000","last_clean":"2023-09-30T09:12:10.063538+0000","last_became_active":"2023-09-30T09:12:07.944737+0000","last_became_peered":"2023-09-30T09:12:07.944737+0000","last_unstale":"2023-09-30T09:12:10.063538+0000","last_undegraded":"2023-09-30T09:12:10.063538+0000","last_fullsized":"2023-09-30T09:12:10.063538+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:28:20.999667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234014,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6604,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738548,"statfs":{"total":95995035648,"available":95988273152,"internally_reserved":0,"allocated":1314816,"data_stored":891677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.48899999999999999,"15min":0.49299999999999999},"min":{"1min":0.30499999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.63300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.51600000000000001,"15min":0.52000000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.69099999999999995,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.53000000000000003},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.5,"5min":0.52000000000000002,"15min":0.52400000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.66300000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.64600000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54200000000000004,"15min":0.54800000000000004},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68300000000000005,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.40000000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.69199999999999995,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.51800000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56899999999999995,"15min":0.57599999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65400000000000003,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56100000000000005,"15min":0.56599999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.72099999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.625}]},{"osd":4,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.46700000000000003,"15min":0.46500000000000002},"min":{"1min":0.313,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.32500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.71699999999999997,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.502,"15min":0.497},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.40799999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.80700000000000005,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.438}]},{"osd":6,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.38400000000000001,"5min":0.378,"15min":0.378},"max":{"1min":0.78400000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.52000000000000002,"15min":0.51400000000000001},"min":{"1min":0.39000000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.504}]}]},{"osd":6,"up_from":46,"seq":197568495651,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6584,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738568,"statfs":{"total":95995035648,"available":95988293632,"internally_reserved":0,"allocated":1294336,"data_stored":890943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56599999999999995,"15min":0.57499999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73499999999999999,"5min":0.872,"15min":0.872},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.57099999999999995,"15min":0.57899999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71999999999999997,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.51500000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.56100000000000005,"15min":0.56999999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.621,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.56599999999999995,"15min":0.56999999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75700000000000001,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.437}]},{"osd":2,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.56799999999999995,"15min":0.57299999999999995},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71999999999999997,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.58299999999999996,"15min":0.58899999999999997},"min":{"1min":0.374,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73899999999999999,"5min":1.002,"15min":1.002},"last":0.58999999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.57199999999999995,"15min":0.57399999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.75800000000000001,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.56200000000000006,"15min":0.57299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.59299999999999997,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.53600000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47499999999999998,"15min":0.47299999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.48899999999999999,"15min":0.48599999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63500000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.51700000000000002,"15min":0.50900000000000001},"min":{"1min":0.33900000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.74299999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.77400000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574906,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6764,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738388,"statfs":{"total":95995035648,"available":95988109312,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:14 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.41399999999999998,"15min":0.38400000000000001},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":2.9409999999999998,"5min":2.9409999999999998,"15min":2.9409999999999998},"last":0.436},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.441,"15min":0.41899999999999998},"min":{"1min":0.26300000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7149999999999999,"5min":2.7149999999999999,"15min":2.7149999999999999},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.442,"15min":0.42999999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.64100000000000001,"5min":2.9769999999999999,"15min":2.9769999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.45200000000000001,"15min":0.45400000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.58699999999999997,"5min":2.6549999999999998,"15min":2.6549999999999998},"last":0.52100000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.48199999999999998,"15min":0.46999999999999997},"min":{"1min":0.29999999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.9969999999999999,"5min":2.9969999999999999,"15min":2.9969999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.54300000000000004,"15min":0.53100000000000003},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.9129999999999998,"5min":2.9129999999999998,"15min":2.9129999999999998},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.622,"15min":0.61599999999999999},"min":{"1min":0.42599999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.78000000000000003,"5min":2.726,"15min":2.726},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.65200000000000002,"15min":0.64400000000000002},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85299999999999998,"5min":2.8570000000000002,"15min":2.8570000000000002},"last":0.78800000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.64300000000000002,"15min":0.63600000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":2.8100000000000001,"5min":2.8100000000000001,"15min":2.8100000000000001},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.64300000000000002,"15min":0.63500000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":2.7469999999999999,"5min":2.7469999999999999,"15min":2.7469999999999999},"last":0.83499999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.64300000000000002,"15min":0.63100000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.0529999999999999,"5min":3.0529999999999999,"15min":3.0529999999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.64700000000000002,"15min":0.63700000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":3.0350000000000001,"5min":3.0350000000000001,"15min":3.0350000000000001},"last":0.745}]},{"osd":7,"last update":"Sat Sep 30 09:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.73699999999999999,"15min":0.755},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.82199999999999995,"5min":2.8849999999999998,"15min":2.8849999999999998},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.67800000000000005,"15min":0.69299999999999995},"min":{"1min":0.497,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.78900000000000003,"5min":2.7709999999999999,"15min":2.7709999999999999},"last":0.86699999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738432,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7416,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737736,"statfs":{"total":95995035648,"available":95987441664,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:12:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.36899999999999999,"15min":0.35599999999999998},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.377,"15min":0.35699999999999998},"min":{"1min":0.247,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.27400000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.63},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.505,"15min":0.45000000000000001},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.246},"max":{"1min":4.9340000000000002,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.66600000000000004}]},{"osd":3,"last update":"Sat Sep 30 09:12:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.47299999999999998,"15min":0.48399999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.48899999999999999,"15min":0.503},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.59799999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.61799999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:12:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.628,"15min":0.65400000000000003},"min":{"1min":0.40799999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.83799999999999997,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.64600000000000002,"15min":0.66000000000000003},"min":{"1min":0.44700000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.236,"5min":1.236,"15min":1.236},"last":0.83599999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.60599999999999998,"15min":0.61099999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.63500000000000001,"15min":0.64100000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.85499999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.45600000000000002,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79700000000000004,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.68100000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:12:34 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.497,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63,"15min":0.622},"min":{"1min":0.496,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.292,"5min":1.292,"15min":1.292},"last":0.70899999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411382,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6636,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738516,"statfs":{"total":95995035648,"available":95988240384,"internally_reserved":0,"allocated":692224,"data_stored":299615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.378,"15min":0.38800000000000001},"min":{"1min":0.254,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.45000000000000001,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.434},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.40999999999999998,"15min":0.41699999999999998},"min":{"1min":0.30199999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56999999999999995,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.40000000000000002,"15min":0.41399999999999998},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.48699999999999999,"5min":0.623,"15min":0.623},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.27400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.7349999999999999,"5min":2.7349999999999999,"15min":2.7349999999999999},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:12:03 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.47899999999999998,"15min":0.495},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.58699999999999997,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.47899999999999998,"15min":0.5},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.57499999999999996,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:12:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.70299999999999996,"15min":0.753},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.79800000000000004,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.66500000000000004,"15min":0.68899999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.89800000000000002,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.70999999999999996,"15min":0.73399999999999999},"min":{"1min":0.41099999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84699999999999998,"5min":2.6419999999999999,"15min":2.6419999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.63800000000000001,"15min":0.65000000000000002},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.77600000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.62,"15min":0.622},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80400000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.621},{"interface":"front","average":{"1min":0.624,"5min":0.61899999999999999,"15min":0.61799999999999999},"min":{"1min":0.47499999999999998,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.72399999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:12:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.62,"15min":0.61799999999999999},"min":{"1min":0.45800000000000002,"5min":0.435,"15min":0.435},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.84199999999999997,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.80600000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6128,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739024,"statfs":{"total":95995035648,"available":95988760576,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.49399999999999999,"15min":0.41799999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":11.856,"5min":11.856,"15min":11.856},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.54900000000000004,"15min":0.48299999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.875999999999999,"5min":11.875999999999999,"15min":11.875999999999999},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.55600000000000005,"15min":0.48899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.917999999999999,"5min":11.917999999999999,"15min":11.917999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.46500000000000002,"15min":0.48099999999999998},"min":{"1min":0.27500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.2390000000000001,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.5,"15min":0.505},"min":{"1min":0.318,"5min":0.311,"15min":0.311},"max":{"1min":1.4430000000000001,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.46600000000000003,"15min":0.46700000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.26500000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59799999999999998,"15min":0.60199999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.442},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.49399999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.61499999999999999,"15min":0.61099999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.622,"15min":0.62},"min":{"1min":0.501,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.64400000000000002,"15min":0.63800000000000001},"min":{"1min":0.48699999999999999,"5min":0.432,"15min":0.432},"max":{"1min":1.125,"5min":1.125,"15min":1.125},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.63900000000000001,"15min":0.64200000000000002},"min":{"1min":0.48599999999999999,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.63,"5min":1.208,"15min":1.304},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.85899999999999999,"5min":11.801,"15min":11.801},"last":0.41899999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018924,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6204,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738948,"statfs":{"total":95995035648,"available":95988682752,"internally_reserved":0,"allocated":708608,"data_stored":300709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.55500000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.78500000000000003,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.54700000000000004,"15min":0.56000000000000005},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.55200000000000005}]},{"osd":1,"last update":"Sat Sep 30 09:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.54100000000000004,"15min":0.55700000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.55300000000000005,"15min":0.56299999999999994},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.69299999999999995}]},{"osd":2,"last update":"Sat Sep 30 09:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.55400000000000005,"15min":0.56599999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55800000000000005,"15min":0.56599999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.77000000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56200000000000006,"15min":0.56299999999999994},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.82899999999999996,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57099999999999995,"15min":0.58699999999999997},"min":{"1min":0.38400000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.41099999999999998,"15min":0.39800000000000002},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.747},{"interface":"front","average":{"1min":0.496,"5min":0.42599999999999999,"15min":0.41399999999999998},"min":{"1min":0.30499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70099999999999996,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 30 09:12:12 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.47399999999999998,"15min":0.46899999999999997},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.497,"15min":0.48799999999999999},"min":{"1min":0.35599999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.78000000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.53300000000000003,"15min":0.53200000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.72399999999999998,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.71399999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757288,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6064,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739088,"statfs":{"total":95995035648,"available":95988826112,"internally_reserved":0,"allocated":696320,"data_stored":299858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.57899999999999996,"15min":0.58899999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69099999999999995,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.60599999999999998,"15min":0.61599999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.879,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.751,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.46700000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.81200000000000006,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.38200000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.91600000000000004,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76900000000000002,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84399999999999997,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.45000000000000001,"15min":0.443},"min":{"1min":0.30399999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.72899999999999998,"5min":0.875,"15min":0.875},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 30 09:12:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.45700000000000002,"15min":0.44800000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.54200000000000004,"15min":0.53500000000000003},"min":{"1min":0.377,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:12:34 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.376,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.72899999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.625},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.58099999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:12:54.434 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:12:54.804 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:54 smithi019 ceph-mon[100214]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:12:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:54 smithi019 ceph-mon[100214]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:54.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:54 smithi019 ceph-mon[118983]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:12:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:54 smithi019 ceph-mon[118983]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:55.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:54 smithi167 ceph-mon[104403]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:12:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:54 smithi167 ceph-mon[104403]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:56.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:12:55 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:12:56.542 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:12:56.544 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:12:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:56 smithi019 ceph-mon[100214]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:12:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:56 smithi019 ceph-mon[100214]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:56.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:56 smithi019 ceph-mon[118983]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:12:56.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:56 smithi019 ceph-mon[118983]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:57.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:56 smithi167 ceph-mon[104403]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:12:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:56 smithi167 ceph-mon[104403]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:57.239 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":66,"stamp":"2023-09-30T09:12:54.535625+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52400,"kb_used_data":7216,"kb_used_omap":0,"kb_used_meta":45120,"kb_avail":749908816,"statfs":{"total":767960285184,"available":767906627584,"internally_reserved":0,"allocated":7389184,"data_stored":4172507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46202880},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002802"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.014962+0000","last_change":"2023-09-30T09:12:09.014492+0000","last_active":"2023-09-30T09:12:09.014962+0000","last_peered":"2023-09-30T09:12:09.014962+0000","last_clean":"2023-09-30T09:12:09.014962+0000","last_became_active":"2023-09-30T09:12:09.008964+0000","last_became_peered":"2023-09-30T09:12:09.008964+0000","last_unstale":"2023-09-30T09:12:09.014962+0000","last_undegraded":"2023-09-30T09:12:09.014962+0000","last_fullsized":"2023-09-30T09:12:09.014962+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:15:25.163331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.872158+0000","last_change":"2023-09-30T09:12:07.941891+0000","last_active":"2023-09-30T09:12:20.872158+0000","last_peered":"2023-09-30T09:12:20.872158+0000","last_clean":"2023-09-30T09:12:20.872158+0000","last_became_active":"2023-09-30T09:12:07.941062+0000","last_became_peered":"2023-09-30T09:12:07.941062+0000","last_unstale":"2023-09-30T09:12:20.872158+0000","last_undegraded":"2023-09-30T09:12:20.872158+0000","last_fullsized":"2023-09-30T09:12:20.872158+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:46:46.904341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.643561+0000","last_change":"2023-09-30T09:12:09.643163+0000","last_active":"2023-09-30T09:12:09.643561+0000","last_peered":"2023-09-30T09:12:09.643561+0000","last_clean":"2023-09-30T09:12:09.643561+0000","last_became_active":"2023-09-30T09:12:09.642880+0000","last_became_peered":"2023-09-30T09:12:09.642880+0000","last_unstale":"2023-09-30T09:12:09.643561+0000","last_undegraded":"2023-09-30T09:12:09.643561+0000","last_fullsized":"2023-09-30T09:12:09.643561+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:08.883041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.063379+0000","last_change":"2023-09-30T09:12:07.950886+0000","last_active":"2023-09-30T09:12:10.063379+0000","last_peered":"2023-09-30T09:12:10.063379+0000","last_clean":"2023-09-30T09:12:10.063379+0000","last_became_active":"2023-09-30T09:12:07.950425+0000","last_became_peered":"2023-09-30T09:12:07.950425+0000","last_unstale":"2023-09-30T09:12:10.063379+0000","last_undegraded":"2023-09-30T09:12:10.063379+0000","last_fullsized":"2023-09-30T09:12:10.063379+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:45:09.519541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.643772+0000","last_change":"2023-09-30T09:12:09.643522+0000","last_active":"2023-09-30T09:12:09.643772+0000","last_peered":"2023-09-30T09:12:09.643772+0000","last_clean":"2023-09-30T09:12:09.643772+0000","last_became_active":"2023-09-30T09:12:09.643195+0000","last_became_peered":"2023-09-30T09:12:09.643195+0000","last_unstale":"2023-09-30T09:12:09.643772+0000","last_undegraded":"2023-09-30T09:12:09.643772+0000","last_fullsized":"2023-09-30T09:12:09.643772+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:22:18.627117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.873279+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:12:20.873279+0000","last_peered":"2023-09-30T09:12:20.873279+0000","last_clean":"2023-09-30T09:12:20.873279+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:12:20.873279+0000","last_undegraded":"2023-09-30T09:12:20.873279+0000","last_fullsized":"2023-09-30T09:12:20.873279+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017454+0000","last_change":"2023-09-30T09:12:09.015680+0000","last_active":"2023-09-30T09:12:09.017454+0000","last_peered":"2023-09-30T09:12:09.017454+0000","last_clean":"2023-09-30T09:12:09.017454+0000","last_became_active":"2023-09-30T09:12:09.015128+0000","last_became_peered":"2023-09-30T09:12:09.015128+0000","last_unstale":"2023-09-30T09:12:09.017454+0000","last_undegraded":"2023-09-30T09:12:09.017454+0000","last_fullsized":"2023-09-30T09:12:09.017454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:14:26.966437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.871183+0000","last_change":"2023-09-30T09:12:07.947839+0000","last_active":"2023-09-30T09:12:20.871183+0000","last_peered":"2023-09-30T09:12:20.871183+0000","last_clean":"2023-09-30T09:12:20.871183+0000","last_became_active":"2023-09-30T09:12:07.947530+0000","last_became_peered":"2023-09-30T09:12:07.947530+0000","last_unstale":"2023-09-30T09:12:20.871183+0000","last_undegraded":"2023-09-30T09:12:20.871183+0000","last_fullsized":"2023-09-30T09:12:20.871183+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:38.822389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.534139+0000","last_change":"2023-09-30T09:12:08.094365+0000","last_active":"2023-09-30T09:12:09.534139+0000","last_peered":"2023-09-30T09:12:09.534139+0000","last_clean":"2023-09-30T09:12:09.534139+0000","last_became_active":"2023-09-30T09:12:08.094023+0000","last_became_peered":"2023-09-30T09:12:08.094023+0000","last_unstale":"2023-09-30T09:12:09.534139+0000","last_undegraded":"2023-09-30T09:12:09.534139+0000","last_fullsized":"2023-09-30T09:12:09.534139+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:07:53.843136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.063110+0000","last_change":"2023-09-30T09:12:07.945701+0000","last_active":"2023-09-30T09:12:10.063110+0000","last_peered":"2023-09-30T09:12:10.063110+0000","last_clean":"2023-09-30T09:12:10.063110+0000","last_became_active":"2023-09-30T09:12:07.945064+0000","last_became_peered":"2023-09-30T09:12:07.945064+0000","last_unstale":"2023-09-30T09:12:10.063110+0000","last_undegraded":"2023-09-30T09:12:10.063110+0000","last_fullsized":"2023-09-30T09:12:10.063110+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:55.000129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017430+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:12:09.017430+0000","last_peered":"2023-09-30T09:12:09.017430+0000","last_clean":"2023-09-30T09:12:09.017430+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:12:09.017430+0000","last_undegraded":"2023-09-30T09:12:09.017430+0000","last_fullsized":"2023-09-30T09:12:09.017430+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.063330+0000","last_change":"2023-09-30T09:12:07.942405+0000","last_active":"2023-09-30T09:12:10.063330+0000","last_peered":"2023-09-30T09:12:10.063330+0000","last_clean":"2023-09-30T09:12:10.063330+0000","last_became_active":"2023-09-30T09:12:07.942066+0000","last_became_peered":"2023-09-30T09:12:07.942066+0000","last_unstale":"2023-09-30T09:12:10.063330+0000","last_undegraded":"2023-09-30T09:12:10.063330+0000","last_fullsized":"2023-09-30T09:12:10.063330+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:19:40.632109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954027+0000","last_change":"2023-09-30T09:11:08.916599+0000","last_active":"2023-09-30T09:12:08.954027+0000","last_peered":"2023-09-30T09:12:08.954027+0000","last_clean":"2023-09-30T09:12:08.954027+0000","last_became_active":"2023-09-30T09:11:08.916301+0000","last_became_peered":"2023-09-30T09:11:08.916301+0000","last_unstale":"2023-09-30T09:12:08.954027+0000","last_undegraded":"2023-09-30T09:12:08.954027+0000","last_fullsized":"2023-09-30T09:12:08.954027+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:19.518596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.760245+0000","last_change":"2023-09-30T09:12:09.759528+0000","last_active":"2023-09-30T09:12:09.760245+0000","last_peered":"2023-09-30T09:12:09.760245+0000","last_clean":"2023-09-30T09:12:09.760245+0000","last_became_active":"2023-09-30T09:12:09.759199+0000","last_became_peered":"2023-09-30T09:12:09.759199+0000","last_unstale":"2023-09-30T09:12:09.760245+0000","last_undegraded":"2023-09-30T09:12:09.760245+0000","last_fullsized":"2023-09-30T09:12:09.760245+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:44.333971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954289+0000","last_change":"2023-09-30T09:11:09.118530+0000","last_active":"2023-09-30T09:12:08.954289+0000","last_peered":"2023-09-30T09:12:08.954289+0000","last_clean":"2023-09-30T09:12:08.954289+0000","last_became_active":"2023-09-30T09:11:09.118152+0000","last_became_peered":"2023-09-30T09:11:09.118152+0000","last_unstale":"2023-09-30T09:12:08.954289+0000","last_undegraded":"2023-09-30T09:12:08.954289+0000","last_fullsized":"2023-09-30T09:12:08.954289+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:29:57.045104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.870634+0000","last_change":"2023-09-30T09:12:07.952075+0000","last_active":"2023-09-30T09:12:20.870634+0000","last_peered":"2023-09-30T09:12:20.870634+0000","last_clean":"2023-09-30T09:12:20.870634+0000","last_became_active":"2023-09-30T09:12:07.951745+0000","last_became_peered":"2023-09-30T09:12:07.951745+0000","last_unstale":"2023-09-30T09:12:20.870634+0000","last_undegraded":"2023-09-30T09:12:20.870634+0000","last_fullsized":"2023-09-30T09:12:20.870634+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:16.750757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642182+0000","last_change":"2023-09-30T09:12:08.658689+0000","last_active":"2023-09-30T09:12:09.642182+0000","last_peered":"2023-09-30T09:12:09.642182+0000","last_clean":"2023-09-30T09:12:09.642182+0000","last_became_active":"2023-09-30T09:12:08.658407+0000","last_became_peered":"2023-09-30T09:12:08.658407+0000","last_unstale":"2023-09-30T09:12:09.642182+0000","last_undegraded":"2023-09-30T09:12:09.642182+0000","last_fullsized":"2023-09-30T09:12:09.642182+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:42.921517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954180+0000","last_change":"2023-09-30T09:12:07.948494+0000","last_active":"2023-09-30T09:12:08.954180+0000","last_peered":"2023-09-30T09:12:08.954180+0000","last_clean":"2023-09-30T09:12:08.954180+0000","last_became_active":"2023-09-30T09:12:07.948128+0000","last_became_peered":"2023-09-30T09:12:07.948128+0000","last_unstale":"2023-09-30T09:12:08.954180+0000","last_undegraded":"2023-09-30T09:12:08.954180+0000","last_fullsized":"2023-09-30T09:12:08.954180+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:25.537779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.758954+0000","last_change":"2023-09-30T09:11:09.118453+0000","last_active":"2023-09-30T09:12:09.758954+0000","last_peered":"2023-09-30T09:12:09.758954+0000","last_clean":"2023-09-30T09:12:09.758954+0000","last_became_active":"2023-09-30T09:11:09.118151+0000","last_became_peered":"2023-09-30T09:11:09.118151+0000","last_unstale":"2023-09-30T09:12:09.758954+0000","last_undegraded":"2023-09-30T09:12:09.758954+0000","last_fullsized":"2023-09-30T09:12:09.758954+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:53:19.764905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642328+0000","last_change":"2023-09-30T09:12:07.961450+0000","last_active":"2023-09-30T09:12:09.642328+0000","last_peered":"2023-09-30T09:12:09.642328+0000","last_clean":"2023-09-30T09:12:09.642328+0000","last_became_active":"2023-09-30T09:12:07.961110+0000","last_became_peered":"2023-09-30T09:12:07.961110+0000","last_unstale":"2023-09-30T09:12:09.642328+0000","last_undegraded":"2023-09-30T09:12:09.642328+0000","last_fullsized":"2023-09-30T09:12:09.642328+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:15.930823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642004+0000","last_change":"2023-09-30T09:12:07.946576+0000","last_active":"2023-09-30T09:12:09.642004+0000","last_peered":"2023-09-30T09:12:09.642004+0000","last_clean":"2023-09-30T09:12:09.642004+0000","last_became_active":"2023-09-30T09:12:07.946090+0000","last_became_peered":"2023-09-30T09:12:07.946090+0000","last_unstale":"2023-09-30T09:12:09.642004+0000","last_undegraded":"2023-09-30T09:12:09.642004+0000","last_fullsized":"2023-09-30T09:12:09.642004+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:59:12.592884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.758689+0000","last_change":"2023-09-30T09:11:08.918926+0000","last_active":"2023-09-30T09:12:09.758689+0000","last_peered":"2023-09-30T09:12:09.758689+0000","last_clean":"2023-09-30T09:12:09.758689+0000","last_became_active":"2023-09-30T09:11:08.918633+0000","last_became_peered":"2023-09-30T09:11:08.918633+0000","last_unstale":"2023-09-30T09:12:09.758689+0000","last_undegraded":"2023-09-30T09:12:09.758689+0000","last_fullsized":"2023-09-30T09:12:09.758689+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:10:07.071958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.761142+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:12:09.761142+0000","last_peered":"2023-09-30T09:12:09.761142+0000","last_clean":"2023-09-30T09:12:09.761142+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:12:09.761142+0000","last_undegraded":"2023-09-30T09:12:09.761142+0000","last_fullsized":"2023-09-30T09:12:09.761142+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953287+0000","last_change":"2023-09-30T09:12:07.953074+0000","last_active":"2023-09-30T09:12:08.953287+0000","last_peered":"2023-09-30T09:12:08.953287+0000","last_clean":"2023-09-30T09:12:08.953287+0000","last_became_active":"2023-09-30T09:12:07.951152+0000","last_became_peered":"2023-09-30T09:12:07.951152+0000","last_unstale":"2023-09-30T09:12:08.953287+0000","last_undegraded":"2023-09-30T09:12:08.953287+0000","last_fullsized":"2023-09-30T09:12:08.953287+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:45.636524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":276,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.758615+0000","last_change":"2023-09-30T09:10:32.609108+0000","last_active":"2023-09-30T09:12:09.758615+0000","last_peered":"2023-09-30T09:12:09.758615+0000","last_clean":"2023-09-30T09:12:09.758615+0000","last_became_active":"2023-09-30T09:10:32.608566+0000","last_became_peered":"2023-09-30T09:10:32.608566+0000","last_unstale":"2023-09-30T09:12:09.758615+0000","last_undegraded":"2023-09-30T09:12:09.758615+0000","last_fullsized":"2023-09-30T09:12:09.758615+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:37:39.870350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950423+0000","last_change":"2023-09-30T09:11:08.916682+0000","last_active":"2023-09-30T09:12:08.950423+0000","last_peered":"2023-09-30T09:12:08.950423+0000","last_clean":"2023-09-30T09:12:08.950423+0000","last_became_active":"2023-09-30T09:11:08.916369+0000","last_became_peered":"2023-09-30T09:11:08.916369+0000","last_unstale":"2023-09-30T09:12:08.950423+0000","last_undegraded":"2023-09-30T09:12:08.950423+0000","last_fullsized":"2023-09-30T09:12:08.950423+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:21.617802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.644302+0000","last_change":"2023-09-30T09:12:09.643472+0000","last_active":"2023-09-30T09:12:09.644302+0000","last_peered":"2023-09-30T09:12:09.644302+0000","last_clean":"2023-09-30T09:12:09.644302+0000","last_became_active":"2023-09-30T09:12:09.643121+0000","last_became_peered":"2023-09-30T09:12:09.643121+0000","last_unstale":"2023-09-30T09:12:09.644302+0000","last_undegraded":"2023-09-30T09:12:09.644302+0000","last_fullsized":"2023-09-30T09:12:09.644302+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:34.435180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953803+0000","last_change":"2023-09-30T09:12:07.945781+0000","last_active":"2023-09-30T09:12:08.953803+0000","last_peered":"2023-09-30T09:12:08.953803+0000","last_clean":"2023-09-30T09:12:08.953803+0000","last_became_active":"2023-09-30T09:12:07.945479+0000","last_became_peered":"2023-09-30T09:12:07.945479+0000","last_unstale":"2023-09-30T09:12:08.953803+0000","last_undegraded":"2023-09-30T09:12:08.953803+0000","last_fullsized":"2023-09-30T09:12:08.953803+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:42.046448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.533853+0000","last_change":"2023-09-30T09:11:08.988754+0000","last_active":"2023-09-30T09:12:09.533853+0000","last_peered":"2023-09-30T09:12:09.533853+0000","last_clean":"2023-09-30T09:12:09.533853+0000","last_became_active":"2023-09-30T09:11:08.988424+0000","last_became_peered":"2023-09-30T09:11:08.988424+0000","last_unstale":"2023-09-30T09:12:09.533853+0000","last_undegraded":"2023-09-30T09:12:09.533853+0000","last_fullsized":"2023-09-30T09:12:09.533853+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:05:03.608949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.535370+0000","last_change":"2023-09-30T09:12:09.534926+0000","last_active":"2023-09-30T09:12:09.535370+0000","last_peered":"2023-09-30T09:12:09.535370+0000","last_clean":"2023-09-30T09:12:09.535370+0000","last_became_active":"2023-09-30T09:12:09.534644+0000","last_became_peered":"2023-09-30T09:12:09.534644+0000","last_unstale":"2023-09-30T09:12:09.535370+0000","last_undegraded":"2023-09-30T09:12:09.535370+0000","last_fullsized":"2023-09-30T09:12:09.535370+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:28:06.343901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.008814+0000","last_change":"2023-09-30T09:12:07.947037+0000","last_active":"2023-09-30T09:12:09.008814+0000","last_peered":"2023-09-30T09:12:09.008814+0000","last_clean":"2023-09-30T09:12:09.008814+0000","last_became_active":"2023-09-30T09:12:07.946797+0000","last_became_peered":"2023-09-30T09:12:07.946797+0000","last_unstale":"2023-09-30T09:12:09.008814+0000","last_undegraded":"2023-09-30T09:12:09.008814+0000","last_fullsized":"2023-09-30T09:12:09.008814+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:56:34.851327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950295+0000","last_change":"2023-09-30T09:11:08.921010+0000","last_active":"2023-09-30T09:12:08.950295+0000","last_peered":"2023-09-30T09:12:08.950295+0000","last_clean":"2023-09-30T09:12:08.950295+0000","last_became_active":"2023-09-30T09:11:08.920716+0000","last_became_peered":"2023-09-30T09:11:08.920716+0000","last_unstale":"2023-09-30T09:12:08.950295+0000","last_undegraded":"2023-09-30T09:12:08.950295+0000","last_fullsized":"2023-09-30T09:12:08.950295+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:33:29.433934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.015746+0000","last_change":"2023-09-30T09:12:09.015260+0000","last_active":"2023-09-30T09:12:09.015746+0000","last_peered":"2023-09-30T09:12:09.015746+0000","last_clean":"2023-09-30T09:12:09.015746+0000","last_became_active":"2023-09-30T09:12:09.014920+0000","last_became_peered":"2023-09-30T09:12:09.014920+0000","last_unstale":"2023-09-30T09:12:09.015746+0000","last_undegraded":"2023-09-30T09:12:09.015746+0000","last_fullsized":"2023-09-30T09:12:09.015746+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:31:53.981691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950645+0000","last_change":"2023-09-30T09:12:07.952887+0000","last_active":"2023-09-30T09:12:08.950645+0000","last_peered":"2023-09-30T09:12:08.950645+0000","last_clean":"2023-09-30T09:12:08.950645+0000","last_became_active":"2023-09-30T09:12:07.952590+0000","last_became_peered":"2023-09-30T09:12:07.952590+0000","last_unstale":"2023-09-30T09:12:08.950645+0000","last_undegraded":"2023-09-30T09:12:08.950645+0000","last_fullsized":"2023-09-30T09:12:08.950645+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:31:44.696202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954067+0000","last_change":"2023-09-30T09:11:08.919069+0000","last_active":"2023-09-30T09:12:08.954067+0000","last_peered":"2023-09-30T09:12:08.954067+0000","last_clean":"2023-09-30T09:12:08.954067+0000","last_became_active":"2023-09-30T09:11:08.918732+0000","last_became_peered":"2023-09-30T09:11:08.918732+0000","last_unstale":"2023-09-30T09:12:08.954067+0000","last_undegraded":"2023-09-30T09:12:08.954067+0000","last_fullsized":"2023-09-30T09:12:08.954067+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:45:47.599770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.015656+0000","last_change":"2023-09-30T09:12:09.015232+0000","last_active":"2023-09-30T09:12:09.015656+0000","last_peered":"2023-09-30T09:12:09.015656+0000","last_clean":"2023-09-30T09:12:09.015656+0000","last_became_active":"2023-09-30T09:12:09.014949+0000","last_became_peered":"2023-09-30T09:12:09.014949+0000","last_unstale":"2023-09-30T09:12:09.015656+0000","last_undegraded":"2023-09-30T09:12:09.015656+0000","last_fullsized":"2023-09-30T09:12:09.015656+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:22:22.362529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953331+0000","last_change":"2023-09-30T09:12:07.952393+0000","last_active":"2023-09-30T09:12:08.953331+0000","last_peered":"2023-09-30T09:12:08.953331+0000","last_clean":"2023-09-30T09:12:08.953331+0000","last_became_active":"2023-09-30T09:12:07.952001+0000","last_became_peered":"2023-09-30T09:12:07.952001+0000","last_unstale":"2023-09-30T09:12:08.953331+0000","last_undegraded":"2023-09-30T09:12:08.953331+0000","last_fullsized":"2023-09-30T09:12:08.953331+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:31.068241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017441+0000","last_change":"2023-09-30T09:12:09.016837+0000","last_active":"2023-09-30T09:12:09.017441+0000","last_peered":"2023-09-30T09:12:09.017441+0000","last_clean":"2023-09-30T09:12:09.017441+0000","last_became_active":"2023-09-30T09:12:09.016418+0000","last_became_peered":"2023-09-30T09:12:09.016418+0000","last_unstale":"2023-09-30T09:12:09.017441+0000","last_undegraded":"2023-09-30T09:12:09.017441+0000","last_fullsized":"2023-09-30T09:12:09.017441+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:55:35.980668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.014757+0000","last_change":"2023-09-30T09:12:07.945521+0000","last_active":"2023-09-30T09:12:09.014757+0000","last_peered":"2023-09-30T09:12:09.014757+0000","last_clean":"2023-09-30T09:12:09.014757+0000","last_became_active":"2023-09-30T09:12:07.944823+0000","last_became_peered":"2023-09-30T09:12:07.944823+0000","last_unstale":"2023-09-30T09:12:09.014757+0000","last_undegraded":"2023-09-30T09:12:09.014757+0000","last_fullsized":"2023-09-30T09:12:09.014757+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:00:14.114953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.535296+0000","last_change":"2023-09-30T09:12:09.535095+0000","last_active":"2023-09-30T09:12:09.535296+0000","last_peered":"2023-09-30T09:12:09.535296+0000","last_clean":"2023-09-30T09:12:09.535296+0000","last_became_active":"2023-09-30T09:12:09.534785+0000","last_became_peered":"2023-09-30T09:12:09.534785+0000","last_unstale":"2023-09-30T09:12:09.535296+0000","last_undegraded":"2023-09-30T09:12:09.535296+0000","last_fullsized":"2023-09-30T09:12:09.535296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:38:46.809605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.954701+0000","last_change":"2023-09-30T09:12:07.948029+0000","last_active":"2023-09-30T09:12:08.954701+0000","last_peered":"2023-09-30T09:12:08.954701+0000","last_clean":"2023-09-30T09:12:08.954701+0000","last_became_active":"2023-09-30T09:12:07.947750+0000","last_became_peered":"2023-09-30T09:12:07.947750+0000","last_unstale":"2023-09-30T09:12:08.954701+0000","last_undegraded":"2023-09-30T09:12:08.954701+0000","last_fullsized":"2023-09-30T09:12:08.954701+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:45.735654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.016499+0000","last_change":"2023-09-30T09:12:09.015235+0000","last_active":"2023-09-30T09:12:09.016499+0000","last_peered":"2023-09-30T09:12:09.016499+0000","last_clean":"2023-09-30T09:12:09.016499+0000","last_became_active":"2023-09-30T09:12:09.014835+0000","last_became_peered":"2023-09-30T09:12:09.014835+0000","last_unstale":"2023-09-30T09:12:09.016499+0000","last_undegraded":"2023-09-30T09:12:09.016499+0000","last_fullsized":"2023-09-30T09:12:09.016499+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:31.079441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.870976+0000","last_change":"2023-09-30T09:12:07.950514+0000","last_active":"2023-09-30T09:12:20.870976+0000","last_peered":"2023-09-30T09:12:20.870976+0000","last_clean":"2023-09-30T09:12:20.870976+0000","last_became_active":"2023-09-30T09:12:07.950178+0000","last_became_peered":"2023-09-30T09:12:07.950178+0000","last_unstale":"2023-09-30T09:12:20.870976+0000","last_undegraded":"2023-09-30T09:12:20.870976+0000","last_fullsized":"2023-09-30T09:12:20.870976+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:28:38.340356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642106+0000","last_change":"2023-09-30T09:12:07.962829+0000","last_active":"2023-09-30T09:12:09.642106+0000","last_peered":"2023-09-30T09:12:09.642106+0000","last_clean":"2023-09-30T09:12:09.642106+0000","last_became_active":"2023-09-30T09:12:07.962534+0000","last_became_peered":"2023-09-30T09:12:07.962534+0000","last_unstale":"2023-09-30T09:12:09.642106+0000","last_undegraded":"2023-09-30T09:12:09.642106+0000","last_fullsized":"2023-09-30T09:12:09.642106+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:32:46.328589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950450+0000","last_change":"2023-09-30T09:12:07.952897+0000","last_active":"2023-09-30T09:12:08.950450+0000","last_peered":"2023-09-30T09:12:08.950450+0000","last_clean":"2023-09-30T09:12:08.950450+0000","last_became_active":"2023-09-30T09:12:07.952622+0000","last_became_peered":"2023-09-30T09:12:07.952622+0000","last_unstale":"2023-09-30T09:12:08.950450+0000","last_undegraded":"2023-09-30T09:12:08.950450+0000","last_fullsized":"2023-09-30T09:12:08.950450+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:03.628387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.759666+0000","last_change":"2023-09-30T09:12:09.759390+0000","last_active":"2023-09-30T09:12:09.759666+0000","last_peered":"2023-09-30T09:12:09.759666+0000","last_clean":"2023-09-30T09:12:09.759666+0000","last_became_active":"2023-09-30T09:12:09.759143+0000","last_became_peered":"2023-09-30T09:12:09.759143+0000","last_unstale":"2023-09-30T09:12:09.759666+0000","last_undegraded":"2023-09-30T09:12:09.759666+0000","last_fullsized":"2023-09-30T09:12:09.759666+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:58:24.272665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.064222+0000","last_change":"2023-09-30T09:12:07.944640+0000","last_active":"2023-09-30T09:12:10.064222+0000","last_peered":"2023-09-30T09:12:10.064222+0000","last_clean":"2023-09-30T09:12:10.064222+0000","last_became_active":"2023-09-30T09:12:07.944362+0000","last_became_peered":"2023-09-30T09:12:07.944362+0000","last_unstale":"2023-09-30T09:12:10.064222+0000","last_undegraded":"2023-09-30T09:12:10.064222+0000","last_fullsized":"2023-09-30T09:12:10.064222+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:14.703475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017547+0000","last_change":"2023-09-30T09:12:09.017229+0000","last_active":"2023-09-30T09:12:09.017547+0000","last_peered":"2023-09-30T09:12:09.017547+0000","last_clean":"2023-09-30T09:12:09.017547+0000","last_became_active":"2023-09-30T09:12:09.016966+0000","last_became_peered":"2023-09-30T09:12:09.016966+0000","last_unstale":"2023-09-30T09:12:09.017547+0000","last_undegraded":"2023-09-30T09:12:09.017547+0000","last_fullsized":"2023-09-30T09:12:09.017547+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:27:48.033742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.872441+0000","last_change":"2023-09-30T09:12:07.947365+0000","last_active":"2023-09-30T09:12:20.872441+0000","last_peered":"2023-09-30T09:12:20.872441+0000","last_clean":"2023-09-30T09:12:20.872441+0000","last_became_active":"2023-09-30T09:12:07.947076+0000","last_became_peered":"2023-09-30T09:12:07.947076+0000","last_unstale":"2023-09-30T09:12:20.872441+0000","last_undegraded":"2023-09-30T09:12:20.872441+0000","last_fullsized":"2023-09-30T09:12:20.872441+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:10:12.657377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.761024+0000","last_change":"2023-09-30T09:12:09.760712+0000","last_active":"2023-09-30T09:12:09.761024+0000","last_peered":"2023-09-30T09:12:09.761024+0000","last_clean":"2023-09-30T09:12:09.761024+0000","last_became_active":"2023-09-30T09:12:09.760435+0000","last_became_peered":"2023-09-30T09:12:09.760435+0000","last_unstale":"2023-09-30T09:12:09.761024+0000","last_undegraded":"2023-09-30T09:12:09.761024+0000","last_fullsized":"2023-09-30T09:12:09.761024+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:03.896756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.064297+0000","last_change":"2023-09-30T09:12:07.947099+0000","last_active":"2023-09-30T09:12:10.064297+0000","last_peered":"2023-09-30T09:12:10.064297+0000","last_clean":"2023-09-30T09:12:10.064297+0000","last_became_active":"2023-09-30T09:12:07.946790+0000","last_became_peered":"2023-09-30T09:12:07.946790+0000","last_unstale":"2023-09-30T09:12:10.064297+0000","last_undegraded":"2023-09-30T09:12:10.064297+0000","last_fullsized":"2023-09-30T09:12:10.064297+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:19.132459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.760204+0000","last_change":"2023-09-30T09:12:09.759547+0000","last_active":"2023-09-30T09:12:09.760204+0000","last_peered":"2023-09-30T09:12:09.760204+0000","last_clean":"2023-09-30T09:12:09.760204+0000","last_became_active":"2023-09-30T09:12:09.759241+0000","last_became_peered":"2023-09-30T09:12:09.759241+0000","last_unstale":"2023-09-30T09:12:09.760204+0000","last_undegraded":"2023-09-30T09:12:09.760204+0000","last_fullsized":"2023-09-30T09:12:09.760204+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:22.119179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.873505+0000","last_change":"2023-09-30T09:12:07.940067+0000","last_active":"2023-09-30T09:12:20.873505+0000","last_peered":"2023-09-30T09:12:20.873505+0000","last_clean":"2023-09-30T09:12:20.873505+0000","last_became_active":"2023-09-30T09:12:07.939797+0000","last_became_peered":"2023-09-30T09:12:07.939797+0000","last_unstale":"2023-09-30T09:12:20.873505+0000","last_undegraded":"2023-09-30T09:12:20.873505+0000","last_fullsized":"2023-09-30T09:12:20.873505+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:28:44.481894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642239+0000","last_change":"2023-09-30T09:12:08.659348+0000","last_active":"2023-09-30T09:12:09.642239+0000","last_peered":"2023-09-30T09:12:09.642239+0000","last_clean":"2023-09-30T09:12:09.642239+0000","last_became_active":"2023-09-30T09:12:08.658925+0000","last_became_peered":"2023-09-30T09:12:08.658925+0000","last_unstale":"2023-09-30T09:12:09.642239+0000","last_undegraded":"2023-09-30T09:12:09.642239+0000","last_fullsized":"2023-09-30T09:12:09.642239+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:40.725283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.758620+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:12:09.758620+0000","last_peered":"2023-09-30T09:12:09.758620+0000","last_clean":"2023-09-30T09:12:09.758620+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:12:09.758620+0000","last_undegraded":"2023-09-30T09:12:09.758620+0000","last_fullsized":"2023-09-30T09:12:09.758620+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.761228+0000","last_change":"2023-09-30T09:12:09.760771+0000","last_active":"2023-09-30T09:12:09.761228+0000","last_peered":"2023-09-30T09:12:09.761228+0000","last_clean":"2023-09-30T09:12:09.761228+0000","last_became_active":"2023-09-30T09:12:09.760514+0000","last_became_peered":"2023-09-30T09:12:09.760514+0000","last_unstale":"2023-09-30T09:12:09.761228+0000","last_undegraded":"2023-09-30T09:12:09.761228+0000","last_fullsized":"2023-09-30T09:12:09.761228+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:19:15.004373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.641960+0000","last_change":"2023-09-30T09:12:07.945845+0000","last_active":"2023-09-30T09:12:09.641960+0000","last_peered":"2023-09-30T09:12:09.641960+0000","last_clean":"2023-09-30T09:12:09.641960+0000","last_became_active":"2023-09-30T09:12:07.945545+0000","last_became_peered":"2023-09-30T09:12:07.945545+0000","last_unstale":"2023-09-30T09:12:09.641960+0000","last_undegraded":"2023-09-30T09:12:09.641960+0000","last_fullsized":"2023-09-30T09:12:09.641960+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:47.395756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.760132+0000","last_change":"2023-09-30T09:12:09.759365+0000","last_active":"2023-09-30T09:12:09.760132+0000","last_peered":"2023-09-30T09:12:09.760132+0000","last_clean":"2023-09-30T09:12:09.760132+0000","last_became_active":"2023-09-30T09:12:09.758986+0000","last_became_peered":"2023-09-30T09:12:09.758986+0000","last_unstale":"2023-09-30T09:12:09.760132+0000","last_undegraded":"2023-09-30T09:12:09.760132+0000","last_fullsized":"2023-09-30T09:12:09.760132+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:19:44.539397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.872841+0000","last_change":"2023-09-30T09:12:07.949827+0000","last_active":"2023-09-30T09:12:20.872841+0000","last_peered":"2023-09-30T09:12:20.872841+0000","last_clean":"2023-09-30T09:12:20.872841+0000","last_became_active":"2023-09-30T09:12:07.949537+0000","last_became_peered":"2023-09-30T09:12:07.949537+0000","last_unstale":"2023-09-30T09:12:20.872841+0000","last_undegraded":"2023-09-30T09:12:20.872841+0000","last_fullsized":"2023-09-30T09:12:20.872841+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:56.783979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.016679+0000","last_change":"2023-09-30T09:12:09.015237+0000","last_active":"2023-09-30T09:12:09.016679+0000","last_peered":"2023-09-30T09:12:09.016679+0000","last_clean":"2023-09-30T09:12:09.016679+0000","last_became_active":"2023-09-30T09:12:09.014839+0000","last_became_peered":"2023-09-30T09:12:09.014839+0000","last_unstale":"2023-09-30T09:12:09.016679+0000","last_undegraded":"2023-09-30T09:12:09.016679+0000","last_fullsized":"2023-09-30T09:12:09.016679+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:03:23.891794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.873893+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:12:20.873893+0000","last_peered":"2023-09-30T09:12:20.873893+0000","last_clean":"2023-09-30T09:12:20.873893+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:12:20.873893+0000","last_undegraded":"2023-09-30T09:12:20.873893+0000","last_fullsized":"2023-09-30T09:12:20.873893+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.535701+0000","last_change":"2023-09-30T09:12:09.535404+0000","last_active":"2023-09-30T09:12:09.535701+0000","last_peered":"2023-09-30T09:12:09.535701+0000","last_clean":"2023-09-30T09:12:09.535701+0000","last_became_active":"2023-09-30T09:12:09.535079+0000","last_became_peered":"2023-09-30T09:12:09.535079+0000","last_unstale":"2023-09-30T09:12:09.535701+0000","last_undegraded":"2023-09-30T09:12:09.535701+0000","last_fullsized":"2023-09-30T09:12:09.535701+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:57:34.552136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:20.872829+0000","last_change":"2023-09-30T09:12:07.952351+0000","last_active":"2023-09-30T09:12:20.872829+0000","last_peered":"2023-09-30T09:12:20.872829+0000","last_clean":"2023-09-30T09:12:20.872829+0000","last_became_active":"2023-09-30T09:12:07.951985+0000","last_became_peered":"2023-09-30T09:12:07.951985+0000","last_unstale":"2023-09-30T09:12:20.872829+0000","last_undegraded":"2023-09-30T09:12:20.872829+0000","last_fullsized":"2023-09-30T09:12:20.872829+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:05:11.694255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.760107+0000","last_change":"2023-09-30T09:12:09.759379+0000","last_active":"2023-09-30T09:12:09.760107+0000","last_peered":"2023-09-30T09:12:09.760107+0000","last_clean":"2023-09-30T09:12:09.760107+0000","last_became_active":"2023-09-30T09:12:09.759063+0000","last_became_peered":"2023-09-30T09:12:09.759063+0000","last_unstale":"2023-09-30T09:12:09.760107+0000","last_undegraded":"2023-09-30T09:12:09.760107+0000","last_fullsized":"2023-09-30T09:12:09.760107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:12:39.717308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.950535+0000","last_change":"2023-09-30T09:12:07.945831+0000","last_active":"2023-09-30T09:12:08.950535+0000","last_peered":"2023-09-30T09:12:08.950535+0000","last_clean":"2023-09-30T09:12:08.950535+0000","last_became_active":"2023-09-30T09:12:07.945557+0000","last_became_peered":"2023-09-30T09:12:07.945557+0000","last_unstale":"2023-09-30T09:12:08.950535+0000","last_undegraded":"2023-09-30T09:12:08.950535+0000","last_fullsized":"2023-09-30T09:12:08.950535+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:19:32.608191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.958975+0000","last_change":"2023-09-30T09:12:08.958737+0000","last_active":"2023-09-30T09:12:08.958975+0000","last_peered":"2023-09-30T09:12:08.958975+0000","last_clean":"2023-09-30T09:12:08.958975+0000","last_became_active":"2023-09-30T09:12:08.958451+0000","last_became_peered":"2023-09-30T09:12:08.958451+0000","last_unstale":"2023-09-30T09:12:08.958975+0000","last_undegraded":"2023-09-30T09:12:08.958975+0000","last_fullsized":"2023-09-30T09:12:08.958975+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:22:36.866388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642366+0000","last_change":"2023-09-30T09:12:07.949148+0000","last_active":"2023-09-30T09:12:09.642366+0000","last_peered":"2023-09-30T09:12:09.642366+0000","last_clean":"2023-09-30T09:12:09.642366+0000","last_became_active":"2023-09-30T09:12:07.948862+0000","last_became_peered":"2023-09-30T09:12:07.948862+0000","last_unstale":"2023-09-30T09:12:09.642366+0000","last_undegraded":"2023-09-30T09:12:09.642366+0000","last_fullsized":"2023-09-30T09:12:09.642366+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:20:35.484179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.957252+0000","last_change":"2023-09-30T09:12:08.957017+0000","last_active":"2023-09-30T09:12:08.957252+0000","last_peered":"2023-09-30T09:12:08.957252+0000","last_clean":"2023-09-30T09:12:08.957252+0000","last_became_active":"2023-09-30T09:12:08.956769+0000","last_became_peered":"2023-09-30T09:12:08.956769+0000","last_unstale":"2023-09-30T09:12:08.957252+0000","last_undegraded":"2023-09-30T09:12:08.957252+0000","last_fullsized":"2023-09-30T09:12:08.957252+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:58.002090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953417+0000","last_change":"2023-09-30T09:12:07.948020+0000","last_active":"2023-09-30T09:12:08.953417+0000","last_peered":"2023-09-30T09:12:08.953417+0000","last_clean":"2023-09-30T09:12:08.953417+0000","last_became_active":"2023-09-30T09:12:07.947546+0000","last_became_peered":"2023-09-30T09:12:07.947546+0000","last_unstale":"2023-09-30T09:12:08.953417+0000","last_undegraded":"2023-09-30T09:12:08.953417+0000","last_fullsized":"2023-09-30T09:12:08.953417+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:39.392899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.642242+0000","last_change":"2023-09-30T09:12:08.659724+0000","last_active":"2023-09-30T09:12:09.642242+0000","last_peered":"2023-09-30T09:12:09.642242+0000","last_clean":"2023-09-30T09:12:09.642242+0000","last_became_active":"2023-09-30T09:12:08.659386+0000","last_became_peered":"2023-09-30T09:12:08.659386+0000","last_unstale":"2023-09-30T09:12:09.642242+0000","last_undegraded":"2023-09-30T09:12:09.642242+0000","last_fullsized":"2023-09-30T09:12:09.642242+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:00.336588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:08.953943+0000","last_change":"2023-09-30T09:12:07.952014+0000","last_active":"2023-09-30T09:12:08.953943+0000","last_peered":"2023-09-30T09:12:08.953943+0000","last_clean":"2023-09-30T09:12:08.953943+0000","last_became_active":"2023-09-30T09:12:07.951700+0000","last_became_peered":"2023-09-30T09:12:07.951700+0000","last_unstale":"2023-09-30T09:12:08.953943+0000","last_undegraded":"2023-09-30T09:12:08.953943+0000","last_fullsized":"2023-09-30T09:12:08.953943+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:53.078385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-30T09:12:09.017429+0000","last_change":"2023-09-30T09:12:09.017177+0000","last_active":"2023-09-30T09:12:09.017429+0000","last_peered":"2023-09-30T09:12:09.017429+0000","last_clean":"2023-09-30T09:12:09.017429+0000","last_became_active":"2023-09-30T09:12:09.016900+0000","last_became_peered":"2023-09-30T09:12:09.016900+0000","last_unstale":"2023-09-30T09:12:09.017429+0000","last_undegraded":"2023-09-30T09:12:09.017429+0000","last_fullsized":"2023-09-30T09:12:09.017429+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:12:51.092516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-30T09:12:10.063538+0000","last_change":"2023-09-30T09:12:07.945009+0000","last_active":"2023-09-30T09:12:10.063538+0000","last_peered":"2023-09-30T09:12:10.063538+0000","last_clean":"2023-09-30T09:12:10.063538+0000","last_became_active":"2023-09-30T09:12:07.944737+0000","last_became_peered":"2023-09-30T09:12:07.944737+0000","last_unstale":"2023-09-30T09:12:10.063538+0000","last_undegraded":"2023-09-30T09:12:10.063538+0000","last_fullsized":"2023-09-30T09:12:10.063538+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:28:20.999667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234015,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6604,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738548,"statfs":{"total":95995035648,"available":95988273152,"internally_reserved":0,"allocated":1314816,"data_stored":891677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.48899999999999999,"15min":0.49299999999999999},"min":{"1min":0.30499999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.63300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.51600000000000001,"15min":0.52000000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.69099999999999995,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.36499999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.53000000000000003},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.439},{"interface":"front","average":{"1min":0.5,"5min":0.52000000000000002,"15min":0.52400000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.66300000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.48999999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54200000000000004,"15min":0.54800000000000004},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68300000000000005,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.40000000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.69199999999999995,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56899999999999995,"15min":0.57599999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65400000000000003,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56100000000000005,"15min":0.56599999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.72099999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.747}]},{"osd":4,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.46700000000000003,"15min":0.46500000000000002},"min":{"1min":0.313,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.32500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.71699999999999997,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.55600000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.502,"15min":0.497},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.40799999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.80700000000000005,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.38400000000000001,"5min":0.378,"15min":0.378},"max":{"1min":0.78400000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.626},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.52000000000000002,"15min":0.51400000000000001},"min":{"1min":0.39000000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.60899999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495652,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6584,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738568,"statfs":{"total":95995035648,"available":95988293632,"internally_reserved":0,"allocated":1294336,"data_stored":890943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56599999999999995,"15min":0.57499999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73499999999999999,"5min":0.872,"15min":0.872},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.57099999999999995,"15min":0.57899999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71999999999999997,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.443}]},{"osd":1,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.56100000000000005,"15min":0.56999999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.621,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.56599999999999995,"15min":0.56999999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75700000000000001,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.54900000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.56799999999999995,"15min":0.57299999999999995},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71999999999999997,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.58299999999999996,"15min":0.58899999999999997},"min":{"1min":0.374,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73899999999999999,"5min":1.002,"15min":1.002},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.57199999999999995,"15min":0.57399999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.75800000000000001,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.56200000000000006,"15min":0.57299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.59299999999999997,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47499999999999998,"15min":0.47299999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.373},{"interface":"front","average":{"1min":0.505,"5min":0.48899999999999999,"15min":0.48599999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.505}]},{"osd":5,"last update":"Sat Sep 30 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63500000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.51700000000000002,"15min":0.50900000000000001},"min":{"1min":0.33900000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.58699999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.65700000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6764,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738388,"statfs":{"total":95995035648,"available":95988109312,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:14 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.41399999999999998,"15min":0.38400000000000001},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":2.9409999999999998,"5min":2.9409999999999998,"15min":2.9409999999999998},"last":0.254},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.441,"15min":0.41899999999999998},"min":{"1min":0.26300000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7149999999999999,"5min":2.7149999999999999,"15min":2.7149999999999999},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.442,"15min":0.42999999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.64100000000000001,"5min":2.9769999999999999,"15min":2.9769999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.45200000000000001,"15min":0.45400000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.58699999999999997,"5min":2.6549999999999998,"15min":2.6549999999999998},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.48199999999999998,"15min":0.46999999999999997},"min":{"1min":0.29999999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.9969999999999999,"5min":2.9969999999999999,"15min":2.9969999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.54300000000000004,"15min":0.53100000000000003},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.9129999999999998,"5min":2.9129999999999998,"15min":2.9129999999999998},"last":0.441}]},{"osd":4,"last update":"Sat Sep 30 09:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.622,"15min":0.61599999999999999},"min":{"1min":0.42599999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.78000000000000003,"5min":2.726,"15min":2.726},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.65200000000000002,"15min":0.64400000000000002},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85299999999999998,"5min":2.8570000000000002,"15min":2.8570000000000002},"last":0.80500000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:12:48 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.64100000000000001,"15min":0.63500000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.93100000000000005,"5min":2.8100000000000001,"15min":2.8100000000000001},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.64800000000000002,"15min":0.63700000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.92100000000000004,"5min":2.7469999999999999,"15min":2.7469999999999999},"last":0.72599999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.64300000000000002,"15min":0.63100000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.0529999999999999,"5min":3.0529999999999999,"15min":3.0529999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.64700000000000002,"15min":0.63700000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":3.0350000000000001,"5min":3.0350000000000001,"15min":3.0350000000000001},"last":0.66300000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.73699999999999999,"15min":0.755},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.82199999999999995,"5min":2.8849999999999998,"15min":2.8849999999999998},"last":0.624},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.67800000000000005,"15min":0.69299999999999995},"min":{"1min":0.497,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.78900000000000003,"5min":2.7709999999999999,"15min":2.7709999999999999},"last":0.79200000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738433,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7416,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737736,"statfs":{"total":95995035648,"available":95987441664,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:12:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.36899999999999999,"15min":0.35599999999999998},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.254},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.377,"15min":0.35699999999999998},"min":{"1min":0.247,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.27400000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.505,"15min":0.45000000000000001},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.246},"max":{"1min":4.9340000000000002,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.316}]},{"osd":3,"last update":"Sat Sep 30 09:12:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.47299999999999998,"15min":0.48399999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.48899999999999999,"15min":0.503},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.59799999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:12:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.628,"15min":0.65400000000000003},"min":{"1min":0.40799999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.83799999999999997,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.64600000000000002,"15min":0.66000000000000003},"min":{"1min":0.44700000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.236,"5min":1.236,"15min":1.236},"last":0.67700000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.65200000000000002,"15min":0.626},"min":{"1min":0.40300000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":4.7050000000000001,"5min":4.7050000000000001,"15min":4.7050000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.63900000000000001,"15min":0.64300000000000002},"min":{"1min":0.39200000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.98199999999999998,"5min":1.333,"15min":1.333},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.45600000000000002,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79700000000000004,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:12:34 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.497,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63,"15min":0.622},"min":{"1min":0.496,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.292,"5min":1.292,"15min":1.292},"last":0.65000000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411383,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6636,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738516,"statfs":{"total":95995035648,"available":95988240384,"internally_reserved":0,"allocated":692224,"data_stored":299615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.378,"15min":0.38800000000000001},"min":{"1min":0.254,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.45000000000000001,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.308},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.40999999999999998,"15min":0.41699999999999998},"min":{"1min":0.30199999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56999999999999995,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.40000000000000002,"15min":0.41399999999999998},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.48699999999999999,"5min":0.623,"15min":0.623},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.27400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.7349999999999999,"5min":2.7349999999999999,"15min":2.7349999999999999},"last":0.32700000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:12:03 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.47899999999999998,"15min":0.495},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.58699999999999997,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.47899999999999998,"15min":0.5},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.57499999999999996,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:12:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.70299999999999996,"15min":0.753},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.79800000000000004,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.66500000000000004,"15min":0.68899999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.89800000000000002,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:12:48 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.67700000000000005,"15min":0.72299999999999998},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83099999999999996,"5min":2.6419999999999999,"15min":2.6419999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.628,"15min":0.64600000000000002},"min":{"1min":0.40600000000000003,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.62,"15min":0.622},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80400000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.61899999999999999,"15min":0.61799999999999999},"min":{"1min":0.47499999999999998,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:12:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.62,"15min":0.61799999999999999},"min":{"1min":0.45800000000000002,"5min":0.435,"15min":0.435},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.84199999999999997,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.436}]}]},{"osd":3,"up_from":25,"seq":107374182450,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6128,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739024,"statfs":{"total":95995035648,"available":95988760576,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.49399999999999999,"15min":0.41799999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":11.856,"5min":11.856,"15min":11.856},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.54900000000000004,"15min":0.48299999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.875999999999999,"5min":11.875999999999999,"15min":11.875999999999999},"last":0.32100000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.55600000000000005,"15min":0.48899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.917999999999999,"5min":11.917999999999999,"15min":11.917999999999999},"last":0.437},{"interface":"front","average":{"1min":0.439,"5min":0.46500000000000002,"15min":0.48099999999999998},"min":{"1min":0.27500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.2390000000000001,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.29699999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.5,"15min":0.505},"min":{"1min":0.318,"5min":0.311,"15min":0.311},"max":{"1min":1.4430000000000001,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.46600000000000003,"15min":0.46700000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.34300000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59799999999999998,"15min":0.60199999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.61499999999999999,"15min":0.61099999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.622,"15min":0.62},"min":{"1min":0.501,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.64400000000000002,"15min":0.63800000000000001},"min":{"1min":0.48699999999999999,"5min":0.432,"15min":0.432},"max":{"1min":1.125,"5min":1.125,"15min":1.125},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.63900000000000001,"15min":0.64200000000000002},"min":{"1min":0.48599999999999999,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.208,"15min":1.304},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.85899999999999999,"5min":11.801,"15min":11.801},"last":0.63200000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6204,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738948,"statfs":{"total":95995035648,"available":95988682752,"internally_reserved":0,"allocated":708608,"data_stored":300709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.55500000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.78500000000000003,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.504},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.54700000000000004,"15min":0.56000000000000005},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.59499999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.54100000000000004,"15min":0.55700000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.55300000000000005,"15min":0.56299999999999994},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.36899999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.55400000000000005,"15min":0.56599999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55800000000000005,"15min":0.56599999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.42599999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56200000000000006,"15min":0.56299999999999994},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.82899999999999996,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57099999999999995,"15min":0.58699999999999997},"min":{"1min":0.38400000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:12:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.436,"15min":0.40600000000000003},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.45200000000000001,"15min":0.42299999999999999},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77200000000000002,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.67500000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:12:12 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.47399999999999998,"15min":0.46899999999999997},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.497,"15min":0.48799999999999999},"min":{"1min":0.35599999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.625}]},{"osd":7,"last update":"Sat Sep 30 09:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.53300000000000003,"15min":0.53200000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.72399999999999998,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.61299999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757289,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6064,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739088,"statfs":{"total":95995035648,"available":95988826112,"internally_reserved":0,"allocated":696320,"data_stored":299858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.55700000000000005,"15min":0.58099999999999996},"min":{"1min":0.34499999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.624,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.56699999999999995,"15min":0.59699999999999998},"min":{"1min":0.33600000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.66200000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.65900000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.58699999999999997,"15min":0.60999999999999999},"min":{"1min":0.378,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.745,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.54700000000000004,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.64400000000000002,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.433}]},{"osd":2,"last update":"Sat Sep 30 09:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.56499999999999995,"15min":0.58199999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64300000000000002,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.58799999999999997,"15min":0.59999999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.82399999999999995,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.52700000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.57399999999999995,"15min":0.58999999999999997},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.68000000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.59599999999999997,"15min":0.60799999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.85899999999999999,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.59099999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.438,"15min":0.42199999999999999},"min":{"1min":0.307,"5min":0.247,"15min":0.247},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.47299999999999998,"15min":0.45100000000000001},"min":{"1min":0.35999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.753,"5min":0.875,"15min":0.875},"last":0.621}]},{"osd":6,"last update":"Sat Sep 30 09:12:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.45700000000000002,"15min":0.44800000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.54200000000000004,"15min":0.53500000000000003},"min":{"1min":0.377,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.72299999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:12:34 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.376,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.72899999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.64300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:12:57.247 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-30T09:12:57.248 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-30T09:12:57.248 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-30T09:12:57.248 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-30T09:12:57.612 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:12:58.914 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:58 smithi019 ceph-mon[100214]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:58.915 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:58 smithi019 ceph-mon[118983]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:58.915 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:12:58 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:12:58] "GET /metrics HTTP/1.1" 200 238600 "" "Prometheus/2.43.0" 2023-09-30T09:12:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:58 smithi167 ceph-mon[104403]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:12:59.381 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:12:59.382 INFO:teuthology.orchestra.run.smithi019.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-30T09:12:59.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:12:59 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1675495233' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:12:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:12:59 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1675495233' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:13:00.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:12:59 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1675495233' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:13:00.096 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-30T09:13:00.097 INFO:teuthology.run_tasks:Running task print... 2023-09-30T09:13:00.110 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-30T09:13:00.110 INFO:teuthology.run_tasks:Running task parallel... 2023-09-30T09:13:00.190 INFO:teuthology.task.parallel:starting parallel... 2023-09-30T09:13:00.190 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-30T09:13:00.191 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-30T09:13:00.194 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-30T09:13:00.195 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-30T09:13:00.195 DEBUG:teuthology.orchestra.run.smithi019:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:13:00.195 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-30T09:13:00.195 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-30T09:13:00.196 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi019.front.sepia.ceph.com 2023-09-30T09:13:00.196 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-30T09:13:00.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T09:13:00.215 INFO:teuthology.orchestra.run.smithi019.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-30T09:13:00.216 DEBUG:teuthology.orchestra.run.smithi019:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:13:00.272 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-30T09:13:00.272 DEBUG:teuthology.orchestra.run.smithi019:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-30T09:13:00.329 INFO:tasks.workunit:timeout=3h 2023-09-30T09:13:00.330 INFO:tasks.workunit:cleanup=True 2023-09-30T09:13:00.330 DEBUG:teuthology.orchestra.run.smithi019:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-30T09:13:00.388 INFO:tasks.workunit.client.0.smithi019.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-30T09:13:00.566 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:13:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:00 smithi019 ceph-mon[100214]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:00 smithi019 ceph-mon[118983]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:01.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:00 smithi167 ceph-mon[104403]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:02.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:02 smithi019 conmon[100636]: [30/Sep/2023:09:13:02] ENGINE Bus STOPPING 2023-09-30T09:13:02.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:02 smithi019 conmon[100636]: [30/Sep/2023:09:13:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:13:02.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:02 smithi019 conmon[100636]: [30/Sep/2023:09:13:02] ENGINE Bus STOPPED 2023-09-30T09:13:02.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:02 smithi019 conmon[100636]: [30/Sep/2023:09:13:02] ENGINE Bus STARTING 2023-09-30T09:13:02.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:02 smithi019 conmon[100636]: [30/Sep/2023:09:13:02] ENGINE Serving on http://:::9283 2023-09-30T09:13:02.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:02 smithi019 conmon[100636]: [30/Sep/2023:09:13:02] ENGINE Bus STARTED 2023-09-30T09:13:02.969 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-30T09:13:03.370 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:13:03.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1442295266' entity='client.admin' 2023-09-30T09:13:03.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:03.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:13:03.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:13:03.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:03.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[100214]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:03.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1442295266' entity='client.admin' 2023-09-30T09:13:03.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:03.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:13:03.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:13:03.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:03.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:03 smithi019 ceph-mon[118983]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:03.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:03 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1442295266' entity='client.admin' 2023-09-30T09:13:03.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:03 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:03.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:03 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:13:03.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:03 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:13:03.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:03 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:03.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:03 smithi167 ceph-mon[104403]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:04.747 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:04 smithi019 ceph-mon[100214]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:04.747 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:04 smithi019 ceph-mon[118983]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:04 smithi167 ceph-mon[104403]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:05.987 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-30T09:13:06.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:13:05 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:13:06.360 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:13:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-30T09:13:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-09-30T09:13:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-30T09:13:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:06 smithi167 ceph-mon[104403]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:07.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-30T09:13:07.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-09-30T09:13:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-30T09:13:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:06 smithi019 ceph-mon[100214]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-30T09:13:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-09-30T09:13:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-30T09:13:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:06 smithi019 ceph-mon[118983]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:07.223 DEBUG:teuthology.orchestra.run.smithi019:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-30T09:13:07.317 INFO:tasks.workunit.client.0.smithi019.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-30T09:13:07.320 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-30T09:13:07.320 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-30T09:13:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:07 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-30T09:13:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:07 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-09-30T09:13:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:07 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-30T09:13:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:07 smithi167 ceph-mon[104403]: osdmap e69: 8 total, 8 up, 8 in 2023-09-30T09:13:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:07 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-30T09:13:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-09-30T09:13:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-30T09:13:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[100214]: osdmap e69: 8 total, 8 up, 8 in 2023-09-30T09:13:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-30T09:13:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-09-30T09:13:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-30T09:13:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[118983]: osdmap e69: 8 total, 8 up, 8 in 2023-09-30T09:13:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:07 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:08 smithi167 ceph-mon[104403]: osdmap e70: 8 total, 8 up, 8 in 2023-09-30T09:13:09.342 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:08 smithi167 ceph-mon[104403]: pgmap v75: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:09.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:08 smithi019 ceph-mon[100214]: osdmap e70: 8 total, 8 up, 8 in 2023-09-30T09:13:09.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:08 smithi019 ceph-mon[100214]: pgmap v75: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:09.343 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:08 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:08] "GET /metrics HTTP/1.1" 200 238582 "" "Prometheus/2.43.0" 2023-09-30T09:13:09.343 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:08 smithi019 ceph-mon[118983]: osdmap e70: 8 total, 8 up, 8 in 2023-09-30T09:13:09.343 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:08 smithi019 ceph-mon[118983]: pgmap v75: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:10.510 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-30T09:13:10.621 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-30T09:13:10.676 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-30T09:13:10.678 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-30T09:13:10.678 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-30T09:13:10.718 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-30T09:13:10.722 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:13:10.722 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-30T09:13:10.739 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-30T09:13:10.740 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-30T09:13:10.740 DEBUG:teuthology.orchestra.run.smithi019:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-30T09:13:10.801 INFO:tasks.workunit.client.0.smithi019.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-30T09:13:10.801 INFO:tasks.workunit.client.0.smithi019.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-30T09:13:10.802 INFO:tasks.workunit.client.0.smithi019.stderr:+ TMPDIR=/tmp/rbd_import_export_192116 2023-09-30T09:13:10.802 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm -rf /tmp/rbd_import_export_192116 2023-09-30T09:13:10.802 INFO:tasks.workunit.client.0.smithi019.stderr:+ mkdir /tmp/rbd_import_export_192116 2023-09-30T09:13:10.803 INFO:tasks.workunit.client.0.smithi019.stderr:+ trap 'rm -rf /tmp/rbd_import_export_192116' INT TERM EXIT 2023-09-30T09:13:10.803 INFO:tasks.workunit.client.0.smithi019.stderr:+ mkdir foo.192116 2023-09-30T09:13:10.804 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import foo.192116 foo.dir 2023-09-30T09:13:10.853 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: cannot import a directory 2023-09-30T09:13:10.853 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 0% complete...failed. 2023-09-30T09:13:10.853 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: import failed: (21) Is a directory 2023-09-30T09:13:10.855 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:13:10.855 INFO:tasks.workunit.client.0.smithi019.stderr:+ rmdir foo.192116 2023-09-30T09:13:10.863 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_192116/img bs=1k count=1 seek=10 2023-09-30T09:13:10.864 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:13:10.864 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:13:10.864 INFO:tasks.workunit.client.0.smithi019.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.6363e-05 s, 22.1 MB/s 2023-09-30T09:13:10.864 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_192116/img bs=1k count=10 seek=100 2023-09-30T09:13:10.864 INFO:tasks.workunit.client.0.smithi019.stderr:10+0 records in 2023-09-30T09:13:10.865 INFO:tasks.workunit.client.0.smithi019.stderr:10+0 records out 2023-09-30T09:13:10.865 INFO:tasks.workunit.client.0.smithi019.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.781e-05 s, 177 MB/s 2023-09-30T09:13:10.865 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_192116/img bs=1k count=100 seek=1000 2023-09-30T09:13:10.865 INFO:tasks.workunit.client.0.smithi019.stderr:70+1 records in 2023-09-30T09:13:10.865 INFO:tasks.workunit.client.0.smithi019.stderr:70+1 records out 2023-09-30T09:13:10.865 INFO:tasks.workunit.client.0.smithi019.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000198821 s, 362 MB/s 2023-09-30T09:13:10.866 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_192116/img bs=1k seek=10000 2023-09-30T09:13:10.866 INFO:tasks.workunit.client.0.smithi019.stderr:139+1 records in 2023-09-30T09:13:10.866 INFO:tasks.workunit.client.0.smithi019.stderr:139+1 records out 2023-09-30T09:13:10.866 INFO:tasks.workunit.client.0.smithi019.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000349042 s, 410 MB/s 2023-09-30T09:13:10.867 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_192116/img bs=1k seek=100000 2023-09-30T09:13:10.870 INFO:tasks.workunit.client.0.smithi019.stderr:70+1 records in 2023-09-30T09:13:10.870 INFO:tasks.workunit.client.0.smithi019.stderr:70+1 records out 2023-09-30T09:13:10.871 INFO:tasks.workunit.client.0.smithi019.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.00327161 s, 22.1 MB/s 2023-09-30T09:13:10.871 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_192116/img bs=1k seek=1000000 2023-09-30T09:13:10.872 INFO:tasks.workunit.client.0.smithi019.stderr:189+1 records in 2023-09-30T09:13:10.872 INFO:tasks.workunit.client.0.smithi019.stderr:189+1 records out 2023-09-30T09:13:10.872 INFO:tasks.workunit.client.0.smithi019.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.00058634 s, 330 MB/s 2023-09-30T09:13:10.872 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm testimg 2023-09-30T09:13:10.912 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 0% complete...failed. 2023-09-30T09:13:10.912 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: delete error: (2) No such file or directory 2023-09-30T09:13:10.915 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:13:10.915 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format /tmp/rbd_import_export_192116/img testimg 2023-09-30T09:13:11.051 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:13:11.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:10 smithi019 ceph-mon[100214]: pgmap v76: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:11.236 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:10 smithi019 ceph-mon[118983]: pgmap v76: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:10 smithi167 ceph-mon[104403]: pgmap v76: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:11.853 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch ps' 2023-09-30T09:13:12.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:13:12.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:13:12.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:12.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:12.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:13:12.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[100214]: osdmap e71: 8 total, 8 up, 8 in 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:13:12.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:12.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:13:12.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:12 smithi019 ceph-mon[118983]: osdmap e71: 8 total, 8 up, 8 in 2023-09-30T09:13:12.264 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:13:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:13:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:13:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:13:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:13:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1d10b71792f3be8887a7631e69851ac2df3585af 2023-09-30T09:13:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:12 smithi167 ceph-mon[104403]: osdmap e71: 8 total, 8 up, 8 in 2023-09-30T09:13:13.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:13 smithi019 ceph-mon[118983]: pgmap v78: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:13 smithi019 ceph-mon[118983]: osdmap e72: 8 total, 8 up, 8 in 2023-09-30T09:13:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:13 smithi019 ceph-mon[100214]: pgmap v78: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:13 smithi019 ceph-mon[100214]: osdmap e72: 8 total, 8 up, 8 in 2023-09-30T09:13:13.404 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-30T09:13:13.405 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:13.408 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg /tmp/rbd_import_export_192116/img2 2023-09-30T09:13:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:13 smithi167 ceph-mon[104403]: pgmap v78: 73 pgs: 3 activating, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:13 smithi167 ceph-mon[104403]: osdmap e72: 8 total, 8 up, 8 in 2023-09-30T09:13:13.741 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:13.744 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg - 2023-09-30T09:13:14.437 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-30T09:13:14.438 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (67s) 22s ago 112s 17.0M - 0.25.0 c8568f914cd2 51fb7ff237b4 2023-09-30T09:13:14.438 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi167 *:3000 running (92s) 41s ago 92s 89.1M - 9.4.7 2c41d148cca3 1e414220bf37 2023-09-30T09:13:14.438 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi019.kcgktg smithi019 running (54s) 22s ago 53s 14.9M - 17.2.6-1161-g191c1869 8497a1260037 ca3d56fcab07 2023-09-30T09:13:14.438 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi167.bnuarp smithi167 running (51s) 41s ago 51s 14.6M - 17.2.6-1161-g191c1869 8497a1260037 a70137fe60d5 2023-09-30T09:13:14.438 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi167 *:8443 running (5m) 41s ago 5m 439M - 17.2.6-1161-g191c1869 8497a1260037 6176c234bba1 2023-09-30T09:13:14.439 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:9283 running (7m) 22s ago 7m 494M - 17.2.6-1161-g191c1869 8497a1260037 08339cd406da 2023-09-30T09:13:14.439 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (7m) 22s ago 7m 82.6M 2048M 17.2.6-1161-g191c1869 8497a1260037 91f2eb5c5520 2023-09-30T09:13:14.439 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi167 running (6m) 41s ago 6m 57.0M 2048M 17.2.6-1161-g191c1869 8497a1260037 b5fffa651c9c 2023-09-30T09:13:14.439 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (6m) 22s ago 6m 59.2M 2048M 17.2.6-1161-g191c1869 8497a1260037 88df4d08fbd4 2023-09-30T09:13:14.439 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (82s) 22s ago 82s 17.0M - 1.5.0 0da6a335fe13 63714dd11e25 2023-09-30T09:13:14.440 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi167 *:9100 running (78s) 41s ago 78s 9303k - 1.5.0 0da6a335fe13 23727543383f 2023-09-30T09:13:14.440 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (5m) 22s ago 5m 57.5M 4037M 17.2.6-1161-g191c1869 8497a1260037 0ff07d0fefc1 2023-09-30T09:13:14.440 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (5m) 22s ago 5m 59.3M 4037M 17.2.6-1161-g191c1869 8497a1260037 4f868828e949 2023-09-30T09:13:14.440 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (4m) 22s ago 4m 53.3M 4037M 17.2.6-1161-g191c1869 8497a1260037 6673f026fd30 2023-09-30T09:13:14.440 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (4m) 22s ago 4m 49.8M 4037M 17.2.6-1161-g191c1869 8497a1260037 8fa7001643e1 2023-09-30T09:13:14.440 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi167 running (3m) 41s ago 3m 51.5M 4036M 17.2.6-1161-g191c1869 8497a1260037 6557c10c6a0b 2023-09-30T09:13:14.441 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi167 running (3m) 41s ago 3m 51.6M 4036M 17.2.6-1161-g191c1869 8497a1260037 30ccc3c97db6 2023-09-30T09:13:14.441 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi167 running (3m) 41s ago 3m 52.1M 4036M 17.2.6-1161-g191c1869 8497a1260037 4a5f3bfc86e4 2023-09-30T09:13:14.441 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi167 running (2m) 41s ago 2m 53.4M 4036M 17.2.6-1161-g191c1869 8497a1260037 92e8410b5190 2023-09-30T09:13:14.441 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi167 *:9095 running (62s) 41s ago 2m 34.7M - 2.43.0 a07b618ecd1d 8187f7a0d1a5 2023-09-30T09:13:14.619 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:14.623 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm testimg 2023-09-30T09:13:14.730 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-30T09:13:14.733 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp /tmp/rbd_import_export_192116/img /tmp/rbd_import_export_192116/img2 2023-09-30T09:13:14.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:14 smithi019 ceph-mon[118983]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:13:14.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:14 smithi019 ceph-mon[118983]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 147 B/s wr, 0 op/s 2023-09-30T09:13:14.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:14 smithi019 ceph-mon[100214]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:13:14.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:14 smithi019 ceph-mon[100214]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 147 B/s wr, 0 op/s 2023-09-30T09:13:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:14 smithi167 ceph-mon[104403]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:13:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:14 smithi167 ceph-mon[104403]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 147 B/s wr, 0 op/s 2023-09-30T09:13:15.514 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp /tmp/rbd_import_export_192116/img /tmp/rbd_import_export_192116/img3 2023-09-30T09:13:16.024 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img2 /tmp/rbd_import_export_192116/img3 2023-09-30T09:13:16.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:13:15 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:13:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:16 smithi167 ceph-mon[104403]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-30T09:13:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:16 smithi019 ceph-mon[100214]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-30T09:13:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:16 smithi019 ceph-mon[118983]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-30T09:13:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:18 smithi167 ceph-mon[104403]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-30T09:13:19.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:18 smithi019 ceph-mon[100214]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-30T09:13:19.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:18 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:18] "GET /metrics HTTP/1.1" 200 238582 "" "Prometheus/2.43.0" 2023-09-30T09:13:19.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:18 smithi019 ceph-mon[118983]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-30T09:13:20.839 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format - testimg 2023-09-30T09:13:21.264 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:21.265 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:21.268 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg /tmp/rbd_import_export_192116/img2 2023-09-30T09:13:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:20 smithi167 ceph-mon[104403]: osdmap e73: 8 total, 8 up, 8 in 2023-09-30T09:13:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:20 smithi167 ceph-mon[104403]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-30T09:13:21.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:20 smithi019 ceph-mon[100214]: osdmap e73: 8 total, 8 up, 8 in 2023-09-30T09:13:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:20 smithi019 ceph-mon[100214]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-30T09:13:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:20 smithi019 ceph-mon[118983]: osdmap e73: 8 total, 8 up, 8 in 2023-09-30T09:13:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:20 smithi019 ceph-mon[118983]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-30T09:13:21.558 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:21.562 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg - 2023-09-30T09:13:22.034 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-30T09:13:22.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:21 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:22.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:21 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:21 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:21 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:22.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:21 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:22.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:21 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:22.375 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:22.379 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm testimg 2023-09-30T09:13:22.456 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:13:22.484 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-30T09:13:22.488 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp /tmp/rbd_import_export_192116/img /tmp/rbd_import_export_192116/img2 2023-09-30T09:13:23.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:22 smithi019 ceph-mon[100214]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 53 KiB/s wr, 31 op/s 2023-09-30T09:13:23.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:22 smithi019 ceph-mon[118983]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 53 KiB/s wr, 31 op/s 2023-09-30T09:13:23.237 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp /tmp/rbd_import_export_192116/img /tmp/rbd_import_export_192116/img3 2023-09-30T09:13:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:22 smithi167 ceph-mon[104403]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 53 KiB/s wr, 31 op/s 2023-09-30T09:13:23.798 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img /tmp/rbd_import_export_192116/img2 /tmp/rbd_import_export_192116/img3 2023-09-30T09:13:25.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:24 smithi167 ceph-mon[104403]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-30T09:13:25.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:24 smithi019 ceph-mon[100214]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-30T09:13:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:24 smithi019 ceph-mon[118983]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-30T09:13:26.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:13:25 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:13:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:26 smithi167 ceph-mon[104403]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-30T09:13:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:26 smithi019 ceph-mon[100214]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-30T09:13:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:26 smithi019 ceph-mon[118983]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-30T09:13:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:28 smithi167 ceph-mon[104403]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-30T09:13:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:28 smithi019 ceph-mon[100214]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-30T09:13:29.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:28 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:28] "GET /metrics HTTP/1.1" 200 239093 "" "Prometheus/2.43.0" 2023-09-30T09:13:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:28 smithi019 ceph-mon[118983]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-30T09:13:29.264 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd help export 2023-09-30T09:13:29.264 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -q export-format 2023-09-30T09:13:29.285 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_192116/img bs=1k count=10 seek=100 2023-09-30T09:13:29.286 INFO:tasks.workunit.client.0.smithi019.stderr:10+0 records in 2023-09-30T09:13:29.286 INFO:tasks.workunit.client.0.smithi019.stderr:10+0 records out 2023-09-30T09:13:29.286 INFO:tasks.workunit.client.0.smithi019.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.2887e-05 s, 140 MB/s 2023-09-30T09:13:29.286 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format /tmp/rbd_import_export_192116/img testimg 2023-09-30T09:13:29.352 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:29.353 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:29.355 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap create testimg@snap 2023-09-30T09:13:29.618 INFO:tasks.workunit.client.0.smithi019.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-30T09:13:29.626 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd image-meta set testimg key1 value1 2023-09-30T09:13:29.677 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd image-meta list testimg 2023-09-30T09:13:29.723 INFO:tasks.workunit.client.0.smithi019.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-30T09:13:29.724 INFO:tasks.workunit.client.0.smithi019.stderr: 2023-09-30T09:13:29.724 INFO:tasks.workunit.client.0.smithi019.stderr:Key Value 2023-09-30T09:13:29.724 INFO:tasks.workunit.client.0.smithi019.stderr:key1 value1' 2023-09-30T09:13:29.724 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192116/img_v2 2023-09-30T09:13:29.762 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:29.765 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192116/img_v2 testimg_import 2023-09-30T09:13:30.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:30 smithi019 ceph-mon[100214]: osdmap e74: 8 total, 8 up, 8 in 2023-09-30T09:13:30.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:30 smithi019 ceph-mon[100214]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-30T09:13:30.631 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:13:30.634 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout:rbd image 'testimg_import': 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout: size 110 KiB in 1 objects 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout: order 22 (4 MiB objects) 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout: snapshot_count: 1 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout: id: 60e9ddeaaf39 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout: block_name_prefix: rbd_data.60e9ddeaaf39 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout: format: 2 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-30T09:13:30.673 INFO:tasks.workunit.client.0.smithi019.stdout: op_features: 2023-09-30T09:13:30.674 INFO:tasks.workunit.client.0.smithi019.stdout: flags: 2023-09-30T09:13:30.674 INFO:tasks.workunit.client.0.smithi019.stdout: create_timestamp: Sat Sep 30 09:13:29 2023 2023-09-30T09:13:30.674 INFO:tasks.workunit.client.0.smithi019.stdout: access_timestamp: Sat Sep 30 09:13:29 2023 2023-09-30T09:13:30.674 INFO:tasks.workunit.client.0.smithi019.stdout: modify_timestamp: Sat Sep 30 09:13:29 2023 2023-09-30T09:13:30.675 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import@snap 2023-09-30T09:13:30.717 INFO:tasks.workunit.client.0.smithi019.stdout:rbd image 'testimg_import': 2023-09-30T09:13:30.717 INFO:tasks.workunit.client.0.smithi019.stdout: size 110 KiB in 1 objects 2023-09-30T09:13:30.717 INFO:tasks.workunit.client.0.smithi019.stdout: order 22 (4 MiB objects) 2023-09-30T09:13:30.717 INFO:tasks.workunit.client.0.smithi019.stdout: snapshot_count: 1 2023-09-30T09:13:30.717 INFO:tasks.workunit.client.0.smithi019.stdout: id: 60e9ddeaaf39 2023-09-30T09:13:30.717 INFO:tasks.workunit.client.0.smithi019.stdout: block_name_prefix: rbd_data.60e9ddeaaf39 2023-09-30T09:13:30.717 INFO:tasks.workunit.client.0.smithi019.stdout: format: 2 2023-09-30T09:13:30.718 INFO:tasks.workunit.client.0.smithi019.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-30T09:13:30.718 INFO:tasks.workunit.client.0.smithi019.stdout: op_features: 2023-09-30T09:13:30.718 INFO:tasks.workunit.client.0.smithi019.stdout: flags: 2023-09-30T09:13:30.718 INFO:tasks.workunit.client.0.smithi019.stdout: create_timestamp: Sat Sep 30 09:13:29 2023 2023-09-30T09:13:30.718 INFO:tasks.workunit.client.0.smithi019.stdout: access_timestamp: Sat Sep 30 09:13:29 2023 2023-09-30T09:13:30.718 INFO:tasks.workunit.client.0.smithi019.stdout: modify_timestamp: Sat Sep 30 09:13:29 2023 2023-09-30T09:13:30.718 INFO:tasks.workunit.client.0.smithi019.stdout: protected: False 2023-09-30T09:13:30.720 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd image-meta list testimg_import 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr: 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr:Key Value 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr:key1 value1' 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr: 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr:Key Value 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-30T09:13:30.762 INFO:tasks.workunit.client.0.smithi019.stderr: 2023-09-30T09:13:30.763 INFO:tasks.workunit.client.0.smithi019.stderr:Key Value 2023-09-30T09:13:30.763 INFO:tasks.workunit.client.0.smithi019.stderr:key1 value1' ']' 2023-09-30T09:13:30.763 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg_import /tmp/rbd_import_export_192116/img_import 2023-09-30T09:13:30.794 INFO:teuthology.orchestra.run.smithi019.stdout:wait for minority of mons to upgrade 2023-09-30T09:13:30.798 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete...done. 2023-09-30T09:13:30.801 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192116/img /tmp/rbd_import_export_192116/img_import 2023-09-30T09:13:30.802 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_192116/img /tmp/rbd_import_export_192116/img_import 2023-09-30T09:13:30.803 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/img --format %b 2023-09-30T09:13:30.804 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=24 2023-09-30T09:13:30.804 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/img_import --format %b 2023-09-30T09:13:30.805 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=224 2023-09-30T09:13:30.805 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=200 2023-09-30T09:13:30.805 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=200 2023-09-30T09:13:30.805 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-30T09:13:30.805 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_192116/img_snap 2023-09-30T09:13:30.848 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete...done. 2023-09-30T09:13:30.851 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_192116/img_snap_import 2023-09-30T09:13:30.890 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete...done. 2023-09-30T09:13:30.893 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192116/img_snap /tmp/rbd_import_export_192116/img_snap_import 2023-09-30T09:13:30.893 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_192116/img_snap /tmp/rbd_import_export_192116/img_snap_import 2023-09-30T09:13:30.894 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/img_snap --format %b 2023-09-30T09:13:30.895 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=224 2023-09-30T09:13:30.895 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/img_snap_import --format %b 2023-09-30T09:13:30.896 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=224 2023-09-30T09:13:30.897 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:30.897 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:30.897 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:13:30.897 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img_v2 2023-09-30T09:13:30.897 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img_import 2023-09-30T09:13:30.898 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img_snap 2023-09-30T09:13:30.898 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img_snap_import 2023-09-30T09:13:30.899 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap rm testimg_import@snap 2023-09-30T09:13:30.949 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:30 smithi019 ceph-mon[118983]: osdmap e74: 8 total, 8 up, 8 in 2023-09-30T09:13:30.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:30 smithi019 ceph-mon[118983]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-30T09:13:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:30 smithi167 ceph-mon[104403]: osdmap e74: 8 total, 8 up, 8 in 2023-09-30T09:13:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:30 smithi167 ceph-mon[104403]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-30T09:13:31.656 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-30T09:13:31.885 INFO:tasks.workunit.client.0.smithi019.stderr: Removing snap: 100% complete...done. 2023-09-30T09:13:31.895 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:13:31.991 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:31.994 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap rm testimg@snap 2023-09-30T09:13:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:31 smithi167 ceph-mon[104403]: osdmap e75: 8 total, 8 up, 8 in 2023-09-30T09:13:32.031 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:13:32.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:31 smithi019 ceph-mon[100214]: osdmap e75: 8 total, 8 up, 8 in 2023-09-30T09:13:32.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:31 smithi019 ceph-mon[118983]: osdmap e75: 8 total, 8 up, 8 in 2023-09-30T09:13:32.897 INFO:tasks.workunit.client.0.smithi019.stderr: Removing snap: 100% complete...done. 2023-09-30T09:13:32.905 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm testimg 2023-09-30T09:13:32.994 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:32.997 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --order 20 /tmp/rbd_import_export_192116/img testimg 2023-09-30T09:13:33.052 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:33.052 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:13:33.054 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192116/img_v2 2023-09-30T09:13:33.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:32 smithi019 ceph-mon[100214]: osdmap e76: 8 total, 8 up, 8 in 2023-09-30T09:13:33.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:32 smithi019 ceph-mon[100214]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:32 smithi019 ceph-mon[118983]: osdmap e76: 8 total, 8 up, 8 in 2023-09-30T09:13:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:32 smithi019 ceph-mon[118983]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:33.093 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:33.096 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192116/img_v2 testimg_import 2023-09-30T09:13:33.156 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:13:33.159 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import 2023-09-30T09:13:33.160 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep order 2023-09-30T09:13:33.160 INFO:tasks.workunit.client.0.smithi019.stderr:+ awk '{print $2}' 2023-09-30T09:13:33.160 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 20 2023-09-30T09:13:33.199 INFO:tasks.workunit.client.0.smithi019.stdout:20 2023-09-30T09:13:33.199 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img_v2 2023-09-30T09:13:33.200 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:13:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:32 smithi167 ceph-mon[104403]: osdmap e76: 8 total, 8 up, 8 in 2023-09-30T09:13:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:32 smithi167 ceph-mon[104403]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:33.303 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:33.307 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg 2023-09-30T09:13:33.407 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:33.410 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_192116/img testimg 2023-09-30T09:13:33.461 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:33.464 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd info testimg 2023-09-30T09:13:33.465 INFO:tasks.workunit.client.0.smithi019.stderr:++ grep features 2023-09-30T09:13:33.517 INFO:tasks.workunit.client.0.smithi019.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-30T09:13:33.518 INFO:tasks.workunit.client.0.smithi019.stderr: op_features: ' 2023-09-30T09:13:33.518 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192116/img_v2 2023-09-30T09:13:33.561 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:33.564 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192116/img_v2 testimg_import 2023-09-30T09:13:33.621 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:13:33.624 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd info testimg_import 2023-09-30T09:13:33.624 INFO:tasks.workunit.client.0.smithi019.stderr:++ grep features 2023-09-30T09:13:33.663 INFO:tasks.workunit.client.0.smithi019.stderr:+ FEATURES_AFTER=' features: layering 2023-09-30T09:13:33.663 INFO:tasks.workunit.client.0.smithi019.stderr: op_features: ' 2023-09-30T09:13:33.663 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' ' features: layering 2023-09-30T09:13:33.663 INFO:tasks.workunit.client.0.smithi019.stderr: op_features: ' '!=' ' features: layering 2023-09-30T09:13:33.663 INFO:tasks.workunit.client.0.smithi019.stderr: op_features: ' ']' 2023-09-30T09:13:33.663 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img_v2 2023-09-30T09:13:33.664 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:13:33.743 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:33.745 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg 2023-09-30T09:13:33.823 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:33.827 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_192116/img testimg 2023-09-30T09:13:33.893 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:33.897 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192116/img_v2 2023-09-30T09:13:33.936 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:33.939 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192116/img_v2 testimg_import 2023-09-30T09:13:34.010 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:13:34.015 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 'stripe unit' 2023-09-30T09:13:34.015 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-30T09:13:34.015 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import 2023-09-30T09:13:34.056 INFO:tasks.workunit.client.0.smithi019.stdout: stripe unit: 4 KiB 2023-09-30T09:13:34.057 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import 2023-09-30T09:13:34.057 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 'stripe count' 2023-09-30T09:13:34.057 INFO:tasks.workunit.client.0.smithi019.stderr:+ awk '{print $3}' 2023-09-30T09:13:34.057 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 1000 2023-09-30T09:13:34.096 INFO:tasks.workunit.client.0.smithi019.stdout:1000 2023-09-30T09:13:34.097 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/img_v2 2023-09-30T09:13:34.097 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:13:34.197 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-30T09:13:34.201 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg 2023-09-30T09:13:34.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:33 smithi167 ceph-mon[104403]: osdmap e77: 8 total, 8 up, 8 in 2023-09-30T09:13:34.299 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-30T09:13:34.303 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_192116/img testimg 2023-09-30T09:13:34.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:33 smithi019 ceph-mon[100214]: osdmap e77: 8 total, 8 up, 8 in 2023-09-30T09:13:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:33 smithi019 ceph-mon[118983]: osdmap e77: 8 total, 8 up, 8 in 2023-09-30T09:13:34.364 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:34.367 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap create testimg@snap1 2023-09-30T09:13:34.907 INFO:tasks.workunit.client.0.smithi019.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-30T09:13:34.913 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap create testimg@snap2 2023-09-30T09:13:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:34 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1254236844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:34 smithi167 ceph-mon[104403]: pgmap v95: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 176 op/s 2023-09-30T09:13:35.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:34 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1254236844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:35.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:34 smithi019 ceph-mon[118983]: pgmap v95: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 176 op/s 2023-09-30T09:13:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:34 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1254236844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:34 smithi019 ceph-mon[100214]: pgmap v95: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 176 op/s 2023-09-30T09:13:35.913 INFO:tasks.workunit.client.0.smithi019.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-30T09:13:35.920 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap protect testimg@snap2 2023-09-30T09:13:35.967 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192116/snap_protect 2023-09-30T09:13:36.011 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:36.014 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192116/snap_protect testimg_import 2023-09-30T09:13:36.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:13:35 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:13:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:35 smithi167 ceph-mon[104403]: osdmap e78: 8 total, 8 up, 8 in 2023-09-30T09:13:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:35 smithi019 ceph-mon[100214]: osdmap e78: 8 total, 8 up, 8 in 2023-09-30T09:13:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:35 smithi019 ceph-mon[118983]: osdmap e78: 8 total, 8 up, 8 in 2023-09-30T09:13:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:36 smithi167 ceph-mon[104403]: osdmap e79: 8 total, 8 up, 8 in 2023-09-30T09:13:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:36 smithi167 ceph-mon[104403]: pgmap v98: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 13 KiB/s wr, 187 op/s 2023-09-30T09:13:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:36 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:36 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3096858076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:36 smithi019 ceph-mon[100214]: osdmap e79: 8 total, 8 up, 8 in 2023-09-30T09:13:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:36 smithi019 ceph-mon[100214]: pgmap v98: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 13 KiB/s wr, 187 op/s 2023-09-30T09:13:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:36 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:36 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3096858076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:36 smithi019 ceph-mon[118983]: osdmap e79: 8 total, 8 up, 8 in 2023-09-30T09:13:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:36 smithi019 ceph-mon[118983]: pgmap v98: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 13 KiB/s wr, 187 op/s 2023-09-30T09:13:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:36 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:36 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3096858076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:37.938 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:13:37.943 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import@snap1 2023-09-30T09:13:37.943 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 'protected: False' 2023-09-30T09:13:37.992 INFO:tasks.workunit.client.0.smithi019.stdout: protected: False 2023-09-30T09:13:37.993 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 'protected: True' 2023-09-30T09:13:37.993 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import@snap2 2023-09-30T09:13:38.039 INFO:tasks.workunit.client.0.smithi019.stdout: protected: True 2023-09-30T09:13:38.039 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/snap_protect 2023-09-30T09:13:38.040 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap unprotect testimg@snap2 2023-09-30T09:13:38.099 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-30T09:13:38.150 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap purge testimg 2023-09-30T09:13:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:37 smithi167 ceph-mon[104403]: osdmap e80: 8 total, 8 up, 8 in 2023-09-30T09:13:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:37 smithi019 ceph-mon[100214]: osdmap e80: 8 total, 8 up, 8 in 2023-09-30T09:13:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:37 smithi019 ceph-mon[118983]: osdmap e80: 8 total, 8 up, 8 in 2023-09-30T09:13:38.921 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:38 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:38] "GET /metrics HTTP/1.1" 200 239248 "" "Prometheus/2.43.0" 2023-09-30T09:13:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:38 smithi167 ceph-mon[104403]: osdmap e81: 8 total, 8 up, 8 in 2023-09-30T09:13:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:38 smithi167 ceph-mon[104403]: pgmap v101: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-30T09:13:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:38 smithi019 ceph-mon[118983]: osdmap e81: 8 total, 8 up, 8 in 2023-09-30T09:13:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:38 smithi019 ceph-mon[118983]: pgmap v101: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-30T09:13:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:38 smithi019 ceph-mon[100214]: osdmap e81: 8 total, 8 up, 8 in 2023-09-30T09:13:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:38 smithi019 ceph-mon[100214]: pgmap v101: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-30T09:13:39.885 INFO:tasks.workunit.client.0.smithi019.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-30T09:13:39.893 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap purge testimg_import 2023-09-30T09:13:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:39 smithi167 ceph-mon[104403]: osdmap e82: 8 total, 8 up, 8 in 2023-09-30T09:13:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:39 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1085966695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:39 smithi167 ceph-mon[104403]: osdmap e83: 8 total, 8 up, 8 in 2023-09-30T09:13:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:39 smithi019 ceph-mon[100214]: osdmap e82: 8 total, 8 up, 8 in 2023-09-30T09:13:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:39 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1085966695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:39 smithi019 ceph-mon[100214]: osdmap e83: 8 total, 8 up, 8 in 2023-09-30T09:13:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:39 smithi019 ceph-mon[118983]: osdmap e82: 8 total, 8 up, 8 in 2023-09-30T09:13:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:39 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1085966695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:39 smithi019 ceph-mon[118983]: osdmap e83: 8 total, 8 up, 8 in 2023-09-30T09:13:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:40 smithi167 ceph-mon[104403]: pgmap v104: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-30T09:13:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:40 smithi019 ceph-mon[118983]: pgmap v104: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-30T09:13:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:40 smithi019 ceph-mon[100214]: pgmap v104: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-30T09:13:41.939 INFO:tasks.workunit.client.0.smithi019.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-30T09:13:41.948 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg 2023-09-30T09:13:42.045 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:42.048 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:13:42.140 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:42.144 INFO:tasks.workunit.client.0.smithi019.stderr:+ tiered=0 2023-09-30T09:13:42.144 INFO:tasks.workunit.client.0.smithi019.stderr:+ ceph osd dump 2023-09-30T09:13:42.144 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep '^pool' 2023-09-30T09:13:42.144 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep tier 2023-09-30T09:13:42.145 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep ''\''rbd'\''' 2023-09-30T09:13:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:41 smithi167 ceph-mon[104403]: osdmap e84: 8 total, 8 up, 8 in 2023-09-30T09:13:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:41 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3905219571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:42.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:41 smithi019 ceph-mon[100214]: osdmap e84: 8 total, 8 up, 8 in 2023-09-30T09:13:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:41 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3905219571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:41 smithi019 ceph-mon[118983]: osdmap e84: 8 total, 8 up, 8 in 2023-09-30T09:13:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:41 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3905219571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:42.693 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_192116/sparse1 2023-09-30T09:13:42.700 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:13:42.700 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:13:42.701 INFO:tasks.workunit.client.0.smithi019.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00601591 s, 174 MB/s 2023-09-30T09:13:42.701 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_192116/sparse2 2023-09-30T09:13:42.707 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:13:42.707 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:13:42.707 INFO:tasks.workunit.client.0.smithi019.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00523543 s, 200 MB/s 2023-09-30T09:13:42.707 INFO:tasks.workunit.client.0.smithi019.stderr:+ truncate /tmp/rbd_import_export_192116/sparse2 -s 2M 2023-09-30T09:13:42.730 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse1 2023-09-30T09:13:42.770 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 0% complete...failed. 2023-09-30T09:13:42.770 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: delete error: (2) No such file or directory 2023-09-30T09:13:42.773 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:13:42.773 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_192116/sparse1 2023-09-30T09:13:42.838 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-30T09:13:42.840 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:42.840 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:13:42.843 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd ls -l 2023-09-30T09:13:42.843 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep sparse1 2023-09-30T09:13:42.843 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-30T09:13:42.890 INFO:tasks.workunit.client.0.smithi019.stdout:sparse1 2 MiB 2 2023-09-30T09:13:42.891 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse1 2023-09-30T09:13:42.891 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse1 2023-09-30T09:13:42.891 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse1 2023-09-30T09:13:42.891 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:13:42.892 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:13:42.942 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.62183f58126c 2023-09-30T09:13:42.943 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.62183f58126c 2023-09-30T09:13:42.943 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse1 2023-09-30T09:13:42.943 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.62183f58126c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:13:42.943 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse1 2023-09-30T09:13:42.943 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:13:42.944 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:13:42.944 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:13:42.945 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse1 2023-09-30T09:13:42.986 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:13:42.986 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:13:42.986 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:13:42.986 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:13:42.987 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:13:43.030 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=1 2023-09-30T09:13:43.030 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 1 2023-09-30T09:13:43.030 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-30T09:13:43.031 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse1 /tmp/rbd_import_export_192116/sparse1.out 2023-09-30T09:13:43.072 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:43.075 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192116/sparse1 /tmp/rbd_import_export_192116/sparse1.out 2023-09-30T09:13:43.075 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_192116/sparse1 /tmp/rbd_import_export_192116/sparse1.out 2023-09-30T09:13:43.078 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse1 --format %b 2023-09-30T09:13:43.079 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=2048 2023-09-30T09:13:43.079 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse1.out --format %b 2023-09-30T09:13:43.081 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=2048 2023-09-30T09:13:43.081 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:43.081 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:43.081 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:13:43.081 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/sparse1.out 2023-09-30T09:13:43.081 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse1 2023-09-30T09:13:43.175 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-30T09:13:43.179 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse2 2023-09-30T09:13:43.221 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 0% complete...failed. 2023-09-30T09:13:43.221 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: delete error: (2) No such file or directory 2023-09-30T09:13:43.224 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:13:43.224 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_192116/sparse2 2023-09-30T09:13:43.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:42 smithi167 ceph-mon[104403]: osdmap e85: 8 total, 8 up, 8 in 2023-09-30T09:13:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:42 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/220618205' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:13:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:42 smithi167 ceph-mon[104403]: pgmap v107: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:42 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/220618205' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:13:43.292 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-30T09:13:43.293 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:43.293 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:13:43.296 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd ls -l 2023-09-30T09:13:43.296 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep sparse2 2023-09-30T09:13:43.296 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-30T09:13:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:42 smithi019 ceph-mon[118983]: osdmap e85: 8 total, 8 up, 8 in 2023-09-30T09:13:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:42 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/220618205' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:13:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:42 smithi019 ceph-mon[118983]: pgmap v107: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:42 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/220618205' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:13:43.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:42 smithi019 ceph-mon[100214]: osdmap e85: 8 total, 8 up, 8 in 2023-09-30T09:13:43.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:42 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/220618205' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:13:43.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:42 smithi019 ceph-mon[100214]: pgmap v107: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:43.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:42 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/220618205' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:13:43.340 INFO:tasks.workunit.client.0.smithi019.stdout:sparse2 2 MiB 2 2023-09-30T09:13:43.341 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse2 2023-09-30T09:13:43.341 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse2 2023-09-30T09:13:43.341 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse2 2023-09-30T09:13:43.341 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:13:43.341 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:13:43.394 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.3b9437a4a7e7 2023-09-30T09:13:43.395 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.3b9437a4a7e7 2023-09-30T09:13:43.395 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.3b9437a4a7e7\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:13:43.395 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse2 2023-09-30T09:13:43.395 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse2 2023-09-30T09:13:43.396 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:13:43.396 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:13:43.398 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse2 2023-09-30T09:13:43.400 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:13:43.443 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:13:43.443 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:13:43.443 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:13:43.444 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:13:43.444 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:13:43.486 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=0 2023-09-30T09:13:43.486 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 0 2023-09-30T09:13:43.486 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-30T09:13:43.486 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse2 /tmp/rbd_import_export_192116/sparse2.out 2023-09-30T09:13:43.525 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:43.528 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192116/sparse2 /tmp/rbd_import_export_192116/sparse2.out 2023-09-30T09:13:43.528 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_192116/sparse2 /tmp/rbd_import_export_192116/sparse2.out 2023-09-30T09:13:43.530 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse2 --format %b 2023-09-30T09:13:43.531 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=2048 2023-09-30T09:13:43.532 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse2.out --format %b 2023-09-30T09:13:43.533 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=2048 2023-09-30T09:13:43.533 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:43.533 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:43.533 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:13:43.533 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/sparse2.out 2023-09-30T09:13:43.533 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse2 2023-09-30T09:13:43.621 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-30T09:13:43.624 INFO:tasks.workunit.client.0.smithi019.stderr:+ truncate /tmp/rbd_import_export_192116/sparse1 -s 10M 2023-09-30T09:13:43.625 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-30T09:13:43.717 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:43.717 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:43.718 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:13:43.721 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd ls -l 2023-09-30T09:13:43.721 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep sparse1 2023-09-30T09:13:43.721 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-30T09:13:43.757 INFO:tasks.workunit.client.0.smithi019.stdout:sparse1 10 MiB 2 2023-09-30T09:13:43.758 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse1 2023-09-30T09:13:43.758 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse1 2023-09-30T09:13:43.758 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse1 2023-09-30T09:13:43.759 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:13:43.759 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:13:43.795 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.3bc1d602f830 2023-09-30T09:13:43.796 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.3bc1d602f830 2023-09-30T09:13:43.797 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse1 2023-09-30T09:13:43.797 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.3bc1d602f830\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:13:43.797 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse1 2023-09-30T09:13:43.797 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:13:43.797 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse1 2023-09-30T09:13:43.798 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:13:43.799 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:13:43.834 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:13:43.835 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:13:43.835 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:13:43.835 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:13:43.835 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:13:43.867 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=1 2023-09-30T09:13:43.867 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 1 2023-09-30T09:13:43.867 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-30T09:13:43.867 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse1 /tmp/rbd_import_export_192116/sparse1.out 2023-09-30T09:13:43.911 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:43.914 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192116/sparse1 /tmp/rbd_import_export_192116/sparse1.out 2023-09-30T09:13:43.914 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_192116/sparse1 /tmp/rbd_import_export_192116/sparse1.out 2023-09-30T09:13:43.921 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse1 --format %b 2023-09-30T09:13:43.922 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=2048 2023-09-30T09:13:43.922 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse1.out --format %b 2023-09-30T09:13:43.924 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=2048 2023-09-30T09:13:43.924 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:43.924 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:43.924 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:13:43.924 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/sparse1.out 2023-09-30T09:13:43.925 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse1 2023-09-30T09:13:44.012 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-30T09:13:44.015 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_192116/sparse2 oflag=append conv=notrunc 2023-09-30T09:13:44.026 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:13:44.026 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:13:44.027 INFO:tasks.workunit.client.0.smithi019.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0101336 s, 207 MB/s 2023-09-30T09:13:44.027 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-30T09:13:44.130 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:44.131 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:44.131 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:13:44.134 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd ls -l 2023-09-30T09:13:44.134 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep sparse2 2023-09-30T09:13:44.134 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-30T09:13:44.176 INFO:tasks.workunit.client.0.smithi019.stdout:sparse2 4 MiB 2 2023-09-30T09:13:44.177 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse2 2023-09-30T09:13:44.177 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse2 2023-09-30T09:13:44.177 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse2 2023-09-30T09:13:44.177 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:13:44.177 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:13:44.217 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.62a2bf9823aa 2023-09-30T09:13:44.218 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.62a2bf9823aa\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:13:44.219 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:13:44.219 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.62a2bf9823aa 2023-09-30T09:13:44.219 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse2 2023-09-30T09:13:44.219 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse2 2023-09-30T09:13:44.219 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse2 2023-09-30T09:13:44.219 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:13:44.219 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:13:44.262 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:13:44.263 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:13:44.263 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:13:44.263 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:13:44.263 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:13:44.295 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects='0 2023-09-30T09:13:44.295 INFO:tasks.workunit.client.0.smithi019.stderr:2 2023-09-30T09:13:44.295 INFO:tasks.workunit.client.0.smithi019.stderr:3' 2023-09-30T09:13:44.296 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 0 2 3 2023-09-30T09:13:44.296 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-30T09:13:44.296 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse2 /tmp/rbd_import_export_192116/sparse2.out 2023-09-30T09:13:44.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:44 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/460524990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:44.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:44 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/460524990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:44.335 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-30T09:13:44.338 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192116/sparse2 /tmp/rbd_import_export_192116/sparse2.out 2023-09-30T09:13:44.338 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_192116/sparse2 /tmp/rbd_import_export_192116/sparse2.out 2023-09-30T09:13:44.342 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse2 --format %b 2023-09-30T09:13:44.342 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=6144 2023-09-30T09:13:44.346 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse2.out --format %b 2023-09-30T09:13:44.346 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=6144 2023-09-30T09:13:44.346 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:44.346 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:13:44.346 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:13:44.346 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/sparse2.out 2023-09-30T09:13:44.346 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse2 2023-09-30T09:13:44.438 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-30T09:13:44.441 INFO:tasks.workunit.client.0.smithi019.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-30T09:13:44.441 INFO:tasks.workunit.client.0.smithi019.stdout:partially-sparse file imports to partially-sparse image 2023-09-30T09:13:44.442 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_192116/sparse1 sparse 2023-09-30T09:13:44.503 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-30T09:13:44.503 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:44.504 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:13:44.506 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse 2023-09-30T09:13:44.506 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse 2023-09-30T09:13:44.507 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse 2023-09-30T09:13:44.507 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:13:44.507 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:13:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:44 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/460524990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:44.543 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.3c21663f3282 2023-09-30T09:13:44.544 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.3c21663f3282 2023-09-30T09:13:44.544 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse 2023-09-30T09:13:44.544 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.3c21663f3282\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:13:44.544 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse 2023-09-30T09:13:44.544 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:13:44.545 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse 2023-09-30T09:13:44.545 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:13:44.545 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:13:44.583 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:13:44.583 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:13:44.583 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:13:44.583 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:13:44.583 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:13:44.616 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=1 2023-09-30T09:13:44.616 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 1 2023-09-30T09:13:44.616 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-30T09:13:44.616 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse 2023-09-30T09:13:44.706 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-30T09:13:44.709 INFO:tasks.workunit.client.0.smithi019.stdout:zeros import through stdin to sparse image 2023-09-30T09:13:44.710 INFO:tasks.workunit.client.0.smithi019.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-30T09:13:44.710 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-30T09:13:44.710 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format - sparse 2023-09-30T09:13:44.758 INFO:tasks.workunit.client.0.smithi019.stderr:4+0 records in 2023-09-30T09:13:44.759 INFO:tasks.workunit.client.0.smithi019.stderr:4+0 records out 2023-09-30T09:13:44.759 INFO:tasks.workunit.client.0.smithi019.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0477193 s, 87.9 MB/s 2023-09-30T09:13:44.769 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:13:44.769 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:44.771 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse 2023-09-30T09:13:44.771 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse 2023-09-30T09:13:44.772 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse 2023-09-30T09:13:44.772 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:13:44.772 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:13:44.807 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.3c426fb64545 2023-09-30T09:13:44.808 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.3c426fb64545 2023-09-30T09:13:44.808 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.3c426fb64545\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:13:44.808 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse 2023-09-30T09:13:44.808 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse 2023-09-30T09:13:44.809 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:13:44.809 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse 2023-09-30T09:13:44.809 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:13:44.809 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:13:44.846 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:13:44.846 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:13:44.846 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:13:44.847 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:13:44.847 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:13:44.881 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects= 2023-09-30T09:13:44.882 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 2023-09-30T09:13:44.882 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-30T09:13:44.882 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse 2023-09-30T09:13:44.966 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:44.969 INFO:tasks.workunit.client.0.smithi019.stdout:zeros export to sparse file 2023-09-30T09:13:44.970 INFO:tasks.workunit.client.0.smithi019.stderr:+ echo 'zeros export to sparse file' 2023-09-30T09:13:44.970 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd create --new-format sparse --size 4 2023-09-30T09:13:45.007 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:13:45.010 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd info sparse 2023-09-30T09:13:45.010 INFO:tasks.workunit.client.0.smithi019.stderr:++ grep block_name_prefix 2023-09-30T09:13:45.010 INFO:tasks.workunit.client.0.smithi019.stderr:++ awk '{print $NF;}' 2023-09-30T09:13:45.046 INFO:tasks.workunit.client.0.smithi019.stderr:+ prefix=rbd_data.3c6352ad2808 2023-09-30T09:13:45.046 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-30T09:13:45.047 INFO:tasks.workunit.client.0.smithi019.stderr:++ get_image_data_pool sparse 2023-09-30T09:13:45.047 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse 2023-09-30T09:13:45.047 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse 2023-09-30T09:13:45.047 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep 'data_pool: ' 2023-09-30T09:13:45.047 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk -F: '{ print $NF }' 2023-09-30T09:13:45.083 INFO:tasks.workunit.client.0.smithi019.stderr:++ data_pool= 2023-09-30T09:13:45.083 INFO:tasks.workunit.client.0.smithi019.stderr:++ '[' -z ']' 2023-09-30T09:13:45.083 INFO:tasks.workunit.client.0.smithi019.stderr:++ data_pool=rbd 2023-09-30T09:13:45.083 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo rbd 2023-09-30T09:13:45.084 INFO:tasks.workunit.client.0.smithi019.stderr:+ rados -p rbd put rbd_data.3c6352ad2808.000000000000 - 2023-09-30T09:13:45.113 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:13:45.113 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:13:45.114 INFO:tasks.workunit.client.0.smithi019.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0665121 s, 63.1 MB/s 2023-09-30T09:13:45.147 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse 2023-09-30T09:13:45.147 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse 2023-09-30T09:13:45.147 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse 2023-09-30T09:13:45.147 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:13:45.147 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:13:45.183 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.3c6352ad2808 2023-09-30T09:13:45.183 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.3c6352ad2808 2023-09-30T09:13:45.183 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse 2023-09-30T09:13:45.184 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse 2023-09-30T09:13:45.184 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.3c6352ad2808\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:13:45.184 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:13:45.184 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse 2023-09-30T09:13:45.184 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:13:45.184 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:13:45.219 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:13:45.219 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:13:45.219 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:13:45.219 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:13:45.219 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:13:45.251 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=0 2023-09-30T09:13:45.251 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 0 2023-09-30T09:13:45.251 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-30T09:13:45.252 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/sparse 2023-09-30T09:13:45.252 INFO:tasks.workunit.client.0.smithi019.stderr:rm: cannot remove '/tmp/rbd_import_export_192116/sparse': No such file or directory 2023-09-30T09:13:45.252 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:13:45.252 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse /tmp/rbd_import_export_192116/sparse 2023-09-30T09:13:45.287 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete...done. 2023-09-30T09:13:45.290 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_192116/sparse --format=%b 2023-09-30T09:13:45.291 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 = 0 ']' 2023-09-30T09:13:45.291 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse 2023-09-30T09:13:45.373 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:13:45.376 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_192116/sparse /tmp/rbd_import_export_192116/sparse1 /tmp/rbd_import_export_192116/sparse2 /tmp/rbd_import_export_192116/sparse3 2023-09-30T09:13:45.378 INFO:tasks.workunit.client.0.smithi019.stderr:rm: cannot remove '/tmp/rbd_import_export_192116/sparse3': No such file or directory 2023-09-30T09:13:45.378 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:13:45.378 INFO:tasks.workunit.client.0.smithi019.stdout:OK 2023-09-30T09:13:45.379 INFO:tasks.workunit.client.0.smithi019.stderr:+ echo OK 2023-09-30T09:13:45.379 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm -rf /tmp/rbd_import_export_192116 2023-09-30T09:13:45.379 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-30T09:13:45.380 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-30T09:13:45.407 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:45 smithi019 ceph-mon[100214]: pgmap v108: 73 pgs: 73 active+clean; 1.6 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s rd, 190 KiB/s wr, 155 op/s 2023-09-30T09:13:45.407 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:45 smithi019 ceph-mon[118983]: pgmap v108: 73 pgs: 73 active+clean; 1.6 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s rd, 190 KiB/s wr, 155 op/s 2023-09-30T09:13:45.411 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-30T09:13:45.411 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-30T09:13:45.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:45 smithi167 ceph-mon[104403]: pgmap v108: 73 pgs: 73 active+clean; 1.6 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s rd, 190 KiB/s wr, 155 op/s 2023-09-30T09:13:45.666 DEBUG:teuthology.parallel:result is None 2023-09-30T09:13:45.667 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:13:45.694 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:13:45.694 DEBUG:teuthology.orchestra.run.smithi019:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:13:45.752 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:13:45.752 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-30T09:13:45.752 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-30T09:13:45.752 DEBUG:teuthology.parallel:result is None 2023-09-30T09:13:46.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:13:45 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:13:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:46 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/128956012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:46 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/128956012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:46.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:46 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/128956012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:47 smithi167 ceph-mon[104403]: pgmap v109: 73 pgs: 73 active+clean; 1.6 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s rd, 161 KiB/s wr, 131 op/s 2023-09-30T09:13:47.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:47 smithi019 ceph-mon[100214]: pgmap v109: 73 pgs: 73 active+clean; 1.6 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s rd, 161 KiB/s wr, 131 op/s 2023-09-30T09:13:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:47 smithi019 ceph-mon[118983]: pgmap v109: 73 pgs: 73 active+clean; 1.6 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s rd, 161 KiB/s wr, 131 op/s 2023-09-30T09:13:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:48 smithi167 ceph-mon[104403]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 386 op/s 2023-09-30T09:13:49.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:48 smithi019 ceph-mon[100214]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 386 op/s 2023-09-30T09:13:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:48 smithi019 ceph-mon[118983]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 386 op/s 2023-09-30T09:13:49.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:48 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:48] "GET /metrics HTTP/1.1" 200 239248 "" "Prometheus/2.43.0" 2023-09-30T09:13:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:49 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1046196275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:50.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:49 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1046196275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:50.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:49 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1046196275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:50 smithi167 ceph-mon[104403]: osdmap e86: 8 total, 8 up, 8 in 2023-09-30T09:13:51.379 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:50 smithi167 ceph-mon[104403]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 981 KiB/s rd, 1.3 MiB/s wr, 359 op/s 2023-09-30T09:13:51.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:50 smithi019 ceph-mon[100214]: osdmap e86: 8 total, 8 up, 8 in 2023-09-30T09:13:51.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:50 smithi019 ceph-mon[100214]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 981 KiB/s rd, 1.3 MiB/s wr, 359 op/s 2023-09-30T09:13:51.380 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:50 smithi019 ceph-mon[118983]: osdmap e86: 8 total, 8 up, 8 in 2023-09-30T09:13:51.380 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:50 smithi019 ceph-mon[118983]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 981 KiB/s rd, 1.3 MiB/s wr, 359 op/s 2023-09-30T09:13:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:51 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1832279619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:51 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:52.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:51 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1832279619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:52.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:51 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:51 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1832279619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:51 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:13:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:52 smithi167 ceph-mon[104403]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 845 KiB/s rd, 1.1 MiB/s wr, 309 op/s 2023-09-30T09:13:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:52 smithi019 ceph-mon[100214]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 845 KiB/s rd, 1.1 MiB/s wr, 309 op/s 2023-09-30T09:13:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:52 smithi019 ceph-mon[118983]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 845 KiB/s rd, 1.1 MiB/s wr, 309 op/s 2023-09-30T09:13:54.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:53 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3655007202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:54.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:53 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3655007202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:54.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:53 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3655007202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:55 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:55 smithi167 ceph-mon[104403]: Upgrade: Target is version 18.0.0-6447-g1d10b717 (unknown) 2023-09-30T09:13:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:55 smithi167 ceph-mon[104403]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0'] 2023-09-30T09:13:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:55 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:55 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:55 smithi167 ceph-mon[104403]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-30T09:13:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:55 smithi167 ceph-mon[104403]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 on smithi167 2023-09-30T09:13:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:55 smithi167 ceph-mon[104403]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 221 op/s 2023-09-30T09:13:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[100214]: Upgrade: Target is version 18.0.0-6447-g1d10b717 (unknown) 2023-09-30T09:13:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[100214]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0'] 2023-09-30T09:13:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[100214]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[100214]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 on smithi167 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[100214]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 221 op/s 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[118983]: Upgrade: Target is version 18.0.0-6447-g1d10b717 (unknown) 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[118983]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0'] 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[118983]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[118983]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 on smithi167 2023-09-30T09:13:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:55 smithi019 ceph-mon[118983]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 221 op/s 2023-09-30T09:13:56.186 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:13:55 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:13:56.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:56 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1031771481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:56 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1031771481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:56.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:56 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1031771481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:57 smithi167 ceph-mon[104403]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 221 op/s 2023-09-30T09:13:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:57 smithi019 ceph-mon[100214]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 221 op/s 2023-09-30T09:13:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:57 smithi019 ceph-mon[118983]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 221 op/s 2023-09-30T09:13:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:58 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3668837430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:58.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:13:58 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:13:58] "GET /metrics HTTP/1.1" 200 239511 "" "Prometheus/2.43.0" 2023-09-30T09:13:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:58 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3668837430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:59.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:58 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3668837430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:13:59.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:13:59 smithi019 ceph-mon[100214]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:13:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:13:59 smithi019 ceph-mon[118983]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:00.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:13:59 smithi167 ceph-mon[104403]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:01.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:00 smithi167 ceph-mon[104403]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:00 smithi019 ceph-mon[100214]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:01.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:00 smithi019 ceph-mon[118983]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:01 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/4235141653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:01 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4235141653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:01 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/4235141653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:03.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:02 smithi167 ceph-mon[104403]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:02 smithi019 ceph-mon[100214]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:02 smithi019 ceph-mon[118983]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:03 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1766478412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:03 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1766478412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:03 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1766478412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:04 smithi167 ceph-mon[104403]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:04 smithi019 ceph-mon[100214]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:04 smithi019 ceph-mon[118983]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:05 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/932142011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:06.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:05 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:14:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:05 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/932142011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:05 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/932142011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:07.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:06 smithi167 ceph-mon[104403]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:06 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:06 smithi019 ceph-mon[100214]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:06 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:07.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:06 smithi019 ceph-mon[118983]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:07.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:06 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:08 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/723582656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:08 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/723582656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:08 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/723582656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:09.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:08 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:08] "GET /metrics HTTP/1.1" 200 239448 "" "Prometheus/2.43.0" 2023-09-30T09:14:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:09 smithi167 ceph-mon[104403]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:09 smithi019 ceph-mon[100214]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:09.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:09 smithi019 ceph-mon[118983]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:10 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/255534470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:10 smithi167 ceph-mon[104403]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:10 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/255534470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:10 smithi019 ceph-mon[100214]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:10 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/255534470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:10 smithi019 ceph-mon[118983]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:12 smithi167 ceph-mon[104403]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:13.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:12 smithi019 ceph-mon[100214]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:12 smithi019 ceph-mon[118983]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:13 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3414881582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:13 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3414881582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:13 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3414881582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:14 smithi167 ceph-mon[104403]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:15.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:14 smithi019 ceph-mon[100214]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:15.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:14 smithi019 ceph-mon[118983]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:16.274 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:15 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:14:16.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:15 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3985176361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:15 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3985176361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:15 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3985176361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:16 smithi167 ceph-mon[104403]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:16 smithi019 ceph-mon[100214]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:16 smithi019 ceph-mon[118983]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:17 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/61973931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:17 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/61973931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:18.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:17 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/61973931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:18.901 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:18 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:18] "GET /metrics HTTP/1.1" 200 239448 "" "Prometheus/2.43.0" 2023-09-30T09:14:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:18 smithi167 ceph-mon[104403]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:18 smithi019 ceph-mon[100214]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:18 smithi019 ceph-mon[118983]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:21.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:20 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2919956192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:20 smithi167 ceph-mon[104403]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:20 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2919956192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:20 smithi019 ceph-mon[100214]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:20 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2919956192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:20 smithi019 ceph-mon[118983]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:22.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:21 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:21 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:22.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:21 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:22 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/621968895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:22 smithi167 ceph-mon[104403]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:23.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:22 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/621968895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:23.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:22 smithi019 ceph-mon[100214]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:22 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/621968895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:22 smithi019 ceph-mon[118983]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:24 smithi167 ceph-mon[104403]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:25.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:24 smithi019 ceph-mon[100214]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:24 smithi019 ceph-mon[118983]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:25 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1241052291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:26.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:25 smithi167 conmon[107623]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:14:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:25 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1241052291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:25 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1241052291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:26 smithi167 ceph-mon[104403]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:27.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:26 smithi019 ceph-mon[100214]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:26 smithi019 ceph-mon[118983]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:27 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3937332203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:28.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:27 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3937332203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:28.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:27 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3937332203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:28.904 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:28 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:28] "GET /metrics HTTP/1.1" 200 239392 "" "Prometheus/2.43.0" 2023-09-30T09:14:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:28 smithi167 ceph-mon[104403]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:28 smithi019 ceph-mon[100214]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:28 smithi019 ceph-mon[118983]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:29 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/15062490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:29 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/15062490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:29 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/15062490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:30 smithi167 ceph-mon[104403]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:30 smithi019 ceph-mon[100214]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:30 smithi019 ceph-mon[118983]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:32.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:32 smithi167 ceph-mon[104403]: Upgrade: Updating mgr.x 2023-09-30T09:14:32.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:32 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:32.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:32 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:14:32.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:32 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:14:32.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:32 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:32.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:32 smithi167 ceph-mon[104403]: Deploying daemon mgr.x on smithi167 2023-09-30T09:14:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[100214]: Upgrade: Updating mgr.x 2023-09-30T09:14:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:14:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:14:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[100214]: Deploying daemon mgr.x on smithi167 2023-09-30T09:14:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[118983]: Upgrade: Updating mgr.x 2023-09-30T09:14:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:14:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:14:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:32 smithi019 ceph-mon[118983]: Deploying daemon mgr.x on smithi167 2023-09-30T09:14:32.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:32 smithi167 systemd[1]: Stopping Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:14:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:33 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4200286190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:33 smithi019 ceph-mon[100214]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:33 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/4200286190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:33 smithi019 ceph-mon[118983]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:33.395 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:33 smithi167 bash[158492]: 6176c234bba154419f88145510827da36ad02476c4b9d5d8326302f81baf7d5c 2023-09-30T09:14:33.395 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:33 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:14:33.395 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:33 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/4200286190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:33.395 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:33 smithi167 ceph-mon[104403]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:33.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:33 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-30T09:14:33.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:33 smithi167 systemd[1]: Stopped Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:14:33.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:33 smithi167 systemd[1]: Starting Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:14:34.433 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:34 smithi167 ceph-mgr[158911]: -- 172.21.15.167:0/1154137059 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55665455f1e0 con 0x556654536c00 2023-09-30T09:14:34.433 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:34 smithi167 bash[158845]: 833bc43a28aa32377d4f0a2b4a50ef2b67991c9b9947f62a2dcf8998fc9b1b60 2023-09-30T09:14:34.433 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:34 smithi167 systemd[1]: Started Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:14:34.433 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:34 smithi167 conmon[158886]: 2023-09-30T09:14:34.249+0000 7f82edf8c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:14:34.433 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:34 smithi167 conmon[158886]: 2023-09-30T09:14:34.336+0000 7f82edf8c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:14:34.433 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:34 smithi167 conmon[158886]: 2023-09-30T09:14:34.432+0000 7f82edf8c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:14:34.744 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:34 smithi167 conmon[158886]: 2023-09-30T09:14:34.539+0000 7f82edf8c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:14:35.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:34 smithi167 conmon[158886]: 2023-09-30T09:14:34.743+0000 7f82edf8c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:14:35.509 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:35 smithi167 conmon[158886]: 2023-09-30T09:14:35.156+0000 7f82edf8c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:14:35.509 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:35 smithi167 conmon[158886]: 2023-09-30T09:14:35.307+0000 7f82edf8c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:14:35.509 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:35 smithi167 conmon[158886]: 2023-09-30T09:14:35.405+0000 7f82edf8c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:14:35.510 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:35 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:35.510 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:35 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:35.510 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:35 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:35.510 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:35 smithi167 ceph-mon[104403]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:35.510 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:35 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1002862047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[118983]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1002862047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[100214]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:35 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1002862047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:35.771 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:35 smithi167 conmon[158886]: 2023-09-30T09:14:35.508+0000 7f82edf8c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:14:36.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:35 smithi167 conmon[158886]: 2023-09-30T09:14:35.995+0000 7f82edf8c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:14:37.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:36 smithi167 conmon[158886]: 2023-09-30T09:14:36.733+0000 7f82edf8c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:14:37.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:36 smithi167 conmon[158886]: 2023-09-30T09:14:36.824+0000 7f82edf8c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:14:37.771 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:37 smithi167 conmon[158886]: 2023-09-30T09:14:37.692+0000 7f82edf8c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:14:37.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:37 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:37.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:37 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:37.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:37 smithi167 ceph-mon[104403]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:37.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:37 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:37.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:37 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/873496711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:37.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:37.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:37.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[100214]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:37.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:37.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/873496711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:37.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:37.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:37.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[118983]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:37.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:37.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:37 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/873496711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:38.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:37 smithi167 conmon[158886]: 2023-09-30T09:14:37.804+0000 7f82edf8c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:14:38.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:37 smithi167 conmon[158886]: 2023-09-30T09:14:37.993+0000 7f82edf8c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:14:38.720 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:38 smithi167 conmon[158886]: 2023-09-30T09:14:38.302+0000 7f82edf8c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:14:38.720 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:38 smithi167 conmon[158886]: 2023-09-30T09:14:38.544+0000 7f82edf8c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:14:38.720 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:38 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.721 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:38 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.721 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:38 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.721 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:38 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.838 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:38 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:38] "GET /metrics HTTP/1.1" 200 239387 "" "Prometheus/2.43.0" 2023-09-30T09:14:38.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:38 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:38 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:38 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:38 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:38 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:38 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:38 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:38.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:38 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:39.021 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:38 smithi167 conmon[158886]: 2023-09-30T09:14:38.719+0000 7f82edf8c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:14:39.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:39 smithi167 conmon[158886]: 2023-09-30T09:14:39.020+0000 7f82edf8c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:14:39.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:39 smithi019 ceph-mon[100214]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:39.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:39 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3615293855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:39 smithi019 ceph-mon[118983]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:39 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3615293855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:39.883 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:39 smithi167 ceph-mon[104403]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:39.883 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:39 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3615293855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:39.884 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:39 smithi167 conmon[158886]: 2023-09-30T09:14:39.763+0000 7f82edf8c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:14:40.179 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:39 smithi167 conmon[158886]: 2023-09-30T09:14:39.905+0000 7f82edf8c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:14:40.180 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:40 smithi167 conmon[158886]: 2023-09-30T09:14:40.120+0000 7f82edf8c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:14:40.496 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:40 smithi167 conmon[158886]: 2023-09-30T09:14:40.248+0000 7f82edf8c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:14:40.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:40 smithi167 conmon[158886]: 2023-09-30T09:14:40.496+0000 7f82edf8c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:14:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:40 smithi167 ceph-mon[104403]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:40 smithi019 ceph-mon[118983]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:40 smithi019 ceph-mon[100214]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:41.552 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: 2023-09-30T09:14:41.432+0000 7f82edf8c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:14:41.552 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: [30/Sep/2023:09:14:41] ENGINE Bus STARTING 2023-09-30T09:14:41.896 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: CherryPy Checker: 2023-09-30T09:14:41.896 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: The Application mounted at '' has an empty config. 2023-09-30T09:14:41.896 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: 2023-09-30T09:14:41.896 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: [30/Sep/2023:09:14:41] ENGINE Serving on http://:::9283 2023-09-30T09:14:41.896 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: [30/Sep/2023:09:14:41] ENGINE Bus STARTED 2023-09-30T09:14:42.150 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: Standby manager daemon x restarted 2023-09-30T09:14:42.150 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: Standby manager daemon x started 2023-09-30T09:14:42.150 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:41 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3894696312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:42.151 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: [30/Sep/2023:09:14:41] ENGINE Bus STOPPING 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: [30/Sep/2023:09:14:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:41 smithi167 conmon[158886]: [30/Sep/2023:09:14:41] ENGINE Bus STOPPED 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: [30/Sep/2023:09:14:42] ENGINE Bus STARTING 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824faf0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824faf0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824faf0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824faf0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824faf0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824caea700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824caea700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.152 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824caea700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.153 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824caea700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.153 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: 2023-09-30T09:14:42.073+0000 7f824caea700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:14:42.153 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: [30/Sep/2023:09:14:42] ENGINE Serving on http://:::9283 2023-09-30T09:14:42.153 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:42 smithi167 conmon[158886]: [30/Sep/2023:09:14:42] ENGINE Bus STARTED 2023-09-30T09:14:42.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:42 smithi019 conmon[100636]: ignoring --setuser ceph since I am not root 2023-09-30T09:14:42.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:42 smithi019 conmon[100636]: ignoring --setgroup ceph since I am not root 2023-09-30T09:14:42.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mgr[100660]: -- 172.21.15.19:0/110426528 <== mon.0 v2:172.21.15.19:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5556610d69c0 con 0x5556610aec00 2023-09-30T09:14:42.236 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:42 smithi019 conmon[100636]: 2023-09-30T09:14:42.139+0000 7fd71a1691c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:14:42.236 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:42 smithi019 conmon[100636]: 2023-09-30T09:14:42.233+0000 7fd71a1691c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:14:42.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: Standby manager daemon x restarted 2023-09-30T09:14:42.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: Standby manager daemon x started 2023-09-30T09:14:42.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:14:42.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:14:42.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:14:42.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:14:42.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:42.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:14:42.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3894696312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:42.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: Standby manager daemon x restarted 2023-09-30T09:14:42.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: Standby manager daemon x started 2023-09-30T09:14:42.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:14:42.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:14:42.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:14:42.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:14:42.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:42.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:14:42.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' 2023-09-30T09:14:42.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:41 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3894696312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:42.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:42 smithi019 conmon[100636]: 2023-09-30T09:14:42.341+0000 7fd71a1691c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:14:42.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:42 smithi019 conmon[100636]: 2023-09-30T09:14:42.454+0000 7fd71a1691c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:14:43.061 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:42 smithi019 conmon[100636]: 2023-09-30T09:14:42.659+0000 7fd71a1691c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:14:43.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: mgrmap e21: y(active, since 3m), standbys: x 2023-09-30T09:14:43.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-30T09:14:43.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:14:43.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:14:43.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:14:43.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:14:43.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: osdmap e87: 8 total, 8 up, 8 in 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: mgrmap e22: x(active, starting, since 0.00821223s) 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:14:43.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: Manager daemon x is now available 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:43.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:14:43.065 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:14:43.065 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-30T09:14:43.065 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-30T09:14:43.065 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.065 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: mgrmap e21: y(active, since 3m), standbys: x 2023-09-30T09:14:43.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: osdmap e87: 8 total, 8 up, 8 in 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:14:43.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: mgrmap e22: x(active, starting, since 0.00821223s) 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:14:43.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: Manager daemon x is now available 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:14:43.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-30T09:14:43.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-30T09:14:43.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:42 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: mgrmap e21: y(active, since 3m), standbys: x 2023-09-30T09:14:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-30T09:14:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: osdmap e87: 8 total, 8 up, 8 in 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.14454 172.21.15.19:0/2521405664' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: mgrmap e22: x(active, starting, since 0.00821223s) 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:14:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: Manager daemon x is now available 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:14:43.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:43.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:14:43.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:14:43.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-30T09:14:43.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-30T09:14:43.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:42 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:43.318 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:43 smithi019 conmon[100636]: 2023-09-30T09:14:43.059+0000 7fd71a1691c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:14:43.319 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:43 smithi019 conmon[100636]: 2023-09-30T09:14:43.166+0000 7fd71a1691c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:14:43.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:43 smithi019 conmon[100636]: 2023-09-30T09:14:43.317+0000 7fd71a1691c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:14:43.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:43 smithi019 conmon[100636]: 2023-09-30T09:14:43.431+0000 7fd71a1691c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:14:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:43 smithi167 ceph-mon[104403]: Deploying cephadm binary to smithi167 2023-09-30T09:14:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:43 smithi167 ceph-mon[104403]: Deploying cephadm binary to smithi019 2023-09-30T09:14:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:43 smithi167 ceph-mon[104403]: mgrmap e23: x(active, since 1.00763s) 2023-09-30T09:14:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:43 smithi167 ceph-mon[104403]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:44.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:43 smithi019 conmon[100636]: 2023-09-30T09:14:43.944+0000 7fd71a1691c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:14:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:43 smithi019 ceph-mon[100214]: Deploying cephadm binary to smithi167 2023-09-30T09:14:44.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:43 smithi019 ceph-mon[100214]: Deploying cephadm binary to smithi019 2023-09-30T09:14:44.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:43 smithi019 ceph-mon[100214]: mgrmap e23: x(active, since 1.00763s) 2023-09-30T09:14:44.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:43 smithi019 ceph-mon[100214]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:44.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:43 smithi019 ceph-mon[118983]: Deploying cephadm binary to smithi167 2023-09-30T09:14:44.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:43 smithi019 ceph-mon[118983]: Deploying cephadm binary to smithi019 2023-09-30T09:14:44.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:43 smithi019 ceph-mon[118983]: mgrmap e23: x(active, since 1.00763s) 2023-09-30T09:14:44.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:43 smithi019 ceph-mon[118983]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:44.959 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:44 smithi019 conmon[100636]: 2023-09-30T09:14:44.675+0000 7fd71a1691c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:14:44.959 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:44 smithi019 conmon[100636]: 2023-09-30T09:14:44.875+0000 7fd71a1691c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:14:44.959 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:44 smithi019 ceph-mon[100214]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:44.959 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:44 smithi019 ceph-mon[100214]: mgrmap e24: x(active, since 2s) 2023-09-30T09:14:44.959 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:44 smithi019 ceph-mon[100214]: [30/Sep/2023:09:14:44] ENGINE Bus STARTING 2023-09-30T09:14:44.960 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:44 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3384194215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:44 smithi167 ceph-mon[104403]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:44 smithi167 ceph-mon[104403]: mgrmap e24: x(active, since 2s) 2023-09-30T09:14:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:44 smithi167 ceph-mon[104403]: [30/Sep/2023:09:14:44] ENGINE Bus STARTING 2023-09-30T09:14:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:44 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3384194215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:44 smithi019 ceph-mon[118983]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:44 smithi019 ceph-mon[118983]: mgrmap e24: x(active, since 2s) 2023-09-30T09:14:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:44 smithi019 ceph-mon[118983]: [30/Sep/2023:09:14:44] ENGINE Bus STARTING 2023-09-30T09:14:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:44 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3384194215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:46.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:45 smithi167 conmon[158886]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:45] "GET /metrics HTTP/1.1" 200 30879 "" "Prometheus/2.43.0" 2023-09-30T09:14:46.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:45 smithi167 ceph-mon[104403]: [30/Sep/2023:09:14:44] ENGINE Serving on http://172.21.15.167:8765 2023-09-30T09:14:46.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:45 smithi167 ceph-mon[104403]: [30/Sep/2023:09:14:44] ENGINE Serving on https://172.21.15.167:7150 2023-09-30T09:14:46.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:45 smithi167 ceph-mon[104403]: [30/Sep/2023:09:14:44] ENGINE Bus STARTED 2023-09-30T09:14:46.295 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:45 smithi019 ceph-mon[100214]: [30/Sep/2023:09:14:44] ENGINE Serving on http://172.21.15.167:8765 2023-09-30T09:14:46.296 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:45 smithi019 ceph-mon[100214]: [30/Sep/2023:09:14:44] ENGINE Serving on https://172.21.15.167:7150 2023-09-30T09:14:46.296 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:45 smithi019 ceph-mon[100214]: [30/Sep/2023:09:14:44] ENGINE Bus STARTED 2023-09-30T09:14:46.296 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:45 smithi019 conmon[100636]: 2023-09-30T09:14:45.939+0000 7fd71a1691c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:14:46.296 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:46 smithi019 conmon[100636]: 2023-09-30T09:14:46.061+0000 7fd71a1691c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:14:46.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:45 smithi019 ceph-mon[118983]: [30/Sep/2023:09:14:44] ENGINE Serving on http://172.21.15.167:8765 2023-09-30T09:14:46.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:45 smithi019 ceph-mon[118983]: [30/Sep/2023:09:14:44] ENGINE Serving on https://172.21.15.167:7150 2023-09-30T09:14:46.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:45 smithi019 ceph-mon[118983]: [30/Sep/2023:09:14:44] ENGINE Bus STARTED 2023-09-30T09:14:46.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:46 smithi019 conmon[100636]: 2023-09-30T09:14:46.294+0000 7fd71a1691c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:14:46.964 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:46 smithi019 conmon[100636]: 2023-09-30T09:14:46.669+0000 7fd71a1691c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:14:46.964 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:46 smithi019 conmon[100636]: 2023-09-30T09:14:46.785+0000 7fd71a1691c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:14:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:46 smithi167 ceph-mon[104403]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:46 smithi167 ceph-mon[104403]: mgrmap e25: x(active, since 4s) 2023-09-30T09:14:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:46 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:46 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:46 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1009436978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:47.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:47 smithi019 conmon[100636]: 2023-09-30T09:14:47.131+0000 7fd71a1691c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:14:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[100214]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[100214]: mgrmap e25: x(active, since 4s) 2023-09-30T09:14:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:47.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1009436978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:47.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[118983]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:47.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[118983]: mgrmap e25: x(active, since 4s) 2023-09-30T09:14:47.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:47.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:47.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:46 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1009436978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:47.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:47 smithi019 conmon[100636]: 2023-09-30T09:14:47.467+0000 7fd71a1691c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:14:48.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi019 to 2501M 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:48 smithi167 ceph-mon[104403]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:48.469 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:48 smithi019 conmon[100636]: 2023-09-30T09:14:48.308+0000 7fd71a1691c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi019 to 2501M 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[100214]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi019 to 2501M 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:48.473 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:48 smithi019 ceph-mon[118983]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:14:48.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:48 smithi019 conmon[100636]: 2023-09-30T09:14:48.468+0000 7fd71a1691c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:14:48.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:48 smithi019 conmon[100636]: 2023-09-30T09:14:48.714+0000 7fd71a1691c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:14:49.147 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:48 smithi019 conmon[100636]: 2023-09-30T09:14:48.859+0000 7fd71a1691c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:14:49.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:49 smithi019 conmon[100636]: 2023-09-30T09:14:49.145+0000 7fd71a1691c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:14:50.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:49 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/722726946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:50.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:49 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/722726946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:49 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/722726946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:50.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:50 smithi019 conmon[100636]: 2023-09-30T09:14:50.202+0000 7fd71a1691c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:14:50.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:50 smithi019 conmon[100636]: [30/Sep/2023:09:14:50] ENGINE Bus STARTING 2023-09-30T09:14:50.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:50 smithi019 conmon[100636]: CherryPy Checker: 2023-09-30T09:14:50.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:50 smithi019 conmon[100636]: The Application mounted at '' has an empty config. 2023-09-30T09:14:50.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:50 smithi019 conmon[100636]: 2023-09-30T09:14:50.586 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:50 smithi019 conmon[100636]: [30/Sep/2023:09:14:50] ENGINE Serving on http://:::9283 2023-09-30T09:14:50.586 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:50 smithi019 conmon[100636]: [30/Sep/2023:09:14:50] ENGINE Bus STARTED 2023-09-30T09:14:51.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:50 smithi167 ceph-mon[104403]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:14:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:50 smithi167 ceph-mon[104403]: Standby manager daemon y started 2023-09-30T09:14:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:50 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:14:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:50 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:14:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:50 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-30T09:14:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:50 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:14:51.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[100214]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:14:51.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[100214]: Standby manager daemon y started 2023-09-30T09:14:51.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:14:51.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:14:51.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-30T09:14:51.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:14:51.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[118983]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:14:51.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[118983]: Standby manager daemon y started 2023-09-30T09:14:51.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:14:51.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:14:51.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-30T09:14:51.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:50 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.19:0/3867266367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: mgrmap e26: x(active, since 8s), standbys: y 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:52.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:14:52.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:51 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2308287296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: mgrmap e26: x(active, since 8s), standbys: y 2023-09-30T09:14:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:14:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:14:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2308287296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: mgrmap e26: x(active, since 8s), standbys: y 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-30T09:14:52.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:52.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:14:52.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:14:52.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:51 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2308287296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Adjusting osd_memory_target on smithi167 to 2500M 2023-09-30T09:14:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:14:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:14:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:14:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:14:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:14:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Adjusting osd_memory_target on smithi167 to 2500M 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:14:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:53 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Adjusting osd_memory_target on smithi167 to 2500M 2023-09-30T09:14:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:14:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:14:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:53.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:53 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:54 smithi019 ceph-mon[118983]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-30T09:14:54.385 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:54 smithi019 ceph-mon[118983]: Reconfiguring daemon alertmanager.a on smithi019 2023-09-30T09:14:54.385 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:54 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/612853325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:54.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:54 smithi019 ceph-mon[100214]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-30T09:14:54.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:54 smithi019 ceph-mon[100214]: Reconfiguring daemon alertmanager.a on smithi019 2023-09-30T09:14:54.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:54 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/612853325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:54 smithi167 ceph-mon[104403]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-30T09:14:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:54 smithi167 ceph-mon[104403]: Reconfiguring daemon alertmanager.a on smithi019 2023-09-30T09:14:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:54 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/612853325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:55.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:55 smithi019 ceph-mon[100214]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:14:55.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:55 smithi019 ceph-mon[118983]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:14:55.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:54 smithi019 systemd[1]: Stopping Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:14:55.381 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:55 smithi019 conmon[177261]: ts=2023-09-30T09:14:55.093Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-30T09:14:55.381 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:55 smithi019 bash[200707]: 51fb7ff237b45584ec3fe8f3ccb2742f25c78682c3b61c68f9c7a6c0ca637203 2023-09-30T09:14:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:55 smithi167 ceph-mon[104403]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:14:56.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:55 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-30T09:14:56.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:55 smithi019 systemd[1]: Stopped Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:14:56.085 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:55 smithi019 systemd[1]: Starting Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:14:56.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:14:55 smithi167 conmon[158886]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:55] "GET /metrics HTTP/1.1" 200 30879 "" "Prometheus/2.43.0" 2023-09-30T09:14:56.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 conmon[201097]: ts=2023-09-30T09:14:56.534Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-30T09:14:56.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 conmon[201097]: ts=2023-09-30T09:14:56.534Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-30T09:14:56.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 conmon[201097]: ts=2023-09-30T09:14:56.536Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.19 port=9094 2023-09-30T09:14:56.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 conmon[201097]: ts=2023-09-30T09:14:56.537Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-30T09:14:56.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 bash[201037]: b16b2fb388da6510e25ce057c250c9fa7f5d7e82ec53d1b96a8573f420b97bc1 2023-09-30T09:14:56.836 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 systemd[1]: Started Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:14:56.836 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 conmon[201097]: ts=2023-09-30T09:14:56.566Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-30T09:14:56.836 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 conmon[201097]: ts=2023-09-30T09:14:56.566Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-30T09:14:56.836 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 conmon[201097]: ts=2023-09-30T09:14:56.569Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-30T09:14:56.836 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:56 smithi019 conmon[201097]: ts=2023-09-30T09:14:56.569Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-30T09:14:57.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[100214]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:14:57.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/930888889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:57.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:57.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:57.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:57.282 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[118983]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:14:57.282 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/930888889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:57.282 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:57.282 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:57.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:57 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:57.283 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:56 smithi019 systemd[1]: Stopping Ceph node-exporter.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:14:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:57 smithi167 ceph-mon[104403]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:14:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:57 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/930888889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:57 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:57 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:57 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:14:58.066 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:57 smithi019 bash[201333]: 63714dd11e25f25601eb6135fdd8e97c1c42466033c264e1c99bb0bfed906da1 2023-09-30T09:14:58.066 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:57 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:14:58.066 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:57 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-30T09:14:58.066 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:57 smithi019 systemd[1]: Stopped Ceph node-exporter.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:14:58.067 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:57 smithi019 systemd[1]: Starting Ceph node-exporter.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:14:58.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:58 smithi019 ceph-mon[100214]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-30T09:14:58.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:58 smithi019 ceph-mon[100214]: Reconfiguring daemon node-exporter.a on smithi019 2023-09-30T09:14:58.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:58 smithi019 ceph-mon[118983]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-30T09:14:58.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:58 smithi019 ceph-mon[118983]: Reconfiguring daemon node-exporter.a on smithi019 2023-09-30T09:14:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:58 smithi167 ceph-mon[104403]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-30T09:14:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:58 smithi167 ceph-mon[104403]: Reconfiguring daemon node-exporter.a on smithi019 2023-09-30T09:14:58.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[100636]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:14:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:14:58.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201097]: ts=2023-09-30T09:14:58.537Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000246374s 2023-09-30T09:14:58.836 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.535Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-30T09:14:58.836 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.535Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-30T09:14:58.836 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.535Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-30T09:14:58.836 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-30T09:14:58.836 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-30T09:14:58.836 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-30T09:14:58.836 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-30T09:14:58.836 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=node_exporter.go:117 level=info collector=arp 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.536Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=edac 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-30T09:14:58.837 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-30T09:14:58.838 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=os 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=stat 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=time 2023-09-30T09:14:58.839 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-30T09:14:58.840 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=uname 2023-09-30T09:14:58.840 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-30T09:14:58.840 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-30T09:14:58.840 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-30T09:14:58.840 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-30T09:14:58.840 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 conmon[201699]: ts=2023-09-30T09:14:58.537Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-30T09:14:58.840 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 bash[201658]: 63ac922663db64aba5fa0bcb9b17921ae749c25ff950ab0599a3051ca205d79f 2023-09-30T09:14:58.840 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:14:58 smithi019 systemd[1]: Started Ceph node-exporter.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:14:59.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:59 smithi019 ceph-mon[100214]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:14:59.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:59 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:59.545 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:59 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:59.545 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:14:59 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2206628241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:59.545 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:59 smithi019 ceph-mon[118983]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:14:59.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:59 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:59.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:59 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:59.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:14:59 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2206628241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:14:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:59 smithi167 ceph-mon[104403]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:14:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:59 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:14:59.547 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:59 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:00.078 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:14:59 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2206628241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:00 smithi019 ceph-mon[100214]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:15:00.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:00 smithi019 ceph-mon[100214]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:15:00.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:00 smithi019 ceph-mon[118983]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:15:00.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:00 smithi019 ceph-mon[118983]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:15:00.342 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:00 smithi167 ceph-mon[104403]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:15:00.343 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:00 smithi167 ceph-mon[104403]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:15:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:01 smithi167 ceph-mon[104403]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:01 smithi019 ceph-mon[100214]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:01 smithi019 ceph-mon[118983]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:01.717 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 systemd[1]: Stopping Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.716Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-30T09:15:02.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-30T09:15:02.023 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-30T09:15:02.023 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-30T09:15:02.023 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:01 smithi167 conmon[153683]: ts=2023-09-30T09:15:01.717Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-30T09:15:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:02 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3108235651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:02 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3108235651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:02.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:02 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3108235651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:02.377 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 bash[171239]: 8187f7a0d1a5802ceb93b826b2500065857624ef8525235754e788a8c8a1c7eb 2023-09-30T09:15:02.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-30T09:15:02.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 systemd[1]: Stopped Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:02.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 systemd[1]: Starting Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:03 smithi167 ceph-mon[104403]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:03 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:03 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:03.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 bash[171567]: 57a4ce72a40439712aab3b73402b68e94dcc376008d77b48c008971f8f2736f0 2023-09-30T09:15:03.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 conmon[171608]: ts=2023-09-30T09:15:02.997Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-30T09:15:03.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 conmon[171608]: ts=2023-09-30T09:15:02.998Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 conmon[171608]: ts=2023-09-30T09:15:02.998Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi167 (none))" 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 conmon[171608]: ts=2023-09-30T09:15:02.998Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:02 smithi167 conmon[171608]: ts=2023-09-30T09:15:02.998Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.000Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.000Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.001Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.001Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.002Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.002Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.946µs 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.002Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 systemd[1]: Started Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:03.273 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.003Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.021Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.022Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.022Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.609µs wal_replay_duration=19.473059ms wbl_replay_duration=218ns total_replay_duration=19.514301ms 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.025Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.025Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.025Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.042Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.486922ms db_storage=918ns remote_storage=1.337µs web_handler=500ns query_engine=751ns scrape=187.176µs scrape_sd=95.598µs notify=10.133µs notify_sd=11.706µs rules=16.861113ms tracing=7.999µs 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.042Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-30T09:15:03.274 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:03 smithi167 conmon[171608]: ts=2023-09-30T09:15:03.042Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-30T09:15:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:03 smithi019 ceph-mon[100214]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:03 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:03 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:03 smithi019 ceph-mon[118983]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:03 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:03 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:04 smithi167 ceph-mon[104403]: Reconfiguring grafana.a (dependencies changed)... 2023-09-30T09:15:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:04 smithi167 ceph-mon[104403]: Reconfiguring daemon grafana.a on smithi167 2023-09-30T09:15:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:04 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1272864640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:04 smithi019 ceph-mon[100214]: Reconfiguring grafana.a (dependencies changed)... 2023-09-30T09:15:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:04 smithi019 ceph-mon[100214]: Reconfiguring daemon grafana.a on smithi167 2023-09-30T09:15:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:04 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1272864640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:04 smithi019 ceph-mon[118983]: Reconfiguring grafana.a (dependencies changed)... 2023-09-30T09:15:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:04 smithi019 ceph-mon[118983]: Reconfiguring daemon grafana.a on smithi167 2023-09-30T09:15:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:04 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1272864640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:05 smithi167 ceph-mon[104403]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:05.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:05 smithi019 ceph-mon[100214]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:05.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:05 smithi019 ceph-mon[118983]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:05.741 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:05 smithi167 systemd[1]: Stopping Ceph grafana.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:06 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/4040511619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:06.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:06 smithi167 bash[172088]: 1e414220bf374cfcc91ff68c39306554766b69236339d14cf7a9f05f0c76d33f 2023-09-30T09:15:06.523 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:06 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:15:06.523 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:06 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-30T09:15:06.523 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:06 smithi167 systemd[1]: Stopped Ceph grafana.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:06.523 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:06 smithi167 systemd[1]: Starting Ceph grafana.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:06.540 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:06 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4040511619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:06.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:06 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/4040511619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:06.835 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:15:06 smithi019 conmon[201097]: ts=2023-09-30T09:15:06.538Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001677291s 2023-09-30T09:15:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:07 smithi167 ceph-mon[104403]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:07 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:07 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:07.272 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:06 smithi167 bash[172416]: f7c9f74b80886c83d45e9a4ce841b90929cdca4f5133696113b2831ed06b1d11 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:06 smithi167 systemd[1]: Started Ceph grafana.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066412379Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.06659233Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066610267Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066617351Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.06662411Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066630821Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.0666365Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066642335Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-30T09:15:07.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066649506Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066655854Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066662224Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066667828Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.06667346Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=settings t=2023-09-30T09:15:07.066679308Z level=info msg="App mode production" 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=sqlstore t=2023-09-30T09:15:07.066734866Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=sqlstore t=2023-09-30T09:15:07.066751774Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=migrator t=2023-09-30T09:15:07.081253379Z level=info msg="Starting DB migrations" 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=migrator t=2023-09-30T09:15:07.085853179Z level=info msg="migrations completed" performed=0 skipped=481 duration=455.792µs 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=sqlstore t=2023-09-30T09:15:07.086214878Z level=info msg="Created default organization" 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=plugin.loader t=2023-09-30T09:15:07.143659674Z level=info msg="Plugin registered" pluginID=input 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=plugin.loader t=2023-09-30T09:15:07.172991707Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-30T09:15:07.274 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=plugin.loader t=2023-09-30T09:15:07.173030407Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-30T09:15:07.275 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=secrets t=2023-09-30T09:15:07.173067941Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-30T09:15:07.275 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=query_data t=2023-09-30T09:15:07.174805537Z level=info msg="Query Service initialization" 2023-09-30T09:15:07.275 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=live.push_http t=2023-09-30T09:15:07.177216274Z level=info msg="Live Push Gateway initialization" 2023-09-30T09:15:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:07 smithi019 ceph-mon[100214]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:07 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:07 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:07.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:07 smithi019 ceph-mon[118983]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:07 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:07 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:07.691 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:07 smithi167 systemd[1]: Stopping Ceph node-exporter.b for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:08 smithi167 ceph-mon[104403]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-30T09:15:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:08 smithi167 ceph-mon[104403]: Reconfiguring daemon node-exporter.b on smithi167 2023-09-30T09:15:08.272 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:15:08.272 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 bash[172688]: 23727543383f3736c6549181ca9f0cdf1763b64852ada7cdd903a919dcb43a54 2023-09-30T09:15:08.272 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:07 smithi167 conmon[172457]: logger=infra.usagestats.collector t=2023-09-30T09:15:07.950553249Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-30T09:15:08.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=provisioning.datasources t=2023-09-30T09:15:08.147215861Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-30T09:15:08.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=provisioning.datasources t=2023-09-30T09:15:08.147252293Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-30T09:15:08.273 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=provisioning.datasources t=2023-09-30T09:15:08.158499408Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-30T09:15:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:08 smithi019 ceph-mon[100214]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-30T09:15:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:08 smithi019 ceph-mon[100214]: Reconfiguring daemon node-exporter.b on smithi167 2023-09-30T09:15:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:08 smithi019 ceph-mon[118983]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-30T09:15:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:08 smithi019 ceph-mon[118983]: Reconfiguring daemon node-exporter.b on smithi167 2023-09-30T09:15:08.772 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-30T09:15:08.772 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 systemd[1]: Stopped Ceph node-exporter.b for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:08.772 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 systemd[1]: Starting Ceph node-exporter.b for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:08.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=provisioning.alerting t=2023-09-30T09:15:08.306691951Z level=info msg="starting to provision alerting" 2023-09-30T09:15:08.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=provisioning.alerting t=2023-09-30T09:15:08.306728371Z level=info msg="finished to provision alerting" 2023-09-30T09:15:08.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=grafanaStorageLogger t=2023-09-30T09:15:08.306876117Z level=info msg="storage starting" 2023-09-30T09:15:08.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=ngalert.state.manager t=2023-09-30T09:15:08.306920698Z level=info msg="Warming state cache for startup" 2023-09-30T09:15:08.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=http.server t=2023-09-30T09:15:08.308847806Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-30T09:15:08.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=ngalert.state.manager t=2023-09-30T09:15:08.355044843Z level=info msg="State cache has been initialized" states=0 duration=48.122654ms 2023-09-30T09:15:08.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=ticker t=2023-09-30T09:15:08.355149815Z level=info msg=starting first_tick=2023-09-30T09:15:10Z 2023-09-30T09:15:08.773 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[172457]: logger=ngalert.multiorg.alertmanager t=2023-09-30T09:15:08.355167068Z level=info msg="starting MultiOrg Alertmanager" 2023-09-30T09:15:09.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:09 smithi167 conmon[158886]: [30/Sep/2023:09:15:09] ENGINE Bus STOPPING 2023-09-30T09:15:09.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:09 smithi167 conmon[158886]: [30/Sep/2023:09:15:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:15:09.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:09 smithi167 conmon[158886]: [30/Sep/2023:09:15:09] ENGINE Bus STOPPED 2023-09-30T09:15:09.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:09 smithi167 conmon[158886]: [30/Sep/2023:09:15:09] ENGINE Bus STARTING 2023-09-30T09:15:09.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:09 smithi167 conmon[158886]: [30/Sep/2023:09:15:09] ENGINE Serving on http://:::9283 2023-09-30T09:15:09.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:09 smithi167 conmon[158886]: [30/Sep/2023:09:15:09] ENGINE Bus STARTED 2023-09-30T09:15:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2303125574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:15:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:09 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.910Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.910Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.910Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.910Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.910Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.910Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=arp 2023-09-30T09:15:09.274 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=edac 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-30T09:15:09.275 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-30T09:15:09.276 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=os 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=stat 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-30T09:15:09.277 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=time 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=uname 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 conmon[173068]: ts=2023-09-30T09:15:08.911Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 bash[173027]: d6c59fa1a46188b98f96b42a4a61170a66ba6b368023c8b05460e73c82932884 2023-09-30T09:15:09.278 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:15:08 smithi167 systemd[1]: Started Ceph node-exporter.b for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:09.537 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:09.538 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2303125574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:09.538 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.538 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.538 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:15:09.538 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2023-09-30T09:15:09.538 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.538 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:09.538 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2303125574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:09.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:09.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:15:09.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:09 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:10.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[100214]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:15:10.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[118983]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:15:10.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[118983]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2023-09-30T09:15:10.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[118983]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:10.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[118983]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:15:10.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[118983]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-30T09:15:10.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[118983]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-30T09:15:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:10 smithi167 ceph-mon[104403]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-30T09:15:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:10 smithi167 ceph-mon[104403]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2023-09-30T09:15:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:10 smithi167 ceph-mon[104403]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:10 smithi167 ceph-mon[104403]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:15:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:10 smithi167 ceph-mon[104403]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-30T09:15:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:10 smithi167 ceph-mon[104403]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-30T09:15:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[100214]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2023-09-30T09:15:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[100214]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[100214]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-30T09:15:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[100214]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-30T09:15:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:10 smithi019 ceph-mon[100214]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-30T09:15:11.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[100214]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:11.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2062657127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[118983]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:11.361 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:11.362 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:15:11.362 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:11.362 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:11 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2062657127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:11.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:11 smithi167 conmon[158886]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:15:11] "GET /metrics HTTP/1.1" 200 33770 "" "Prometheus/2.43.0" 2023-09-30T09:15:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:11 smithi167 ceph-mon[104403]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:11 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:11 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:11 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:11 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:15:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:11 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:11 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2062657127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:12.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:12 smithi019 systemd[1]: Stopping Ceph mgr.y for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:12 smithi019 ceph-mon[100214]: Upgrade: Updating mgr.y 2023-09-30T09:15:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:12 smithi019 ceph-mon[100214]: Deploying daemon mgr.y on smithi019 2023-09-30T09:15:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:12 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:12 smithi019 ceph-mon[118983]: Upgrade: Updating mgr.y 2023-09-30T09:15:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:12 smithi019 ceph-mon[118983]: Deploying daemon mgr.y on smithi019 2023-09-30T09:15:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:12 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:12 smithi167 ceph-mon[104403]: Upgrade: Updating mgr.y 2023-09-30T09:15:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:12 smithi167 ceph-mon[104403]: Deploying daemon mgr.y on smithi019 2023-09-30T09:15:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:12 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:13.005 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:13 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:15:13.317 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:13 smithi019 ceph-mon[100214]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:13.318 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:13 smithi019 ceph-mon[118983]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:13.318 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:13 smithi019 bash[202484]: 08339cd406da9636bc25fbc7ec0177b9ada7c27042f99ae70b336a271243e40a 2023-09-30T09:15:13.318 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:13 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-30T09:15:13.318 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:13 smithi019 systemd[1]: Stopped Ceph mgr.y for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:13 smithi167 ceph-mon[104403]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:13.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:13 smithi019 systemd[1]: Starting Ceph mgr.y for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:14.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:14 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/949387716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:14.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:14 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:14.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:14 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:14.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:14 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:14.183 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:13 smithi019 ceph-mgr[202923]: -- 172.21.15.19:0/4276625357 <== mon.0 v2:172.21.15.19:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5632d31bf1e0 con 0x5632d3197000 2023-09-30T09:15:14.183 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:13 smithi019 bash[202858]: f13d86332ace2022f4b7a9b9112a957f3f25a08c135233b1ac86be3ee2e43a41 2023-09-30T09:15:14.183 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:13 smithi019 systemd[1]: Started Ceph mgr.y for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:14.184 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:14 smithi019 conmon[202899]: 2023-09-30T09:15:14.079+0000 7fd52b7dc200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:15:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:14 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/949387716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:14 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:14 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:14 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:14 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/949387716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:14 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:14 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:14 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:14.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:14 smithi019 conmon[202899]: 2023-09-30T09:15:14.181+0000 7fd52b7dc200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:15:14.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:14 smithi019 conmon[202899]: 2023-09-30T09:15:14.298+0000 7fd52b7dc200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:15:14.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:14 smithi019 conmon[202899]: 2023-09-30T09:15:14.426+0000 7fd52b7dc200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:15:15.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:14 smithi019 conmon[202899]: 2023-09-30T09:15:14.653+0000 7fd52b7dc200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:15:15.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:15 smithi019 ceph-mon[100214]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:15.467 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:15 smithi019 conmon[202899]: 2023-09-30T09:15:15.169+0000 7fd52b7dc200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:15:15.467 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:15 smithi019 conmon[202899]: 2023-09-30T09:15:15.330+0000 7fd52b7dc200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:15:15.467 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:15 smithi019 conmon[202899]: 2023-09-30T09:15:15.446+0000 7fd52b7dc200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:15:15.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:15 smithi019 ceph-mon[118983]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:15 smithi167 ceph-mon[104403]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:15.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:15 smithi019 conmon[202899]: 2023-09-30T09:15:15.575+0000 7fd52b7dc200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:15:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:16 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3504446413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:16.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:16 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3504446413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:16.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:16 smithi019 conmon[202899]: 2023-09-30T09:15:16.167+0000 7fd52b7dc200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:15:16.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:16 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3504446413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:17 smithi019 ceph-mon[100214]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:17 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:17 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:17.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:17 smithi019 conmon[202899]: 2023-09-30T09:15:17.023+0000 7fd52b7dc200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:15:17.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:17 smithi019 conmon[202899]: 2023-09-30T09:15:17.130+0000 7fd52b7dc200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:15:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:17 smithi019 ceph-mon[118983]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:17 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:17 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:17 smithi167 ceph-mon[104403]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:17 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:17 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.519 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.519 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.519 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.519 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.519 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[100214]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:18.519 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2632356392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:18.519 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:18 smithi019 conmon[202899]: 2023-09-30T09:15:18.189+0000 7fd52b7dc200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:18.520 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:18 smithi019 conmon[202899]: 2023-09-30T09:15:18.311+0000 7fd52b7dc200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:15:18.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[118983]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:18.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:18 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2632356392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:18.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:18 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:18 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:18 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:18 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:18 smithi167 ceph-mon[104403]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:18 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2632356392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:18.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:18 smithi019 conmon[202899]: 2023-09-30T09:15:18.547+0000 7fd52b7dc200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:19.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:18 smithi167 conmon[171608]: ts=2023-09-30T09:15:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:15:19.178 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:18 smithi019 conmon[202899]: 2023-09-30T09:15:18.912+0000 7fd52b7dc200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:15:19.434 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:19 smithi019 conmon[202899]: 2023-09-30T09:15:19.176+0000 7fd52b7dc200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:15:19.434 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:19 smithi019 conmon[202899]: 2023-09-30T09:15:19.374+0000 7fd52b7dc200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:15:20.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:19 smithi019 conmon[202899]: 2023-09-30T09:15:19.729+0000 7fd52b7dc200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:15:20.782 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:20 smithi019 conmon[202899]: 2023-09-30T09:15:20.610+0000 7fd52b7dc200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:15:21.039 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:20 smithi019 conmon[202899]: 2023-09-30T09:15:20.780+0000 7fd52b7dc200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:15:21.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:20 smithi019 ceph-mon[100214]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:21.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:20 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2656071650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:21.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:20 smithi019 ceph-mon[118983]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:21.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:20 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2656071650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:20 smithi167 ceph-mon[104403]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:20 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2656071650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:21.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:21 smithi019 conmon[202899]: 2023-09-30T09:15:21.037+0000 7fd52b7dc200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:15:21.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:21 smithi019 conmon[202899]: 2023-09-30T09:15:21.180+0000 7fd52b7dc200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:15:21.629 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 conmon[158886]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:15:21] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.43.0" 2023-09-30T09:15:21.630 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 conmon[158886]: 2023-09-30T09:15:21.462+0000 7f82e15f3700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-30T09:15:21.630 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 conmon[158886]: ignoring --setuser ceph since I am not root 2023-09-30T09:15:21.630 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 conmon[158886]: ignoring --setgroup ceph since I am not root 2023-09-30T09:15:21.630 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 ceph-mgr[158911]: -- 172.21.15.167:0/478363376 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5557731cd1e0 con 0x5557731a4c00 2023-09-30T09:15:21.630 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 conmon[158886]: 2023-09-30T09:15:21.628+0000 7f56aa933200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:15:21.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:21 smithi019 conmon[202899]: 2023-09-30T09:15:21.472+0000 7fd52b7dc200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:15:21.894 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 conmon[158886]: 2023-09-30T09:15:21.709+0000 7f56aa933200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:15:21.894 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 conmon[158886]: 2023-09-30T09:15:21.799+0000 7f56aa933200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:15:22.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:21 smithi167 conmon[158886]: 2023-09-30T09:15:21.893+0000 7f56aa933200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:15:22.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:22 smithi167 conmon[158886]: 2023-09-30T09:15:22.064+0000 7f56aa933200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:15:22.558 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.558 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.558 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:22.558 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: osdmap e88: 8 total, 8 up, 8 in 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: from='mgr.25426 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[100214]: mgrmap e27: y(active, starting, since 0.00661397s) 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:22.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:22.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:22.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-30T09:15:22.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-30T09:15:22.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: osdmap e88: 8 total, 8 up, 8 in 2023-09-30T09:15:22.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: from='mgr.25426 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-30T09:15:22.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:22 smithi019 ceph-mon[118983]: mgrmap e27: y(active, starting, since 0.00661397s) 2023-09-30T09:15:22.562 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:22 smithi167 conmon[158886]: 2023-09-30T09:15:22.430+0000 7f56aa933200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:15:22.562 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.562 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.562 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:22.562 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:22.562 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' 2023-09-30T09:15:22.562 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:22.562 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 172.21.15.167:0/3365789255' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-30T09:15:22.563 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-30T09:15:22.563 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: osdmap e88: 8 total, 8 up, 8 in 2023-09-30T09:15:22.563 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: from='mgr.25426 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-30T09:15:22.563 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:22 smithi167 ceph-mon[104403]: mgrmap e27: y(active, starting, since 0.00661397s) 2023-09-30T09:15:22.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.556+0000 7fd52b7dc200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:15:22.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE Bus STARTING 2023-09-30T09:15:22.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: CherryPy Checker: 2023-09-30T09:15:22.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: The Application mounted at '' has an empty config. 2023-09-30T09:15:22.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE Serving on http://:::9283 2023-09-30T09:15:22.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE Bus STARTED 2023-09-30T09:15:22.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE Bus STOPPING 2023-09-30T09:15:22.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:15:22.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE Bus STOPPED 2023-09-30T09:15:22.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE Bus STARTING 2023-09-30T09:15:22.868 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:22 smithi167 conmon[158886]: 2023-09-30T09:15:22.561+0000 7f56aa933200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:15:22.868 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:22 smithi167 conmon[158886]: 2023-09-30T09:15:22.652+0000 7f56aa933200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:15:22.868 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:22 smithi167 conmon[158886]: 2023-09-30T09:15:22.747+0000 7f56aa933200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:15:23.294 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd48cb3f700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.294 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd48cb3f700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.294 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd48cb3f700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.294 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd48cb3f700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.294 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd48cb3f700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.294 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd489b39700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.294 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd489b39700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.294 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd489b39700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.295 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd489b39700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.295 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: 2023-09-30T09:15:22.857+0000 7fd489b39700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:15:23.295 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE Serving on http://:::9283 2023-09-30T09:15:23.295 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:22 smithi019 conmon[202899]: [30/Sep/2023:09:15:22] ENGINE Bus STARTED 2023-09-30T09:15:23.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:23 smithi167 conmon[158886]: 2023-09-30T09:15:23.186+0000 7f56aa933200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:15:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: Active manager daemon y restarted 2023-09-30T09:15:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: Activating manager daemon y 2023-09-30T09:15:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: osdmap e89: 8 total, 8 up, 8 in 2023-09-30T09:15:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: mgrmap e28: y(active, starting, since 0.00579721s) 2023-09-30T09:15:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:15:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: Manager daemon y is now available 2023-09-30T09:15:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:23.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:23.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:23.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:15:23.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:15:23.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:23 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/67144538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: Active manager daemon y restarted 2023-09-30T09:15:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: Activating manager daemon y 2023-09-30T09:15:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: osdmap e89: 8 total, 8 up, 8 in 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: mgrmap e28: y(active, starting, since 0.00579721s) 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:15:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:23.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: Manager daemon y is now available 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/67144538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: Active manager daemon y restarted 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: Activating manager daemon y 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: osdmap e89: 8 total, 8 up, 8 in 2023-09-30T09:15:23.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: mgrmap e28: y(active, starting, since 0.00579721s) 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:15:23.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:23.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: Manager daemon y is now available 2023-09-30T09:15:23.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:23.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:23.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:23.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:15:23.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:15:23.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:23 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/67144538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:24.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:23 smithi167 conmon[158886]: 2023-09-30T09:15:23.940+0000 7f56aa933200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:15:24.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:24 smithi167 conmon[158886]: 2023-09-30T09:15:24.034+0000 7f56aa933200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:15:24.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[100214]: [30/Sep/2023:09:15:23] ENGINE Bus STARTING 2023-09-30T09:15:24.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[100214]: [30/Sep/2023:09:15:23] ENGINE Serving on http://172.21.15.19:8765 2023-09-30T09:15:24.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[100214]: mgrmap e29: y(active, since 1.00846s) 2023-09-30T09:15:24.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[100214]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:24.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[100214]: [30/Sep/2023:09:15:23] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:15:24.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[100214]: [30/Sep/2023:09:15:23] ENGINE Bus STARTED 2023-09-30T09:15:24.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[118983]: [30/Sep/2023:09:15:23] ENGINE Bus STARTING 2023-09-30T09:15:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[118983]: [30/Sep/2023:09:15:23] ENGINE Serving on http://172.21.15.19:8765 2023-09-30T09:15:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[118983]: mgrmap e29: y(active, since 1.00846s) 2023-09-30T09:15:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[118983]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[118983]: [30/Sep/2023:09:15:23] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:15:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:24 smithi019 ceph-mon[118983]: [30/Sep/2023:09:15:23] ENGINE Bus STARTED 2023-09-30T09:15:24.936 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:24 smithi167 ceph-mon[104403]: [30/Sep/2023:09:15:23] ENGINE Bus STARTING 2023-09-30T09:15:25.030 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:24 smithi167 ceph-mon[104403]: [30/Sep/2023:09:15:23] ENGINE Serving on http://172.21.15.19:8765 2023-09-30T09:15:25.030 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:24 smithi167 ceph-mon[104403]: mgrmap e29: y(active, since 1.00846s) 2023-09-30T09:15:25.031 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:24 smithi167 ceph-mon[104403]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:25.031 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:24 smithi167 ceph-mon[104403]: [30/Sep/2023:09:15:23] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:15:25.031 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:24 smithi167 ceph-mon[104403]: [30/Sep/2023:09:15:23] ENGINE Bus STARTED 2023-09-30T09:15:25.233 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:24 smithi167 conmon[158886]: 2023-09-30T09:15:24.934+0000 7f56aa933200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:25.233 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:25 smithi167 conmon[158886]: 2023-09-30T09:15:25.044+0000 7f56aa933200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:15:25.233 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:25 smithi167 conmon[158886]: 2023-09-30T09:15:25.232+0000 7f56aa933200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:25.900 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:25 smithi167 conmon[158886]: 2023-09-30T09:15:25.549+0000 7f56aa933200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:15:25.901 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:25 smithi167 conmon[158886]: 2023-09-30T09:15:25.798+0000 7f56aa933200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:15:26.256 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:25 smithi167 conmon[158886]: 2023-09-30T09:15:25.973+0000 7f56aa933200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:15:26.256 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:25 smithi167 ceph-mon[104403]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:26.256 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:25 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3881833537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:26.256 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:25 smithi167 ceph-mon[104403]: mgrmap e30: y(active, since 3s) 2023-09-30T09:15:26.256 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:25 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:26.256 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:25 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[118983]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3881833537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[118983]: mgrmap e30: y(active, since 3s) 2023-09-30T09:15:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[100214]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3881833537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[100214]: mgrmap e30: y(active, since 3s) 2023-09-30T09:15:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:25 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:26.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:26 smithi167 conmon[158886]: 2023-09-30T09:15:26.255+0000 7f56aa933200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:15:27.391 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:27 smithi167 conmon[158886]: 2023-09-30T09:15:27.050+0000 7f56aa933200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:15:27.392 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:27 smithi167 conmon[158886]: 2023-09-30T09:15:27.192+0000 7f56aa933200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:15:27.739 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:27 smithi167 conmon[158886]: 2023-09-30T09:15:27.394+0000 7f56aa933200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:15:27.740 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:27 smithi167 conmon[158886]: 2023-09-30T09:15:27.516+0000 7f56aa933200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:15:27.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:27 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:27 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:27 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:27 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:27 smithi167 ceph-mon[104403]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:27.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:27 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.741 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:27 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[100214]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[118983]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:27 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:28.021 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:27 smithi167 conmon[158886]: 2023-09-30T09:15:27.738+0000 7f56aa933200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:15:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:28 smithi019 ceph-mon[100214]: mgrmap e31: y(active, since 4s) 2023-09-30T09:15:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:28 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3753320865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:28 smithi019 ceph-mon[118983]: mgrmap e31: y(active, since 4s) 2023-09-30T09:15:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:28 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3753320865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:28.743 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:28 smithi167 conmon[158886]: 2023-09-30T09:15:28.623+0000 7f56aa933200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:15:28.743 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:28 smithi167 conmon[158886]: [30/Sep/2023:09:15:28] ENGINE Bus STARTING 2023-09-30T09:15:28.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:28 smithi167 ceph-mon[104403]: mgrmap e31: y(active, since 4s) 2023-09-30T09:15:28.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:28 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3753320865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:28.744 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:28 smithi167 conmon[171608]: ts=2023-09-30T09:15:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:15:29.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:28 smithi167 conmon[158886]: CherryPy Checker: 2023-09-30T09:15:29.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:28 smithi167 conmon[158886]: The Application mounted at '' has an empty config. 2023-09-30T09:15:29.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:28 smithi167 conmon[158886]: 2023-09-30T09:15:29.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:28 smithi167 conmon[158886]: [30/Sep/2023:09:15:28] ENGINE Serving on http://:::9283 2023-09-30T09:15:29.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:28 smithi167 conmon[158886]: [30/Sep/2023:09:15:28] ENGINE Bus STARTED 2023-09-30T09:15:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:29 smithi167 ceph-mon[104403]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:29 smithi167 ceph-mon[104403]: Standby manager daemon x started 2023-09-30T09:15:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:29 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:15:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:29 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:29 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:15:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:29 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:29.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[100214]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[100214]: Standby manager daemon x started 2023-09-30T09:15:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:15:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:15:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:29.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[118983]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:29.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[118983]: Standby manager daemon x started 2023-09-30T09:15:29.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:15:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:15:29.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:29 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/2110990175' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:30 smithi167 ceph-mon[104403]: mgrmap e32: y(active, since 6s), standbys: x 2023-09-30T09:15:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:30 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:15:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:30 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:30 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:30 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:30 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:30 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/287058932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[118983]: mgrmap e32: y(active, since 6s), standbys: x 2023-09-30T09:15:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:15:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/287058932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[100214]: mgrmap e32: y(active, since 6s), standbys: x 2023-09-30T09:15:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:15:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:30 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/287058932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:31.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:31 smithi167 conmon[158886]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:15:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-30T09:15:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:15:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:15:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:15:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:15:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:15:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:15:31.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:15:31.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:15:31.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:15:31.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:31 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:15:31.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:15:31.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:15:31.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:15:31.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:31.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:31 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:32 smithi167 ceph-mon[104403]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:15:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:32 smithi167 ceph-mon[104403]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:15:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:32 smithi019 ceph-mon[100214]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:15:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:32 smithi019 ceph-mon[100214]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:15:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:32 smithi019 ceph-mon[118983]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-30T09:15:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:32 smithi019 ceph-mon[118983]: Reconfiguring daemon prometheus.a on smithi167 2023-09-30T09:15:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:33 smithi167 ceph-mon[104403]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:15:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:33 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/4278203052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:33.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:33 smithi019 ceph-mon[100214]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:15:33.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:33 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4278203052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:33.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:33 smithi019 ceph-mon[118983]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:15:33.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:33 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/4278203052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:34.353 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 systemd[1]: Stopping Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-30T09:15:34.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.352Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-30T09:15:34.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.353Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-30T09:15:34.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.353Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-30T09:15:34.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 conmon[171608]: ts=2023-09-30T09:15:34.353Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-30T09:15:35.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:34 smithi167 bash[177028]: 57a4ce72a40439712aab3b73402b68e94dcc376008d77b48c008971f8f2736f0 2023-09-30T09:15:35.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-30T09:15:35.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 systemd[1]: Stopped Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:35.272 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 systemd[1]: Starting Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:35.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.643Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-30T09:15:35.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.643Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-30T09:15:35.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.643Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi167 (none))" 2023-09-30T09:15:35.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.643Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-30T09:15:35.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.643Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-30T09:15:35.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.645Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-30T09:15:35.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.645Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.646Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.646Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.648Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.648Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.872µs 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.648Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.649Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 bash[177359]: c625e147e7f2f8b7279faf9702fa57160092fd76d680f2597c355a5ae04907fb 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.663Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 systemd[1]: Started Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.665Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.665Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-30T09:15:35.773 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.665Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.304µs wal_replay_duration=16.947453ms wbl_replay_duration=116ns total_replay_duration=16.991078ms 2023-09-30T09:15:35.774 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.668Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-30T09:15:35.774 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.668Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-30T09:15:35.774 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.668Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-30T09:15:35.774 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.687Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.436537ms db_storage=897ns remote_storage=1.141µs web_handler=432ns query_engine=662ns scrape=196.399µs scrape_sd=76.308µs notify=14.181µs notify_sd=9.656µs rules=17.917268ms tracing=7.184µs 2023-09-30T09:15:35.774 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.687Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-30T09:15:35.774 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:35 smithi167 conmon[177400]: ts=2023-09-30T09:15:35.687Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-30T09:15:36.244 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:35 smithi167 ceph-mon[104403]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:15:36.244 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:35 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2370691834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:36.245 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:35 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:36.245 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:35 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:36.245 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:35 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:36.245 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:35 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[118983]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:15:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2370691834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:36.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[100214]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:15:36.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2370691834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:36.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:36.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:36.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:36.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:35 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:36 smithi167 ceph-mon[104403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:36 smithi019 ceph-mon[100214]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:36 smithi019 ceph-mon[118983]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-30T09:15:38.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:37 smithi167 ceph-mon[104403]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:38.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:37 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:38.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:37 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:15:38.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:37 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:38.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:37 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3689405719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:38.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:37 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[100214]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:15:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3689405719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[118983]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:15:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:15:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3689405719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:37 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:38.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:38 smithi167 systemd[1]: Stopping Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:38 smithi167 ceph-mon[104403]: Upgrade: Updating mgr.x 2023-09-30T09:15:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:38 smithi167 ceph-mon[104403]: Deploying daemon mgr.x on smithi167 2023-09-30T09:15:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:38 smithi019 ceph-mon[100214]: Upgrade: Updating mgr.x 2023-09-30T09:15:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:38 smithi019 ceph-mon[100214]: Deploying daemon mgr.x on smithi167 2023-09-30T09:15:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:38 smithi019 ceph-mon[118983]: Upgrade: Updating mgr.x 2023-09-30T09:15:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:38 smithi019 ceph-mon[118983]: Deploying daemon mgr.x on smithi167 2023-09-30T09:15:39.830 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:39 smithi167 bash[178038]: 833bc43a28aa32377d4f0a2b4a50ef2b67991c9b9947f62a2dcf8998fc9b1b60 2023-09-30T09:15:39.830 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:39 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:15:39.830 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:39 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-30T09:15:39.830 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:39 smithi167 systemd[1]: Stopped Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:40.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:39 smithi167 systemd[1]: Starting Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:39 smithi167 ceph-mon[104403]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:39 smithi019 ceph-mon[100214]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:39 smithi019 ceph-mon[118983]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:40.737 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:40 smithi167 bash[178399]: 3468c121edb90fafd517f14a61dd20ac0faf64e3d34cfbb6f2283443215b194e 2023-09-30T09:15:40.738 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:40 smithi167 systemd[1]: Started Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:40.738 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:40 smithi167 ceph-mgr[178464]: -- 172.21.15.167:0/2313766310 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557a0f9471e0 con 0x557a0f91ec00 2023-09-30T09:15:40.738 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:40 smithi167 conmon[178440]: 2023-09-30T09:15:40.553+0000 7f58ce735200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:15:40.738 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:40 smithi167 conmon[178440]: 2023-09-30T09:15:40.637+0000 7f58ce735200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:15:40.738 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:40 smithi167 conmon[178440]: 2023-09-30T09:15:40.736+0000 7f58ce735200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:15:41.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:40 smithi167 conmon[178440]: 2023-09-30T09:15:40.848+0000 7f58ce735200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:15:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:40 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/4191802690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:40 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:40 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:40 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:40 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/4191802690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:40 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:40 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:40 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:40 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/4191802690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:40 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:40 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:40 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:41.452 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:41 smithi167 conmon[178440]: 2023-09-30T09:15:41.046+0000 7f58ce735200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:15:41.771 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:41 smithi167 conmon[178440]: 2023-09-30T09:15:41.451+0000 7f58ce735200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:15:41.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:41 smithi167 conmon[178440]: 2023-09-30T09:15:41.594+0000 7f58ce735200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:15:41.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:41 smithi167 conmon[178440]: 2023-09-30T09:15:41.690+0000 7f58ce735200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:15:42.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:41 smithi167 conmon[178440]: 2023-09-30T09:15:41.793+0000 7f58ce735200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:15:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:41 smithi167 ceph-mon[104403]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:41 smithi019 ceph-mon[100214]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:41 smithi019 ceph-mon[118983]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:15:42.771 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:42 smithi167 conmon[178440]: 2023-09-30T09:15:42.285+0000 7f58ce735200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:15:43.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:43 smithi167 conmon[178440]: 2023-09-30T09:15:43.037+0000 7f58ce735200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:15:43.272 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:43 smithi167 conmon[178440]: 2023-09-30T09:15:43.121+0000 7f58ce735200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:15:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:42 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2578699401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:43.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:42 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2578699401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:42 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2578699401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:44.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:15:43] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.43.0" 2023-09-30T09:15:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:43 smithi019 ceph-mon[100214]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:43 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:43 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:43 smithi019 ceph-mon[118983]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:43 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:43 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:44.261 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:43 smithi167 conmon[178440]: 2023-09-30T09:15:43.976+0000 7f58ce735200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:44.261 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:44 smithi167 conmon[178440]: 2023-09-30T09:15:44.074+0000 7f58ce735200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:15:44.262 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:44 smithi167 conmon[178440]: 2023-09-30T09:15:44.260+0000 7f58ce735200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:44.262 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:43 smithi167 ceph-mon[104403]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:44.262 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:43 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:44.262 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:43 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:44.818 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:44 smithi167 conmon[178440]: 2023-09-30T09:15:44.566+0000 7f58ce735200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:15:45.076 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:44 smithi167 conmon[178440]: 2023-09-30T09:15:44.817+0000 7f58ce735200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:15:45.076 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:44 smithi167 conmon[178440]: 2023-09-30T09:15:44.980+0000 7f58ce735200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:15:45.343 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:45 smithi167 conmon[178440]: 2023-09-30T09:15:45.273+0000 7f58ce735200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:15:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:45 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:45 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:45 smithi167 ceph-mon[104403]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:45 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3379736920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:45 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:45 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:45 smithi019 ceph-mon[118983]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:45 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3379736920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:45.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:45 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:45 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:45 smithi019 ceph-mon[100214]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:45 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/3379736920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:46.172 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:46 smithi167 conmon[178440]: 2023-09-30T09:15:46.025+0000 7f58ce735200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:15:46.510 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:46 smithi167 conmon[178440]: 2023-09-30T09:15:46.171+0000 7f58ce735200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:15:46.511 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:46 smithi167 conmon[178440]: 2023-09-30T09:15:46.386+0000 7f58ce735200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:15:46.771 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:46 smithi167 conmon[178440]: 2023-09-30T09:15:46.510+0000 7f58ce735200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:15:46.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:46 smithi167 conmon[178440]: 2023-09-30T09:15:46.758+0000 7f58ce735200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:15:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:47 smithi167 ceph-mon[104403]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:47 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/2803868093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:48.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:47 smithi167 conmon[178440]: 2023-09-30T09:15:47.678+0000 7f58ce735200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:15:48.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:47 smithi167 conmon[178440]: [30/Sep/2023:09:15:47] ENGINE Bus STARTING 2023-09-30T09:15:48.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:47 smithi167 conmon[178440]: CherryPy Checker: 2023-09-30T09:15:48.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:47 smithi167 conmon[178440]: The Application mounted at '' has an empty config. 2023-09-30T09:15:48.023 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:47 smithi167 conmon[178440]: 2023-09-30T09:15:48.023 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:47 smithi167 conmon[178440]: [30/Sep/2023:09:15:47] ENGINE Serving on http://:::9283 2023-09-30T09:15:48.023 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:47 smithi167 conmon[178440]: [30/Sep/2023:09:15:47] ENGINE Bus STARTED 2023-09-30T09:15:48.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:47 smithi019 ceph-mon[100214]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:48.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:47 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/2803868093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:48.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:47 smithi019 ceph-mon[118983]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:48.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:47 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/2803868093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: Standby manager daemon x restarted 2023-09-30T09:15:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: Standby manager daemon x started 2023-09-30T09:15:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:15:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:15:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:15:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-30T09:15:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-30T09:15:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-30T09:15:48.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-30T09:15:48.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:15:48.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-30T09:15:48.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: Standby manager daemon x restarted 2023-09-30T09:15:48.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: Standby manager daemon x started 2023-09-30T09:15:48.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:15:48.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:48.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:15:48.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:48.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:48.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:48.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:48.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:48.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:48.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:48.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:15:48.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:48.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-30T09:15:48.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-30T09:15:48.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-30T09:15:48.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-30T09:15:48.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:15:48.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:48 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-30T09:15:49.023 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:48 smithi167 conmon[177400]: ts=2023-09-30T09:15:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:15:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: Standby manager daemon x restarted 2023-09-30T09:15:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: Standby manager daemon x started 2023-09-30T09:15:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:15:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:15:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:15:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1326997147' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-30T09:15:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-30T09:15:49.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:15:49.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:48 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-30T09:15:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[118983]: Upgrade: Setting container_image for all mgr 2023-09-30T09:15:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[118983]: Upgrade: It appears safe to stop mon.a 2023-09-30T09:15:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[118983]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[118983]: mgrmap e33: y(active, since 26s), standbys: x 2023-09-30T09:15:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:15:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[100214]: Upgrade: Setting container_image for all mgr 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[100214]: Upgrade: It appears safe to stop mon.a 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[100214]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[100214]: mgrmap e33: y(active, since 26s), standbys: x 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:15:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:49 smithi019 ceph-mon[100214]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:49 smithi167 ceph-mon[104403]: Upgrade: Setting container_image for all mgr 2023-09-30T09:15:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:49 smithi167 ceph-mon[104403]: Upgrade: It appears safe to stop mon.a 2023-09-30T09:15:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:49 smithi167 ceph-mon[104403]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:49 smithi167 ceph-mon[104403]: mgrmap e33: y(active, since 26s), standbys: x 2023-09-30T09:15:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:49 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' 2023-09-30T09:15:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:49 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:15:50.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:49 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:15:50.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:49 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:15:50.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:50 smithi019 ceph-mon[118983]: Upgrade: Updating mon.a 2023-09-30T09:15:50.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:50 smithi019 ceph-mon[118983]: Deploying daemon mon.a on smithi019 2023-09-30T09:15:50.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:50 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/1145727921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:50.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:50 smithi019 ceph-mon[100214]: Upgrade: Updating mon.a 2023-09-30T09:15:50.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:50 smithi019 ceph-mon[100214]: Deploying daemon mon.a on smithi019 2023-09-30T09:15:50.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:50 smithi019 ceph-mon[100214]: from='client.? 172.21.15.19:0/1145727921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:50 smithi167 ceph-mon[104403]: Upgrade: Updating mon.a 2023-09-30T09:15:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:50 smithi167 ceph-mon[104403]: Deploying daemon mon.a on smithi019 2023-09-30T09:15:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:50 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/1145727921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:51.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:51 smithi019 systemd[1]: Stopping Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:51.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:51 smithi019 conmon[100190]: 2023-09-30T09:15:51.293+0000 7f51c2e69700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:15:51.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:51 smithi019 conmon[100190]: 2023-09-30T09:15:51.294+0000 7f51c2e69700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-30T09:15:51.585 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:51 smithi019 conmon[202899]: [30/Sep/2023:09:15:51] ENGINE Bus STOPPING 2023-09-30T09:15:51.586 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:51 smithi019 conmon[202899]: [30/Sep/2023:09:15:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:15:51.586 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:51 smithi019 conmon[202899]: [30/Sep/2023:09:15:51] ENGINE Bus STOPPED 2023-09-30T09:15:51.586 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:51 smithi019 conmon[202899]: [30/Sep/2023:09:15:51] ENGINE Bus STARTING 2023-09-30T09:15:51.586 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:51 smithi019 conmon[202899]: [30/Sep/2023:09:15:51] ENGINE Serving on http://:::9283 2023-09-30T09:15:51.586 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:51 smithi019 conmon[202899]: [30/Sep/2023:09:15:51] ENGINE Bus STARTED 2023-09-30T09:15:52.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:51 smithi019 bash[216351]: 91f2eb5c552094bf4653affa7296c1bd6a5e866412d2bc337bc706732be19a5a 2023-09-30T09:15:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-30T09:15:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 systemd[1]: Stopped Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 systemd[1]: Starting Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:15:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: set uid:gid to 167:167 (ceph:ceph) 2023-09-30T09:15:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev), process ceph-mon, pid 7 2023-09-30T09:15:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: pidfile_write: ignore empty --pid-file 2023-09-30T09:15:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: load: jerasure load: lrc 2023-09-30T09:15:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: RocksDB version: 7.9.2 2023-09-30T09:15:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Git sha 0 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Compile date 2023-09-29 16:35:50 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: DB SUMMARY 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: DB Session ID: SQ5372C7CYCTJYH0O7JD 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: CURRENT file: CURRENT 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: IDENTITY file: IDENTITY 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1171 Bytes 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 579591 ; 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.error_if_exists: 0 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.create_if_missing: 0 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.paranoid_checks: 1 2023-09-30T09:15:53.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.env: 0x5569492fada0 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.fs: PosixFileSystem 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.info_log: 0x55694b0bb5a0 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_file_opening_threads: 16 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.statistics: (nil) 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.use_fsync: 0 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_log_file_size: 0 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-30T09:15:53.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.keep_log_file_num: 1000 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.recycle_log_file_num: 0 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.allow_fallocate: 1 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.allow_mmap_reads: 0 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.allow_mmap_writes: 0 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.use_direct_reads: 0 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.create_missing_column_families: 0 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.db_log_dir: 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.wal_dir: 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-30T09:15:53.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.advise_random_on_open: 1 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.db_write_buffer_size: 0 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.write_buffer_manager: 0x55694b070fa0 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.rate_limiter: (nil) 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.wal_recovery_mode: 2 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.enable_thread_tracking: 0 2023-09-30T09:15:53.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.enable_pipelined_write: 0 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.unordered_write: 0 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.row_cache: None 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.wal_filter: None 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.allow_ingest_behind: 0 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.two_write_queues: 0 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.manual_wal_flush: 0 2023-09-30T09:15:53.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.wal_compression: 0 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.atomic_flush: 0 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.log_readahead_size: 0 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.best_efforts_recovery: 0 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.allow_data_in_errors: 0 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.db_host_id: __hostname__ 2023-09-30T09:15:53.343 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_background_jobs: 2 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_background_compactions: -1 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_subcompactions: 1 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_total_wal_size: 0 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_open_files: -1 2023-09-30T09:15:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bytes_per_sync: 0 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_readahead_size: 0 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_background_flushes: -1 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Compression algorithms supported: 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: kZSTD supported: 0 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: kXpressCompression supported: 0 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: kLZ4HCCompression supported: 1 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: kZlibCompression supported: 1 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: kSnappyCompression supported: 1 2023-09-30T09:15:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: kLZ4Compression supported: 1 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: kBZip2Compression supported: 0 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.merge_operator: 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_filter: None 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_filter_factory: None 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.sst_partitioner_factory: None 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-30T09:15:53.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55694b0bb6a0) 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: cache_index_and_filter_blocks: 1 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: pin_top_level_index_and_filter: 1 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_type: 0 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_block_index_type: 0 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_shortening: 1 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: checksum: 4 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: no_block_cache: 0 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache: 0x55694b1111f0 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_name: BinnedLRUCache 2023-09-30T09:15:53.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_options: 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: capacity : 536870912 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: num_shard_bits : 4 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: strict_capacity_limit : 0 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: high_pri_pool_ratio: 0.000 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_compressed: (nil) 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: persistent_cache: (nil) 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_size: 4096 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_size_deviation: 10 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_restart_interval: 16 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_block_restart_interval: 1 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: metadata_block_size: 4096 2023-09-30T09:15:53.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: partition_filters: 0 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: use_delta_encoding: 1 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: filter_policy: bloomfilter 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: whole_key_filtering: 1 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: verify_compression: 0 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: read_amp_bytes_per_bit: 0 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: format_version: 5 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: enable_index_compression: 1 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_align: 0 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: max_auto_readahead_size: 262144 2023-09-30T09:15:53.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: prepopulate_block_cache: 0 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: initial_auto_readahead_size: 8192 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: num_file_reads_for_auto_readahead: 2 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.write_buffer_size: 33554432 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_write_buffer_number: 2 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression: NoCompression 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression: Disabled 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.prefix_extractor: nullptr 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.num_levels: 7 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-30T09:15:53.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.level: 32767 2023-09-30T09:15:53.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.strategy: 0 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.enabled: false 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.target_file_size_base: 67108864 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-30T09:15:53.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-30T09:15:53.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.arena_block_size: 1048576 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.disable_auto_compactions: 0 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-30T09:15:53.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.table_properties_collectors: 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.inplace_update_support: 0 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.bloom_locality: 0 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.max_successive_merges: 0 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.paranoid_file_checks: 0 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.force_consistency_checks: 1 2023-09-30T09:15:53.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.report_bg_io_stats: 0 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.ttl: 2592000 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.enable_blob_files: false 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.min_blob_size: 0 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.blob_file_size: 268435456 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-30T09:15:53.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.blob_file_starting_level: 0 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12504, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 93c200b6-f7d8-4377-8c33-cd7077544374 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065352986816, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065352989803, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 529631, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12496, "largest_seqno": 12862, "table_properties": {"data_size": 527171, "index_size": 998, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 4291, "raw_average_key_size": 24, "raw_value_size": 523347, "raw_average_value_size": 2990, "num_data_blocks": 45, "num_entries": 175, "num_filter_entries": 175, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696065352, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "93c200b6-f7d8-4377-8c33-cd7077544374", "db_session_id": "SQ5372C7CYCTJYH0O7JD", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065352989881, "job": 1, "event": "recovery_finished"} 2023-09-30T09:15:53.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55694b1ac000 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: DB pointer 0x55694b19a000 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** DB Stats ** 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:15:53.358 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: L0 1/0 517.22 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 284.9 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: L6 1/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: Sum 2/0 7.03 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 284.9 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 284.9 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.359 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 284.9 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:15:53.360 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative compaction: 0.00 GB write, 88.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval compaction: 0.00 GB write, 88.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout: Block cache BinnedLRUCache@0x55694b1111f0#7 capacity: 512.00 MB usage: 1.58 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.50 KB,9.53674e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: starting mon.a rank 0 at public addrs [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] at bind addrs [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: mon.a@-1(???) e3 preinit fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: mon.a@-1(???).mds e9 new map 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: mon.a@-1(???).mds e9 print_map 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout: e9 2023-09-30T09:15:53.361 INFO:journalctl@ceph.mon.a.smithi019.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: legacy client fscid: 1 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: Filesystem 'foo' (1) 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: fs_name foo 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: epoch 8 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: created 2023-09-30T09:12:07.940181+0000 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: modified 2023-09-30T09:12:24.930975+0000 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: tableserver 0 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: root 0 2023-09-30T09:15:53.362 INFO:journalctl@ceph.mon.a.smithi019.stdout: session_timeout 60 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: session_autoclose 300 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: max_file_size 1099511627776 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: max_xattr_size 65536 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: required_client_features {} 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: last_failure 0 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: last_failure_osd_epoch 0 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: max_mds 1 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: in 0 2023-09-30T09:15:53.363 INFO:journalctl@ceph.mon.a.smithi019.stdout: up {0=14670} 2023-09-30T09:15:53.364 INFO:journalctl@ceph.mon.a.smithi019.stdout: failed 2023-09-30T09:15:53.364 INFO:journalctl@ceph.mon.a.smithi019.stdout: damaged 2023-09-30T09:15:53.364 INFO:journalctl@ceph.mon.a.smithi019.stdout: stopped 2023-09-30T09:15:53.364 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_pools [4] 2023-09-30T09:15:53.364 INFO:journalctl@ceph.mon.a.smithi019.stdout: metadata_pool 3 2023-09-30T09:15:53.364 INFO:journalctl@ceph.mon.a.smithi019.stdout: inline_data disabled 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: balancer 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: bal_rank_mask -1 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: standby_count_wanted 1 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: [mds.foo.smithi019.kcgktg{0:14670} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] compat {c=[1],r=[1],i=[7ff]}] 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: Standby daemons: 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout: [mds.foo.smithi167.bnuarp{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] compat {c=[1],r=[1],i=[7ff]}] 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: mon.a@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:15:53.365 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 bash[216750]: c1584e4cc7f31463fd0eb275db2ada84011832d67c7f751cb1fee23ed1708808 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta expand map: {default=false} 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta from 'false' to 'false' 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta expanded map: {default=false} 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta expand map: {default=info} 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta from 'info' to 'info' 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta expanded map: {default=info} 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta expand map: {default=daemon} 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta expanded map: {default=daemon} 2023-09-30T09:15:53.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta expand map: {default=debug} 2023-09-30T09:15:53.367 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta from 'debug' to 'debug' 2023-09-30T09:15:53.367 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 ceph-mon[216815]: expand_channel_meta expanded map: {default=debug} 2023-09-30T09:15:53.367 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:52 smithi019 systemd[1]: Started Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:15:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:15:53] "GET /metrics HTTP/1.1" 200 33767 "" "Prometheus/2.43.0" 2023-09-30T09:15:54.586 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)": 1 2023-09-30T09:15:54.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:54 smithi019 conmon[202899]: ignoring --setuser ceph since I am not root 2023-09-30T09:15:54.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:54 smithi019 conmon[202899]: ignoring --setgroup ceph since I am not root 2023-09-30T09:15:54.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mgr[202923]: -- 172.21.15.19:0/3014405790 <== mon.2 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557edb9db1e0 con 0x557edb9b2c00 2023-09-30T09:15:54.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:54 smithi019 conmon[202899]: 2023-09-30T09:15:54.647+0000 7f1dd8b3a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:15:54.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:54 smithi019 conmon[202899]: 2023-09-30T09:15:54.759+0000 7f1dd8b3a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/493538923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: mon.a calling monitor election 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:15:54.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: osdmap e89: 8 total, 8 up, 8 in 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: mgrmap e33: y(active, since 31s), standbys: x 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: from='mgr.15780 ' entity='' 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: mgrmap e34: y(active, since 31s), standbys: x 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[216815]: overall HEALTH_OK 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:15:54.764 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/493538923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: mon.a calling monitor election 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: osdmap e89: 8 total, 8 up, 8 in 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: mgrmap e33: y(active, since 31s), standbys: x 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: from='mgr.15780 ' entity='' 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: mgrmap e34: y(active, since 31s), standbys: x 2023-09-30T09:15:54.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:54 smithi019 ceph-mon[118983]: overall HEALTH_OK 2023-09-30T09:15:54.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:54 smithi167 conmon[178440]: ignoring --setuser ceph since I am not root 2023-09-30T09:15:54.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:54 smithi167 conmon[178440]: ignoring --setgroup ceph since I am not root 2023-09-30T09:15:54.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mgr[178464]: -- 172.21.15.167:0/2130343527 <== mon.1 v2:172.21.15.167:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x559c1200f1e0 con 0x559c11fd2800 2023-09-30T09:15:54.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mgr[178464]: -- 172.21.15.167:0/2130343527 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559c11ffb1e0 con 0x559c11fd2800 2023-09-30T09:15:54.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:54 smithi167 conmon[178440]: 2023-09-30T09:15:54.610+0000 7f5ae605c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-30T09:15:54.772 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:54 smithi167 conmon[178440]: 2023-09-30T09:15:54.691+0000 7f5ae605c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-30T09:15:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:15:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/493538923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: from='mgr.15780 172.21.15.19:0/3685447301' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: mon.a calling monitor election 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: osdmap e89: 8 total, 8 up, 8 in 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: mgrmap e33: y(active, since 31s), standbys: x 2023-09-30T09:15:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: from='mgr.15780 ' entity='' 2023-09-30T09:15:54.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: mgrmap e34: y(active, since 31s), standbys: x 2023-09-30T09:15:54.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:54 smithi167 ceph-mon[104403]: overall HEALTH_OK 2023-09-30T09:15:55.046 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:54 smithi167 conmon[178440]: 2023-09-30T09:15:54.780+0000 7f5ae605c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:15:55.046 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:54 smithi167 conmon[178440]: 2023-09-30T09:15:54.874+0000 7f5ae605c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:15:55.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:54 smithi019 conmon[202899]: 2023-09-30T09:15:54.872+0000 7f1dd8b3a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-30T09:15:55.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:54 smithi019 conmon[202899]: 2023-09-30T09:15:54.986+0000 7f1dd8b3a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-30T09:15:55.315 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch ps' 2023-09-30T09:15:55.410 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:55 smithi167 conmon[178440]: 2023-09-30T09:15:55.045+0000 7f5ae605c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:15:55.489 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:55 smithi019 conmon[202899]: 2023-09-30T09:15:55.198+0000 7f1dd8b3a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-30T09:15:55.683 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:15:55.726 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:15:55 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/889428920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:55.726 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:55 smithi167 conmon[178440]: 2023-09-30T09:15:55.409+0000 7f5ae605c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:15:55.726 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:55 smithi167 conmon[178440]: 2023-09-30T09:15:55.542+0000 7f5ae605c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:15:55.727 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:55 smithi167 conmon[178440]: 2023-09-30T09:15:55.630+0000 7f5ae605c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:15:55.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:15:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/889428920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:55.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:55 smithi019 conmon[202899]: 2023-09-30T09:15:55.701+0000 7f1dd8b3a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-30T09:15:55.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:15:55 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/889428920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:15:56.022 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:55 smithi167 conmon[178440]: 2023-09-30T09:15:55.725+0000 7f5ae605c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:15:56.264 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:55 smithi019 conmon[202899]: 2023-09-30T09:15:55.869+0000 7f1dd8b3a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-30T09:15:56.264 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:55 smithi019 conmon[202899]: 2023-09-30T09:15:55.984+0000 7f1dd8b3a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-30T09:15:56.264 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:56 smithi019 conmon[202899]: 2023-09-30T09:15:56.104+0000 7f1dd8b3a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-30T09:15:56.521 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:56 smithi167 conmon[178440]: 2023-09-30T09:15:56.150+0000 7f5ae605c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:15:56.992 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:56 smithi019 conmon[202899]: 2023-09-30T09:15:56.632+0000 7f1dd8b3a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-30T09:15:57.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:56 smithi167 conmon[178440]: 2023-09-30T09:15:56.807+0000 7f5ae605c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:15:57.342 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:56 smithi167 conmon[178440]: 2023-09-30T09:15:56.890+0000 7f5ae605c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:15:57.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:57 smithi019 conmon[202899]: 2023-09-30T09:15:57.465+0000 7f1dd8b3a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-30T09:15:57.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:57 smithi019 conmon[202899]: 2023-09-30T09:15:57.567+0000 7f1dd8b3a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-30T09:15:57.937 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:57 smithi167 conmon[178440]: 2023-09-30T09:15:57.667+0000 7f5ae605c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:57.937 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:57 smithi167 conmon[178440]: 2023-09-30T09:15:57.761+0000 7f5ae605c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:15:58.223 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:57 smithi167 conmon[178440]: 2023-09-30T09:15:57.936+0000 7f5ae605c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:58.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:58 smithi167 conmon[178440]: 2023-09-30T09:15:58.222+0000 7f5ae605c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:15:58.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:58 smithi167 conmon[178440]: 2023-09-30T09:15:58.450+0000 7f5ae605c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:15:58.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:58 smithi019 conmon[202899]: 2023-09-30T09:15:58.561+0000 7f1dd8b3a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:58.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:58 smithi019 conmon[202899]: 2023-09-30T09:15:58.676+0000 7f1dd8b3a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-30T09:15:58.873 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:58 smithi167 conmon[178440]: 2023-09-30T09:15:58.601+0000 7f5ae605c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:15:58.873 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:15:58 smithi167 conmon[177400]: ts=2023-09-30T09:15:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:15:59.266 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:58 smithi019 conmon[202899]: 2023-09-30T09:15:58.903+0000 7f1dd8b3a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-30T09:15:59.271 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:58 smithi167 conmon[178440]: 2023-09-30T09:15:58.872+0000 7f5ae605c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:15:59.524 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:59 smithi019 conmon[202899]: 2023-09-30T09:15:59.264+0000 7f1dd8b3a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-30T09:15:59.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:59 smithi019 conmon[202899]: 2023-09-30T09:15:59.523+0000 7f1dd8b3a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-30T09:15:59.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:15:59 smithi019 conmon[202899]: 2023-09-30T09:15:59.709+0000 7f1dd8b3a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-30T09:15:59.923 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:59 smithi167 conmon[178440]: 2023-09-30T09:15:59.588+0000 7f5ae605c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:15:59.923 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:59 smithi167 conmon[178440]: 2023-09-30T09:15:59.723+0000 7f5ae605c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:16:00.270 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:15:59 smithi167 conmon[178440]: 2023-09-30T09:15:59.922+0000 7f5ae605c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:16:00.270 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:00 smithi167 conmon[178440]: 2023-09-30T09:16:00.042+0000 7f5ae605c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:16:00.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:00 smithi019 conmon[202899]: 2023-09-30T09:16:00.053+0000 7f1dd8b3a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-30T09:16:00.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:00 smithi167 conmon[178440]: 2023-09-30T09:16:00.269+0000 7f5ae605c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:16:01.205 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:00 smithi019 conmon[202899]: 2023-09-30T09:16:00.895+0000 7f1dd8b3a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-30T09:16:01.205 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:01 smithi019 conmon[202899]: 2023-09-30T09:16:01.060+0000 7f1dd8b3a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-30T09:16:01.463 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:01 smithi019 conmon[202899]: 2023-09-30T09:16:01.321+0000 7f1dd8b3a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[216815]: Standby manager daemon x restarted 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[216815]: Standby manager daemon x started 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[216815]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[216815]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[216815]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[216815]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[118983]: Standby manager daemon x restarted 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[118983]: Standby manager daemon x started 2023-09-30T09:16:01.464 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:16:01.465 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:16:01.465 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:16:01.465 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:01 smithi019 ceph-mon[118983]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:16:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:01 smithi167 ceph-mon[104403]: Standby manager daemon x restarted 2023-09-30T09:16:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:01 smithi167 ceph-mon[104403]: Standby manager daemon x started 2023-09-30T09:16:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:01 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-30T09:16:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:01 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-30T09:16:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:01 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-30T09:16:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:01 smithi167 ceph-mon[104403]: from='mgr.? 172.21.15.167:0/1226871445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-30T09:16:01.522 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:01 smithi167 conmon[178440]: 2023-09-30T09:16:01.143+0000 7f5ae605c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:16:01.523 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:01 smithi167 conmon[178440]: [30/Sep/2023:09:16:01] ENGINE Bus STARTING 2023-09-30T09:16:01.523 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:01 smithi167 conmon[178440]: CherryPy Checker: 2023-09-30T09:16:01.523 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:01 smithi167 conmon[178440]: The Application mounted at '' has an empty config. 2023-09-30T09:16:01.523 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:01 smithi167 conmon[178440]: 2023-09-30T09:16:01.523 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:01 smithi167 conmon[178440]: [30/Sep/2023:09:16:01] ENGINE Serving on http://:::9283 2023-09-30T09:16:01.523 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:16:01 smithi167 conmon[178440]: [30/Sep/2023:09:16:01] ENGINE Bus STARTED 2023-09-30T09:16:01.748 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:01 smithi019 conmon[202899]: 2023-09-30T09:16:01.462+0000 7f1dd8b3a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-30T09:16:02.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:01 smithi019 conmon[202899]: 2023-09-30T09:16:01.747+0000 7f1dd8b3a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-30T09:16:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:02 smithi167 ceph-mon[104403]: mgrmap e35: y(active, since 38s), standbys: x 2023-09-30T09:16:02.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:02 smithi019 ceph-mon[216815]: mgrmap e35: y(active, since 38s), standbys: x 2023-09-30T09:16:02.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:02 smithi019 ceph-mon[118983]: mgrmap e35: y(active, since 38s), standbys: x 2023-09-30T09:16:03.076 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:02 smithi019 conmon[118958]: 2023-09-30T09:16:02.839+0000 7fe413f78700 -1 mon.c@2(peon).osd e90 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:02 smithi019 conmon[202899]: 2023-09-30T09:16:02.822+0000 7f1dd8b3a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:02 smithi019 conmon[202899]: [30/Sep/2023:09:16:02] ENGINE Bus STARTING 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d37582700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d37582700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d37582700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d37582700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d37582700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d3e590700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d3e590700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d3e590700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.078 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d3e590700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.078 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:02.999+0000 7f1d3e590700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-30T09:16:03.078 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: CherryPy Checker: 2023-09-30T09:16:03.078 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: The Application mounted at '' has an empty config. 2023-09-30T09:16:03.078 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:03.078 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: [30/Sep/2023:09:16:03] ENGINE Serving on http://:::9283 2023-09-30T09:16:03.208 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:02 smithi167 conmon[104379]: 2023-09-30T09:16:02.840+0000 7fc56ad27700 -1 mon.b@1(peon).osd e90 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-30T09:16:03.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: [30/Sep/2023:09:16:03] ENGINE Bus STARTED 2023-09-30T09:16:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: Active manager daemon y restarted 2023-09-30T09:16:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: Activating manager daemon y 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: mgrmap e36: y(active, starting, since 0.00576991s), standbys: x 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: failed to encode map e90 with expected crc 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: failed to encode map e90 with expected crc 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:16:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: Manager daemon y is now available 2023-09-30T09:16:03.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: Active manager daemon y restarted 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: Activating manager daemon y 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:16:03.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: mgrmap e36: y(active, starting, since 0.00576991s), standbys: x 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: failed to encode map e90 with expected crc 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: failed to encode map e90 with expected crc 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:16:03.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: Manager daemon y is now available 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:03.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:16:03.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:16:03.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:16:03.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:03 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:16:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: Active manager daemon y restarted 2023-09-30T09:16:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: Activating manager daemon y 2023-09-30T09:16:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:16:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: mgrmap e36: y(active, starting, since 0.00576991s), standbys: x 2023-09-30T09:16:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: failed to encode map e90 with expected crc 2023-09-30T09:16:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: failed to encode map e90 with expected crc 2023-09-30T09:16:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi019.kcgktg"}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi167.bnuarp"}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:16:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: Manager daemon y is now available 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:16:03.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:03 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-30T09:16:03.868 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-30T09:16:03.868 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (67s) - 4m 21.0M - 0.25.0 c8568f914cd2 b16b2fb388da 2023-09-30T09:16:03.868 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi167 *:3000 running (57s) - 4m 84.9M - 9.4.7 2c41d148cca3 f7c9f74b8088 2023-09-30T09:16:03.868 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi019.kcgktg smithi019 running (3m) - 3m 16.4M - 17.2.6-1161-g191c1869 8497a1260037 ca3d56fcab07 2023-09-30T09:16:03.868 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi167.bnuarp smithi167 running (3m) - 3m 16.0M - 17.2.6-1161-g191c1869 8497a1260037 a70137fe60d5 2023-09-30T09:16:03.868 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi167 *:8443,9283,8765 running (23s) - 8m 45.4M - 18.0.0-6447-g1d10b717 dda7fab67785 3468c121edb9 2023-09-30T09:16:03.868 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (50s) - 10m 476M - 18.0.0-6447-g1d10b717 dda7fab67785 f13d86332ace 2023-09-30T09:16:03.868 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (10m) - 10m 101M 2048M 17.2.6-1161-g191c1869 8497a1260037 91f2eb5c5520 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi167 running (9m) - 9m 69.9M 2048M 17.2.6-1161-g191c1869 8497a1260037 b5fffa651c9c 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (8m) - 8m 69.6M 2048M 17.2.6-1161-g191c1869 8497a1260037 88df4d08fbd4 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (65s) - 4m 9101k - 1.5.0 0da6a335fe13 63ac922663db 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi167 *:9100 running (55s) - 4m 8808k - 1.5.0 0da6a335fe13 d6c59fa1a461 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (8m) - 8m 75.7M 2501M 17.2.6-1161-g191c1869 8497a1260037 0ff07d0fefc1 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (7m) - 7m 99.0M 2501M 17.2.6-1161-g191c1869 8497a1260037 4f868828e949 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (7m) - 7m 76.3M 2501M 17.2.6-1161-g191c1869 8497a1260037 6673f026fd30 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (7m) - 7m 64.1M 2501M 17.2.6-1161-g191c1869 8497a1260037 8fa7001643e1 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi167 running (6m) - 6m 72.9M 2500M 17.2.6-1161-g191c1869 8497a1260037 6557c10c6a0b 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi167 running (6m) - 6m 78.8M 2500M 17.2.6-1161-g191c1869 8497a1260037 30ccc3c97db6 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi167 running (6m) - 6m 74.3M 2500M 17.2.6-1161-g191c1869 8497a1260037 4a5f3bfc86e4 2023-09-30T09:16:03.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi167 running (5m) - 5m 95.5M 2500M 17.2.6-1161-g191c1869 8497a1260037 92e8410b5190 2023-09-30T09:16:03.870 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi167 *:9095 running (28s) - 5m 40.7M - 2.43.0 a07b618ecd1d c625e147e7f2 2023-09-30T09:16:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:16:03] "GET /metrics HTTP/1.1" 200 31081 "" "Prometheus/2.43.0" 2023-09-30T09:16:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:03 smithi019 conmon[202899]: 2023-09-30T09:16:03.852+0000 7f1d921ed700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-30T09:16:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:04 smithi167 ceph-mon[104403]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:04 smithi167 ceph-mon[104403]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:04 smithi167 ceph-mon[104403]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:04 smithi167 ceph-mon[104403]: mgrmap e37: y(active, since 1.0095s), standbys: x 2023-09-30T09:16:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:04 smithi019 ceph-mon[216815]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:04 smithi019 ceph-mon[216815]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:04 smithi019 ceph-mon[216815]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:04 smithi019 ceph-mon[216815]: mgrmap e37: y(active, since 1.0095s), standbys: x 2023-09-30T09:16:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:04 smithi019 ceph-mon[118983]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:04 smithi019 ceph-mon[118983]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:04 smithi019 ceph-mon[118983]: failed to encode map e90 with expected crc 2023-09-30T09:16:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:04 smithi019 ceph-mon[118983]: mgrmap e37: y(active, since 1.0095s), standbys: x 2023-09-30T09:16:04.873 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch upgrade pause' 2023-09-30T09:16:05.466 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:16:05.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:05 smithi167 ceph-mon[104403]: [30/Sep/2023:09:16:04] ENGINE Bus STARTING 2023-09-30T09:16:05.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:05 smithi019 ceph-mon[216815]: [30/Sep/2023:09:16:04] ENGINE Bus STARTING 2023-09-30T09:16:05.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:05 smithi019 ceph-mon[118983]: [30/Sep/2023:09:16:04] ENGINE Bus STARTING 2023-09-30T09:16:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[216815]: [30/Sep/2023:09:16:04] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:16:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[216815]: [30/Sep/2023:09:16:04] ENGINE Serving on http://172.21.15.19:8765 2023-09-30T09:16:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[216815]: [30/Sep/2023:09:16:04] ENGINE Bus STARTED 2023-09-30T09:16:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[216815]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[216815]: mgrmap e38: y(active, since 2s), standbys: x 2023-09-30T09:16:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[118983]: [30/Sep/2023:09:16:04] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:16:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[118983]: [30/Sep/2023:09:16:04] ENGINE Serving on http://172.21.15.19:8765 2023-09-30T09:16:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[118983]: [30/Sep/2023:09:16:04] ENGINE Bus STARTED 2023-09-30T09:16:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[118983]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:06 smithi019 ceph-mon[118983]: mgrmap e38: y(active, since 2s), standbys: x 2023-09-30T09:16:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:06 smithi167 ceph-mon[104403]: [30/Sep/2023:09:16:04] ENGINE Serving on https://172.21.15.19:7150 2023-09-30T09:16:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:06 smithi167 ceph-mon[104403]: [30/Sep/2023:09:16:04] ENGINE Serving on http://172.21.15.19:8765 2023-09-30T09:16:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:06 smithi167 ceph-mon[104403]: [30/Sep/2023:09:16:04] ENGINE Bus STARTED 2023-09-30T09:16:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:06 smithi167 ceph-mon[104403]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:06 smithi167 ceph-mon[104403]: mgrmap e38: y(active, since 2s), standbys: x 2023-09-30T09:16:07.604 INFO:teuthology.orchestra.run.smithi019.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:16:07.762 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:07 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.763 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:07 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.763 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:07 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.763 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:07 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:07 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:07 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:07 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:07 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:07 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:07 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.830 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:07 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:07.830 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:07 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.354 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'sleep 60' 2023-09-30T09:16:08.726 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:16:08.759 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:08 smithi167 ceph-mon[104403]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:08.759 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:08 smithi167 ceph-mon[104403]: mgrmap e39: y(active, since 4s), standbys: x 2023-09-30T09:16:08.760 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:08 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.760 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:08 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.760 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:08 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.760 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:16:08 smithi167 conmon[177400]: ts=2023-09-30T09:16:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:16:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[216815]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[216815]: mgrmap e39: y(active, since 4s), standbys: x 2023-09-30T09:16:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[118983]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[118983]: mgrmap e39: y(active, since 4s), standbys: x 2023-09-30T09:16:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:08 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:09.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:09 smithi167 ceph-mon[104403]: from='client.25877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:16:09.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:09 smithi167 ceph-mon[104403]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:16:09.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:09 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:09.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:09 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:09.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:09 smithi019 ceph-mon[216815]: from='client.25877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:16:09.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:09 smithi019 ceph-mon[216815]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:16:09.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:09 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:09.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:09 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:09.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:09 smithi019 ceph-mon[118983]: from='client.25877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:16:09.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:09 smithi019 ceph-mon[118983]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:16:09.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:09 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:09.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:09 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:10 smithi167 ceph-mon[104403]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:10 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:10 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:10 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:16:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:10 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:16:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[216815]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:16:10.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:16:10.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[118983]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:10.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:10.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:10.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:16:10.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:10 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:16:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:11 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:16:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:11 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:16:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:11 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:11 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:11 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:11 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:11 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[216815]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:16:11.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[216815]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:16:11.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.conf 2023-09-30T09:16:11.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.conf 2023-09-30T09:16:11.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:11.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:11 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:16:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:16:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:16:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:16:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:16:12.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:16:12.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:16:12.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:12.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:16:12.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:16:12.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:12 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:16:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:16:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:16:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:16:12.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:16:12.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:16:12.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:16:12.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:12.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:16:12.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:16:12.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:12.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:16:12.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: Updating smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: Updating smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:16:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:12 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:16:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:14 smithi019 ceph-mon[216815]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:16:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:16:13] "GET /metrics HTTP/1.1" 200 31081 "" "Prometheus/2.43.0" 2023-09-30T09:16:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:14 smithi019 ceph-mon[118983]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:16:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:14 smithi167 ceph-mon[104403]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2023-09-30T09:16:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:16 smithi167 ceph-mon[104403]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:16:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:16 smithi019 ceph-mon[216815]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:16:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:16 smithi019 ceph-mon[118983]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2023-09-30T09:16:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:18 smithi167 ceph-mon[104403]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:18.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:18 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:18 smithi019 ceph-mon[216815]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:18.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:18 smithi019 ceph-mon[118983]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:18.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:18 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:19.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:16:18 smithi167 conmon[177400]: ts=2023-09-30T09:16:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:16:20.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:20 smithi019 ceph-mon[216815]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:20 smithi019 ceph-mon[118983]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:20 smithi167 ceph-mon[104403]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:22 smithi019 ceph-mon[216815]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:22.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:22 smithi019 ceph-mon[118983]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:22 smithi167 ceph-mon[104403]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2023-09-30T09:16:24.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:24 smithi019 ceph-mon[216815]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:16:23] "GET /metrics HTTP/1.1" 200 33762 "" "Prometheus/2.43.0" 2023-09-30T09:16:24.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:24 smithi019 ceph-mon[118983]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:24 smithi167 ceph-mon[104403]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:26 smithi019 ceph-mon[216815]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:26 smithi019 ceph-mon[118983]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:26 smithi167 ceph-mon[104403]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:28.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:28 smithi019 ceph-mon[216815]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:28.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:28 smithi019 ceph-mon[118983]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:28 smithi167 ceph-mon[104403]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:29.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:16:28 smithi167 conmon[177400]: ts=2023-09-30T09:16:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:16:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:30 smithi019 ceph-mon[216815]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:30.394 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:30 smithi019 ceph-mon[118983]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:30 smithi167 ceph-mon[104403]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:32 smithi019 ceph-mon[216815]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:32.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:32 smithi019 ceph-mon[118983]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:32 smithi167 ceph-mon[104403]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:33 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:33 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:34.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:34 smithi019 ceph-mon[216815]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:16:33] "GET /metrics HTTP/1.1" 200 33756 "" "Prometheus/2.43.0" 2023-09-30T09:16:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:34 smithi019 ceph-mon[118983]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:34 smithi167 ceph-mon[104403]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:36 smithi019 ceph-mon[216815]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:36 smithi019 ceph-mon[118983]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:36 smithi167 ceph-mon[104403]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:38 smithi019 ceph-mon[216815]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:38 smithi019 ceph-mon[118983]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:38 smithi167 ceph-mon[104403]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:39.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:16:38 smithi167 conmon[177400]: ts=2023-09-30T09:16:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:16:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:40 smithi019 ceph-mon[216815]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:40 smithi019 ceph-mon[118983]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:40 smithi167 ceph-mon[104403]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:42 smithi019 ceph-mon[216815]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:42 smithi019 ceph-mon[118983]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:42 smithi167 ceph-mon[104403]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:44.064 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:16:43] "GET /metrics HTTP/1.1" 200 33756 "" "Prometheus/2.43.0" 2023-09-30T09:16:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:44 smithi019 ceph-mon[216815]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:44.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:44 smithi019 ceph-mon[118983]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:44 smithi167 ceph-mon[104403]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:46.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:46 smithi019 ceph-mon[216815]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:46.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:46 smithi019 ceph-mon[118983]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:46 smithi167 ceph-mon[104403]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:48 smithi019 ceph-mon[216815]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:48 smithi019 ceph-mon[118983]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:48 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:48 smithi167 ceph-mon[104403]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:48 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:16:49.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:16:48 smithi167 conmon[177400]: ts=2023-09-30T09:16:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:16:50.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:50 smithi019 ceph-mon[216815]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:50.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:50 smithi019 ceph-mon[118983]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:50 smithi167 ceph-mon[104403]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:52 smithi019 ceph-mon[216815]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:52.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:52 smithi019 ceph-mon[118983]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:52 smithi167 ceph-mon[104403]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:54.080 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:16:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:16:53] "GET /metrics HTTP/1.1" 200 33758 "" "Prometheus/2.43.0" 2023-09-30T09:16:54.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:54 smithi019 ceph-mon[216815]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:54 smithi019 ceph-mon[118983]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:54 smithi167 ceph-mon[104403]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:56.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:56 smithi019 ceph-mon[216815]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:56.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:56 smithi019 ceph-mon[118983]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:56 smithi167 ceph-mon[104403]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:16:58 smithi167 ceph-mon[104403]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:58.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:16:58 smithi019 ceph-mon[216815]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:58.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:16:58 smithi019 ceph-mon[118983]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:16:59.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:16:58 smithi167 conmon[177400]: ts=2023-09-30T09:16:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:17:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:00 smithi167 ceph-mon[104403]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:00.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:00 smithi019 ceph-mon[216815]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:00.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:00 smithi019 ceph-mon[118983]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:02 smithi167 ceph-mon[104403]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:02.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:02 smithi019 ceph-mon[216815]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:02.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:02 smithi019 ceph-mon[118983]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:03 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:03.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:03 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:17:03] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-30T09:17:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:04 smithi167 ceph-mon[104403]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:04 smithi019 ceph-mon[118983]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:04 smithi019 ceph-mon[216815]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:06.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:06 smithi167 ceph-mon[104403]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:06.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:06 smithi019 ceph-mon[216815]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:06 smithi019 ceph-mon[118983]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:08 smithi167 ceph-mon[104403]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:08 smithi019 ceph-mon[216815]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:08 smithi019 ceph-mon[118983]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:09.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:17:08 smithi167 conmon[177400]: ts=2023-09-30T09:17:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:17:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:10 smithi167 ceph-mon[104403]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:10 smithi019 ceph-mon[216815]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:10.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:10 smithi019 ceph-mon[118983]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:10.698 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch upgrade resume' 2023-09-30T09:17:11.063 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:17:12.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[216815]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:12.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:12.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[118983]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:12.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:12.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:12 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:12 smithi167 ceph-mon[104403]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:12 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:12 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:12 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:12 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:12 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:12.878 INFO:teuthology.orchestra.run.smithi019.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:17:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:17:13] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-30T09:17:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:13 smithi019 ceph-mon[216815]: from='client.25883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:13 smithi019 ceph-mon[216815]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:13 smithi019 ceph-mon[216815]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:17:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:13 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:14.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:13 smithi019 ceph-mon[118983]: from='client.25883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:14.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:13 smithi019 ceph-mon[118983]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:14.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:13 smithi019 ceph-mon[118983]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:17:14.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:13 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:14.123 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-30T09:17:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:13 smithi167 ceph-mon[104403]: from='client.25883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:13 smithi167 ceph-mon[104403]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:13 smithi167 ceph-mon[104403]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:17:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:13 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:14.530 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:17:16.385 INFO:teuthology.orchestra.run.smithi019.stdout:wait for majority of mons to upgrade 2023-09-30T09:17:16.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:16 smithi019 ceph-mon[216815]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:16.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:16 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:16.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:16 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:16.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:16 smithi019 ceph-mon[118983]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:16.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:16 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:16.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:16 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:16 smithi167 ceph-mon[104403]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:16 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:16 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:16.977 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-30T09:17:17.354 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:17:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:18.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:17:18.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:17:18.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-30T09:17:18.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:17 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:18.325 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-30T09:17:18.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:17:18.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:17:18.328 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-30T09:17:18.328 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:17 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:19.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:17:18 smithi167 conmon[177400]: ts=2023-09-30T09:17:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:17:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:19 smithi019 ceph-mon[216815]: Upgrade: It appears safe to stop mon.c 2023-09-30T09:17:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:19 smithi019 ceph-mon[118983]: Upgrade: It appears safe to stop mon.c 2023-09-30T09:17:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:19 smithi167 ceph-mon[104403]: Upgrade: It appears safe to stop mon.c 2023-09-30T09:17:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[118983]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[118983]: from='client.? 172.21.15.19:0/3555611036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[118983]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:20.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:20.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[118983]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[216815]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3555611036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[216815]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:20 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:20 smithi167 ceph-mon[104403]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:20.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:20 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3555611036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:20.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:20 smithi167 ceph-mon[104403]: from='mgr.25835 ' entity='mgr.y' 2023-09-30T09:17:20.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:20 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:20.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:20 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:20.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:20 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:21 smithi019 ceph-mon[216815]: Upgrade: Updating mon.c 2023-09-30T09:17:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:21 smithi019 ceph-mon[216815]: Deploying daemon mon.c on smithi019 2023-09-30T09:17:21.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:21 smithi019 ceph-mon[118983]: Upgrade: Updating mon.c 2023-09-30T09:17:21.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:21 smithi019 ceph-mon[118983]: Deploying daemon mon.c on smithi019 2023-09-30T09:17:21.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:21 smithi019 systemd[1]: Stopping Ceph mon.c for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:17:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:21 smithi167 ceph-mon[104403]: Upgrade: Updating mon.c 2023-09-30T09:17:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:21 smithi167 ceph-mon[104403]: Deploying daemon mon.c on smithi019 2023-09-30T09:17:21.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:21 smithi019 conmon[118958]: 2023-09-30T09:17:21.365+0000 7fe419783700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:17:21.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:21 smithi019 conmon[118958]: 2023-09-30T09:17:21.365+0000 7fe419783700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-30T09:17:21.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:21 smithi019 conmon[202899]: [30/Sep/2023:09:17:21] ENGINE Bus STOPPING 2023-09-30T09:17:21.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:21 smithi019 conmon[202899]: [30/Sep/2023:09:17:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:17:21.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:21 smithi019 conmon[202899]: [30/Sep/2023:09:17:21] ENGINE Bus STOPPED 2023-09-30T09:17:21.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:21 smithi019 conmon[202899]: [30/Sep/2023:09:17:21] ENGINE Bus STARTING 2023-09-30T09:17:21.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:21 smithi019 conmon[202899]: [30/Sep/2023:09:17:21] ENGINE Serving on http://:::9283 2023-09-30T09:17:21.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:21 smithi019 conmon[202899]: [30/Sep/2023:09:17:21] ENGINE Bus STARTED 2023-09-30T09:17:22.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:21 smithi019 bash[223590]: 88df4d08fbd44b1cd0de427cc3e180e257a54a2a4418417b0652ce3153b6bf82 2023-09-30T09:17:22.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-30T09:17:22.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 systemd[1]: Stopped Ceph mon.c for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:17:22.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 systemd[1]: Starting Ceph mon.c for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:17:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: set uid:gid to 167:167 (ceph:ceph) 2023-09-30T09:17:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev), process ceph-mon, pid 7 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: pidfile_write: ignore empty --pid-file 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: load: jerasure load: lrc 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: RocksDB version: 7.9.2 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Git sha 0 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Compile date 2023-09-29 16:35:50 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: DB SUMMARY 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: DB Session ID: UBGK8A77CVZJPS1WVL9B 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 bash[223978]: 84a54501b1347aa8a838d8f22865e949156d01946a1140b5dd3ca72e71180ead 2023-09-30T09:17:23.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: CURRENT file: CURRENT 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: IDENTITY file: IDENTITY 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1096 Bytes 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 4609275 ; 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.error_if_exists: 0 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.create_if_missing: 0 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.paranoid_checks: 1 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.env: 0x55afc6d32da0 2023-09-30T09:17:23.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.fs: PosixFileSystem 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.info_log: 0x55afc946f5a0 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_file_opening_threads: 16 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.statistics: (nil) 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.use_fsync: 0 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_log_file_size: 0 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.keep_log_file_num: 1000 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.recycle_log_file_num: 0 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.allow_fallocate: 1 2023-09-30T09:17:23.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.allow_mmap_reads: 0 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.allow_mmap_writes: 0 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.use_direct_reads: 0 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.create_missing_column_families: 0 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.db_log_dir: 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.wal_dir: 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-30T09:17:23.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.advise_random_on_open: 1 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.db_write_buffer_size: 0 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.write_buffer_manager: 0x55afc9424fa0 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.rate_limiter: (nil) 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.wal_recovery_mode: 2 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.enable_thread_tracking: 0 2023-09-30T09:17:23.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.enable_pipelined_write: 0 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.unordered_write: 0 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.row_cache: None 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.wal_filter: None 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.allow_ingest_behind: 0 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.two_write_queues: 0 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.manual_wal_flush: 0 2023-09-30T09:17:23.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.wal_compression: 0 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.atomic_flush: 0 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.log_readahead_size: 0 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.best_efforts_recovery: 0 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-30T09:17:23.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.allow_data_in_errors: 0 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.db_host_id: __hostname__ 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_background_jobs: 2 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_background_compactions: -1 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_subcompactions: 1 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_total_wal_size: 0 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-30T09:17:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_open_files: -1 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bytes_per_sync: 0 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_readahead_size: 0 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_background_flushes: -1 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Compression algorithms supported: 2023-09-30T09:17:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: kZSTD supported: 0 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: kXpressCompression supported: 0 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: kLZ4HCCompression supported: 1 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: kZlibCompression supported: 1 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: kSnappyCompression supported: 1 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: kLZ4Compression supported: 1 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: kBZip2Compression supported: 0 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-30T09:17:23.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.merge_operator: 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_filter: None 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_filter_factory: None 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.sst_partitioner_factory: None 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55afc946f6a0) 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: cache_index_and_filter_blocks: 1 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-30T09:17:23.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: pin_top_level_index_and_filter: 1 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_type: 0 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_block_index_type: 0 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_shortening: 1 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: checksum: 4 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: no_block_cache: 0 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache: 0x55afc94c51f0 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_name: BinnedLRUCache 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_options: 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: capacity : 536870912 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: num_shard_bits : 4 2023-09-30T09:17:23.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: strict_capacity_limit : 0 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: high_pri_pool_ratio: 0.000 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_compressed: (nil) 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: persistent_cache: (nil) 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_size: 4096 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_size_deviation: 10 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_restart_interval: 16 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_block_restart_interval: 1 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: metadata_block_size: 4096 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: partition_filters: 0 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: use_delta_encoding: 1 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: filter_policy: bloomfilter 2023-09-30T09:17:23.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: whole_key_filtering: 1 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: verify_compression: 0 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: read_amp_bytes_per_bit: 0 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: format_version: 5 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: enable_index_compression: 1 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_align: 0 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: max_auto_readahead_size: 262144 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: prepopulate_block_cache: 0 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: initial_auto_readahead_size: 8192 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: num_file_reads_for_auto_readahead: 2 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.write_buffer_size: 33554432 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_write_buffer_number: 2 2023-09-30T09:17:23.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression: NoCompression 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression: Disabled 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.prefix_extractor: nullptr 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.num_levels: 7 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-30T09:17:23.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.level: 32767 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.strategy: 0 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:17:23.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.enabled: false 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.target_file_size_base: 67108864 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-30T09:17:23.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.arena_block_size: 1048576 2023-09-30T09:17:23.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.disable_auto_compactions: 0 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-30T09:17:23.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.table_properties_collectors: 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.inplace_update_support: 0 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.bloom_locality: 0 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.max_successive_merges: 0 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.paranoid_file_checks: 0 2023-09-30T09:17:23.356 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.force_consistency_checks: 1 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.report_bg_io_stats: 0 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.ttl: 2592000 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.enable_blob_files: false 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.min_blob_size: 0 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.blob_file_size: 268435456 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-30T09:17:23.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.blob_file_starting_level: 0 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13057, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: abb82521-27c9-4c96-88f3-aef79f7e762d 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065442852326, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-30T09:17:23.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 systemd[1]: Started Ceph mon.c for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065442865662, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2881084, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13052, "largest_seqno": 14807, "table_properties": {"data_size": 2874039, "index_size": 4426, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 14400, "raw_average_key_size": 22, "raw_value_size": 2860269, "raw_average_value_size": 4483, "num_data_blocks": 202, "num_entries": 638, "num_filter_entries": 638, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696065442, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "abb82521-27c9-4c96-88f3-aef79f7e762d", "db_session_id": "UBGK8A77CVZJPS1WVL9B", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065442865759, "job": 1, "event": "recovery_finished"} 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55afc9560000 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: DB pointer 0x55afc954e000 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-30T09:17:23.359 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** DB Stats ** 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-30T09:17:23.360 INFO:journalctl@ceph.mon.c.smithi019.stdout: L0 1/0 2.75 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 625.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: L6 1/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: Sum 2/0 9.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 625.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 625.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 625.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-30T09:17:23.361 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative compaction: 0.00 GB write, 168.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval compaction: 0.00 GB write, 168.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: Block cache BinnedLRUCache@0x55afc94c51f0#7 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2023-09-30T09:17:23.362 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: starting mon.c rank 2 at public addrs [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] at bind addrs [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: mon.c@-1(???) e3 preinit fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: mon.c@-1(???).mds e9 new map 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: mon.c@-1(???).mds e9 print_map 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout: e9 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout: legacy client fscid: 1 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout: Filesystem 'foo' (1) 2023-09-30T09:17:23.363 INFO:journalctl@ceph.mon.c.smithi019.stdout: fs_name foo 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: epoch 8 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: created 2023-09-30T09:12:07.940181+0000 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: modified 2023-09-30T09:12:24.930975+0000 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: tableserver 0 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: root 0 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: session_timeout 60 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: session_autoclose 300 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: max_file_size 1099511627776 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: max_xattr_size 65536 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: required_client_features {} 2023-09-30T09:17:23.364 INFO:journalctl@ceph.mon.c.smithi019.stdout: last_failure 0 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: last_failure_osd_epoch 0 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: max_mds 1 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: in 0 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: up {0=14670} 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: failed 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: damaged 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: stopped 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_pools [4] 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: metadata_pool 3 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: inline_data disabled 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: balancer 2023-09-30T09:17:23.365 INFO:journalctl@ceph.mon.c.smithi019.stdout: bal_rank_mask -1 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout: standby_count_wanted 1 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout: [mds.foo.smithi019.kcgktg{0:14670} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] compat {c=[1],r=[1],i=[7ff]}] 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout: Standby daemons: 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout: [mds.foo.smithi167.bnuarp{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] compat {c=[1],r=[1],i=[7ff]}] 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: mon.c@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:17:23.366 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta expand map: {default=false} 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta from 'false' to 'false' 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta expanded map: {default=false} 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta expand map: {default=info} 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta from 'info' to 'info' 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta expanded map: {default=info} 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta expand map: {default=daemon} 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta expanded map: {default=daemon} 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta expand map: {default=debug} 2023-09-30T09:17:23.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta from 'debug' to 'debug' 2023-09-30T09:17:23.368 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:22 smithi019 ceph-mon[224043]: expand_channel_meta expanded map: {default=debug} 2023-09-30T09:17:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:17:23] "GET /metrics HTTP/1.1" 200 33755 "" "Prometheus/2.43.0" 2023-09-30T09:17:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:28 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:17:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:28 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:17:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:28 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:17:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:28 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/185735706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:28 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:17:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:28 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:17:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:28 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:17:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/185735706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:29.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:17:28 smithi167 conmon[177400]: ts=2023-09-30T09:17:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:17:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: mon.c calling monitor election 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: mon.a calling monitor election 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: mon.b calling monitor election 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: mgrmap e39: y(active, since 86s), standbys: x 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-30T09:17:29.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: Cluster is now healthy 2023-09-30T09:17:29.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:29 smithi167 ceph-mon[104403]: overall HEALTH_OK 2023-09-30T09:17:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: mon.c calling monitor election 2023-09-30T09:17:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: mon.a calling monitor election 2023-09-30T09:17:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: mon.b calling monitor election 2023-09-30T09:17:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:17:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:17:29.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:17:29.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:17:29.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: mgrmap e39: y(active, since 86s), standbys: x 2023-09-30T09:17:29.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-30T09:17:29.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:17:29.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[216815]: overall HEALTH_OK 2023-09-30T09:17:29.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:17:29.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:17:29.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:17:29.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/185735706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:29.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: mon.c calling monitor election 2023-09-30T09:17:29.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: mon.a calling monitor election 2023-09-30T09:17:29.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: mon.b calling monitor election 2023-09-30T09:17:29.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:17:29.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:17:29.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:17:29.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:17:29.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: mgrmap e39: y(active, since 86s), standbys: x 2023-09-30T09:17:29.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-30T09:17:29.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:17:29.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:29 smithi019 ceph-mon[224043]: overall HEALTH_OK 2023-09-30T09:17:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:30 smithi167 ceph-mon[104403]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:30.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:30 smithi019 ceph-mon[216815]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:30.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:30 smithi019 ceph-mon[224043]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:31.025 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)": 2 2023-09-30T09:17:31.937 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch ps' 2023-09-30T09:17:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:31 smithi167 ceph-mon[104403]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:31 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:31 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:31 smithi167 ceph-mon[104403]: from='client.? 172.21.15.19:0/3538984223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:31 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:31 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[216815]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3538984223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[224043]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3538984223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:17:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:32.337 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:17:33.201 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:32 smithi019 conmon[202899]: 2023-09-30T09:17:32.876+0000 7f1d921ed700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-30T09:17:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:33 smithi019 ceph-mon[216815]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:34.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:17:33] "GET /metrics HTTP/1.1" 200 33753 "" "Prometheus/2.43.0" 2023-09-30T09:17:34.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:33 smithi019 ceph-mon[224043]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:34.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:33 smithi167 ceph-mon[104403]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:33 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:35.005 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (2m) - 6m 29.1M - 0.25.0 c8568f914cd2 b16b2fb388da 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi167 *:3000 running (2m) - 5m 85.0M - 9.4.7 2c41d148cca3 f7c9f74b8088 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi019.kcgktg smithi019 running (5m) - 5m 17.8M - 17.2.6-1161-g191c1869 8497a1260037 ca3d56fcab07 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi167.bnuarp smithi167 running (5m) - 5m 16.5M - 17.2.6-1161-g191c1869 8497a1260037 a70137fe60d5 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi167 *:8443,9283,8765 running (114s) - 10m 448M - 18.0.0-6447-g1d10b717 dda7fab67785 3468c121edb9 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (2m) - 11m 502M - 18.0.0-6447-g1d10b717 dda7fab67785 f13d86332ace 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (102s) - 12m 49.3M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 c1584e4cc7f3 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi167 running (10m) - 10m 81.9M 2048M 17.2.6-1161-g191c1869 8497a1260037 b5fffa651c9c 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (12s) - 10m 25.6M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 84a54501b134 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (2m) - 5m 17.8M - 1.5.0 0da6a335fe13 63ac922663db 2023-09-30T09:17:35.006 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi167 *:9100 running (2m) - 5m 8669k - 1.5.0 0da6a335fe13 d6c59fa1a461 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (9m) - 9m 81.7M 2501M 17.2.6-1161-g191c1869 8497a1260037 0ff07d0fefc1 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (9m) - 9m 105M 2501M 17.2.6-1161-g191c1869 8497a1260037 4f868828e949 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (9m) - 9m 80.3M 2501M 17.2.6-1161-g191c1869 8497a1260037 6673f026fd30 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (8m) - 8m 70.6M 2501M 17.2.6-1161-g191c1869 8497a1260037 8fa7001643e1 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi167 running (8m) - 8m 77.8M 2500M 17.2.6-1161-g191c1869 8497a1260037 6557c10c6a0b 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi167 running (7m) - 7m 82.6M 2500M 17.2.6-1161-g191c1869 8497a1260037 30ccc3c97db6 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi167 running (7m) - 7m 78.6M 2500M 17.2.6-1161-g191c1869 8497a1260037 4a5f3bfc86e4 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi167 running (7m) - 7m 100M 2500M 17.2.6-1161-g191c1869 8497a1260037 92e8410b5190 2023-09-30T09:17:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi167 *:9095 running (119s) - 6m 49.8M - 2.43.0 a07b618ecd1d c625e147e7f2 2023-09-30T09:17:35.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:35 smithi019 ceph-mon[224043]: mgrmap e40: y(active, since 91s), standbys: x 2023-09-30T09:17:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:35 smithi019 ceph-mon[216815]: mgrmap e40: y(active, since 91s), standbys: x 2023-09-30T09:17:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:35 smithi167 ceph-mon[104403]: mgrmap e40: y(active, since 91s), standbys: x 2023-09-30T09:17:35.956 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch upgrade pause' 2023-09-30T09:17:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:36 smithi019 ceph-mon[216815]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:36 smithi019 ceph-mon[216815]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:36 smithi019 ceph-mon[224043]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:36 smithi019 ceph-mon[224043]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:36.377 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:17:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:36 smithi167 ceph-mon[104403]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:36 smithi167 ceph-mon[104403]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:37.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:17:37.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:17:37.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-30T09:17:37.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:37 smithi167 ceph-mon[104403]: Upgrade: It appears safe to stop mon.b 2023-09-30T09:17:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[216815]: Upgrade: It appears safe to stop mon.b 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:37.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:37.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:37.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:17:37.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-30T09:17:37.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-30T09:17:37.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:37 smithi019 ceph-mon[224043]: Upgrade: It appears safe to stop mon.b 2023-09-30T09:17:38.194 INFO:teuthology.orchestra.run.smithi019.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:17:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:38 smithi167 ceph-mon[104403]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:38 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:38 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:38 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:38 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:38 smithi167 ceph-mon[104403]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[216815]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:38.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:38.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[224043]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:38.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:38.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:38.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:38.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:38.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:38 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:38.950 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'sleep 60' 2023-09-30T09:17:39.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:17:38 smithi167 conmon[177400]: ts=2023-09-30T09:17:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:17:39.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:39 smithi019 ceph-mon[216815]: Upgrade: Updating mon.b 2023-09-30T09:17:39.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:39 smithi019 ceph-mon[216815]: Deploying daemon mon.b on smithi167 2023-09-30T09:17:39.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:39 smithi019 ceph-mon[216815]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:39.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:39 smithi019 ceph-mon[216815]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:17:39.312 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:17:39.478 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:39 smithi167 systemd[1]: Stopping Ceph mon.b for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:17:39.479 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:39 smithi167 ceph-mon[104403]: Upgrade: Updating mon.b 2023-09-30T09:17:39.479 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:39 smithi167 ceph-mon[104403]: Deploying daemon mon.b on smithi167 2023-09-30T09:17:39.479 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:39 smithi167 ceph-mon[104403]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:39.479 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:39 smithi167 ceph-mon[104403]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:17:39.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:39 smithi019 ceph-mon[224043]: Upgrade: Updating mon.b 2023-09-30T09:17:39.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:39 smithi019 ceph-mon[224043]: Deploying daemon mon.b on smithi167 2023-09-30T09:17:39.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:39 smithi019 ceph-mon[224043]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:17:39.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:39 smithi019 ceph-mon[224043]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:17:39.730 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:39 smithi167 conmon[104379]: 2023-09-30T09:17:39.475+0000 7fc570532700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:17:39.731 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:39 smithi167 conmon[104379]: 2023-09-30T09:17:39.475+0000 7fc570532700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-30T09:17:39.731 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:39 smithi167 bash[192452]: b5fffa651c9c3c7a6ab65360244bd148fd7e32bda77558c4813f2c0d212aa919 2023-09-30T09:17:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-30T09:17:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 systemd[1]: Stopped Ceph mon.b for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:17:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 systemd[1]: Starting Ceph mon.b for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:17:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: set uid:gid to 167:167 (ceph:ceph) 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev), process ceph-mon, pid 7 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: pidfile_write: ignore empty --pid-file 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: load: jerasure load: lrc 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: RocksDB version: 7.9.2 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Git sha 0 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Compile date 2023-09-29 16:35:50 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: DB SUMMARY 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: DB Session ID: 8IHIMADSCSIN0B6MQ903 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: CURRENT file: CURRENT 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: IDENTITY file: IDENTITY 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1096 Bytes 2023-09-30T09:17:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 5357659 ; 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.error_if_exists: 0 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.create_if_missing: 0 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.paranoid_checks: 1 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.env: 0x555962ab1da0 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.fs: PosixFileSystem 2023-09-30T09:17:41.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.info_log: 0x555964ceb5a0 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_file_opening_threads: 16 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.statistics: (nil) 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.use_fsync: 0 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_log_file_size: 0 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.keep_log_file_num: 1000 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.recycle_log_file_num: 0 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.allow_fallocate: 1 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.allow_mmap_reads: 0 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.allow_mmap_writes: 0 2023-09-30T09:17:41.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.use_direct_reads: 0 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.create_missing_column_families: 0 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.db_log_dir: 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.wal_dir: 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.advise_random_on_open: 1 2023-09-30T09:17:41.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.db_write_buffer_size: 0 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.write_buffer_manager: 0x555964ca0fa0 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.rate_limiter: (nil) 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.wal_recovery_mode: 2 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.enable_thread_tracking: 0 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.enable_pipelined_write: 0 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.unordered_write: 0 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-30T09:17:41.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.row_cache: None 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.wal_filter: None 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.allow_ingest_behind: 0 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.two_write_queues: 0 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.manual_wal_flush: 0 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.wal_compression: 0 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.atomic_flush: 0 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-30T09:17:41.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.log_readahead_size: 0 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.best_efforts_recovery: 0 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.allow_data_in_errors: 0 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.db_host_id: __hostname__ 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_background_jobs: 2 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_background_compactions: -1 2023-09-30T09:17:41.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_subcompactions: 1 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_total_wal_size: 0 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_open_files: -1 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bytes_per_sync: 0 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-30T09:17:41.280 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_readahead_size: 0 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_background_flushes: -1 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Compression algorithms supported: 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: kZSTD supported: 0 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: kXpressCompression supported: 0 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: kLZ4HCCompression supported: 1 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: kZlibCompression supported: 1 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: kSnappyCompression supported: 1 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: kLZ4Compression supported: 1 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 bash[192815]: fd31a48646fc58ae18b1f0decc619094459267f8e567d306c9cfab2c5e991a9c 2023-09-30T09:17:41.281 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: kBZip2Compression supported: 0 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.merge_operator: 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_filter: None 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_filter_factory: None 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.sst_partitioner_factory: None 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555964ceb6a0) 2023-09-30T09:17:41.282 INFO:journalctl@ceph.mon.b.smithi167.stdout: cache_index_and_filter_blocks: 1 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: pin_top_level_index_and_filter: 1 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: index_type: 0 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: data_block_index_type: 0 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: index_shortening: 1 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: checksum: 4 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: no_block_cache: 0 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_cache: 0x555964d411f0 2023-09-30T09:17:41.283 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_cache_name: BinnedLRUCache 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_cache_options: 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: capacity : 536870912 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: num_shard_bits : 4 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: strict_capacity_limit : 0 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: high_pri_pool_ratio: 0.000 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_cache_compressed: (nil) 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: persistent_cache: (nil) 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_size: 4096 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_size_deviation: 10 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_restart_interval: 16 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: index_block_restart_interval: 1 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: metadata_block_size: 4096 2023-09-30T09:17:41.284 INFO:journalctl@ceph.mon.b.smithi167.stdout: partition_filters: 0 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: use_delta_encoding: 1 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: filter_policy: bloomfilter 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: whole_key_filtering: 1 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: verify_compression: 0 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: read_amp_bytes_per_bit: 0 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: format_version: 5 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: enable_index_compression: 1 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: block_align: 0 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: max_auto_readahead_size: 262144 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: prepopulate_block_cache: 0 2023-09-30T09:17:41.285 INFO:journalctl@ceph.mon.b.smithi167.stdout: initial_auto_readahead_size: 8192 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout: num_file_reads_for_auto_readahead: 2 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.write_buffer_size: 33554432 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_write_buffer_number: 2 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression: NoCompression 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression: Disabled 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.prefix_extractor: nullptr 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.num_levels: 7 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-30T09:17:41.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-30T09:17:41.287 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-30T09:17:41.287 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-30T09:17:41.287 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-30T09:17:41.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-30T09:17:41.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:17:41.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-30T09:17:41.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-30T09:17:41.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-30T09:17:41.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-30T09:17:41.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.level: 32767 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.strategy: 0 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.enabled: false 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-30T09:17:41.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.target_file_size_base: 67108864 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-30T09:17:41.290 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.arena_block_size: 1048576 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.disable_auto_compactions: 0 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-30T09:17:41.291 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.table_properties_collectors: 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.inplace_update_support: 0 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-30T09:17:41.292 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.bloom_locality: 0 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.max_successive_merges: 0 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.paranoid_file_checks: 0 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.force_consistency_checks: 1 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.report_bg_io_stats: 0 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.ttl: 2592000 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-30T09:17:41.293 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.enable_blob_files: false 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.min_blob_size: 0 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.blob_file_size: 268435456 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.blob_file_starting_level: 0 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-30T09:17:41.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13133, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 04629313-cfe7-4259-b5a8-54bfc3bd1673 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065460897903, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 systemd[1]: Started Ceph mon.b for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065460911213, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3317705, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13128, "largest_seqno": 15228, "table_properties": {"data_size": 3309592, "index_size": 5173, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1925, "raw_key_size": 16913, "raw_average_key_size": 22, "raw_value_size": 3293501, "raw_average_value_size": 4391, "num_data_blocks": 237, "num_entries": 750, "num_filter_entries": 750, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696065460, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "04629313-cfe7-4259-b5a8-54bfc3bd1673", "db_session_id": "8IHIMADSCSIN0B6MQ903", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696065460911338, "job": 1, "event": "recovery_finished"} 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-30T09:17:41.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555964ddc000 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: DB pointer 0x555964dca000 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** DB Stats ** 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-30T09:17:41.296 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** Compaction Stats [default] ** 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: L0 1/0 3.16 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 557.2 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: L6 1/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: Sum 2/0 9.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 557.2 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 557.2 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** Compaction Stats [default] ** 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-30T09:17:41.297 INFO:journalctl@ceph.mon.b.smithi167.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 557.2 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: Cumulative compaction: 0.00 GB write, 189.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: Interval compaction: 0.00 GB write, 189.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-30T09:17:41.298 INFO:journalctl@ceph.mon.b.smithi167.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout: Block cache BinnedLRUCache@0x555964d411f0#7 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: starting mon.b rank 1 at public addrs [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] at bind addrs [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: mon.b@-1(???) e3 preinit fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: mon.b@-1(???).mds e9 new map 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: mon.b@-1(???).mds e9 print_map 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout: e9 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:17:41.299 INFO:journalctl@ceph.mon.b.smithi167.stdout: legacy client fscid: 1 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: Filesystem 'foo' (1) 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: fs_name foo 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: epoch 8 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: created 2023-09-30T09:12:07.940181+0000 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: modified 2023-09-30T09:12:24.930975+0000 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: tableserver 0 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: root 0 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: session_timeout 60 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: session_autoclose 300 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: max_file_size 1099511627776 2023-09-30T09:17:41.300 INFO:journalctl@ceph.mon.b.smithi167.stdout: max_xattr_size 65536 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: required_client_features {} 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: last_failure 0 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: last_failure_osd_epoch 0 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: max_mds 1 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: in 0 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: up {0=14670} 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: failed 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: damaged 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: stopped 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: data_pools [4] 2023-09-30T09:17:41.301 INFO:journalctl@ceph.mon.b.smithi167.stdout: metadata_pool 3 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: inline_data disabled 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: balancer 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: bal_rank_mask -1 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: standby_count_wanted 1 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: [mds.foo.smithi019.kcgktg{0:14670} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.19:6834/2755192324,v1:172.21.15.19:6835/2755192324] compat {c=[1],r=[1],i=[7ff]}] 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: Standby daemons: 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout: [mds.foo.smithi167.bnuarp{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.167:6832/2526610055,v1:172.21.15.167:6833/2526610055] compat {c=[1],r=[1],i=[7ff]}] 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: mon.b@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-30T09:17:41.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta expand map: {default=false} 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta from 'false' to 'false' 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta expanded map: {default=false} 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta expand map: {default=info} 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta from 'info' to 'info' 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta expanded map: {default=info} 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta expand map: {default=daemon} 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta expanded map: {default=daemon} 2023-09-30T09:17:41.303 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta expand map: {default=debug} 2023-09-30T09:17:41.304 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta from 'debug' to 'debug' 2023-09-30T09:17:41.304 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:40 smithi167 ceph-mon[192880]: expand_channel_meta expanded map: {default=debug} 2023-09-30T09:17:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:41 smithi167 ceph-mon[192880]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:42.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:41 smithi019 ceph-mon[216815]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:41 smithi019 ceph-mon[224043]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: overall HEALTH_OK 2023-09-30T09:17:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:17:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: mon.c calling monitor election 2023-09-30T09:17:43.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: mon.b calling monitor election 2023-09-30T09:17:43.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:17:43.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: mon.a calling monitor election 2023-09-30T09:17:43.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:17:43.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:17:43.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:17:43.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:17:43.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:17:43.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: mgrmap e40: y(active, since 98s), standbys: x 2023-09-30T09:17:43.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: overall HEALTH_OK 2023-09-30T09:17:43.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:43.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:43.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: overall HEALTH_OK 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: mon.c calling monitor election 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: mon.b calling monitor election 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: mon.a calling monitor election 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:17:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: mgrmap e40: y(active, since 98s), standbys: x 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: overall HEALTH_OK 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: overall HEALTH_OK 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: mon.c calling monitor election 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: mon.b calling monitor election 2023-09-30T09:17:43.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: mon.a calling monitor election 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: fsmap foo:1 {0=foo.smithi019.kcgktg=up:active} 1 up:standby 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: osdmap e90: 8 total, 8 up, 8 in 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: mgrmap e40: y(active, since 98s), standbys: x 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: overall HEALTH_OK 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:43.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:17:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:44 smithi019 ceph-mon[216815]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:44.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:17:43] "GET /metrics HTTP/1.1" 200 33753 "" "Prometheus/2.43.0" 2023-09-30T09:17:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:44 smithi019 ceph-mon[224043]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:44 smithi167 ceph-mon[192880]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:46.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:46 smithi167 ceph-mon[192880]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:46.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.296 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.296 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.337 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:45 smithi019 conmon[202899]: 2023-09-30T09:17:45.922+0000 7f1d921ed700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-30T09:17:46.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[216815]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:46.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[224043]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-30T09:17:46.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:46.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:48 smithi019 ceph-mon[216815]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:48 smithi019 ceph-mon[224043]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:48.376 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:48 smithi167 ceph-mon[192880]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:48.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:17:48.998 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:17:48 smithi167 conmon[177400]: ts=2023-09-30T09:17:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:17:50.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:50.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: Reconfiguring mon.a (monmap changed)... 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[224043]: Reconfiguring daemon mon.a on smithi019 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: Reconfiguring mon.a (monmap changed)... 2023-09-30T09:17:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:50 smithi019 ceph-mon[216815]: Reconfiguring daemon mon.a on smithi019 2023-09-30T09:17:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:17:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: Reconfiguring mon.a (monmap changed)... 2023-09-30T09:17:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:50.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:50.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:50.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:50 smithi167 ceph-mon[192880]: Reconfiguring daemon mon.a on smithi019 2023-09-30T09:17:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[216815]: Reconfiguring mgr.y (monmap changed)... 2023-09-30T09:17:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:17:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:17:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[216815]: Reconfiguring daemon mgr.y on smithi019 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[216815]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[224043]: Reconfiguring mgr.y (monmap changed)... 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[224043]: Reconfiguring daemon mgr.y on smithi019 2023-09-30T09:17:51.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:51 smithi019 ceph-mon[224043]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:51 smithi167 ceph-mon[192880]: Reconfiguring mgr.y (monmap changed)... 2023-09-30T09:17:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:17:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:17:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:51 smithi167 ceph-mon[192880]: Reconfiguring daemon mgr.y on smithi019 2023-09-30T09:17:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:51 smithi167 ceph-mon[192880]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[224043]: Reconfiguring mon.c (monmap changed)... 2023-09-30T09:17:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[224043]: Reconfiguring daemon mon.c on smithi019 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[216815]: Reconfiguring mon.c (monmap changed)... 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:53 smithi019 ceph-mon[216815]: Reconfiguring daemon mon.c on smithi019 2023-09-30T09:17:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:53 smithi167 ceph-mon[192880]: Reconfiguring mon.c (monmap changed)... 2023-09-30T09:17:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:17:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:17:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:53 smithi167 ceph-mon[192880]: Reconfiguring daemon mon.c on smithi019 2023-09-30T09:17:54.054 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:17:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:17:53] "GET /metrics HTTP/1.1" 200 33750 "" "Prometheus/2.43.0" 2023-09-30T09:17:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:54 smithi167 ceph-mon[192880]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:54 smithi167 ceph-mon[192880]: Reconfiguring osd.0 (monmap changed)... 2023-09-30T09:17:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:17:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[216815]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[216815]: Reconfiguring osd.0 (monmap changed)... 2023-09-30T09:17:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:17:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[224043]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[224043]: Reconfiguring osd.0 (monmap changed)... 2023-09-30T09:17:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:17:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:55 smithi167 ceph-mon[192880]: Reconfiguring daemon osd.0 on smithi019 2023-09-30T09:17:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:55 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:55 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:55 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:17:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:55 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:55.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[216815]: Reconfiguring daemon osd.0 on smithi019 2023-09-30T09:17:55.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:55.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:55.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:17:55.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:55.790 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[224043]: Reconfiguring daemon osd.0 on smithi019 2023-09-30T09:17:55.790 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:55.790 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:55.790 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:17:55.791 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:55 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[216815]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[216815]: Reconfiguring osd.1 (monmap changed)... 2023-09-30T09:17:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[216815]: Reconfiguring daemon osd.1 on smithi019 2023-09-30T09:17:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:17:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[224043]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[224043]: Reconfiguring osd.1 (monmap changed)... 2023-09-30T09:17:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[224043]: Reconfiguring daemon osd.1 on smithi019 2023-09-30T09:17:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:17:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:56 smithi167 ceph-mon[192880]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:56 smithi167 ceph-mon[192880]: Reconfiguring osd.1 (monmap changed)... 2023-09-30T09:17:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:56 smithi167 ceph-mon[192880]: Reconfiguring daemon osd.1 on smithi019 2023-09-30T09:17:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:17:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:57 smithi167 ceph-mon[192880]: Reconfiguring osd.2 (monmap changed)... 2023-09-30T09:17:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:57 smithi167 ceph-mon[192880]: Reconfiguring daemon osd.2 on smithi019 2023-09-30T09:17:57.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:57 smithi019 ceph-mon[216815]: Reconfiguring osd.2 (monmap changed)... 2023-09-30T09:17:57.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:57 smithi019 ceph-mon[216815]: Reconfiguring daemon osd.2 on smithi019 2023-09-30T09:17:57.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:57 smithi019 ceph-mon[224043]: Reconfiguring osd.2 (monmap changed)... 2023-09-30T09:17:57.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:57 smithi019 ceph-mon[224043]: Reconfiguring daemon osd.2 on smithi019 2023-09-30T09:17:58.739 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[216815]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[224043]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:17:58.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:58 smithi167 ceph-mon[192880]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:17:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:17:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:58.772 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:17:58 smithi167 conmon[177400]: ts=2023-09-30T09:17:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:17:59.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[216815]: Reconfiguring osd.3 (monmap changed)... 2023-09-30T09:17:59.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[216815]: Reconfiguring daemon osd.3 on smithi019 2023-09-30T09:17:59.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:59.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:59.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:17:59.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:59.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[224043]: Reconfiguring osd.3 (monmap changed)... 2023-09-30T09:17:59.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[224043]: Reconfiguring daemon osd.3 on smithi019 2023-09-30T09:17:59.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:59.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:59.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:17:59.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:17:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:17:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:59 smithi167 ceph-mon[192880]: Reconfiguring osd.3 (monmap changed)... 2023-09-30T09:17:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:59 smithi167 ceph-mon[192880]: Reconfiguring daemon osd.3 on smithi019 2023-09-30T09:17:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:17:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi019.kcgktg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:17:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:17:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:00.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:00 smithi167 ceph-mon[192880]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:00.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:00 smithi167 ceph-mon[192880]: Reconfiguring mds.foo.smithi019.kcgktg (monmap changed)... 2023-09-30T09:18:00.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:00 smithi167 ceph-mon[192880]: Reconfiguring daemon mds.foo.smithi019.kcgktg on smithi019 2023-09-30T09:18:00.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:00 smithi019 ceph-mon[216815]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:00.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:00 smithi019 ceph-mon[216815]: Reconfiguring mds.foo.smithi019.kcgktg (monmap changed)... 2023-09-30T09:18:00.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:00 smithi019 ceph-mon[216815]: Reconfiguring daemon mds.foo.smithi019.kcgktg on smithi019 2023-09-30T09:18:00.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:00 smithi019 ceph-mon[224043]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:00.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:00 smithi019 ceph-mon[224043]: Reconfiguring mds.foo.smithi019.kcgktg (monmap changed)... 2023-09-30T09:18:00.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:00 smithi019 ceph-mon[224043]: Reconfiguring daemon mds.foo.smithi019.kcgktg on smithi019 2023-09-30T09:18:01.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:01 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:01.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:01 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:01.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:01 smithi167 ceph-mon[192880]: Reconfiguring mon.b (monmap changed)... 2023-09-30T09:18:01.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:01 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:18:01.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:01 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:18:01.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:01 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:01.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:01 smithi167 ceph-mon[192880]: Reconfiguring daemon mon.b on smithi167 2023-09-30T09:18:01.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:01 smithi167 ceph-mon[192880]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:02.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:02.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:02.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[216815]: Reconfiguring mon.b (monmap changed)... 2023-09-30T09:18:02.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:18:02.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:18:02.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[216815]: Reconfiguring daemon mon.b on smithi167 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[216815]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[224043]: Reconfiguring mon.b (monmap changed)... 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[224043]: Reconfiguring daemon mon.b on smithi167 2023-09-30T09:18:02.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:01 smithi019 ceph-mon[224043]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:03.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:03.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:03.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:02 smithi167 ceph-mon[192880]: Reconfiguring mgr.x (monmap changed)... 2023-09-30T09:18:03.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:18:03.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:18:03.231 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:03.231 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:02 smithi167 ceph-mon[192880]: Reconfiguring daemon mgr.x on smithi167 2023-09-30T09:18:03.231 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[216815]: Reconfiguring mgr.x (monmap changed)... 2023-09-30T09:18:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:18:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:18:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[216815]: Reconfiguring daemon mgr.x on smithi167 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[224043]: Reconfiguring mgr.x (monmap changed)... 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[224043]: Reconfiguring daemon mgr.x on smithi167 2023-09-30T09:18:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:18:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:18:03] "GET /metrics HTTP/1.1" 200 33747 "" "Prometheus/2.43.0" 2023-09-30T09:18:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:04 smithi167 ceph-mon[192880]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:04 smithi167 ceph-mon[192880]: Reconfiguring osd.4 (monmap changed)... 2023-09-30T09:18:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-30T09:18:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:04.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:04 smithi167 ceph-mon[192880]: Reconfiguring daemon osd.4 on smithi167 2023-09-30T09:18:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[216815]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[216815]: Reconfiguring osd.4 (monmap changed)... 2023-09-30T09:18:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-30T09:18:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:04.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[216815]: Reconfiguring daemon osd.4 on smithi167 2023-09-30T09:18:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[224043]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[224043]: Reconfiguring osd.4 (monmap changed)... 2023-09-30T09:18:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-30T09:18:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:04.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:04 smithi019 ceph-mon[224043]: Reconfiguring daemon osd.4 on smithi167 2023-09-30T09:18:05.963 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:05.963 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:05.963 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:05 smithi167 ceph-mon[192880]: Reconfiguring osd.5 (monmap changed)... 2023-09-30T09:18:05.963 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-30T09:18:05.963 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:05.963 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:05 smithi167 ceph-mon[192880]: Reconfiguring daemon osd.5 on smithi167 2023-09-30T09:18:05.963 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:05 smithi167 ceph-mon[192880]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[216815]: Reconfiguring osd.5 (monmap changed)... 2023-09-30T09:18:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-30T09:18:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[216815]: Reconfiguring daemon osd.5 on smithi167 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[216815]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[224043]: Reconfiguring osd.5 (monmap changed)... 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[224043]: Reconfiguring daemon osd.5 on smithi167 2023-09-30T09:18:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:05 smithi019 ceph-mon[224043]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:07 smithi167 ceph-mon[192880]: Reconfiguring osd.6 (monmap changed)... 2023-09-30T09:18:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-30T09:18:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:07 smithi167 ceph-mon[192880]: Reconfiguring daemon osd.6 on smithi167 2023-09-30T09:18:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[216815]: Reconfiguring osd.6 (monmap changed)... 2023-09-30T09:18:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-30T09:18:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[216815]: Reconfiguring daemon osd.6 on smithi167 2023-09-30T09:18:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[224043]: Reconfiguring osd.6 (monmap changed)... 2023-09-30T09:18:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-30T09:18:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:07 smithi019 ceph-mon[224043]: Reconfiguring daemon osd.6 on smithi167 2023-09-30T09:18:08.661 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:08 smithi167 ceph-mon[192880]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:08.661 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:08.661 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:08.661 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:08 smithi167 ceph-mon[192880]: Reconfiguring osd.7 (monmap changed)... 2023-09-30T09:18:08.661 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-30T09:18:08.662 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:08.662 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:08 smithi167 ceph-mon[192880]: Reconfiguring daemon osd.7 on smithi167 2023-09-30T09:18:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[216815]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:08.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[216815]: Reconfiguring osd.7 (monmap changed)... 2023-09-30T09:18:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-30T09:18:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:08.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[216815]: Reconfiguring daemon osd.7 on smithi167 2023-09-30T09:18:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[224043]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:08.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[224043]: Reconfiguring osd.7 (monmap changed)... 2023-09-30T09:18:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-30T09:18:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:08 smithi019 ceph-mon[224043]: Reconfiguring daemon osd.7 on smithi167 2023-09-30T09:18:09.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:18:08 smithi167 conmon[177400]: ts=2023-09-30T09:18:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:18:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:09 smithi167 ceph-mon[192880]: Reconfiguring mds.foo.smithi167.bnuarp (monmap changed)... 2023-09-30T09:18:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:18:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:10.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:09 smithi167 ceph-mon[192880]: Reconfiguring daemon mds.foo.smithi167.bnuarp on smithi167 2023-09-30T09:18:10.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:09 smithi167 ceph-mon[192880]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[216815]: Reconfiguring mds.foo.smithi167.bnuarp (monmap changed)... 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[216815]: Reconfiguring daemon mds.foo.smithi167.bnuarp on smithi167 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[216815]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[224043]: Reconfiguring mds.foo.smithi167.bnuarp (monmap changed)... 2023-09-30T09:18:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi167.bnuarp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-30T09:18:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[224043]: Reconfiguring daemon mds.foo.smithi167.bnuarp on smithi167 2023-09-30T09:18:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:09 smithi019 ceph-mon[224043]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:18:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:18:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:11.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:11.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:11.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:11.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:11.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:18:11.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:12 smithi167 ceph-mon[192880]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:12.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:12 smithi019 ceph-mon[216815]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:12 smithi019 ceph-mon[224043]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:18:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:18:13] "GET /metrics HTTP/1.1" 200 33747 "" "Prometheus/2.43.0" 2023-09-30T09:18:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:14 smithi167 ceph-mon[192880]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:14 smithi019 ceph-mon[216815]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:14 smithi019 ceph-mon[224043]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:16 smithi167 ceph-mon[192880]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:16.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:16 smithi019 ceph-mon[216815]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:16 smithi019 ceph-mon[224043]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:18 smithi167 ceph-mon[192880]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:18 smithi019 ceph-mon[224043]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:18 smithi019 ceph-mon[216815]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:19.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:18:18 smithi167 conmon[177400]: ts=2023-09-30T09:18:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:18:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:20 smithi167 ceph-mon[192880]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:20 smithi019 ceph-mon[216815]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:20.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:20 smithi019 ceph-mon[224043]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:22 smithi167 ceph-mon[192880]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:22 smithi019 ceph-mon[216815]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:22.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:22 smithi019 ceph-mon[224043]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:18:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:18:23] "GET /metrics HTTP/1.1" 200 33686 "" "Prometheus/2.43.0" 2023-09-30T09:18:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:24 smithi167 ceph-mon[192880]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:24 smithi019 ceph-mon[216815]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:24.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:24 smithi019 ceph-mon[224043]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:26 smithi167 ceph-mon[192880]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:26 smithi019 ceph-mon[216815]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:26 smithi019 ceph-mon[224043]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:28 smithi167 ceph-mon[192880]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:28 smithi019 ceph-mon[216815]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:28 smithi019 ceph-mon[224043]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:29.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:18:28 smithi167 conmon[177400]: ts=2023-09-30T09:18:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:18:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:30 smithi167 ceph-mon[192880]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:30.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:30 smithi019 ceph-mon[216815]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:30.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:30 smithi019 ceph-mon[224043]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:32 smithi167 ceph-mon[192880]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:32.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:32 smithi019 ceph-mon[216815]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:32.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:32 smithi019 ceph-mon[224043]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:33.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:18:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:18:33] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-30T09:18:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:34 smithi167 ceph-mon[192880]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:34 smithi019 ceph-mon[216815]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:34 smithi019 ceph-mon[224043]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:36 smithi167 ceph-mon[192880]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:36 smithi019 ceph-mon[216815]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:36 smithi019 ceph-mon[224043]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:38 smithi167 ceph-mon[192880]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:38 smithi019 ceph-mon[216815]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:38.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:38 smithi019 ceph-mon[224043]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:39.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:18:38 smithi167 conmon[177400]: ts=2023-09-30T09:18:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:18:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:40 smithi167 ceph-mon[192880]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:40.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:40 smithi019 ceph-mon[216815]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:40 smithi019 ceph-mon[224043]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:41.282 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch upgrade resume' 2023-09-30T09:18:41.642 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:18:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:42 smithi167 ceph-mon[192880]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:42 smithi019 ceph-mon[216815]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:42 smithi019 ceph-mon[224043]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:43.333 INFO:teuthology.orchestra.run.smithi019.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:18:43.932 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'echo wait for all mons to upgrade' 2023-09-30T09:18:44.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:18:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:18:43] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-30T09:18:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[216815]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:44.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[216815]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:44.730 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[216815]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:18:44.730 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:44.730 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:44.731 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[224043]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:44.731 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[224043]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:44.731 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[224043]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:18:44.731 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:44.731 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:44 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:44.731 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:18:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:44 smithi167 ceph-mon[192880]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:44 smithi167 ceph-mon[192880]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:44 smithi167 ceph-mon[192880]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:18:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:44 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:44 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:46.401 INFO:teuthology.orchestra.run.smithi019.stdout:wait for all mons to upgrade 2023-09-30T09:18:46.566 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:46 smithi019 ceph-mon[216815]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:46.567 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:46 smithi019 ceph-mon[224043]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:46 smithi167 ceph-mon[192880]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:47.173 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-30T09:18:47.722 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:18:48.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:48 smithi019 ceph-mon[216815]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:48.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:48.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:48.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:48.605 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:48 smithi019 ceph-mon[224043]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:48.605 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:48.605 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:48.605 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:48.660 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:48 smithi167 ceph-mon[192880]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:48.661 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:18:48.661 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:48.661 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:18:48 smithi167 conmon[177400]: ts=2023-09-30T09:18:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:18:49.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:49.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:18:49.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:49.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-30T09:18:49.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-30T09:18:49.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-30T09:18:49.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-30T09:18:49.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-30T09:18:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:18:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:18:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-30T09:18:49.807 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)": 3 2023-09-30T09:18:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:50 smithi167 ceph-mon[192880]: Upgrade: Setting container_image for all mon 2023-09-30T09:18:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:50 smithi167 ceph-mon[192880]: Upgrade: Setting container_image for all crash 2023-09-30T09:18:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:50 smithi167 ceph-mon[192880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-30T09:18:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:50 smithi167 ceph-mon[192880]: Upgrade: osd.0 is safe to restart 2023-09-30T09:18:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:50 smithi167 ceph-mon[192880]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2800644349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:18:50.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[216815]: Upgrade: Setting container_image for all mon 2023-09-30T09:18:50.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[216815]: Upgrade: Setting container_image for all crash 2023-09-30T09:18:50.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[216815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-30T09:18:50.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[216815]: Upgrade: osd.0 is safe to restart 2023-09-30T09:18:50.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[216815]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:50.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2800644349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:18:50.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[224043]: Upgrade: Setting container_image for all mon 2023-09-30T09:18:50.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[224043]: Upgrade: Setting container_image for all crash 2023-09-30T09:18:50.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[224043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-30T09:18:50.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[224043]: Upgrade: osd.0 is safe to restart 2023-09-30T09:18:50.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[224043]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:50.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2800644349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-30T09:18:51.018 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch ps' 2023-09-30T09:18:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[224043]: Upgrade: Updating osd.0 2023-09-30T09:18:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:18:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[224043]: Deploying daemon osd.0 on smithi019 2023-09-30T09:18:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[216815]: Upgrade: Updating osd.0 2023-09-30T09:18:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:18:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:51 smithi019 ceph-mon[216815]: Deploying daemon osd.0 on smithi019 2023-09-30T09:18:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:51 smithi167 ceph-mon[192880]: Upgrade: Updating osd.0 2023-09-30T09:18:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-30T09:18:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:18:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:51 smithi167 ceph-mon[192880]: Deploying daemon osd.0 on smithi019 2023-09-30T09:18:52.139 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:18:52.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:52 smithi019 ceph-mon[216815]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:52 smithi019 ceph-mon[224043]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:53.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:52 smithi167 ceph-mon[192880]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:18:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:18:53] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-30T09:18:54.648 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-30T09:18:54.648 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (3m) - 7m 29.1M - 0.25.0 c8568f914cd2 b16b2fb388da 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi167 *:3000 running (3m) - 7m 91.2M - 9.4.7 2c41d148cca3 f7c9f74b8088 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi019.kcgktg smithi019 running (6m) - 6m 18.5M - 17.2.6-1161-g191c1869 8497a1260037 ca3d56fcab07 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi167.bnuarp smithi167 running (6m) - 6m 16.6M - 17.2.6-1161-g191c1869 8497a1260037 a70137fe60d5 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi167 *:8443,9283,8765 running (3m) - 11m 449M - 18.0.0-6447-g1d10b717 dda7fab67785 3468c121edb9 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (3m) - 13m 506M - 18.0.0-6447-g1d10b717 dda7fab67785 f13d86332ace 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (3m) - 13m 60.1M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 c1584e4cc7f3 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi167 running (74s) - 11m 26.7M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 fd31a48646fc 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (92s) - 11m 45.3M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 84a54501b134 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (3m) - 7m 18.4M - 1.5.0 0da6a335fe13 63ac922663db 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi167 *:9100 running (3m) - 6m 8488k - 1.5.0 0da6a335fe13 d6c59fa1a461 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (11m) - 11m 85.4M 2501M 17.2.6-1161-g191c1869 8497a1260037 0ff07d0fefc1 2023-09-30T09:18:54.649 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (10m) - 10m 108M 2501M 17.2.6-1161-g191c1869 8497a1260037 4f868828e949 2023-09-30T09:18:54.650 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (10m) - 10m 82.8M 2501M 17.2.6-1161-g191c1869 8497a1260037 6673f026fd30 2023-09-30T09:18:54.650 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (10m) - 9m 74.0M 2501M 17.2.6-1161-g191c1869 8497a1260037 8fa7001643e1 2023-09-30T09:18:54.650 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi167 running (9m) - 9m 79.5M 2500M 17.2.6-1161-g191c1869 8497a1260037 6557c10c6a0b 2023-09-30T09:18:54.650 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi167 running (9m) - 9m 83.7M 2500M 17.2.6-1161-g191c1869 8497a1260037 30ccc3c97db6 2023-09-30T09:18:54.650 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi167 running (8m) - 8m 80.1M 2500M 17.2.6-1161-g191c1869 8497a1260037 4a5f3bfc86e4 2023-09-30T09:18:54.650 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi167 running (8m) - 8m 102M 2500M 17.2.6-1161-g191c1869 8497a1260037 92e8410b5190 2023-09-30T09:18:54.650 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi167 *:9095 running (3m) - 7m 54.7M - 2.43.0 a07b618ecd1d c625e147e7f2 2023-09-30T09:18:54.723 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:54 smithi019 ceph-mon[216815]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:54.723 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:54 smithi019 ceph-mon[224043]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:54.724 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:18:54 smithi019 systemd[1]: Stopping Ceph osd.0 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:18:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:54 smithi167 ceph-mon[192880]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:55.085 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:18:54 smithi019 conmon[125233]: 2023-09-30T09:18:54.721+0000 7fbd4ca16700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:18:55.085 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:18:54 smithi019 conmon[125233]: 2023-09-30T09:18:54.721+0000 7fbd4ca16700 -1 osd.0 90 *** Got signal Terminated *** 2023-09-30T09:18:55.085 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:18:54 smithi019 conmon[125233]: 2023-09-30T09:18:54.721+0000 7fbd4ca16700 -1 osd.0 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:18:55.406 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch upgrade pause' 2023-09-30T09:18:55.767 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:18:55.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:55 smithi019 ceph-mon[216815]: osd.0 marked itself down and dead 2023-09-30T09:18:55.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:55 smithi019 ceph-mon[224043]: osd.0 marked itself down and dead 2023-09-30T09:18:56.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:55 smithi167 ceph-mon[192880]: osd.0 marked itself down and dead 2023-09-30T09:18:56.274 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:18:56 smithi019 bash[235747]: 0ff07d0fefc1a070325aae239d0312dee78aa5b1db25f7b0510193e430012d35 2023-09-30T09:18:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[216815]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[216815]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[216815]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:18:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[216815]: failed to encode map e91 with expected crc 2023-09-30T09:18:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[216815]: failed to encode map e91 with expected crc 2023-09-30T09:18:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[216815]: osdmap e91: 8 total, 7 up, 8 in 2023-09-30T09:18:56.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[224043]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:56.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[224043]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:56.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[224043]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:18:56.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[224043]: failed to encode map e91 with expected crc 2023-09-30T09:18:56.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[224043]: failed to encode map e91 with expected crc 2023-09-30T09:18:56.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:56 smithi019 ceph-mon[224043]: osdmap e91: 8 total, 7 up, 8 in 2023-09-30T09:18:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:56 smithi167 ceph-mon[192880]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:56 smithi167 ceph-mon[192880]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:56 smithi167 ceph-mon[192880]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:18:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:56 smithi167 ceph-mon[192880]: failed to encode map e91 with expected crc 2023-09-30T09:18:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:56 smithi167 ceph-mon[192880]: failed to encode map e91 with expected crc 2023-09-30T09:18:57.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:56 smithi167 ceph-mon[192880]: osdmap e91: 8 total, 7 up, 8 in 2023-09-30T09:18:57.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: osdmap e92: 8 total, 7 up, 8 in 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[216815]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:57.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: osdmap e92: 8 total, 7 up, 8 in 2023-09-30T09:18:57.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: failed to encode map e92 with expected crc 2023-09-30T09:18:57.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:57 smithi019 ceph-mon[224043]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:57.954 INFO:teuthology.orchestra.run.smithi019.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:18:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: osdmap e92: 8 total, 7 up, 8 in 2023-09-30T09:18:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: failed to encode map e92 with expected crc 2023-09-30T09:18:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:57 smithi167 ceph-mon[192880]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:18:58.952 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:18:58 smithi167 conmon[177400]: ts=2023-09-30T09:18:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:18:59.106 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'sleep 60' 2023-09-30T09:18:59.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:58 smithi019 ceph-mon[216815]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:59.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:58 smithi019 ceph-mon[216815]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:18:59.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:59.252 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:58 smithi019 ceph-mon[224043]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:59.252 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:58 smithi019 ceph-mon[224043]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:18:59.252 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:58 smithi167 ceph-mon[192880]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:18:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:58 smithi167 ceph-mon[192880]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:18:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:18:59.559 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:18:59.585 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:18:59 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-30T09:18:59.585 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:18:59 smithi019 systemd[1]: Stopped Ceph osd.0 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:18:59.585 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:18:59 smithi019 systemd[1]: Starting Ceph osd.0 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:19:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:59 smithi019 ceph-mon[216815]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:18:59 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:19:00.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:59 smithi019 ceph-mon[224043]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:00.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:18:59 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:19:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:59 smithi167 ceph-mon[192880]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:18:59 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:19:01.034 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:00 smithi019 bash[237043]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-30T09:19:01.034 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:00 smithi019 bash[237043]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-30T09:19:01.034 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:00 smithi019 bash[237043]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-30T09:19:01.035 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:00 smithi019 bash[237043]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-30T09:19:01.035 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:00 smithi019 bash[237043]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-30T09:19:01.035 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:00 smithi019 bash[237043]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-30T09:19:01.035 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:00 smithi019 bash[237043]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-30T09:19:01.035 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:00 smithi019 bash[237043]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-30T09:19:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:01 smithi167 ceph-mon[192880]: pgmap v94: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:01 smithi019 ceph-mon[216815]: pgmap v94: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:01 smithi019 ceph-mon[224043]: pgmap v94: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:03.085 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:02 smithi019 ceph-osd[237660]: -- 172.21.15.19:0/3019910510 <== mon.2 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55be5958e4e0 con 0x55be59561400 2023-09-30T09:19:03.085 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:02 smithi019 bash[237595]: 0112d2fb79a7e5df14657ee7f1b14d117480bcd776678714ee681127d0c04d9f 2023-09-30T09:19:03.085 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:02 smithi019 systemd[1]: Started Ceph osd.0 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:19:03.585 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:03 smithi019 conmon[237636]: 2023-09-30T09:19:03.129+0000 7fbdacbba640 -1 Falling back to public interface 2023-09-30T09:19:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:19:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:03 smithi167 ceph-mon[192880]: pgmap v95: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:19:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:19:03] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.43.0" 2023-09-30T09:19:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:19:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[216815]: pgmap v95: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:19:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[224043]: pgmap v95: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:06.335 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:05 smithi019 conmon[237636]: 2023-09-30T09:19:05.882+0000 7fbdacbba640 -1 osd.0 0 read_superblock omap replica is missing. 2023-09-30T09:19:06.335 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:05 smithi019 conmon[237636]: 2023-09-30T09:19:05.911+0000 7fbdacbba640 -1 osd.0 90 log_to_monitors true 2023-09-30T09:19:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: pgmap v96: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: from='osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:19:06.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:19:06.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: pgmap v96: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:06.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: from='osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:19:06.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:19:06.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:06.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: pgmap v96: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: from='osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:19:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:19:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:07.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:07.281 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:19:06 smithi019 conmon[237636]: 2023-09-30T09:19:06.922+0000 7fbd9b821700 -1 osd.0 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:19:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[216815]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:19:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[216815]: failed to encode map e93 with expected crc 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[216815]: failed to encode map e93 with expected crc 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[216815]: from='osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[216815]: osdmap e93: 8 total, 7 up, 8 in 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[216815]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[216815]: pgmap v98: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[224043]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[224043]: failed to encode map e93 with expected crc 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[224043]: failed to encode map e93 with expected crc 2023-09-30T09:19:08.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[224043]: from='osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:19:08.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[224043]: osdmap e93: 8 total, 7 up, 8 in 2023-09-30T09:19:08.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[224043]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:19:08.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:07 smithi019 ceph-mon[224043]: pgmap v98: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:07 smithi167 ceph-mon[192880]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:19:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:07 smithi167 ceph-mon[192880]: failed to encode map e93 with expected crc 2023-09-30T09:19:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:07 smithi167 ceph-mon[192880]: failed to encode map e93 with expected crc 2023-09-30T09:19:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:07 smithi167 ceph-mon[192880]: from='osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:19:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:07 smithi167 ceph-mon[192880]: osdmap e93: 8 total, 7 up, 8 in 2023-09-30T09:19:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:07 smithi167 ceph-mon[192880]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:19:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:07 smithi167 ceph-mon[192880]: pgmap v98: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-30T09:19:08.923 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:19:08 smithi167 conmon[177400]: ts=2023-09-30T09:19:08.660Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-09-30T09:19:08.923 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:19:08 smithi167 conmon[177400]: ts=2023-09-30T09:19:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591] boot 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: osdmap e94: 8 total, 8 up, 8 in 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: failed to encode map e94 with expected crc 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:08 smithi167 ceph-mon[192880]: osdmap e95: 8 total, 8 up, 8 in 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591] boot 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: osdmap e94: 8 total, 8 up, 8 in 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: failed to encode map e94 with expected crc 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[224043]: osdmap e95: 8 total, 8 up, 8 in 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: osd.0 [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591] boot 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: osdmap e94: 8 total, 8 up, 8 in 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: failed to encode map e94 with expected crc 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: failed to encode map e93 with expected crc 2023-09-30T09:19:09.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:08 smithi019 ceph-mon[216815]: osdmap e95: 8 total, 8 up, 8 in 2023-09-30T09:19:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: pgmap v101: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:19:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:19:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:19:10.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: pgmap v101: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: pgmap v101: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: failed to encode map e95 with expected crc 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:19:10.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:19:10.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:10.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:19:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:10 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:19:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:10 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:19:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:10 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:19:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:10 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:10 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:19:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:10 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:19:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:10 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:19:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:10 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:10 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:19:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:10 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:19:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:10 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:19:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:10 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:12 smithi019 ceph-mon[216815]: pgmap v102: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:19:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:12 smithi019 ceph-mon[224043]: pgmap v102: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:19:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:12 smithi167 ceph-mon[192880]: pgmap v102: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:19:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:14 smithi019 ceph-mon[216815]: pgmap v103: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:14 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-30T09:19:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:14 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:19:14.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:19:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:19:13] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.43.0" 2023-09-30T09:19:14.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:14 smithi019 ceph-mon[224043]: pgmap v103: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:14.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:14 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-30T09:19:14.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:14 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:19:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:14 smithi167 ceph-mon[192880]: pgmap v103: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:14 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-30T09:19:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:14 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:19:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:16 smithi019 ceph-mon[216815]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:16 smithi019 ceph-mon[224043]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:16 smithi167 ceph-mon[192880]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:18 smithi019 ceph-mon[216815]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:18 smithi019 ceph-mon[224043]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:18 smithi167 ceph-mon[192880]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:19.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:19:18 smithi167 conmon[177400]: ts=2023-09-30T09:19:18.660Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-09-30T09:19:19.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:19:18 smithi167 conmon[177400]: ts=2023-09-30T09:19:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:19:20.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:20 smithi019 ceph-mon[216815]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:20 smithi019 ceph-mon[224043]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:20 smithi167 ceph-mon[192880]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:22 smithi019 ceph-mon[216815]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:22.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:22 smithi019 ceph-mon[224043]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:22 smithi167 ceph-mon[192880]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:19:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:19:23] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2023-09-30T09:19:24.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:23 smithi019 ceph-mon[216815]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:24.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:23 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:24.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:23 smithi019 ceph-mon[224043]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:24.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:23 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:23 smithi167 ceph-mon[192880]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:23 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:19:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:25 smithi167 ceph-mon[192880]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:25 smithi019 ceph-mon[216815]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:26.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:25 smithi019 ceph-mon[224043]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:27 smithi167 ceph-mon[192880]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:28.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:27 smithi019 ceph-mon[216815]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:28.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:27 smithi019 ceph-mon[224043]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:29.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:19:28 smithi167 conmon[177400]: ts=2023-09-30T09:19:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:19:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:30 smithi019 ceph-mon[216815]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:30 smithi019 ceph-mon[224043]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:30 smithi167 ceph-mon[192880]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:32 smithi019 ceph-mon[216815]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:32.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:32 smithi019 ceph-mon[224043]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:32 smithi167 ceph-mon[192880]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:34.059 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:19:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:19:33] "GET /metrics HTTP/1.1" 200 33884 "" "Prometheus/2.43.0" 2023-09-30T09:19:34.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:34 smithi019 ceph-mon[216815]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:34 smithi019 ceph-mon[224043]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:34.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:34 smithi167 ceph-mon[192880]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:36 smithi019 ceph-mon[216815]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:36 smithi019 ceph-mon[224043]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:36 smithi167 ceph-mon[192880]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:38 smithi019 ceph-mon[216815]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:38 smithi019 ceph-mon[224043]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:38 smithi167 ceph-mon[192880]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:39.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:19:38 smithi167 conmon[177400]: ts=2023-09-30T09:19:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:19:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:40 smithi019 ceph-mon[216815]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:40 smithi019 ceph-mon[224043]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:40 smithi167 ceph-mon[192880]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:42 smithi019 ceph-mon[216815]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:42 smithi019 ceph-mon[224043]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:42 smithi167 ceph-mon[192880]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:44.075 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:19:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:19:43] "GET /metrics HTTP/1.1" 200 33884 "" "Prometheus/2.43.0" 2023-09-30T09:19:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:44 smithi019 ceph-mon[216815]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:44.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:44 smithi019 ceph-mon[224043]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:44 smithi167 ceph-mon[192880]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:46.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:46 smithi019 ceph-mon[216815]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:46.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:46 smithi019 ceph-mon[224043]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:46 smithi167 ceph-mon[192880]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:48 smithi019 ceph-mon[216815]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:48 smithi019 ceph-mon[224043]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:48 smithi167 ceph-mon[192880]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:19:49.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:19:48 smithi167 conmon[177400]: ts=2023-09-30T09:19:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi019:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi167.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-30T09:19:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:50 smithi019 ceph-mon[216815]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:50.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:50 smithi019 ceph-mon[224043]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:50 smithi167 ceph-mon[192880]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:52 smithi167 ceph-mon[192880]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:52 smithi019 ceph-mon[216815]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:52 smithi019 ceph-mon[224043]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:19:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:19:53] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2023-09-30T09:19:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:54 smithi167 ceph-mon[192880]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:54.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:54 smithi019 ceph-mon[216815]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:54.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:54 smithi019 ceph-mon[224043]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:56 smithi167 ceph-mon[192880]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:56 smithi019 ceph-mon[216815]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:56.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:56 smithi019 ceph-mon[224043]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:19:58 smithi167 ceph-mon[192880]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:58.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:19:58 smithi019 ceph-mon[216815]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:19:58.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:19:58 smithi019 ceph-mon[224043]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:00 smithi167 ceph-mon[192880]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:00 smithi167 ceph-mon[192880]: overall HEALTH_OK 2023-09-30T09:20:00.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:00 smithi019 ceph-mon[216815]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:00.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:00 smithi019 ceph-mon[216815]: overall HEALTH_OK 2023-09-30T09:20:00.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:00 smithi019 ceph-mon[224043]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:00.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:00 smithi019 ceph-mon[224043]: overall HEALTH_OK 2023-09-30T09:20:02.158 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch upgrade resume' 2023-09-30T09:20:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:02 smithi167 ceph-mon[192880]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:02.522 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:20:02.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:02 smithi019 ceph-mon[216815]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:02.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:02 smithi019 ceph-mon[224043]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:03.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:03.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:20:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:20:03] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2023-09-30T09:20:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:04 smithi019 ceph-mon[224043]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:04 smithi019 ceph-mon[216815]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:04.346 INFO:teuthology.orchestra.run.smithi019.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:20:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:04 smithi167 ceph-mon[192880]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:05.134 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'echo wait for half of osds to upgrade' 2023-09-30T09:20:05.365 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:20:05.365 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-30T09:20:05.366 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[216815]: Upgrade: osd.1 is safe to restart 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.368 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.368 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-30T09:20:05.368 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-30T09:20:05.368 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:05 smithi019 ceph-mon[224043]: Upgrade: osd.1 is safe to restart 2023-09-30T09:20:05.606 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:20:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:20:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:20:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-30T09:20:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-30T09:20:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:05 smithi167 ceph-mon[192880]: Upgrade: osd.1 is safe to restart 2023-09-30T09:20:06.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:06 smithi019 ceph-mon[224043]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:06.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:06 smithi019 ceph-mon[216815]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:06 smithi167 ceph-mon[192880]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:07.962 INFO:teuthology.orchestra.run.smithi019.stdout:wait for half of osds to upgrade 2023-09-30T09:20:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:07 smithi167 ceph-mon[192880]: Upgrade: Updating osd.1 2023-09-30T09:20:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:20:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:07 smithi167 ceph-mon[192880]: Deploying daemon osd.1 on smithi019 2023-09-30T09:20:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:07 smithi167 ceph-mon[192880]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[224043]: Upgrade: Updating osd.1 2023-09-30T09:20:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:20:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[224043]: Deploying daemon osd.1 on smithi019 2023-09-30T09:20:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[224043]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[216815]: Upgrade: Updating osd.1 2023-09-30T09:20:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-30T09:20:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[216815]: Deploying daemon osd.1 on smithi019 2023-09-30T09:20:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:07 smithi019 ceph-mon[216815]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:09.035 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-30T09:20:09.569 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:20:10.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:10 smithi019 ceph-mon[216815]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:10.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:10 smithi019 ceph-mon[224043]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:10 smithi167 ceph-mon[192880]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:11.541 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:11 smithi019 systemd[1]: Stopping Ceph osd.1 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:20:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:12 smithi019 ceph-mon[216815]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:12.561 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:12 smithi019 ceph-mon[216815]: osd.1 marked itself down and dead 2023-09-30T09:20:12.561 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3497089564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:12.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:12 smithi019 ceph-mon[224043]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:12.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:12 smithi019 ceph-mon[224043]: osd.1 marked itself down and dead 2023-09-30T09:20:12.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3497089564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:12.562 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:11 smithi019 conmon[134349]: 2023-09-30T09:20:11.842+0000 7f3174f70700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:20:12.562 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:11 smithi019 conmon[134349]: 2023-09-30T09:20:11.842+0000 7f3174f70700 -1 osd.1 95 *** Got signal Terminated *** 2023-09-30T09:20:12.563 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:11 smithi019 conmon[134349]: 2023-09-30T09:20:11.842+0000 7f3174f70700 -1 osd.1 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:20:12.566 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:12 smithi167 ceph-mon[192880]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:12.566 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:12 smithi167 ceph-mon[192880]: osd.1 marked itself down and dead 2023-09-30T09:20:12.566 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3497089564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:13.335 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:13 smithi019 bash[245082]: 4f868828e94965436d2fcef4544f48a637b6dd1308deb285e41a5ede75ddafa4 2023-09-30T09:20:13.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:13 smithi019 ceph-mon[216815]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:20:13.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:13 smithi019 ceph-mon[216815]: failed to encode map e96 with expected crc 2023-09-30T09:20:13.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:13 smithi019 ceph-mon[216815]: osdmap e96: 8 total, 7 up, 8 in 2023-09-30T09:20:13.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:13 smithi019 ceph-mon[224043]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:20:13.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:13 smithi019 ceph-mon[224043]: failed to encode map e96 with expected crc 2023-09-30T09:20:13.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:13 smithi019 ceph-mon[224043]: osdmap e96: 8 total, 7 up, 8 in 2023-09-30T09:20:13.991 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:13 smithi167 ceph-mon[192880]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:20:14.069 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:13 smithi167 ceph-mon[192880]: failed to encode map e96 with expected crc 2023-09-30T09:20:14.069 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:13 smithi167 ceph-mon[192880]: osdmap e96: 8 total, 7 up, 8 in 2023-09-30T09:20:14.083 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:20:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:20:13] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2023-09-30T09:20:14.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: pgmap v134: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:14.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: osdmap e97: 8 total, 7 up, 8 in 2023-09-30T09:20:14.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.373 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.373 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.373 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[224043]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: pgmap v134: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:14.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: osdmap e97: 8 total, 7 up, 8 in 2023-09-30T09:20:14.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.374 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.374 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.374 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.374 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.374 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:14 smithi019 ceph-mon[216815]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: pgmap v134: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:14.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: osdmap e97: 8 total, 7 up, 8 in 2023-09-30T09:20:14.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:14.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:14 smithi167 ceph-mon[192880]: failed to encode map e97 with expected crc 2023-09-30T09:20:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1697843680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1697843680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:15.336 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:14 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-30T09:20:15.336 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:14 smithi019 systemd[1]: Stopped Ceph osd.1 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:20:15.336 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:15 smithi019 systemd[1]: Starting Ceph osd.1 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:20:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1697843680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:16 smithi019 ceph-mon[216815]: pgmap v136: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:20:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:16 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:16 smithi019 ceph-mon[224043]: pgmap v136: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:20:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:16 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:16.336 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:16 smithi019 bash[245760]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-30T09:20:16.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:16 smithi019 bash[245760]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-30T09:20:16.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:16 smithi019 bash[245760]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-30T09:20:16.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:16 smithi019 bash[245760]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-30T09:20:16.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:16 smithi019 bash[245760]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-30T09:20:16.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:16 smithi019 bash[245760]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-30T09:20:16.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:16 smithi019 bash[245760]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-30T09:20:16.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:16 smithi019 bash[245760]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-30T09:20:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:16 smithi167 ceph-mon[192880]: pgmap v136: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:20:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:16 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2806027997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:17.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2806027997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:17.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2806027997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:17.773 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:17 smithi019 ceph-osd[246250]: -- 172.21.15.19:0/3189830009 <== mon.2 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564b08ca44e0 con 0x564b08c76c00 2023-09-30T09:20:17.773 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:17 smithi019 bash[246184]: 5db49d11f877fec7503c3d8c7920fd6b72cd872dc53bce943268a1d1cc209175 2023-09-30T09:20:17.773 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:17 smithi019 systemd[1]: Started Ceph osd.1 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:20:18.085 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:17 smithi019 conmon[246226]: 2023-09-30T09:20:17.810+0000 7ffb7170b640 -1 Falling back to public interface 2023-09-30T09:20:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:18 smithi167 ceph-mon[192880]: pgmap v137: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:20:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[216815]: pgmap v137: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:20:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[224043]: pgmap v137: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:20:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3861164716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:19.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3861164716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:19.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3861164716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:20.359 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:20 smithi167 ceph-mon[192880]: pgmap v138: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:20.359 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:20 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:20.835 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:20 smithi019 conmon[246226]: 2023-09-30T09:20:20.575+0000 7ffb7170b640 -1 osd.1 0 read_superblock omap replica is missing. 2023-09-30T09:20:20.835 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:20 smithi019 conmon[246226]: 2023-09-30T09:20:20.599+0000 7ffb7170b640 -1 osd.1 95 log_to_monitors true 2023-09-30T09:20:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:20 smithi019 ceph-mon[216815]: pgmap v138: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:20 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:20 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:20 smithi019 ceph-mon[224043]: pgmap v138: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:20 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:20 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.835 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:20:21 smithi019 conmon[246226]: 2023-09-30T09:20:21.570+0000 7ffb60b73700 -1 osd.1 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:20:21.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[216815]: from='osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[216815]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/297977127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[224043]: from='osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[224043]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:20:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:21.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/297977127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:20 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:21 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:21 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:21 smithi167 ceph-mon[192880]: from='osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:20:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:21 smithi167 ceph-mon[192880]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:20:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:21 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:21 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:22.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/297977127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:22.664 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[224043]: pgmap v139: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:22.664 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[224043]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:20:22.664 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[224043]: failed to encode map e98 with expected crc 2023-09-30T09:20:22.664 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[224043]: osdmap e98: 8 total, 7 up, 8 in 2023-09-30T09:20:22.665 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[224043]: from='osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:22.665 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[224043]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:22.665 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[216815]: pgmap v139: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:22.665 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[216815]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:20:22.665 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[216815]: failed to encode map e98 with expected crc 2023-09-30T09:20:22.665 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[216815]: osdmap e98: 8 total, 7 up, 8 in 2023-09-30T09:20:22.665 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[216815]: from='osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:22.665 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:22 smithi019 ceph-mon[216815]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:22 smithi167 ceph-mon[192880]: pgmap v139: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:22 smithi167 ceph-mon[192880]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:20:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:22 smithi167 ceph-mon[192880]: failed to encode map e98 with expected crc 2023-09-30T09:20:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:22 smithi167 ceph-mon[192880]: osdmap e98: 8 total, 7 up, 8 in 2023-09-30T09:20:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:22 smithi167 ceph-mon[192880]: from='osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:22 smithi167 ceph-mon[192880]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:23.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:20:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:20:23] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.43.0" 2023-09-30T09:20:23.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404] boot 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: osdmap e99: 8 total, 8 up, 8 in 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: pgmap v142: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:23.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[216815]: osdmap e100: 8 total, 8 up, 8 in 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404] boot 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: osdmap e99: 8 total, 8 up, 8 in 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: failed to encode map e99 with expected crc 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: pgmap v142: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:23.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:23.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:23 smithi019 ceph-mon[224043]: osdmap e100: 8 total, 8 up, 8 in 2023-09-30T09:20:24.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:20:24.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: failed to encode map e99 with expected crc 2023-09-30T09:20:24.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: failed to encode map e99 with expected crc 2023-09-30T09:20:24.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: failed to encode map e99 with expected crc 2023-09-30T09:20:24.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: failed to encode map e99 with expected crc 2023-09-30T09:20:24.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: osd.1 [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404] boot 2023-09-30T09:20:24.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: osdmap e99: 8 total, 8 up, 8 in 2023-09-30T09:20:24.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: failed to encode map e99 with expected crc 2023-09-30T09:20:24.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:20:24.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: pgmap v142: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-30T09:20:24.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:24.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:23 smithi167 ceph-mon[192880]: osdmap e100: 8 total, 8 up, 8 in 2023-09-30T09:20:24.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:20:24 smithi019 conmon[202899]: 2023-09-30T09:20:24.614+0000 7f1d921ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-30T09:20:24.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/890983491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: failed to encode map e100 with expected crc 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/890983491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:24.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: failed to encode map e100 with expected crc 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/890983491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:25.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:25.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:25.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:25.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:25.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:25.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:25.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[216815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[216815]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[216815]: pgmap v144: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[224043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[224043]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-30T09:20:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:25 smithi019 ceph-mon[224043]: pgmap v144: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:20:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:25 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:25 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:25 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:25 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:25 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:25 smithi167 ceph-mon[192880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:25 smithi167 ceph-mon[192880]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-30T09:20:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:25 smithi167 ceph-mon[192880]: pgmap v144: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:20:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3375613358' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3375613358' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3375613358' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:27 smithi167 ceph-mon[192880]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:28.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:27 smithi019 ceph-mon[216815]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:28.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:27 smithi019 ceph-mon[224043]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:28 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-30T09:20:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:28 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:20:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:28 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4046444218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:28 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-30T09:20:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:28 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:20:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:28 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4046444218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:29.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:28 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-30T09:20:29.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:28 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:20:29.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:28 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:29.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4046444218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:29 smithi167 ceph-mon[192880]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:29 smithi019 ceph-mon[216815]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:30.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:29 smithi019 ceph-mon[224043]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1817362405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:31.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1817362405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:31.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1817362405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:31 smithi167 ceph-mon[192880]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:32.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:31 smithi019 ceph-mon[216815]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:32.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:31 smithi019 ceph-mon[224043]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:20:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:20:33] "GET /metrics HTTP/1.1" 200 33877 "" "Prometheus/2.43.0" 2023-09-30T09:20:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:33 smithi019 ceph-mon[224043]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:34.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:34.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:33 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4116143150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:33 smithi019 ceph-mon[216815]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:33 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4116143150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:33 smithi167 ceph-mon[192880]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:33 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4116143150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:35 smithi167 ceph-mon[192880]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3742843073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:35 smithi019 ceph-mon[216815]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3742843073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:35 smithi019 ceph-mon[224043]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3742843073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:38.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:37 smithi167 ceph-mon[192880]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:38.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3800986170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:37 smithi019 ceph-mon[216815]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3800986170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:37 smithi019 ceph-mon[224043]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3800986170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:40 smithi019 ceph-mon[224043]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:40 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2298474324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:40 smithi019 ceph-mon[216815]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:40 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:40.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2298474324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:40 smithi167 ceph-mon[192880]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:40 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2298474324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:41 smithi019 ceph-mon[216815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:41 smithi019 ceph-mon[216815]: Upgrade: osd.2 is safe to restart 2023-09-30T09:20:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:41 smithi019 ceph-mon[224043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:41 smithi019 ceph-mon[224043]: Upgrade: osd.2 is safe to restart 2023-09-30T09:20:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:41 smithi167 ceph-mon[192880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-30T09:20:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:41 smithi167 ceph-mon[192880]: Upgrade: osd.2 is safe to restart 2023-09-30T09:20:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[216815]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[216815]: Upgrade: Updating osd.2 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[216815]: Deploying daemon osd.2 on smithi019 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[224043]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[224043]: Upgrade: Updating osd.2 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:42.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:20:42.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:42.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:42 smithi019 ceph-mon[224043]: Deploying daemon osd.2 on smithi019 2023-09-30T09:20:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:42 smithi167 ceph-mon[192880]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:42 smithi167 ceph-mon[192880]: Upgrade: Updating osd.2 2023-09-30T09:20:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-30T09:20:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:42.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:42 smithi167 ceph-mon[192880]: Deploying daemon osd.2 on smithi019 2023-09-30T09:20:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4055149264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4055149264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4055149264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:44.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:20:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:20:43] "GET /metrics HTTP/1.1" 200 33877 "" "Prometheus/2.43.0" 2023-09-30T09:20:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:44 smithi167 ceph-mon[192880]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:44 smithi019 ceph-mon[216815]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:44.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:44 smithi019 ceph-mon[224043]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:44.585 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:44 smithi019 systemd[1]: Stopping Ceph osd.2 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:20:44.586 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:44 smithi019 conmon[143980]: 2023-09-30T09:20:44.329+0000 7efdca2fd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:20:44.586 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:44 smithi019 conmon[143980]: 2023-09-30T09:20:44.329+0000 7efdca2fd700 -1 osd.2 100 *** Got signal Terminated *** 2023-09-30T09:20:44.586 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:44 smithi019 conmon[143980]: 2023-09-30T09:20:44.329+0000 7efdca2fd700 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:20:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:45 smithi167 ceph-mon[192880]: osd.2 marked itself down and dead 2023-09-30T09:20:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:45 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3183536514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:45 smithi019 ceph-mon[216815]: osd.2 marked itself down and dead 2023-09-30T09:20:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:45 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3183536514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:45 smithi019 ceph-mon[224043]: osd.2 marked itself down and dead 2023-09-30T09:20:45.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:45 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3183536514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:46.085 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:45 smithi019 bash[252659]: 6673f026fd30d0e30303c9740f105c58e834011565b8a69aa077d180067d7e0d 2023-09-30T09:20:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:46 smithi167 ceph-mon[192880]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:46 smithi167 ceph-mon[192880]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:20:46.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:46 smithi167 ceph-mon[192880]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:46 smithi167 ceph-mon[192880]: osdmap e101: 8 total, 7 up, 8 in 2023-09-30T09:20:46.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:46 smithi167 ceph-mon[192880]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:46 smithi167 ceph-mon[192880]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:46 smithi167 ceph-mon[192880]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:46 smithi167 ceph-mon[192880]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[216815]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[216815]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[216815]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[216815]: osdmap e101: 8 total, 7 up, 8 in 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[216815]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[216815]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[216815]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[216815]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[224043]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:20:46.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[224043]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:20:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[224043]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[224043]: osdmap e101: 8 total, 7 up, 8 in 2023-09-30T09:20:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[224043]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[224043]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[224043]: failed to encode map e101 with expected crc 2023-09-30T09:20:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:46 smithi019 ceph-mon[224043]: failed to encode map e101 with expected crc 2023-09-30T09:20:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: osdmap e102: 8 total, 7 up, 8 in 2023-09-30T09:20:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1624510987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:47.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: osdmap e102: 8 total, 7 up, 8 in 2023-09-30T09:20:47.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1624510987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: osdmap e102: 8 total, 7 up, 8 in 2023-09-30T09:20:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: failed to encode map e102 with expected crc 2023-09-30T09:20:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1624510987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:48.213 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:47 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-30T09:20:48.213 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:47 smithi019 systemd[1]: Stopped Ceph osd.2 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:20:48.213 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:47 smithi019 systemd[1]: Starting Ceph osd.2 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:20:48.476 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:48 smithi019 ceph-mon[216815]: pgmap v157: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 5 active+undersized, 66 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:20:48.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:48 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:20:48.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:48.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:48.477 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:48 smithi019 ceph-mon[224043]: pgmap v157: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 5 active+undersized, 66 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:20:48.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:48 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:20:48.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:48.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:48 smithi167 ceph-mon[192880]: pgmap v157: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 5 active+undersized, 66 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:20:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:48 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:20:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:20:49.335 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:48 smithi019 bash[253347]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-30T09:20:49.335 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:48 smithi019 bash[253347]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-30T09:20:49.335 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:48 smithi019 bash[253347]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-30T09:20:49.335 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:48 smithi019 bash[253347]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-30T09:20:49.335 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:48 smithi019 bash[253347]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-30T09:20:49.335 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:48 smithi019 bash[253347]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-30T09:20:49.335 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:48 smithi019 bash[253347]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-30T09:20:49.336 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:48 smithi019 bash[253347]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-30T09:20:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:50 smithi167 ceph-mon[192880]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/720015279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:50 smithi019 ceph-mon[216815]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/720015279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:50.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:50 smithi019 ceph-mon[224043]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/720015279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:50.586 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:50 smithi019 ceph-osd[253844]: -- 172.21.15.19:0/1799964418 <== mon.2 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555b721864e0 con 0x555b72159000 2023-09-30T09:20:50.586 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:50 smithi019 bash[253778]: 328715512fcdddec7a437219a7ac01e77c06f1cf7540a0f364ff1112e009e3ba 2023-09-30T09:20:50.586 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:50 smithi019 systemd[1]: Started Ceph osd.2 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:20:51.085 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:50 smithi019 conmon[253819]: 2023-09-30T09:20:50.792+0000 7fb3284e6640 -1 Falling back to public interface 2023-09-30T09:20:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:52 smithi167 ceph-mon[192880]: pgmap v159: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3662449729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:52.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:52 smithi019 ceph-mon[216815]: pgmap v159: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:52.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3662449729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:52.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:52 smithi019 ceph-mon[224043]: pgmap v159: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:52.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3662449729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:53.803 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:53.803 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:53.803 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:53.804 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:53.804 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:53.804 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:53.804 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:53 smithi019 conmon[253819]: 2023-09-30T09:20:53.574+0000 7fb3284e6640 -1 osd.2 0 read_superblock omap replica is missing. 2023-09-30T09:20:53.804 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:53 smithi019 conmon[253819]: 2023-09-30T09:20:53.594+0000 7fb3284e6640 -1 osd.2 100 log_to_monitors true 2023-09-30T09:20:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:20:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:20:53] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.43.0" 2023-09-30T09:20:54.684 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:20:54 smithi019 conmon[253819]: 2023-09-30T09:20:54.380+0000 7fb319952700 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:20:54.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[216815]: pgmap v160: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:54.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[216815]: from='osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:20:54.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[216815]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:20:54.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:54.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:54.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2303577910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:54.685 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[224043]: pgmap v160: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:54.685 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[224043]: from='osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:20:54.685 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[224043]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:20:54.685 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:54.685 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:54.685 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:54 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2303577910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:54 smithi167 ceph-mon[192880]: pgmap v160: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:54 smithi167 ceph-mon[192880]: from='osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:20:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:54 smithi167 ceph-mon[192880]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-30T09:20:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:54 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2303577910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:55.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[216815]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-30T09:20:55.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[216815]: from='osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:55.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[216815]: osdmap e103: 8 total, 7 up, 8 in 2023-09-30T09:20:55.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[216815]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:55.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[216815]: failed to encode map e103 with expected crc 2023-09-30T09:20:55.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[216815]: failed to encode map e103 with expected crc 2023-09-30T09:20:55.736 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[224043]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-30T09:20:55.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[224043]: from='osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:55.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[224043]: osdmap e103: 8 total, 7 up, 8 in 2023-09-30T09:20:55.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[224043]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:55.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[224043]: failed to encode map e103 with expected crc 2023-09-30T09:20:55.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:55 smithi019 ceph-mon[224043]: failed to encode map e103 with expected crc 2023-09-30T09:20:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:55 smithi167 ceph-mon[192880]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-30T09:20:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:55 smithi167 ceph-mon[192880]: from='osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:55 smithi167 ceph-mon[192880]: osdmap e103: 8 total, 7 up, 8 in 2023-09-30T09:20:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:55 smithi167 ceph-mon[192880]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:20:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:55 smithi167 ceph-mon[192880]: failed to encode map e103 with expected crc 2023-09-30T09:20:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:55 smithi167 ceph-mon[192880]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.699 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: pgmap v162: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:56.699 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.699 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.699 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:20:56.699 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: failed to encode map e104 with expected crc 2023-09-30T09:20:56.699 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465] boot 2023-09-30T09:20:56.699 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: osdmap e104: 8 total, 8 up, 8 in 2023-09-30T09:20:56.699 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1542135684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: pgmap v162: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: failed to encode map e104 with expected crc 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465] boot 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: osdmap e104: 8 total, 8 up, 8 in 2023-09-30T09:20:56.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:20:56.701 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.701 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1542135684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: pgmap v162: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-30T09:20:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:20:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: failed to encode map e104 with expected crc 2023-09-30T09:20:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: osd.2 [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465] boot 2023-09-30T09:20:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: osdmap e104: 8 total, 8 up, 8 in 2023-09-30T09:20:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-30T09:20:56.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: failed to encode map e103 with expected crc 2023-09-30T09:20:56.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1542135684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: osdmap e105: 8 total, 8 up, 8 in 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:57 smithi167 ceph-mon[192880]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:20:57 smithi019 conmon[202899]: 2023-09-30T09:20:57.638+0000 7f1d921ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-30T09:20:57.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: osdmap e105: 8 total, 8 up, 8 in 2023-09-30T09:20:57.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[224043]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: osdmap e105: 8 total, 8 up, 8 in 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:57.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:57 smithi019 ceph-mon[216815]: failed to encode map e105 with expected crc 2023-09-30T09:20:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: pgmap v165: 73 pgs: 2 active+undersized+degraded, 12 active+undersized, 59 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:20:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:20:58.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: pgmap v165: 73 pgs: 2 active+undersized+degraded, 12 active+undersized, 59 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:20:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: pgmap v165: 73 pgs: 2 active+undersized+degraded, 12 active+undersized, 59 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:20:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:20:58.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:20:58.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:20:58.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:20:58.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:20:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:59 smithi167 ceph-mon[192880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:20:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:59 smithi167 ceph-mon[192880]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-30T09:20:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:20:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1474214178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:59 smithi019 ceph-mon[224043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:20:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:59 smithi019 ceph-mon[224043]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-30T09:20:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:20:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1474214178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:20:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:59 smithi019 ceph-mon[216815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:20:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:59 smithi019 ceph-mon[216815]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-30T09:20:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:20:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1474214178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:00.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:00 smithi167 ceph-mon[192880]: pgmap v166: 73 pgs: 2 active+undersized, 71 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:01.397 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:00 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-30T09:21:01.397 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:00 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:21:01.398 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:00 smithi019 ceph-mon[224043]: pgmap v166: 73 pgs: 2 active+undersized, 71 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:01.398 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:00 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-30T09:21:01.398 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:00 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:21:01.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:00 smithi019 ceph-mon[216815]: pgmap v166: 73 pgs: 2 active+undersized, 71 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:01.399 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:00 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-30T09:21:01.399 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:00 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:21:01.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3472470742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:01.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3472470742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:02.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3472470742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:02 smithi167 ceph-mon[192880]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:02.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:02 smithi019 ceph-mon[216815]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:02.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:02 smithi019 ceph-mon[224043]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:21:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:21:03] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2023-09-30T09:21:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:03 smithi019 ceph-mon[216815]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3469171472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:03 smithi019 ceph-mon[224043]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3469171472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:03 smithi167 ceph-mon[192880]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3469171472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:05 smithi167 ceph-mon[192880]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1440600159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:05 smithi019 ceph-mon[216815]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1440600159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:06.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:05 smithi019 ceph-mon[224043]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:06.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1440600159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:07 smithi167 ceph-mon[192880]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:07 smithi019 ceph-mon[216815]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:07 smithi019 ceph-mon[224043]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3749398133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3749398133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3749398133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:10 smithi019 ceph-mon[216815]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:10 smithi019 ceph-mon[224043]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:10 smithi167 ceph-mon[192880]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4254922883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4254922883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4254922883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:12 smithi019 ceph-mon[216815]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:12 smithi019 ceph-mon[224043]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:12 smithi167 ceph-mon[192880]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:13 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:21:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2259796468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:13 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:21:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2259796468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:13 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:21:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2259796468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:14.081 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:21:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:21:13] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2023-09-30T09:21:14.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:14 smithi019 ceph-mon[216815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:21:14.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:14 smithi019 ceph-mon[216815]: Upgrade: osd.3 is safe to restart 2023-09-30T09:21:14.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:14 smithi019 ceph-mon[216815]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:14.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:14 smithi019 ceph-mon[224043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:21:14.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:14 smithi019 ceph-mon[224043]: Upgrade: osd.3 is safe to restart 2023-09-30T09:21:14.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:14 smithi019 ceph-mon[224043]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:14 smithi167 ceph-mon[192880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-30T09:21:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:14 smithi167 ceph-mon[192880]: Upgrade: osd.3 is safe to restart 2023-09-30T09:21:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:14 smithi167 ceph-mon[192880]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[216815]: Upgrade: Updating osd.3 2023-09-30T09:21:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:21:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[216815]: Deploying daemon osd.3 on smithi019 2023-09-30T09:21:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1453496652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[224043]: Upgrade: Updating osd.3 2023-09-30T09:21:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:21:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[224043]: Deploying daemon osd.3 on smithi019 2023-09-30T09:21:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1453496652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:15 smithi167 ceph-mon[192880]: Upgrade: Updating osd.3 2023-09-30T09:21:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:15 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:15 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-30T09:21:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:15 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:15 smithi167 ceph-mon[192880]: Deploying daemon osd.3 on smithi019 2023-09-30T09:21:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1453496652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:16 smithi167 ceph-mon[192880]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:16.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:16 smithi019 ceph-mon[216815]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:16 smithi019 ceph-mon[224043]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:17.300 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:17 smithi019 systemd[1]: Stopping Ceph osd.3 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:21:17.585 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:17 smithi019 conmon[153794]: 2023-09-30T09:21:17.297+0000 7f5460b14700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:21:17.585 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:17 smithi019 conmon[153794]: 2023-09-30T09:21:17.297+0000 7f5460b14700 -1 osd.3 105 *** Got signal Terminated *** 2023-09-30T09:21:17.585 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:17 smithi019 conmon[153794]: 2023-09-30T09:21:17.297+0000 7f5460b14700 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:21:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:18 smithi167 ceph-mon[192880]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:18 smithi167 ceph-mon[192880]: osd.3 marked itself down and dead 2023-09-30T09:21:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4145762403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:18 smithi019 ceph-mon[216815]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:18 smithi019 ceph-mon[216815]: osd.3 marked itself down and dead 2023-09-30T09:21:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4145762403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:18 smithi019 ceph-mon[224043]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:18 smithi019 ceph-mon[224043]: osd.3 marked itself down and dead 2023-09-30T09:21:18.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4145762403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:18.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:19.085 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:18 smithi019 bash[260304]: 8fa7001643e13d58420c8a27d8aac69ff696e052d2b8495fb916cb15e605df22 2023-09-30T09:21:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:21:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: osdmap e106: 8 total, 7 up, 8 in 2023-09-30T09:21:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:19 smithi167 ceph-mon[192880]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:21:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: osdmap e106: 8 total, 7 up, 8 in 2023-09-30T09:21:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[224043]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: osdmap e106: 8 total, 7 up, 8 in 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:19.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:19 smithi019 ceph-mon[216815]: failed to encode map e106 with expected crc 2023-09-30T09:21:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: pgmap v177: 73 pgs: 15 peering, 7 stale+active+clean, 51 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-30T09:21:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: osdmap e107: 8 total, 7 up, 8 in 2023-09-30T09:21:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2478941345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: pgmap v177: 73 pgs: 15 peering, 7 stale+active+clean, 51 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-30T09:21:20.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: osdmap e107: 8 total, 7 up, 8 in 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2478941345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: pgmap v177: 73 pgs: 15 peering, 7 stale+active+clean, 51 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-30T09:21:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: osdmap e107: 8 total, 7 up, 8 in 2023-09-30T09:21:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: failed to encode map e107 with expected crc 2023-09-30T09:21:20.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2478941345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:21.085 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:20 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-30T09:21:21.085 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:20 smithi019 systemd[1]: Stopped Ceph osd.3 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:21:21.085 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:20 smithi019 systemd[1]: Starting Ceph osd.3 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:21:22.190 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:21 smithi019 bash[260977]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-30T09:21:22.190 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:21 smithi019 bash[260977]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-30T09:21:22.190 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:21 smithi019 bash[260977]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-30T09:21:22.190 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:21 smithi019 bash[260977]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-30T09:21:22.190 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:21 smithi019 bash[260977]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-30T09:21:22.190 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:21 smithi019 bash[260977]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-30T09:21:22.191 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:21 smithi019 bash[260977]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-30T09:21:22.191 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:21 smithi019 bash[260977]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-30T09:21:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:22 smithi167 ceph-mon[192880]: pgmap v179: 73 pgs: 15 peering, 5 stale+active+clean, 4 active+undersized, 49 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/254917928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:22 smithi019 ceph-mon[216815]: pgmap v179: 73 pgs: 15 peering, 5 stale+active+clean, 4 active+undersized, 49 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/254917928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:22.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:22 smithi019 ceph-mon[224043]: pgmap v179: 73 pgs: 15 peering, 5 stale+active+clean, 4 active+undersized, 49 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:22.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/254917928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:23.335 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:23 smithi019 ceph-osd[261465]: -- 172.21.15.19:0/3853841273 <== mon.2 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5621c34484e0 con 0x5621c341b000 2023-09-30T09:21:25.153 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:23 smithi019 bash[261400]: 1257456223c880652b760ebb4683b5062a3777de7df18bda0f7be5a07bbb92a0 2023-09-30T09:21:25.153 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:23 smithi019 systemd[1]: Started Ceph osd.3 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:21:25.154 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:24 smithi167 ceph-mon[192880]: pgmap v180: 73 pgs: 1 active+undersized+degraded, 15 peering, 15 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-30T09:21:25.154 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:25.154 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:25.154 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:25.155 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:24 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:21:25.155 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:23 smithi019 conmon[261441]: 2023-09-30T09:21:23.530+0000 7f8c29cf2640 -1 Falling back to public interface 2023-09-30T09:21:25.180 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:21:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:21:23] "GET /metrics HTTP/1.1" 200 33870 "" "Prometheus/2.43.0" 2023-09-30T09:21:25.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[216815]: pgmap v180: 73 pgs: 1 active+undersized+degraded, 15 peering, 15 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-30T09:21:25.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:25.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:25.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:25.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:21:25.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[224043]: pgmap v180: 73 pgs: 1 active+undersized+degraded, 15 peering, 15 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-30T09:21:25.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:25.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:25.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:25.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:24 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:21:25.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:25 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3505390405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:25 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3505390405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:25 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3505390405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:26.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:26 smithi019 ceph-mon[216815]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:21:26.411 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:26 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-30T09:21:26.411 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:26 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:26.411 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:26 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:26.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:26 smithi019 ceph-mon[224043]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:21:26.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:26 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-30T09:21:26.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:26 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:26.412 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:26 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:26.412 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:26 smithi019 conmon[261441]: 2023-09-30T09:21:26.291+0000 7f8c29cf2640 -1 osd.3 0 read_superblock omap replica is missing. 2023-09-30T09:21:26.412 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:26 smithi019 conmon[261441]: 2023-09-30T09:21:26.309+0000 7f8c29cf2640 -1 osd.3 105 log_to_monitors true 2023-09-30T09:21:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:26 smithi167 ceph-mon[192880]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:21:26.556 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:26 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-30T09:21:26.556 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:26 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:26.556 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:26 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.320 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[216815]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:21:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2531230144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:27.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[224043]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:21:27.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2531230144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:27.346 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:21:27 smithi019 conmon[261441]: 2023-09-30T09:21:27.106+0000 7f8c17156700 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:21:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:27 smithi167 ceph-mon[192880]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-30T09:21:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2531230144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:28.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[224043]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:21:28.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[224043]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-30T09:21:28.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[224043]: osdmap e108: 8 total, 7 up, 8 in 2023-09-30T09:21:28.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[224043]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:21:28.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[224043]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[224043]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[224043]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[224043]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.482 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[216815]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:21:28.482 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[216815]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-30T09:21:28.482 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[216815]: osdmap e108: 8 total, 7 up, 8 in 2023-09-30T09:21:28.482 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[216815]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:21:28.482 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[216815]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.482 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[216815]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.482 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[216815]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.482 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:28 smithi019 ceph-mon[216815]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:28 smithi167 ceph-mon[192880]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:21:28.625 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:28 smithi167 ceph-mon[192880]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-30T09:21:28.625 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:28 smithi167 ceph-mon[192880]: osdmap e108: 8 total, 7 up, 8 in 2023-09-30T09:21:28.625 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:28 smithi167 ceph-mon[192880]: from='osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:21:28.626 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:28 smithi167 ceph-mon[192880]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.626 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:28 smithi167 ceph-mon[192880]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.626 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:28 smithi167 ceph-mon[192880]: failed to encode map e108 with expected crc 2023-09-30T09:21:28.626 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:28 smithi167 ceph-mon[192880]: failed to encode map e108 with expected crc 2023-09-30T09:21:29.072 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)": 4 2023-09-30T09:21:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:29 smithi167 ceph-mon[192880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:21:29.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:29 smithi167 ceph-mon[192880]: failed to encode map e109 with expected crc 2023-09-30T09:21:29.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:29 smithi167 ceph-mon[192880]: osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409] boot 2023-09-30T09:21:29.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:29 smithi167 ceph-mon[192880]: osdmap e109: 8 total, 8 up, 8 in 2023-09-30T09:21:29.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:29 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:21:29.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:29 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2306896429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:29.537 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:29 smithi167 ceph-mon[192880]: osdmap e110: 8 total, 8 up, 8 in 2023-09-30T09:21:29.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[216815]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:21:29.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[216815]: failed to encode map e109 with expected crc 2023-09-30T09:21:29.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[216815]: osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409] boot 2023-09-30T09:21:29.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[216815]: osdmap e109: 8 total, 8 up, 8 in 2023-09-30T09:21:29.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:21:29.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2306896429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:29.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[216815]: osdmap e110: 8 total, 8 up, 8 in 2023-09-30T09:21:29.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[224043]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:21:29.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[224043]: failed to encode map e109 with expected crc 2023-09-30T09:21:29.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[224043]: osd.3 [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409] boot 2023-09-30T09:21:29.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[224043]: osdmap e109: 8 total, 8 up, 8 in 2023-09-30T09:21:29.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-30T09:21:29.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2306896429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-30T09:21:29.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:29 smithi019 ceph-mon[224043]: osdmap e110: 8 total, 8 up, 8 in 2023-09-30T09:21:29.937 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch upgrade pause' 2023-09-30T09:21:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: pgmap v185: 73 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:21:30.413 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.413 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:21:30.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[216815]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: pgmap v185: 73 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:21:30.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:30 smithi019 ceph-mon[224043]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.468 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: pgmap v185: 73 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:21:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:30 smithi167 ceph-mon[192880]: failed to encode map e110 with expected crc 2023-09-30T09:21:30.669 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:21:30 smithi019 conmon[202899]: 2023-09-30T09:21:30.588+0000 7f1d921ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-30T09:21:31.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-30T09:21:31.765 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-30T09:21:32.201 INFO:teuthology.orchestra.run.smithi019.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:21:32.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:32 smithi019 ceph-mon[216815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-30T09:21:32.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:32 smithi019 ceph-mon[216815]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-30T09:21:32.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:32 smithi019 ceph-mon[216815]: pgmap v187: 73 pgs: 9 peering, 2 active+undersized+degraded, 18 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:21:32.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:32.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:32 smithi019 ceph-mon[224043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-30T09:21:32.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:32 smithi019 ceph-mon[224043]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-30T09:21:32.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:32 smithi019 ceph-mon[224043]: pgmap v187: 73 pgs: 9 peering, 2 active+undersized+degraded, 18 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:21:32.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:32 smithi167 ceph-mon[192880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-30T09:21:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:32 smithi167 ceph-mon[192880]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-30T09:21:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:32 smithi167 ceph-mon[192880]: pgmap v187: 73 pgs: 9 peering, 2 active+undersized+degraded, 18 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-30T09:21:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:32.923 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch ps' 2023-09-30T09:21:33.289 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:33.727 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:33 smithi019 ceph-mon[216815]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:33.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:33 smithi019 ceph-mon[216815]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:21:33.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:33.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:33.728 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:33 smithi019 ceph-mon[224043]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:33.728 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:33 smithi019 ceph-mon[224043]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:21:33.728 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:33.728 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:33 smithi167 ceph-mon[192880]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:33 smithi167 ceph-mon[192880]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0 2023-09-30T09:21:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:21:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:21:33] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.43.0" 2023-09-30T09:21:34.621 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:34 smithi019 ceph-mon[224043]: pgmap v188: 73 pgs: 9 peering, 64 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:34.621 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:34 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-30T09:21:34.621 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:34 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:21:34.621 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:34 smithi019 ceph-mon[216815]: pgmap v188: 73 pgs: 9 peering, 64 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:34.621 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:34 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-30T09:21:34.621 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:34 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:21:34.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:34 smithi167 ceph-mon[192880]: pgmap v188: 73 pgs: 9 peering, 64 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:34.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:34 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-30T09:21:34.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:34 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:21:35.057 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (6m) - 10m 29.1M - 0.25.0 c8568f914cd2 b16b2fb388da 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi167 *:3000 running (6m) - 9m 91.2M - 9.4.7 2c41d148cca3 f7c9f74b8088 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi019.kcgktg smithi019 running (9m) - 9m 19.2M - 17.2.6-1161-g191c1869 8497a1260037 ca3d56fcab07 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi167.bnuarp smithi167 running (9m) - 9m 17.4M - 17.2.6-1161-g191c1869 8497a1260037 a70137fe60d5 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi167 *:8443,9283,8765 running (5m) - 14m 451M - 18.0.0-6447-g1d10b717 dda7fab67785 3468c121edb9 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (6m) - 15m 516M - 18.0.0-6447-g1d10b717 dda7fab67785 f13d86332ace 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (5m) - 16m 81.7M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 c1584e4cc7f3 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi167 running (3m) - 14m 54.8M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 fd31a48646fc 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (4m) - 14m 61.3M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 84a54501b134 2023-09-30T09:21:35.058 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (6m) - 9m 18.5M - 1.5.0 0da6a335fe13 63ac922663db 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi167 *:9100 running (6m) - 9m 17.2M - 1.5.0 0da6a335fe13 d6c59fa1a461 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (2m) - 13m 61.2M 2501M 18.0.0-6447-g1d10b717 dda7fab67785 0112d2fb79a7 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (78s) - 13m 57.3M 2501M 18.0.0-6447-g1d10b717 dda7fab67785 5db49d11f877 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (45s) - 13m 49.2M 2501M 18.0.0-6447-g1d10b717 dda7fab67785 328715512fcd 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (12s) - 12m 12.4M 2501M 18.0.0-6447-g1d10b717 dda7fab67785 1257456223c8 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi167 running (12m) - 12m 101M 2500M 17.2.6-1161-g191c1869 8497a1260037 6557c10c6a0b 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi167 running (11m) - 11m 99.9M 2500M 17.2.6-1161-g191c1869 8497a1260037 30ccc3c97db6 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi167 running (11m) - 11m 96.7M 2500M 17.2.6-1161-g191c1869 8497a1260037 4a5f3bfc86e4 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi167 running (11m) - 11m 122M 2500M 17.2.6-1161-g191c1869 8497a1260037 92e8410b5190 2023-09-30T09:21:35.059 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi167 *:9095 running (5m) - 10m 64.2M - 2.43.0 a07b618ecd1d c625e147e7f2 2023-09-30T09:21:35.735 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph orch ps' 2023-09-30T09:21:36.096 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:36 smithi019 ceph-mon[216815]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:36 smithi019 ceph-mon[216815]: from='client.44378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:36.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:36 smithi019 ceph-mon[224043]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:36.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:36 smithi019 ceph-mon[224043]: from='client.44378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:36.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:36 smithi167 ceph-mon[192880]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:36.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:36 smithi167 ceph-mon[192880]: from='client.44378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:37.777 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-30T09:21:37.777 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (6m) - 10m 29.1M - 0.25.0 c8568f914cd2 b16b2fb388da 2023-09-30T09:21:37.777 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi167 *:3000 running (6m) - 9m 91.2M - 9.4.7 2c41d148cca3 f7c9f74b8088 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi019.kcgktg smithi019 running (9m) - 9m 19.2M - 17.2.6-1161-g191c1869 8497a1260037 ca3d56fcab07 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:mds.foo.smithi167.bnuarp smithi167 running (9m) - 9m 17.4M - 17.2.6-1161-g191c1869 8497a1260037 a70137fe60d5 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi167 *:8443,9283,8765 running (5m) - 14m 451M - 18.0.0-6447-g1d10b717 dda7fab67785 3468c121edb9 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (6m) - 16m 516M - 18.0.0-6447-g1d10b717 dda7fab67785 f13d86332ace 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (5m) - 16m 81.7M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 c1584e4cc7f3 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi167 running (3m) - 14m 54.8M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 fd31a48646fc 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (4m) - 14m 61.3M 2048M 18.0.0-6447-g1d10b717 dda7fab67785 84a54501b134 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (6m) - 9m 18.5M - 1.5.0 0da6a335fe13 63ac922663db 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi167 *:9100 running (6m) - 9m 17.2M - 1.5.0 0da6a335fe13 d6c59fa1a461 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (2m) - 13m 61.2M 2501M 18.0.0-6447-g1d10b717 dda7fab67785 0112d2fb79a7 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (80s) - 13m 57.3M 2501M 18.0.0-6447-g1d10b717 dda7fab67785 5db49d11f877 2023-09-30T09:21:37.778 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (47s) - 13m 49.2M 2501M 18.0.0-6447-g1d10b717 dda7fab67785 328715512fcd 2023-09-30T09:21:37.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (15s) - 12m 12.4M 2501M 18.0.0-6447-g1d10b717 dda7fab67785 1257456223c8 2023-09-30T09:21:37.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi167 running (12m) - 12m 101M 2500M 17.2.6-1161-g191c1869 8497a1260037 6557c10c6a0b 2023-09-30T09:21:37.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi167 running (11m) - 11m 99.9M 2500M 17.2.6-1161-g191c1869 8497a1260037 30ccc3c97db6 2023-09-30T09:21:37.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi167 running (11m) - 11m 96.7M 2500M 17.2.6-1161-g191c1869 8497a1260037 4a5f3bfc86e4 2023-09-30T09:21:37.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi167 running (11m) - 11m 122M 2500M 17.2.6-1161-g191c1869 8497a1260037 92e8410b5190 2023-09-30T09:21:37.779 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi167 *:9095 running (6m) - 10m 64.2M - 2.43.0 a07b618ecd1d c625e147e7f2 2023-09-30T09:21:38.485 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -e sha1=1d10b71792f3be8887a7631e69851ac2df3585af -- bash -c 'ceph versions' 2023-09-30T09:21:38.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:38 smithi167 ceph-mon[192880]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:38.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:38 smithi019 ceph-mon[216815]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:38.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:38 smithi019 ceph-mon[224043]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:38.844 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:39.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:39 smithi167 ceph-mon[192880]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:39.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:39 smithi019 ceph-mon[216815]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:39 smithi019 ceph-mon[224043]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:21:40.609 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)": 3 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)": 2 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)": 4, 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)": 4 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:21:40.610 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": { 2023-09-30T09:21:40.611 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)": 2 2023-09-30T09:21:40.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:21:40.611 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2023-09-30T09:21:40.611 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.6-1161-g191c1869 (191c18699e384ce04d90fb4045d6dce3223c4681) quincy (stable)": 6, 2023-09-30T09:21:40.611 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 18.0.0-6447-g1d10b717 (1d10b71792f3be8887a7631e69851ac2df3585af) reef (dev)": 9 2023-09-30T09:21:40.611 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:21:40.611 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:21:40.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:40 smithi167 ceph-mon[192880]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:40.818 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:40 smithi019 ceph-mon[216815]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:40.818 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:40 smithi019 ceph-mon[224043]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:41.336 DEBUG:teuthology.parallel:result is None 2023-09-30T09:21:41.337 INFO:teuthology.run_tasks:Running task print... 2023-09-30T09:21:41.346 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-30T09:21:41.346 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-30T09:21:41.376 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-30T09:21:41.376 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-30T09:21:41.376 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-30T09:21:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/373459357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/373459357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:41.742 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:41.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/373459357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-30T09:21:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:42 smithi167 ceph-mon[192880]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:42.828 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:42 smithi019 ceph-mon[216815]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:42.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:42 smithi019 ceph-mon[224043]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:43.617 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:21:44.021 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:21:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:21:43] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.43.0" 2023-09-30T09:21:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:44 smithi167 ceph-mon[192880]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:44 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2857734993' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:21:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:44 smithi019 ceph-mon[216815]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:44 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2857734993' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:21:44.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:44 smithi019 ceph-mon[224043]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:44.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:44 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:44.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2857734993' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-30T09:21:44.369 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":40,"active_gid":25835,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6800","nonce":452239528},{"type":"v1","addr":"172.21.15.19:6801","nonce":452239528}]},"active_addr":"172.21.15.19:6801/452239528","active_change":"2023-09-30T09:16:02.837071+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25693,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.3","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.3","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":90,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":3104859627}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":127117933}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":1588000460}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":1346613679}]}]} 2023-09-30T09:21:44.374 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-30T09:21:44.374 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-30T09:21:44.374 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:21:44.739 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:46.422 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:21:46.971 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":110,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:21:29.074381+0000","last_up_change":"2023-09-30T09:21:28.072487+0000","last_in_change":"2023-09-30T09:10:11.880926+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-30T09:11:07.846610+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-30T09:12:06.877159+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-30T09:12:07.385950+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":2904823591},{"type":"v1","addr":"172.21.15.19:6803","nonce":2904823591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":2904823591},{"type":"v1","addr":"172.21.15.19:6805","nonce":2904823591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":2904823591},{"type":"v1","addr":"172.21.15.19:6809","nonce":2904823591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":2904823591},{"type":"v1","addr":"172.21.15.19:6807","nonce":2904823591}]},"public_addr":"172.21.15.19:6803/2904823591","cluster_addr":"172.21.15.19:6805/2904823591","heartbeat_back_addr":"172.21.15.19:6809/2904823591","heartbeat_front_addr":"172.21.15.19:6807/2904823591","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":1402281404},{"type":"v1","addr":"172.21.15.19:6811","nonce":1402281404}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":1402281404},{"type":"v1","addr":"172.21.15.19:6813","nonce":1402281404}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":1402281404},{"type":"v1","addr":"172.21.15.19:6817","nonce":1402281404}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":1402281404},{"type":"v1","addr":"172.21.15.19:6815","nonce":1402281404}]},"public_addr":"172.21.15.19:6811/1402281404","cluster_addr":"172.21.15.19:6813/1402281404","heartbeat_back_addr":"172.21.15.19:6817/1402281404","heartbeat_front_addr":"172.21.15.19:6815/1402281404","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6819","nonce":2623064465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6821","nonce":2623064465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6825","nonce":2623064465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6823","nonce":2623064465}]},"public_addr":"172.21.15.19:6819/2623064465","cluster_addr":"172.21.15.19:6821/2623064465","heartbeat_back_addr":"172.21.15.19:6825/2623064465","heartbeat_front_addr":"172.21.15.19:6823/2623064465","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6827","nonce":3712654409}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6829","nonce":3712654409}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6833","nonce":3712654409}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6831","nonce":3712654409}]},"public_addr":"172.21.15.19:6827/3712654409","cluster_addr":"172.21.15.19:6829/3712654409","heartbeat_back_addr":"172.21.15.19:6833/3712654409","heartbeat_front_addr":"172.21.15.19:6831/3712654409","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6809","nonce":2834758136}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6811","nonce":2834758136}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6815","nonce":2834758136}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6813","nonce":2834758136}]},"public_addr":"172.21.15.167:6809/2834758136","cluster_addr":"172.21.15.167:6811/2834758136","heartbeat_back_addr":"172.21.15.167:6815/2834758136","heartbeat_front_addr":"172.21.15.167:6813/2834758136","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/4266586334":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/1275272106":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.167:0/625642582":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/956155":"2023-10-01T09:14:41.945226+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.167:6835/1123065418":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/861352263":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6800/605197989":"2023-10-01T09:16:02.836873+0000","172.21.15.167:6834/1123065418":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/3324367156":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/1856357891":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3152350528":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/1638946842":"2023-10-01T09:14:41.945226+0000","172.21.15.167:0/798457358":"2023-10-01T09:15:21.453516+0000","172.21.15.167:0/320118286":"2023-10-01T09:15:21.453516+0000","172.21.15.19:6800/2650629233":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/238111816":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6800/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3931489393":"2023-10-01T09:14:41.945226+0000","172.21.15.19:6801/2650629233":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/936416645":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/2022890892":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/1679148187":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/309324931":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/2218856933":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3289282363":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/554243229":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6801/605197989":"2023-10-01T09:16:02.836873+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:21:46.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:46.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:46.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:46.972 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:47.032 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:47.032 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:21:47.032 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:21:47.032 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:21:47.032 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:47.144 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-30T09:21:47.194 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:21:47.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3525692872' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:21:47.362 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3525692872' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:21:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3525692872' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:21:47.554 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:48 smithi019 ceph-mon[216815]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:48 smithi019 ceph-mon[224043]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:48 smithi167 ceph-mon[192880]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:21:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:21:49.217 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:21:49.217 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":110,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:21:29.074381+0000","last_up_change":"2023-09-30T09:21:28.072487+0000","last_in_change":"2023-09-30T09:10:11.880926+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-30T09:11:07.846610+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-30T09:12:06.877159+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-30T09:12:07.385950+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":2904823591},{"type":"v1","addr":"172.21.15.19:6803","nonce":2904823591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":2904823591},{"type":"v1","addr":"172.21.15.19:6805","nonce":2904823591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":2904823591},{"type":"v1","addr":"172.21.15.19:6809","nonce":2904823591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":2904823591},{"type":"v1","addr":"172.21.15.19:6807","nonce":2904823591}]},"public_addr":"172.21.15.19:6803/2904823591","cluster_addr":"172.21.15.19:6805/2904823591","heartbeat_back_addr":"172.21.15.19:6809/2904823591","heartbeat_front_addr":"172.21.15.19:6807/2904823591","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":1402281404},{"type":"v1","addr":"172.21.15.19:6811","nonce":1402281404}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":1402281404},{"type":"v1","addr":"172.21.15.19:6813","nonce":1402281404}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":1402281404},{"type":"v1","addr":"172.21.15.19:6817","nonce":1402281404}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":1402281404},{"type":"v1","addr":"172.21.15.19:6815","nonce":1402281404}]},"public_addr":"172.21.15.19:6811/1402281404","cluster_addr":"172.21.15.19:6813/1402281404","heartbeat_back_addr":"172.21.15.19:6817/1402281404","heartbeat_front_addr":"172.21.15.19:6815/1402281404","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6819","nonce":2623064465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6821","nonce":2623064465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6825","nonce":2623064465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6823","nonce":2623064465}]},"public_addr":"172.21.15.19:6819/2623064465","cluster_addr":"172.21.15.19:6821/2623064465","heartbeat_back_addr":"172.21.15.19:6825/2623064465","heartbeat_front_addr":"172.21.15.19:6823/2623064465","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6827","nonce":3712654409}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6829","nonce":3712654409}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6833","nonce":3712654409}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6831","nonce":3712654409}]},"public_addr":"172.21.15.19:6827/3712654409","cluster_addr":"172.21.15.19:6829/3712654409","heartbeat_back_addr":"172.21.15.19:6833/3712654409","heartbeat_front_addr":"172.21.15.19:6831/3712654409","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6809","nonce":2834758136}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6811","nonce":2834758136}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6815","nonce":2834758136}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":2834758136},{"type":"v1","addr":"172.21.15.167:6813","nonce":2834758136}]},"public_addr":"172.21.15.167:6809/2834758136","cluster_addr":"172.21.15.167:6811/2834758136","heartbeat_back_addr":"172.21.15.167:6815/2834758136","heartbeat_front_addr":"172.21.15.167:6813/2834758136","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/956155":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.167:0/625642582":"2023-10-01T09:15:21.453516+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.167:6835/1123065418":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/861352263":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/605197989":"2023-10-01T09:16:02.836873+0000","172.21.15.167:6834/1123065418":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/4266586334":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/3289282363":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/3324367156":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/1856357891":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3152350528":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/1638946842":"2023-10-01T09:14:41.945226+0000","172.21.15.167:0/798457358":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/1275272106":"2023-10-01T09:16:02.836873+0000","172.21.15.167:0/320118286":"2023-10-01T09:15:21.453516+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/238111816":"2023-10-01T09:14:41.945226+0000","172.21.15.19:6800/2650629233":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6800/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3931489393":"2023-10-01T09:14:41.945226+0000","172.21.15.19:6801/2650629233":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/936416645":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/2022890892":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/1679148187":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/554243229":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6801/605197989":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/309324931":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/2218856933":"2023-10-01T09:15:21.453516+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:21:49.993 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-30T09:21:49.993 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-30T09:21:49.994 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-30T09:21:49.997 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-30T09:21:49.997 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-30T09:21:49.997 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-30T09:21:49.998 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-30T09:21:49.998 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-30T09:21:50.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:50 smithi019 ceph-mon[216815]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:50.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3171573519' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:21:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:50 smithi019 ceph-mon[224043]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3171573519' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:21:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:50 smithi167 ceph-mon[192880]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3171573519' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:21:50.749 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:50.861 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:51.057 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:51.152 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:51.404 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:51.506 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:52.242 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:52 smithi019 ceph-mon[216815]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:52.243 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:52 smithi019 ceph-mon[224043]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:52 smithi167 ceph-mon[192880]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:53.517 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:53.913 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:21:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:21:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:21:53] "GET /metrics HTTP/1.1" 200 33867 "" "Prometheus/2.43.0" 2023-09-30T09:21:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:54 smithi167 ceph-mon[192880]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:54.530 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:54 smithi019 ceph-mon[216815]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:54.531 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:54 smithi019 ceph-mon[224043]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:56.442 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:56 smithi019 ceph-mon[216815]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:56.442 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:56 smithi019 ceph-mon[224043]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:56 smithi167 ceph-mon[192880]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:56.779 INFO:teuthology.orchestra.run.smithi019.stdout:163208757397 2023-09-30T09:21:56.779 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-30T09:21:58.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:21:58 smithi019 ceph-mon[224043]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:58.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:21:58 smithi019 ceph-mon[216815]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:21:58 smithi167 ceph-mon[192880]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:21:58.733 INFO:teuthology.orchestra.run.smithi019.stdout:468151435271 2023-09-30T09:21:58.733 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-30T09:22:00.215 INFO:teuthology.orchestra.run.smithi019.stdout:403726925860 2023-09-30T09:22:00.215 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-30T09:22:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:00 smithi019 ceph-mon[216815]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:00 smithi019 ceph-mon[224043]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:00 smithi167 ceph-mon[192880]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:00.732 INFO:teuthology.orchestra.run.smithi019.stdout:446676598798 2023-09-30T09:22:00.732 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-30T09:22:00.765 INFO:teuthology.orchestra.run.smithi019.stdout:197568495760 2023-09-30T09:22:00.765 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-30T09:22:01.991 INFO:teuthology.orchestra.run.smithi019.stdout:128849019033 2023-09-30T09:22:01.992 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-30T09:22:02.137 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:02.219 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:02 smithi019 ceph-mon[216815]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:02.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:02 smithi019 ceph-mon[224043]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:02 smithi167 ceph-mon[192880]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:22:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:22:03] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.43.0" 2023-09-30T09:22:04.128 INFO:teuthology.orchestra.run.smithi019.stdout:425201762326 2023-09-30T09:22:04.128 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-30T09:22:04.172 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:04.195 INFO:teuthology.orchestra.run.smithi019.stdout:231928234124 2023-09-30T09:22:04.195 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-30T09:22:04.224 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:04.291 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:04.347 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:04.420 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:04 smithi019 ceph-mon[216815]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:04.420 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:04 smithi019 ceph-mon[224043]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:04 smithi167 ceph-mon[192880]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:04.642 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:04.913 INFO:teuthology.orchestra.run.smithi019.stdout:163208757398 2023-09-30T09:22:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1556739053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:22:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1556739053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:22:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1556739053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:22:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:06 smithi019 ceph-mon[216815]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:06.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:06 smithi019 ceph-mon[224043]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:06.504 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757397 got 163208757398 for osd.5 2023-09-30T09:22:06.505 DEBUG:teuthology.parallel:result is None 2023-09-30T09:22:06.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:06 smithi167 ceph-mon[192880]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:06.992 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:08.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:08 smithi019 ceph-mon[216815]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:08.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:08 smithi019 ceph-mon[224043]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:08 smithi167 ceph-mon[192880]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:09.105 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:09.940 INFO:teuthology.orchestra.run.smithi019.stdout:403726925862 2023-09-30T09:22:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:10 smithi167 ceph-mon[192880]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/559021183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:10 smithi019 ceph-mon[216815]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/559021183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:22:10.571 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:10 smithi019 ceph-mon[224043]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:10.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/559021183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:22:11.238 INFO:teuthology.orchestra.run.smithi019.stdout:128849019036 2023-09-30T09:22:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4108844482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:22:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4108844482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:22:11.395 INFO:teuthology.orchestra.run.smithi019.stdout:446676598801 2023-09-30T09:22:11.423 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925860 got 403726925862 for osd.0 2023-09-30T09:22:11.424 DEBUG:teuthology.parallel:result is None 2023-09-30T09:22:11.517 INFO:teuthology.orchestra.run.smithi019.stdout:468151435274 2023-09-30T09:22:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4108844482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:22:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:12 smithi019 ceph-mon[224043]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1364135363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:22:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/164166194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:22:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:12 smithi019 ceph-mon[216815]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1364135363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:22:12.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/164166194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:22:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:12 smithi167 ceph-mon[192880]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1364135363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:22:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/164166194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:22:13.305 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598798 got 446676598801 for osd.2 2023-09-30T09:22:13.305 DEBUG:teuthology.parallel:result is None 2023-09-30T09:22:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2959875723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:22:13.532 INFO:teuthology.orchestra.run.smithi019.stdout:197568495763 2023-09-30T09:22:13.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2959875723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:22:13.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2959875723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:22:13.866 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019033 got 128849019036 for osd.4 2023-09-30T09:22:13.866 DEBUG:teuthology.parallel:result is None 2023-09-30T09:22:13.903 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435271 got 468151435274 for osd.3 2023-09-30T09:22:13.903 DEBUG:teuthology.parallel:result is None 2023-09-30T09:22:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:22:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:22:13] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.43.0" 2023-09-30T09:22:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:14 smithi167 ceph-mon[192880]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:14 smithi019 ceph-mon[216815]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:14 smithi019 ceph-mon[224043]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:15.169 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495760 got 197568495763 for osd.6 2023-09-30T09:22:15.169 DEBUG:teuthology.parallel:result is None 2023-09-30T09:22:15.572 INFO:teuthology.orchestra.run.smithi019.stdout:425201762328 2023-09-30T09:22:15.832 INFO:teuthology.orchestra.run.smithi019.stdout:231928234127 2023-09-30T09:22:16.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:16 smithi019 ceph-mon[216815]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:16.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2253741275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:22:16.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/727024136' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:22:16.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:16 smithi019 ceph-mon[224043]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:16.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2253741275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:22:16.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/727024136' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:22:16.472 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762326 got 425201762328 for osd.1 2023-09-30T09:22:16.472 DEBUG:teuthology.parallel:result is None 2023-09-30T09:22:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:16 smithi167 ceph-mon[192880]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2253741275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:22:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/727024136' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:22:16.923 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234124 got 231928234127 for osd.7 2023-09-30T09:22:16.923 DEBUG:teuthology.parallel:result is None 2023-09-30T09:22:16.923 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-30T09:22:16.923 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:22:17.293 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:18 smithi167 ceph-mon[192880]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:18 smithi019 ceph-mon[216815]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:18.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:18 smithi019 ceph-mon[224043]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:18.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:19.109 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:22:19.110 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:22:19.842 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":211,"stamp":"2023-09-30T09:22:18.953759+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790183,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6480,"num_read_kb":14131,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":167880,"kb_used_data":23068,"kb_used_omap":19,"kb_used_meta":144684,"kb_avail":749793336,"statfs":{"total":767960285184,"available":767788376064,"internally_reserved":0,"allocated":23621632,"data_stored":19336637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":148156839},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003016"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085795+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:21:28.085795+0000","last_peered":"2023-09-30T09:21:28.085795+0000","last_clean":"2023-09-30T09:21:28.085795+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:21:28.085795+0000","last_undegraded":"2023-09-30T09:21:28.085795+0000","last_fullsized":"2023-09-30T09:21:28.085795+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082738+0000","last_change":"2023-09-30T09:20:56.404258+0000","last_active":"2023-09-30T09:21:28.082738+0000","last_peered":"2023-09-30T09:21:28.082738+0000","last_clean":"2023-09-30T09:21:28.082738+0000","last_became_active":"2023-09-30T09:20:56.403848+0000","last_became_peered":"2023-09-30T09:20:56.403848+0000","last_unstale":"2023-09-30T09:21:28.082738+0000","last_undegraded":"2023-09-30T09:21:28.082738+0000","last_fullsized":"2023-09-30T09:21:28.082738+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:01.175436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093237+0000","last_change":"2023-09-30T09:21:29.093237+0000","last_active":"2023-09-30T09:21:29.093237+0000","last_peered":"2023-09-30T09:21:29.093237+0000","last_clean":"2023-09-30T09:21:29.093237+0000","last_became_active":"2023-09-30T09:21:29.092638+0000","last_became_peered":"2023-09-30T09:21:29.092638+0000","last_unstale":"2023-09-30T09:21:29.093237+0000","last_undegraded":"2023-09-30T09:21:29.093237+0000","last_fullsized":"2023-09-30T09:21:29.093237+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:11:54.139992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104648+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:21:29.104648+0000","last_peered":"2023-09-30T09:21:29.104648+0000","last_clean":"2023-09-30T09:21:29.104648+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:21:29.104648+0000","last_undegraded":"2023-09-30T09:21:29.104648+0000","last_fullsized":"2023-09-30T09:21:29.104648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.097503+0000","last_change":"2023-09-30T09:21:29.097503+0000","last_active":"2023-09-30T09:21:29.097503+0000","last_peered":"2023-09-30T09:21:29.097503+0000","last_clean":"2023-09-30T09:21:29.097503+0000","last_became_active":"2023-09-30T09:21:29.097151+0000","last_became_peered":"2023-09-30T09:21:29.097151+0000","last_unstale":"2023-09-30T09:21:29.097503+0000","last_undegraded":"2023-09-30T09:21:29.097503+0000","last_fullsized":"2023-09-30T09:21:29.097503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:55.978501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081849+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:21:28.081849+0000","last_peered":"2023-09-30T09:21:28.081849+0000","last_clean":"2023-09-30T09:21:28.081849+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:21:28.081849+0000","last_undegraded":"2023-09-30T09:21:28.081849+0000","last_fullsized":"2023-09-30T09:21:28.081849+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096090+0000","last_change":"2023-09-30T09:21:29.096090+0000","last_active":"2023-09-30T09:21:29.096090+0000","last_peered":"2023-09-30T09:21:29.096090+0000","last_clean":"2023-09-30T09:21:29.096090+0000","last_became_active":"2023-09-30T09:21:29.095664+0000","last_became_peered":"2023-09-30T09:21:29.095664+0000","last_unstale":"2023-09-30T09:21:29.096090+0000","last_undegraded":"2023-09-30T09:21:29.096090+0000","last_fullsized":"2023-09-30T09:21:29.096090+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:58.213312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082945+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:21:28.082945+0000","last_peered":"2023-09-30T09:21:28.082945+0000","last_clean":"2023-09-30T09:21:28.082945+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:21:28.082945+0000","last_undegraded":"2023-09-30T09:21:28.082945+0000","last_fullsized":"2023-09-30T09:21:28.082945+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096256+0000","last_change":"2023-09-30T09:21:29.096256+0000","last_active":"2023-09-30T09:21:29.096256+0000","last_peered":"2023-09-30T09:21:29.096256+0000","last_clean":"2023-09-30T09:21:29.096256+0000","last_became_active":"2023-09-30T09:21:29.095930+0000","last_became_peered":"2023-09-30T09:21:29.095930+0000","last_unstale":"2023-09-30T09:21:29.096256+0000","last_undegraded":"2023-09-30T09:21:29.096256+0000","last_fullsized":"2023-09-30T09:21:29.096256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:25.270756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.082734+0000","last_change":"2023-09-30T09:19:08.915504+0000","last_active":"2023-09-30T09:21:29.082734+0000","last_peered":"2023-09-30T09:21:29.082734+0000","last_clean":"2023-09-30T09:21:29.082734+0000","last_became_active":"2023-09-30T09:19:08.915150+0000","last_became_peered":"2023-09-30T09:19:08.915150+0000","last_unstale":"2023-09-30T09:21:29.082734+0000","last_undegraded":"2023-09-30T09:21:29.082734+0000","last_fullsized":"2023-09-30T09:21:29.082734+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:56:15.982749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089369+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:21:28.089369+0000","last_peered":"2023-09-30T09:21:28.089369+0000","last_clean":"2023-09-30T09:21:28.089369+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:21:28.089369+0000","last_undegraded":"2023-09-30T09:21:28.089369+0000","last_fullsized":"2023-09-30T09:21:28.089369+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082758+0000","last_change":"2023-09-30T09:20:23.568788+0000","last_active":"2023-09-30T09:21:28.082758+0000","last_peered":"2023-09-30T09:21:28.082758+0000","last_clean":"2023-09-30T09:21:28.082758+0000","last_became_active":"2023-09-30T09:20:23.568454+0000","last_became_peered":"2023-09-30T09:20:23.568454+0000","last_unstale":"2023-09-30T09:21:28.082758+0000","last_undegraded":"2023-09-30T09:21:28.082758+0000","last_fullsized":"2023-09-30T09:21:28.082758+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:47:26.911269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":363,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083198+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:21:28.083198+0000","last_peered":"2023-09-30T09:21:28.083198+0000","last_clean":"2023-09-30T09:21:28.083198+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:21:28.083198+0000","last_undegraded":"2023-09-30T09:21:28.083198+0000","last_fullsized":"2023-09-30T09:21:28.083198+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094736+0000","last_change":"2023-09-30T09:21:29.094736+0000","last_active":"2023-09-30T09:21:29.094736+0000","last_peered":"2023-09-30T09:21:29.094736+0000","last_clean":"2023-09-30T09:21:29.094736+0000","last_became_active":"2023-09-30T09:21:29.094220+0000","last_became_peered":"2023-09-30T09:21:29.094220+0000","last_unstale":"2023-09-30T09:21:29.094736+0000","last_undegraded":"2023-09-30T09:21:29.094736+0000","last_fullsized":"2023-09-30T09:21:29.094736+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:48:58.067703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1055,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.084527+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:21:28.084527+0000","last_peered":"2023-09-30T09:21:28.084527+0000","last_clean":"2023-09-30T09:21:28.084527+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:21:28.084527+0000","last_undegraded":"2023-09-30T09:21:28.084527+0000","last_fullsized":"2023-09-30T09:21:28.084527+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092954+0000","last_change":"2023-09-30T09:21:29.092954+0000","last_active":"2023-09-30T09:21:29.092954+0000","last_peered":"2023-09-30T09:21:29.092954+0000","last_clean":"2023-09-30T09:21:29.092954+0000","last_became_active":"2023-09-30T09:21:29.092563+0000","last_became_peered":"2023-09-30T09:21:29.092563+0000","last_unstale":"2023-09-30T09:21:29.092954+0000","last_undegraded":"2023-09-30T09:21:29.092954+0000","last_fullsized":"2023-09-30T09:21:29.092954+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:54.133382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082795+0000","last_change":"2023-09-30T09:19:08.911812+0000","last_active":"2023-09-30T09:21:28.082795+0000","last_peered":"2023-09-30T09:21:28.082795+0000","last_clean":"2023-09-30T09:21:28.082795+0000","last_became_active":"2023-09-30T09:19:08.911477+0000","last_became_peered":"2023-09-30T09:19:08.911477+0000","last_unstale":"2023-09-30T09:21:28.082795+0000","last_undegraded":"2023-09-30T09:21:28.082795+0000","last_fullsized":"2023-09-30T09:21:28.082795+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:04.525132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":45,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082420+0000","last_change":"2023-09-30T09:20:56.402640+0000","last_active":"2023-09-30T09:21:28.082420+0000","last_peered":"2023-09-30T09:21:28.082420+0000","last_clean":"2023-09-30T09:21:28.082420+0000","last_became_active":"2023-09-30T09:20:56.402324+0000","last_became_peered":"2023-09-30T09:20:56.402324+0000","last_unstale":"2023-09-30T09:21:28.082420+0000","last_undegraded":"2023-09-30T09:21:28.082420+0000","last_fullsized":"2023-09-30T09:21:28.082420+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:27:18.825926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":650,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086457+0000","last_change":"2023-09-30T09:19:08.914400+0000","last_active":"2023-09-30T09:21:28.086457+0000","last_peered":"2023-09-30T09:21:28.086457+0000","last_clean":"2023-09-30T09:21:28.086457+0000","last_became_active":"2023-09-30T09:19:08.913855+0000","last_became_peered":"2023-09-30T09:19:08.913855+0000","last_unstale":"2023-09-30T09:21:28.086457+0000","last_undegraded":"2023-09-30T09:21:28.086457+0000","last_fullsized":"2023-09-30T09:21:28.086457+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:27.706105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":64,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082854+0000","last_change":"2023-09-30T09:20:56.397596+0000","last_active":"2023-09-30T09:21:28.082854+0000","last_peered":"2023-09-30T09:21:28.082854+0000","last_clean":"2023-09-30T09:21:28.082854+0000","last_became_active":"2023-09-30T09:20:56.397283+0000","last_became_peered":"2023-09-30T09:20:56.397283+0000","last_unstale":"2023-09-30T09:21:28.082854+0000","last_undegraded":"2023-09-30T09:21:28.082854+0000","last_fullsized":"2023-09-30T09:21:28.082854+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:28.695903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094162+0000","last_change":"2023-09-30T09:21:29.094162+0000","last_active":"2023-09-30T09:21:29.094162+0000","last_peered":"2023-09-30T09:21:29.094162+0000","last_clean":"2023-09-30T09:21:29.094162+0000","last_became_active":"2023-09-30T09:21:29.093839+0000","last_became_peered":"2023-09-30T09:21:29.093839+0000","last_unstale":"2023-09-30T09:21:29.094162+0000","last_undegraded":"2023-09-30T09:21:29.094162+0000","last_fullsized":"2023-09-30T09:21:29.094162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:38:47.314823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":590,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086978+0000","last_change":"2023-09-30T09:20:23.569228+0000","last_active":"2023-09-30T09:21:28.086978+0000","last_peered":"2023-09-30T09:21:28.086978+0000","last_clean":"2023-09-30T09:21:28.086978+0000","last_became_active":"2023-09-30T09:20:23.568481+0000","last_became_peered":"2023-09-30T09:20:23.568481+0000","last_unstale":"2023-09-30T09:21:28.086978+0000","last_undegraded":"2023-09-30T09:21:28.086978+0000","last_fullsized":"2023-09-30T09:21:28.086978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:31:06.051903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082973+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:21:28.082973+0000","last_peered":"2023-09-30T09:21:28.082973+0000","last_clean":"2023-09-30T09:21:28.082973+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:21:28.082973+0000","last_undegraded":"2023-09-30T09:21:28.082973+0000","last_fullsized":"2023-09-30T09:21:28.082973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104912+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:21:29.104912+0000","last_peered":"2023-09-30T09:21:29.104912+0000","last_clean":"2023-09-30T09:21:29.104912+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:21:29.104912+0000","last_undegraded":"2023-09-30T09:21:29.104912+0000","last_fullsized":"2023-09-30T09:21:29.104912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":451,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086908+0000","last_change":"2023-09-30T09:19:08.915371+0000","last_active":"2023-09-30T09:21:28.086908+0000","last_peered":"2023-09-30T09:21:28.086908+0000","last_clean":"2023-09-30T09:21:28.086908+0000","last_became_active":"2023-09-30T09:19:08.914936+0000","last_became_peered":"2023-09-30T09:19:08.914936+0000","last_unstale":"2023-09-30T09:21:28.086908+0000","last_undegraded":"2023-09-30T09:21:28.086908+0000","last_fullsized":"2023-09-30T09:21:28.086908+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:41.291423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":517,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080969+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:21:28.080969+0000","last_peered":"2023-09-30T09:21:28.080969+0000","last_clean":"2023-09-30T09:21:28.080969+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:21:28.080969+0000","last_undegraded":"2023-09-30T09:21:28.080969+0000","last_fullsized":"2023-09-30T09:21:28.080969+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085377+0000","last_change":"2023-09-30T09:20:23.574542+0000","last_active":"2023-09-30T09:21:28.085377+0000","last_peered":"2023-09-30T09:21:28.085377+0000","last_clean":"2023-09-30T09:21:28.085377+0000","last_became_active":"2023-09-30T09:20:23.574196+0000","last_became_peered":"2023-09-30T09:20:23.574196+0000","last_unstale":"2023-09-30T09:21:28.085377+0000","last_undegraded":"2023-09-30T09:21:28.085377+0000","last_fullsized":"2023-09-30T09:21:28.085377+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:24:08.605395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":45,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082236+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:21:28.082236+0000","last_peered":"2023-09-30T09:21:28.082236+0000","last_clean":"2023-09-30T09:21:28.082236+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:21:28.082236+0000","last_undegraded":"2023-09-30T09:21:28.082236+0000","last_fullsized":"2023-09-30T09:21:28.082236+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":413,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094131+0000","last_change":"2023-09-30T09:21:29.094131+0000","last_active":"2023-09-30T09:21:29.094131+0000","last_peered":"2023-09-30T09:21:29.094131+0000","last_clean":"2023-09-30T09:21:29.094131+0000","last_became_active":"2023-09-30T09:21:29.093405+0000","last_became_peered":"2023-09-30T09:21:29.093405+0000","last_unstale":"2023-09-30T09:21:29.094131+0000","last_undegraded":"2023-09-30T09:21:29.094131+0000","last_fullsized":"2023-09-30T09:21:29.094131+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:14.014784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092892+0000","last_change":"2023-09-30T09:21:29.092892+0000","last_active":"2023-09-30T09:21:29.092892+0000","last_peered":"2023-09-30T09:21:29.092892+0000","last_clean":"2023-09-30T09:21:29.092892+0000","last_became_active":"2023-09-30T09:21:29.092476+0000","last_became_peered":"2023-09-30T09:21:29.092476+0000","last_unstale":"2023-09-30T09:21:29.092892+0000","last_undegraded":"2023-09-30T09:21:29.092892+0000","last_fullsized":"2023-09-30T09:21:29.092892+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:38.113124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090919+0000","last_change":"2023-09-30T09:19:08.918284+0000","last_active":"2023-09-30T09:21:28.090919+0000","last_peered":"2023-09-30T09:21:28.090919+0000","last_clean":"2023-09-30T09:21:28.090919+0000","last_became_active":"2023-09-30T09:19:08.917861+0000","last_became_peered":"2023-09-30T09:19:08.917861+0000","last_unstale":"2023-09-30T09:21:28.090919+0000","last_undegraded":"2023-09-30T09:21:28.090919+0000","last_fullsized":"2023-09-30T09:21:28.090919+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:51.339869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":331,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081772+0000","last_change":"2023-09-30T09:20:23.574312+0000","last_active":"2023-09-30T09:21:28.081772+0000","last_peered":"2023-09-30T09:21:28.081772+0000","last_clean":"2023-09-30T09:21:28.081772+0000","last_became_active":"2023-09-30T09:20:23.573886+0000","last_became_peered":"2023-09-30T09:20:23.573886+0000","last_unstale":"2023-09-30T09:21:28.081772+0000","last_undegraded":"2023-09-30T09:21:28.081772+0000","last_fullsized":"2023-09-30T09:21:28.081772+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:45:27.269020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":219,"num_read_kb":2265,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081571+0000","last_change":"2023-09-30T09:20:23.572662+0000","last_active":"2023-09-30T09:21:28.081571+0000","last_peered":"2023-09-30T09:21:28.081571+0000","last_clean":"2023-09-30T09:21:28.081571+0000","last_became_active":"2023-09-30T09:20:23.572374+0000","last_became_peered":"2023-09-30T09:20:23.572374+0000","last_unstale":"2023-09-30T09:21:28.081571+0000","last_undegraded":"2023-09-30T09:21:28.081571+0000","last_fullsized":"2023-09-30T09:21:28.081571+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:11.881192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092844+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:21:29.092844+0000","last_peered":"2023-09-30T09:21:29.092844+0000","last_clean":"2023-09-30T09:21:29.092844+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:21:29.092844+0000","last_undegraded":"2023-09-30T09:21:29.092844+0000","last_fullsized":"2023-09-30T09:21:29.092844+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":395,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082530+0000","last_change":"2023-09-30T09:20:23.576133+0000","last_active":"2023-09-30T09:21:28.082530+0000","last_peered":"2023-09-30T09:21:28.082530+0000","last_clean":"2023-09-30T09:21:28.082530+0000","last_became_active":"2023-09-30T09:20:23.575571+0000","last_became_peered":"2023-09-30T09:20:23.575571+0000","last_unstale":"2023-09-30T09:21:28.082530+0000","last_undegraded":"2023-09-30T09:21:28.082530+0000","last_fullsized":"2023-09-30T09:21:28.082530+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:14:08.450502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092042+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:21:29.092042+0000","last_peered":"2023-09-30T09:21:29.092042+0000","last_clean":"2023-09-30T09:21:29.092042+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:21:29.092042+0000","last_undegraded":"2023-09-30T09:21:29.092042+0000","last_fullsized":"2023-09-30T09:21:29.092042+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.089363+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:21:29.089363+0000","last_peered":"2023-09-30T09:21:29.089363+0000","last_clean":"2023-09-30T09:21:29.089363+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:21:29.089363+0000","last_undegraded":"2023-09-30T09:21:29.089363+0000","last_fullsized":"2023-09-30T09:21:29.089363+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086050+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:21:28.086050+0000","last_peered":"2023-09-30T09:21:28.086050+0000","last_clean":"2023-09-30T09:21:28.086050+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:21:28.086050+0000","last_undegraded":"2023-09-30T09:21:28.086050+0000","last_fullsized":"2023-09-30T09:21:28.086050+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090767+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:21:28.090767+0000","last_peered":"2023-09-30T09:21:28.090767+0000","last_clean":"2023-09-30T09:21:28.090767+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:21:28.090767+0000","last_undegraded":"2023-09-30T09:21:28.090767+0000","last_fullsized":"2023-09-30T09:21:28.090767+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089286+0000","last_change":"2023-09-30T09:20:56.406202+0000","last_active":"2023-09-30T09:21:28.089286+0000","last_peered":"2023-09-30T09:21:28.089286+0000","last_clean":"2023-09-30T09:21:28.089286+0000","last_became_active":"2023-09-30T09:20:56.405232+0000","last_became_peered":"2023-09-30T09:20:56.405232+0000","last_unstale":"2023-09-30T09:21:28.089286+0000","last_undegraded":"2023-09-30T09:21:28.089286+0000","last_fullsized":"2023-09-30T09:21:28.089286+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:40:27.385688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081936+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:21:28.081936+0000","last_peered":"2023-09-30T09:21:28.081936+0000","last_clean":"2023-09-30T09:21:28.081936+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:21:28.081936+0000","last_undegraded":"2023-09-30T09:21:28.081936+0000","last_fullsized":"2023-09-30T09:21:28.081936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104691+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:21:29.104691+0000","last_peered":"2023-09-30T09:21:29.104691+0000","last_clean":"2023-09-30T09:21:29.104691+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:21:29.104691+0000","last_undegraded":"2023-09-30T09:21:29.104691+0000","last_fullsized":"2023-09-30T09:21:29.104691+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093267+0000","last_change":"2023-09-30T09:21:29.093267+0000","last_active":"2023-09-30T09:21:29.093267+0000","last_peered":"2023-09-30T09:21:29.093267+0000","last_clean":"2023-09-30T09:21:29.093267+0000","last_became_active":"2023-09-30T09:21:29.092904+0000","last_became_peered":"2023-09-30T09:21:29.092904+0000","last_unstale":"2023-09-30T09:21:29.093267+0000","last_undegraded":"2023-09-30T09:21:29.093267+0000","last_fullsized":"2023-09-30T09:21:29.093267+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:31.930457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083274+0000","last_change":"2023-09-30T09:20:23.567941+0000","last_active":"2023-09-30T09:21:28.083274+0000","last_peered":"2023-09-30T09:21:28.083274+0000","last_clean":"2023-09-30T09:21:28.083274+0000","last_became_active":"2023-09-30T09:20:23.567652+0000","last_became_peered":"2023-09-30T09:20:23.567652+0000","last_unstale":"2023-09-30T09:21:28.083274+0000","last_undegraded":"2023-09-30T09:21:28.083274+0000","last_fullsized":"2023-09-30T09:21:28.083274+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:41:04.133820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104438+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:21:29.104438+0000","last_peered":"2023-09-30T09:21:29.104438+0000","last_clean":"2023-09-30T09:21:29.104438+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:21:29.104438+0000","last_undegraded":"2023-09-30T09:21:29.104438+0000","last_fullsized":"2023-09-30T09:21:29.104438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081779+0000","last_change":"2023-09-30T09:20:23.580109+0000","last_active":"2023-09-30T09:21:28.081779+0000","last_peered":"2023-09-30T09:21:28.081779+0000","last_clean":"2023-09-30T09:21:28.081779+0000","last_became_active":"2023-09-30T09:20:23.579747+0000","last_became_peered":"2023-09-30T09:20:23.579747+0000","last_unstale":"2023-09-30T09:21:28.081779+0000","last_undegraded":"2023-09-30T09:21:28.081779+0000","last_fullsized":"2023-09-30T09:21:28.081779+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:04.192497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086557+0000","last_change":"2023-09-30T09:12:07.944640+0000","last_active":"2023-09-30T09:21:28.086557+0000","last_peered":"2023-09-30T09:21:28.086557+0000","last_clean":"2023-09-30T09:21:28.086557+0000","last_became_active":"2023-09-30T09:12:07.944362+0000","last_became_peered":"2023-09-30T09:12:07.944362+0000","last_unstale":"2023-09-30T09:21:28.086557+0000","last_undegraded":"2023-09-30T09:21:28.086557+0000","last_fullsized":"2023-09-30T09:21:28.086557+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:14.703475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.097658+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:21:29.097658+0000","last_peered":"2023-09-30T09:21:29.097658+0000","last_clean":"2023-09-30T09:21:29.097658+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:21:29.097658+0000","last_undegraded":"2023-09-30T09:21:29.097658+0000","last_fullsized":"2023-09-30T09:21:29.097658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087158+0000","last_change":"2023-09-30T09:20:23.568846+0000","last_active":"2023-09-30T09:21:28.087158+0000","last_peered":"2023-09-30T09:21:28.087158+0000","last_clean":"2023-09-30T09:21:28.087158+0000","last_became_active":"2023-09-30T09:20:23.567910+0000","last_became_peered":"2023-09-30T09:20:23.567910+0000","last_unstale":"2023-09-30T09:21:28.087158+0000","last_undegraded":"2023-09-30T09:21:28.087158+0000","last_fullsized":"2023-09-30T09:21:28.087158+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:48:49.264877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080900+0000","last_change":"2023-09-30T09:12:09.760712+0000","last_active":"2023-09-30T09:21:28.080900+0000","last_peered":"2023-09-30T09:21:28.080900+0000","last_clean":"2023-09-30T09:21:28.080900+0000","last_became_active":"2023-09-30T09:12:09.760435+0000","last_became_peered":"2023-09-30T09:12:09.760435+0000","last_unstale":"2023-09-30T09:21:28.080900+0000","last_undegraded":"2023-09-30T09:21:28.080900+0000","last_fullsized":"2023-09-30T09:21:28.080900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:03.896756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087175+0000","last_change":"2023-09-30T09:19:08.916779+0000","last_active":"2023-09-30T09:21:28.087175+0000","last_peered":"2023-09-30T09:21:28.087175+0000","last_clean":"2023-09-30T09:21:28.087175+0000","last_became_active":"2023-09-30T09:19:08.916333+0000","last_became_peered":"2023-09-30T09:19:08.916333+0000","last_unstale":"2023-09-30T09:21:28.087175+0000","last_undegraded":"2023-09-30T09:21:28.087175+0000","last_fullsized":"2023-09-30T09:21:28.087175+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:25.205240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092140+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:21:29.092140+0000","last_peered":"2023-09-30T09:21:29.092140+0000","last_clean":"2023-09-30T09:21:29.092140+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:21:29.092140+0000","last_undegraded":"2023-09-30T09:21:29.092140+0000","last_fullsized":"2023-09-30T09:21:29.092140+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.084198+0000","last_change":"2023-09-30T09:19:08.913366+0000","last_active":"2023-09-30T09:21:28.084198+0000","last_peered":"2023-09-30T09:21:28.084198+0000","last_clean":"2023-09-30T09:21:28.084198+0000","last_became_active":"2023-09-30T09:19:08.912978+0000","last_became_peered":"2023-09-30T09:19:08.912978+0000","last_unstale":"2023-09-30T09:21:28.084198+0000","last_undegraded":"2023-09-30T09:21:28.084198+0000","last_fullsized":"2023-09-30T09:21:28.084198+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:47:20.546906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096658+0000","last_change":"2023-09-30T09:21:29.094783+0000","last_active":"2023-09-30T09:21:29.096658+0000","last_peered":"2023-09-30T09:21:29.096658+0000","last_clean":"2023-09-30T09:21:29.096658+0000","last_became_active":"2023-09-30T09:21:29.094482+0000","last_became_peered":"2023-09-30T09:21:29.094482+0000","last_unstale":"2023-09-30T09:21:29.096658+0000","last_undegraded":"2023-09-30T09:21:29.096658+0000","last_fullsized":"2023-09-30T09:21:29.096658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:59:14.875660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087317+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:21:28.087317+0000","last_peered":"2023-09-30T09:21:28.087317+0000","last_clean":"2023-09-30T09:21:28.087317+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:21:28.087317+0000","last_undegraded":"2023-09-30T09:21:28.087317+0000","last_fullsized":"2023-09-30T09:21:28.087317+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092932+0000","last_change":"2023-09-30T09:21:29.092932+0000","last_active":"2023-09-30T09:21:29.092932+0000","last_peered":"2023-09-30T09:21:29.092932+0000","last_clean":"2023-09-30T09:21:29.092932+0000","last_became_active":"2023-09-30T09:21:29.092450+0000","last_became_peered":"2023-09-30T09:21:29.092450+0000","last_unstale":"2023-09-30T09:21:29.092932+0000","last_undegraded":"2023-09-30T09:21:29.092932+0000","last_fullsized":"2023-09-30T09:21:29.092932+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:39:06.199633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093775+0000","last_change":"2023-09-30T09:21:29.093775+0000","last_active":"2023-09-30T09:21:29.093775+0000","last_peered":"2023-09-30T09:21:29.093775+0000","last_clean":"2023-09-30T09:21:29.093775+0000","last_became_active":"2023-09-30T09:21:29.093399+0000","last_became_peered":"2023-09-30T09:21:29.093399+0000","last_unstale":"2023-09-30T09:21:29.093775+0000","last_undegraded":"2023-09-30T09:21:29.093775+0000","last_fullsized":"2023-09-30T09:21:29.093775+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:50.192497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092287+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:21:29.092287+0000","last_peered":"2023-09-30T09:21:29.092287+0000","last_clean":"2023-09-30T09:21:29.092287+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:21:29.092287+0000","last_undegraded":"2023-09-30T09:21:29.092287+0000","last_fullsized":"2023-09-30T09:21:29.092287+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085471+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:21:28.085471+0000","last_peered":"2023-09-30T09:21:28.085471+0000","last_clean":"2023-09-30T09:21:28.085471+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:21:28.085471+0000","last_undegraded":"2023-09-30T09:21:28.085471+0000","last_fullsized":"2023-09-30T09:21:28.085471+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090741+0000","last_change":"2023-09-30T09:20:23.574453+0000","last_active":"2023-09-30T09:21:28.090741+0000","last_peered":"2023-09-30T09:21:28.090741+0000","last_clean":"2023-09-30T09:21:28.090741+0000","last_became_active":"2023-09-30T09:20:23.573953+0000","last_became_peered":"2023-09-30T09:20:23.573953+0000","last_unstale":"2023-09-30T09:21:28.090741+0000","last_undegraded":"2023-09-30T09:21:28.090741+0000","last_fullsized":"2023-09-30T09:21:28.090741+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:23:21.634624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089127+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:21:28.089127+0000","last_peered":"2023-09-30T09:21:28.089127+0000","last_clean":"2023-09-30T09:21:28.089127+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:21:28.089127+0000","last_undegraded":"2023-09-30T09:21:28.089127+0000","last_fullsized":"2023-09-30T09:21:28.089127+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081654+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:21:28.081654+0000","last_peered":"2023-09-30T09:21:28.081654+0000","last_clean":"2023-09-30T09:21:28.081654+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:21:28.081654+0000","last_undegraded":"2023-09-30T09:21:28.081654+0000","last_fullsized":"2023-09-30T09:21:28.081654+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092097+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:21:29.092097+0000","last_peered":"2023-09-30T09:21:29.092097+0000","last_clean":"2023-09-30T09:21:29.092097+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:21:29.092097+0000","last_undegraded":"2023-09-30T09:21:29.092097+0000","last_fullsized":"2023-09-30T09:21:29.092097+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104391+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:21:29.104391+0000","last_peered":"2023-09-30T09:21:29.104391+0000","last_clean":"2023-09-30T09:21:29.104391+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:21:29.104391+0000","last_undegraded":"2023-09-30T09:21:29.104391+0000","last_fullsized":"2023-09-30T09:21:29.104391+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080851+0000","last_change":"2023-09-30T09:20:23.573657+0000","last_active":"2023-09-30T09:21:28.080851+0000","last_peered":"2023-09-30T09:21:28.080851+0000","last_clean":"2023-09-30T09:21:28.080851+0000","last_became_active":"2023-09-30T09:20:23.573337+0000","last_became_peered":"2023-09-30T09:20:23.573337+0000","last_unstale":"2023-09-30T09:21:28.080851+0000","last_undegraded":"2023-09-30T09:21:28.080851+0000","last_fullsized":"2023-09-30T09:21:28.080851+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:00.644205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104648+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:21:29.104648+0000","last_peered":"2023-09-30T09:21:29.104648+0000","last_clean":"2023-09-30T09:21:29.104648+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:21:29.104648+0000","last_undegraded":"2023-09-30T09:21:29.104648+0000","last_fullsized":"2023-09-30T09:21:29.104648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.098232+0000","last_change":"2023-09-30T09:21:29.097434+0000","last_active":"2023-09-30T09:21:29.098232+0000","last_peered":"2023-09-30T09:21:29.098232+0000","last_clean":"2023-09-30T09:21:29.098232+0000","last_became_active":"2023-09-30T09:21:29.097116+0000","last_became_peered":"2023-09-30T09:21:29.097116+0000","last_unstale":"2023-09-30T09:21:29.098232+0000","last_undegraded":"2023-09-30T09:21:29.098232+0000","last_fullsized":"2023-09-30T09:21:29.098232+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:41.904331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083111+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:21:28.083111+0000","last_peered":"2023-09-30T09:21:28.083111+0000","last_clean":"2023-09-30T09:21:28.083111+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:21:28.083111+0000","last_undegraded":"2023-09-30T09:21:28.083111+0000","last_fullsized":"2023-09-30T09:21:28.083111+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094532+0000","last_change":"2023-09-30T09:21:29.094532+0000","last_active":"2023-09-30T09:21:29.094532+0000","last_peered":"2023-09-30T09:21:29.094532+0000","last_clean":"2023-09-30T09:21:29.094532+0000","last_became_active":"2023-09-30T09:21:29.093807+0000","last_became_peered":"2023-09-30T09:21:29.093807+0000","last_unstale":"2023-09-30T09:21:29.094532+0000","last_undegraded":"2023-09-30T09:21:29.094532+0000","last_fullsized":"2023-09-30T09:21:29.094532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:39:32.517645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.098125+0000","last_change":"2023-09-30T09:21:29.098125+0000","last_active":"2023-09-30T09:21:29.098125+0000","last_peered":"2023-09-30T09:21:29.098125+0000","last_clean":"2023-09-30T09:21:29.098125+0000","last_became_active":"2023-09-30T09:21:29.097837+0000","last_became_peered":"2023-09-30T09:21:29.097837+0000","last_unstale":"2023-09-30T09:21:29.098125+0000","last_undegraded":"2023-09-30T09:21:29.098125+0000","last_fullsized":"2023-09-30T09:21:29.098125+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:19:36.896416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092203+0000","last_change":"2023-09-30T09:21:29.092203+0000","last_active":"2023-09-30T09:21:29.092203+0000","last_peered":"2023-09-30T09:21:29.092203+0000","last_clean":"2023-09-30T09:21:29.092203+0000","last_became_active":"2023-09-30T09:21:29.091882+0000","last_became_peered":"2023-09-30T09:21:29.091882+0000","last_unstale":"2023-09-30T09:21:29.092203+0000","last_undegraded":"2023-09-30T09:21:29.092203+0000","last_fullsized":"2023-09-30T09:21:29.092203+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:56.866973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093480+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:21:29.093480+0000","last_peered":"2023-09-30T09:21:29.093480+0000","last_clean":"2023-09-30T09:21:29.093480+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:21:29.093480+0000","last_undegraded":"2023-09-30T09:21:29.093480+0000","last_fullsized":"2023-09-30T09:21:29.093480+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085932+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:21:28.085932+0000","last_peered":"2023-09-30T09:21:28.085932+0000","last_clean":"2023-09-30T09:21:28.085932+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:21:28.085932+0000","last_undegraded":"2023-09-30T09:21:28.085932+0000","last_fullsized":"2023-09-30T09:21:28.085932+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234128,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17072,"kb_used_data":5800,"kb_used_omap":0,"kb_used_meta":11264,"kb_avail":93728080,"statfs":{"total":95995035648,"available":95977553920,"internally_reserved":0,"allocated":5939200,"data_stored":5405566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11534336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.60199999999999998,"15min":0.62},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.61199999999999999,"15min":0.64400000000000002},"min":{"1min":0.42599999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.51900000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63500000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:21:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.57699999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.30199999999999999},"max":{"1min":0.68600000000000005,"5min":0.89700000000000002,"15min":1.5429999999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.61199999999999999,"15min":0.58999999999999997},"min":{"1min":0.35799999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.69499999999999995,"5min":0.85899999999999999,"15min":2.359},"last":0.629}]},{"osd":4,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.48399999999999999,"15min":0.46899999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":0.68799999999999994,"5min":0.77800000000000002,"15min":0.878},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.497},"min":{"1min":0.33000000000000002,"5min":0.317,"15min":0.28799999999999998},"max":{"1min":0.73899999999999999,"5min":0.78000000000000003,"15min":1.111},"last":0.44800000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.53500000000000003,"15min":0.51100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":0.754,"5min":0.82499999999999996,"15min":0.86099999999999999},"last":0.498},{"interface":"front","average":{"1min":0.625,"5min":0.58399999999999996,"15min":0.55600000000000005},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.32600000000000001},"max":{"1min":0.81599999999999995,"5min":0.82299999999999995,"15min":0.89400000000000002},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59899999999999998,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34100000000000003},"max":{"1min":0.83899999999999997,"5min":0.85099999999999998,"15min":1.01},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.60799999999999998,"15min":0.55500000000000005},"min":{"1min":0.41599999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.81899999999999995,"15min":1.595},"last":0.65100000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495765,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8224,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":6528,"kb_avail":93736928,"statfs":{"total":95995035648,"available":95986614272,"internally_reserved":0,"allocated":1728512,"data_stored":1210547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6684672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.622,"15min":0.63},"min":{"1min":0.372,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.82899999999999996,"5min":2.673,"15min":2.673},"last":0.75},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.64600000000000002,"15min":0.65300000000000002},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73699999999999999,"5min":3.7450000000000001,"15min":3.7450000000000001},"last":0.64300000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.74199999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.79900000000000004}]},{"osd":3,"last update":"Sat Sep 30 09:20:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.61099999999999999,"15min":0.58399999999999996},"min":{"1min":0.45600000000000002,"5min":0.36899999999999999,"15min":0.29199999999999998},"max":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.58699999999999997,"15min":0.57499999999999996},"min":{"1min":0.441,"5min":0.37,"15min":0.34200000000000003},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":3.6200000000000001},"last":0.69799999999999995}]},{"osd":4,"last update":"Sat Sep 30 09:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.49399999999999999,"15min":0.48699999999999999},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.22700000000000001},"max":{"1min":0.69999999999999996,"5min":0.82599999999999996,"15min":0.83299999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52700000000000002,"15min":0.50800000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.80300000000000005,"5min":0.85299999999999998,"15min":0.92300000000000004},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.50900000000000001,"15min":0.48699999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.24399999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.82099999999999995},"last":0.625},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.57399999999999995,"15min":0.54700000000000004},"min":{"1min":0.39700000000000002,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.95199999999999996},"last":0.66000000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.58599999999999997,"15min":0.57399999999999995},"min":{"1min":0.39900000000000002,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":0.79100000000000004,"5min":0.85599999999999998,"15min":0.877},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.623,"15min":0.60799999999999998},"min":{"1min":0.45000000000000001,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":0.80200000000000005,"5min":0.878,"15min":1.0269999999999999},"last":0.71999999999999997}]}]},{"osd":1,"up_from":99,"seq":425201762329,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32728,"kb_used_data":5252,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93712424,"statfs":{"total":95995035648,"available":95961522176,"internally_reserved":0,"allocated":5378048,"data_stored":4821802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":28108982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.374},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.435}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.52900000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.624}]},{"osd":7,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.504},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.58799999999999997}]}]},{"osd":0,"up_from":94,"seq":403726925864,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33328,"kb_used_data":5856,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93711824,"statfs":{"total":95995035648,"available":95960907776,"internally_reserved":0,"allocated":5996544,"data_stored":5417259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":28109053},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.54200000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:21:12 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.50700000000000001,"15min":0.51800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.625,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.45700000000000002,"15min":0.46100000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.54900000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.67100000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.68000000000000005,"15min":0.72399999999999998},"min":{"1min":0.41599999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.78200000000000003,"5min":2.363,"15min":2.363},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.72399999999999998,"15min":0.76500000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.399,"5min":2.399,"15min":2.399},"last":0.629}]},{"osd":5,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.76100000000000001,"15min":0.80700000000000005},"min":{"1min":0.36599999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.3610000000000002,"5min":2.3610000000000002,"15min":2.3610000000000002},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.76000000000000001,"15min":0.81200000000000006},"min":{"1min":0.5,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84199999999999997,"5min":2.4260000000000002,"15min":2.4260000000000002},"last":0.78100000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.74399999999999999,"15min":0.79100000000000004},"min":{"1min":0.45200000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93600000000000005,"5min":2.5369999999999999,"15min":2.5369999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.78600000000000003,"15min":0.84599999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89700000000000002,"5min":2.6139999999999999,"15min":2.6139999999999999},"last":0.72399999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.71799999999999997,"15min":0.75900000000000001},"min":{"1min":0.46200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.879,"5min":2.3039999999999998,"15min":2.3039999999999998},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.77100000000000002,"15min":0.83699999999999997},"min":{"1min":0.44,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76500000000000001,"5min":2.492,"15min":2.492},"last":0.68999999999999995}]}]},{"osd":2,"up_from":104,"seq":446676598802,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28612,"kb_used_data":1136,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93716540,"statfs":{"total":95995035648,"available":95965736960,"internally_reserved":0,"allocated":1163264,"data_stored":624404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":28109453},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.48099999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.52300000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]},{"osd":4,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":0.751}]},{"osd":6,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.139,"5min":1.139,"15min":1.139},"last":0.83499999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.71899999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435276,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28532,"kb_used_data":1120,"kb_used_omap":2,"kb_used_meta":27389,"kb_avail":93716620,"statfs":{"total":95995035648,"available":95965818880,"internally_reserved":0,"allocated":1146880,"data_stored":620489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":28046695},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25900000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.379}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.43099999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019038,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12000,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":10880,"kb_avail":93733152,"statfs":{"total":95995035648,"available":95982747648,"internally_reserved":0,"allocated":1138688,"data_stored":620294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11141120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.60399999999999998,"15min":0.63100000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77200000000000002,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.60399999999999998,"15min":0.63300000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65100000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.52900000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59299999999999997,"15min":0.57399999999999995},"min":{"1min":0.41499999999999998,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":0.72099999999999997,"5min":0.81999999999999995,"15min":1.6839999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.63,"5min":0.61399999999999999,"15min":0.58499999999999996},"min":{"1min":0.40600000000000003,"5min":0.38200000000000001,"15min":0.379},"max":{"1min":0.79400000000000004,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.69699999999999995}]},{"osd":5,"last update":"Sat Sep 30 09:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.49299999999999999,"15min":0.46300000000000002},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.27200000000000002},"max":{"1min":0.68500000000000005,"5min":0.74399999999999999,"15min":0.98699999999999999},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54100000000000004,"15min":0.49399999999999999},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":0.70299999999999996,"5min":0.80200000000000005,"15min":0.84099999999999997},"last":0.77100000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54200000000000004,"15min":0.501},"min":{"1min":0.35099999999999998,"5min":0.33100000000000002,"15min":0.30099999999999999},"max":{"1min":0.84399999999999997,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.748},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.59499999999999997,"15min":0.54100000000000004},"min":{"1min":0.47699999999999998,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.78700000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.59399999999999997,"15min":0.56399999999999995},"min":{"1min":0.38100000000000001,"5min":0.375,"15min":0.32200000000000001},"max":{"1min":0.81999999999999995,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.628,"15min":0.57899999999999996},"min":{"1min":0.51600000000000001,"5min":0.36399999999999999,"15min":0.33700000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.85199999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757402,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7384,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":6272,"kb_avail":93737768,"statfs":{"total":95995035648,"available":95987474432,"internally_reserved":0,"allocated":1130496,"data_stored":616276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6422528},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38900000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.501},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.61499999999999999,"15min":0.61299999999999999},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":0.46700000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57899999999999996,"15min":0.57399999999999995},"min":{"1min":0.39000000000000001,"5min":0.34799999999999998,"15min":0.33700000000000002},"max":{"1min":0.77100000000000002,"5min":0.83099999999999996,"15min":2.1110000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.63800000000000001,"15min":0.60599999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.33000000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.53600000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.495,"15min":0.45700000000000002},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":0.63800000000000001,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52500000000000002,"15min":0.502},"min":{"1min":0.371,"5min":0.316,"15min":0.25900000000000001},"max":{"1min":0.66700000000000004,"5min":0.89700000000000002,"15min":0.91000000000000003},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.54900000000000004,"15min":0.501},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.28699999999999998},"max":{"1min":0.78100000000000003,"5min":0.872,"15min":0.872},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63400000000000001,"15min":0.58599999999999997},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.622}]},{"osd":7,"last update":"Sat Sep 30 09:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60699999999999998,"15min":0.58099999999999996},"min":{"1min":0.41099999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.87,"5min":0.88800000000000001,"15min":0.91400000000000003},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":0.57499999999999996},"min":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":0.83799999999999997,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:22:19.853 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:22:20.221 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:20.421 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:20 smithi019 ceph-mon[224043]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:20.421 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:20 smithi019 ceph-mon[224043]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:20.421 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:20 smithi019 ceph-mon[216815]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:20.421 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:20 smithi019 ceph-mon[216815]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:20 smithi167 ceph-mon[192880]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:20 smithi167 ceph-mon[192880]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:21.961 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:22:21.962 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:22:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:22 smithi019 ceph-mon[216815]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:22.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:22 smithi019 ceph-mon[224043]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:22 smithi167 ceph-mon[192880]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:22.680 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-09-30T09:22:20.954397+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790183,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6480,"num_read_kb":14131,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":167880,"kb_used_data":23068,"kb_used_omap":19,"kb_used_meta":144684,"kb_avail":749793336,"statfs":{"total":767960285184,"available":767788376064,"internally_reserved":0,"allocated":23621632,"data_stored":19336637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":148156839},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003169"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085795+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:21:28.085795+0000","last_peered":"2023-09-30T09:21:28.085795+0000","last_clean":"2023-09-30T09:21:28.085795+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:21:28.085795+0000","last_undegraded":"2023-09-30T09:21:28.085795+0000","last_fullsized":"2023-09-30T09:21:28.085795+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082738+0000","last_change":"2023-09-30T09:20:56.404258+0000","last_active":"2023-09-30T09:21:28.082738+0000","last_peered":"2023-09-30T09:21:28.082738+0000","last_clean":"2023-09-30T09:21:28.082738+0000","last_became_active":"2023-09-30T09:20:56.403848+0000","last_became_peered":"2023-09-30T09:20:56.403848+0000","last_unstale":"2023-09-30T09:21:28.082738+0000","last_undegraded":"2023-09-30T09:21:28.082738+0000","last_fullsized":"2023-09-30T09:21:28.082738+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:01.175436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093237+0000","last_change":"2023-09-30T09:21:29.093237+0000","last_active":"2023-09-30T09:21:29.093237+0000","last_peered":"2023-09-30T09:21:29.093237+0000","last_clean":"2023-09-30T09:21:29.093237+0000","last_became_active":"2023-09-30T09:21:29.092638+0000","last_became_peered":"2023-09-30T09:21:29.092638+0000","last_unstale":"2023-09-30T09:21:29.093237+0000","last_undegraded":"2023-09-30T09:21:29.093237+0000","last_fullsized":"2023-09-30T09:21:29.093237+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:11:54.139992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104648+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:21:29.104648+0000","last_peered":"2023-09-30T09:21:29.104648+0000","last_clean":"2023-09-30T09:21:29.104648+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:21:29.104648+0000","last_undegraded":"2023-09-30T09:21:29.104648+0000","last_fullsized":"2023-09-30T09:21:29.104648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.097503+0000","last_change":"2023-09-30T09:21:29.097503+0000","last_active":"2023-09-30T09:21:29.097503+0000","last_peered":"2023-09-30T09:21:29.097503+0000","last_clean":"2023-09-30T09:21:29.097503+0000","last_became_active":"2023-09-30T09:21:29.097151+0000","last_became_peered":"2023-09-30T09:21:29.097151+0000","last_unstale":"2023-09-30T09:21:29.097503+0000","last_undegraded":"2023-09-30T09:21:29.097503+0000","last_fullsized":"2023-09-30T09:21:29.097503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:55.978501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081849+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:21:28.081849+0000","last_peered":"2023-09-30T09:21:28.081849+0000","last_clean":"2023-09-30T09:21:28.081849+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:21:28.081849+0000","last_undegraded":"2023-09-30T09:21:28.081849+0000","last_fullsized":"2023-09-30T09:21:28.081849+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096090+0000","last_change":"2023-09-30T09:21:29.096090+0000","last_active":"2023-09-30T09:21:29.096090+0000","last_peered":"2023-09-30T09:21:29.096090+0000","last_clean":"2023-09-30T09:21:29.096090+0000","last_became_active":"2023-09-30T09:21:29.095664+0000","last_became_peered":"2023-09-30T09:21:29.095664+0000","last_unstale":"2023-09-30T09:21:29.096090+0000","last_undegraded":"2023-09-30T09:21:29.096090+0000","last_fullsized":"2023-09-30T09:21:29.096090+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:58.213312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082945+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:21:28.082945+0000","last_peered":"2023-09-30T09:21:28.082945+0000","last_clean":"2023-09-30T09:21:28.082945+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:21:28.082945+0000","last_undegraded":"2023-09-30T09:21:28.082945+0000","last_fullsized":"2023-09-30T09:21:28.082945+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096256+0000","last_change":"2023-09-30T09:21:29.096256+0000","last_active":"2023-09-30T09:21:29.096256+0000","last_peered":"2023-09-30T09:21:29.096256+0000","last_clean":"2023-09-30T09:21:29.096256+0000","last_became_active":"2023-09-30T09:21:29.095930+0000","last_became_peered":"2023-09-30T09:21:29.095930+0000","last_unstale":"2023-09-30T09:21:29.096256+0000","last_undegraded":"2023-09-30T09:21:29.096256+0000","last_fullsized":"2023-09-30T09:21:29.096256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:25.270756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.082734+0000","last_change":"2023-09-30T09:19:08.915504+0000","last_active":"2023-09-30T09:21:29.082734+0000","last_peered":"2023-09-30T09:21:29.082734+0000","last_clean":"2023-09-30T09:21:29.082734+0000","last_became_active":"2023-09-30T09:19:08.915150+0000","last_became_peered":"2023-09-30T09:19:08.915150+0000","last_unstale":"2023-09-30T09:21:29.082734+0000","last_undegraded":"2023-09-30T09:21:29.082734+0000","last_fullsized":"2023-09-30T09:21:29.082734+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:56:15.982749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089369+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:21:28.089369+0000","last_peered":"2023-09-30T09:21:28.089369+0000","last_clean":"2023-09-30T09:21:28.089369+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:21:28.089369+0000","last_undegraded":"2023-09-30T09:21:28.089369+0000","last_fullsized":"2023-09-30T09:21:28.089369+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082758+0000","last_change":"2023-09-30T09:20:23.568788+0000","last_active":"2023-09-30T09:21:28.082758+0000","last_peered":"2023-09-30T09:21:28.082758+0000","last_clean":"2023-09-30T09:21:28.082758+0000","last_became_active":"2023-09-30T09:20:23.568454+0000","last_became_peered":"2023-09-30T09:20:23.568454+0000","last_unstale":"2023-09-30T09:21:28.082758+0000","last_undegraded":"2023-09-30T09:21:28.082758+0000","last_fullsized":"2023-09-30T09:21:28.082758+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:47:26.911269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":363,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083198+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:21:28.083198+0000","last_peered":"2023-09-30T09:21:28.083198+0000","last_clean":"2023-09-30T09:21:28.083198+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:21:28.083198+0000","last_undegraded":"2023-09-30T09:21:28.083198+0000","last_fullsized":"2023-09-30T09:21:28.083198+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094736+0000","last_change":"2023-09-30T09:21:29.094736+0000","last_active":"2023-09-30T09:21:29.094736+0000","last_peered":"2023-09-30T09:21:29.094736+0000","last_clean":"2023-09-30T09:21:29.094736+0000","last_became_active":"2023-09-30T09:21:29.094220+0000","last_became_peered":"2023-09-30T09:21:29.094220+0000","last_unstale":"2023-09-30T09:21:29.094736+0000","last_undegraded":"2023-09-30T09:21:29.094736+0000","last_fullsized":"2023-09-30T09:21:29.094736+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:48:58.067703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1055,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.084527+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:21:28.084527+0000","last_peered":"2023-09-30T09:21:28.084527+0000","last_clean":"2023-09-30T09:21:28.084527+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:21:28.084527+0000","last_undegraded":"2023-09-30T09:21:28.084527+0000","last_fullsized":"2023-09-30T09:21:28.084527+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092954+0000","last_change":"2023-09-30T09:21:29.092954+0000","last_active":"2023-09-30T09:21:29.092954+0000","last_peered":"2023-09-30T09:21:29.092954+0000","last_clean":"2023-09-30T09:21:29.092954+0000","last_became_active":"2023-09-30T09:21:29.092563+0000","last_became_peered":"2023-09-30T09:21:29.092563+0000","last_unstale":"2023-09-30T09:21:29.092954+0000","last_undegraded":"2023-09-30T09:21:29.092954+0000","last_fullsized":"2023-09-30T09:21:29.092954+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:54.133382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082795+0000","last_change":"2023-09-30T09:19:08.911812+0000","last_active":"2023-09-30T09:21:28.082795+0000","last_peered":"2023-09-30T09:21:28.082795+0000","last_clean":"2023-09-30T09:21:28.082795+0000","last_became_active":"2023-09-30T09:19:08.911477+0000","last_became_peered":"2023-09-30T09:19:08.911477+0000","last_unstale":"2023-09-30T09:21:28.082795+0000","last_undegraded":"2023-09-30T09:21:28.082795+0000","last_fullsized":"2023-09-30T09:21:28.082795+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:04.525132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":45,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082420+0000","last_change":"2023-09-30T09:20:56.402640+0000","last_active":"2023-09-30T09:21:28.082420+0000","last_peered":"2023-09-30T09:21:28.082420+0000","last_clean":"2023-09-30T09:21:28.082420+0000","last_became_active":"2023-09-30T09:20:56.402324+0000","last_became_peered":"2023-09-30T09:20:56.402324+0000","last_unstale":"2023-09-30T09:21:28.082420+0000","last_undegraded":"2023-09-30T09:21:28.082420+0000","last_fullsized":"2023-09-30T09:21:28.082420+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:27:18.825926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":650,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086457+0000","last_change":"2023-09-30T09:19:08.914400+0000","last_active":"2023-09-30T09:21:28.086457+0000","last_peered":"2023-09-30T09:21:28.086457+0000","last_clean":"2023-09-30T09:21:28.086457+0000","last_became_active":"2023-09-30T09:19:08.913855+0000","last_became_peered":"2023-09-30T09:19:08.913855+0000","last_unstale":"2023-09-30T09:21:28.086457+0000","last_undegraded":"2023-09-30T09:21:28.086457+0000","last_fullsized":"2023-09-30T09:21:28.086457+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:27.706105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":64,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082854+0000","last_change":"2023-09-30T09:20:56.397596+0000","last_active":"2023-09-30T09:21:28.082854+0000","last_peered":"2023-09-30T09:21:28.082854+0000","last_clean":"2023-09-30T09:21:28.082854+0000","last_became_active":"2023-09-30T09:20:56.397283+0000","last_became_peered":"2023-09-30T09:20:56.397283+0000","last_unstale":"2023-09-30T09:21:28.082854+0000","last_undegraded":"2023-09-30T09:21:28.082854+0000","last_fullsized":"2023-09-30T09:21:28.082854+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:28.695903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094162+0000","last_change":"2023-09-30T09:21:29.094162+0000","last_active":"2023-09-30T09:21:29.094162+0000","last_peered":"2023-09-30T09:21:29.094162+0000","last_clean":"2023-09-30T09:21:29.094162+0000","last_became_active":"2023-09-30T09:21:29.093839+0000","last_became_peered":"2023-09-30T09:21:29.093839+0000","last_unstale":"2023-09-30T09:21:29.094162+0000","last_undegraded":"2023-09-30T09:21:29.094162+0000","last_fullsized":"2023-09-30T09:21:29.094162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:38:47.314823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":590,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086978+0000","last_change":"2023-09-30T09:20:23.569228+0000","last_active":"2023-09-30T09:21:28.086978+0000","last_peered":"2023-09-30T09:21:28.086978+0000","last_clean":"2023-09-30T09:21:28.086978+0000","last_became_active":"2023-09-30T09:20:23.568481+0000","last_became_peered":"2023-09-30T09:20:23.568481+0000","last_unstale":"2023-09-30T09:21:28.086978+0000","last_undegraded":"2023-09-30T09:21:28.086978+0000","last_fullsized":"2023-09-30T09:21:28.086978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:31:06.051903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082973+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:21:28.082973+0000","last_peered":"2023-09-30T09:21:28.082973+0000","last_clean":"2023-09-30T09:21:28.082973+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:21:28.082973+0000","last_undegraded":"2023-09-30T09:21:28.082973+0000","last_fullsized":"2023-09-30T09:21:28.082973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104912+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:21:29.104912+0000","last_peered":"2023-09-30T09:21:29.104912+0000","last_clean":"2023-09-30T09:21:29.104912+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:21:29.104912+0000","last_undegraded":"2023-09-30T09:21:29.104912+0000","last_fullsized":"2023-09-30T09:21:29.104912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":451,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086908+0000","last_change":"2023-09-30T09:19:08.915371+0000","last_active":"2023-09-30T09:21:28.086908+0000","last_peered":"2023-09-30T09:21:28.086908+0000","last_clean":"2023-09-30T09:21:28.086908+0000","last_became_active":"2023-09-30T09:19:08.914936+0000","last_became_peered":"2023-09-30T09:19:08.914936+0000","last_unstale":"2023-09-30T09:21:28.086908+0000","last_undegraded":"2023-09-30T09:21:28.086908+0000","last_fullsized":"2023-09-30T09:21:28.086908+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:41.291423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":517,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080969+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:21:28.080969+0000","last_peered":"2023-09-30T09:21:28.080969+0000","last_clean":"2023-09-30T09:21:28.080969+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:21:28.080969+0000","last_undegraded":"2023-09-30T09:21:28.080969+0000","last_fullsized":"2023-09-30T09:21:28.080969+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085377+0000","last_change":"2023-09-30T09:20:23.574542+0000","last_active":"2023-09-30T09:21:28.085377+0000","last_peered":"2023-09-30T09:21:28.085377+0000","last_clean":"2023-09-30T09:21:28.085377+0000","last_became_active":"2023-09-30T09:20:23.574196+0000","last_became_peered":"2023-09-30T09:20:23.574196+0000","last_unstale":"2023-09-30T09:21:28.085377+0000","last_undegraded":"2023-09-30T09:21:28.085377+0000","last_fullsized":"2023-09-30T09:21:28.085377+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:24:08.605395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":45,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082236+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:21:28.082236+0000","last_peered":"2023-09-30T09:21:28.082236+0000","last_clean":"2023-09-30T09:21:28.082236+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:21:28.082236+0000","last_undegraded":"2023-09-30T09:21:28.082236+0000","last_fullsized":"2023-09-30T09:21:28.082236+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":413,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094131+0000","last_change":"2023-09-30T09:21:29.094131+0000","last_active":"2023-09-30T09:21:29.094131+0000","last_peered":"2023-09-30T09:21:29.094131+0000","last_clean":"2023-09-30T09:21:29.094131+0000","last_became_active":"2023-09-30T09:21:29.093405+0000","last_became_peered":"2023-09-30T09:21:29.093405+0000","last_unstale":"2023-09-30T09:21:29.094131+0000","last_undegraded":"2023-09-30T09:21:29.094131+0000","last_fullsized":"2023-09-30T09:21:29.094131+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:14.014784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092892+0000","last_change":"2023-09-30T09:21:29.092892+0000","last_active":"2023-09-30T09:21:29.092892+0000","last_peered":"2023-09-30T09:21:29.092892+0000","last_clean":"2023-09-30T09:21:29.092892+0000","last_became_active":"2023-09-30T09:21:29.092476+0000","last_became_peered":"2023-09-30T09:21:29.092476+0000","last_unstale":"2023-09-30T09:21:29.092892+0000","last_undegraded":"2023-09-30T09:21:29.092892+0000","last_fullsized":"2023-09-30T09:21:29.092892+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:38.113124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090919+0000","last_change":"2023-09-30T09:19:08.918284+0000","last_active":"2023-09-30T09:21:28.090919+0000","last_peered":"2023-09-30T09:21:28.090919+0000","last_clean":"2023-09-30T09:21:28.090919+0000","last_became_active":"2023-09-30T09:19:08.917861+0000","last_became_peered":"2023-09-30T09:19:08.917861+0000","last_unstale":"2023-09-30T09:21:28.090919+0000","last_undegraded":"2023-09-30T09:21:28.090919+0000","last_fullsized":"2023-09-30T09:21:28.090919+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:51.339869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":331,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081772+0000","last_change":"2023-09-30T09:20:23.574312+0000","last_active":"2023-09-30T09:21:28.081772+0000","last_peered":"2023-09-30T09:21:28.081772+0000","last_clean":"2023-09-30T09:21:28.081772+0000","last_became_active":"2023-09-30T09:20:23.573886+0000","last_became_peered":"2023-09-30T09:20:23.573886+0000","last_unstale":"2023-09-30T09:21:28.081772+0000","last_undegraded":"2023-09-30T09:21:28.081772+0000","last_fullsized":"2023-09-30T09:21:28.081772+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:45:27.269020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":219,"num_read_kb":2265,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081571+0000","last_change":"2023-09-30T09:20:23.572662+0000","last_active":"2023-09-30T09:21:28.081571+0000","last_peered":"2023-09-30T09:21:28.081571+0000","last_clean":"2023-09-30T09:21:28.081571+0000","last_became_active":"2023-09-30T09:20:23.572374+0000","last_became_peered":"2023-09-30T09:20:23.572374+0000","last_unstale":"2023-09-30T09:21:28.081571+0000","last_undegraded":"2023-09-30T09:21:28.081571+0000","last_fullsized":"2023-09-30T09:21:28.081571+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:11.881192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092844+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:21:29.092844+0000","last_peered":"2023-09-30T09:21:29.092844+0000","last_clean":"2023-09-30T09:21:29.092844+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:21:29.092844+0000","last_undegraded":"2023-09-30T09:21:29.092844+0000","last_fullsized":"2023-09-30T09:21:29.092844+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":395,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082530+0000","last_change":"2023-09-30T09:20:23.576133+0000","last_active":"2023-09-30T09:21:28.082530+0000","last_peered":"2023-09-30T09:21:28.082530+0000","last_clean":"2023-09-30T09:21:28.082530+0000","last_became_active":"2023-09-30T09:20:23.575571+0000","last_became_peered":"2023-09-30T09:20:23.575571+0000","last_unstale":"2023-09-30T09:21:28.082530+0000","last_undegraded":"2023-09-30T09:21:28.082530+0000","last_fullsized":"2023-09-30T09:21:28.082530+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:14:08.450502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092042+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:21:29.092042+0000","last_peered":"2023-09-30T09:21:29.092042+0000","last_clean":"2023-09-30T09:21:29.092042+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:21:29.092042+0000","last_undegraded":"2023-09-30T09:21:29.092042+0000","last_fullsized":"2023-09-30T09:21:29.092042+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.089363+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:21:29.089363+0000","last_peered":"2023-09-30T09:21:29.089363+0000","last_clean":"2023-09-30T09:21:29.089363+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:21:29.089363+0000","last_undegraded":"2023-09-30T09:21:29.089363+0000","last_fullsized":"2023-09-30T09:21:29.089363+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086050+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:21:28.086050+0000","last_peered":"2023-09-30T09:21:28.086050+0000","last_clean":"2023-09-30T09:21:28.086050+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:21:28.086050+0000","last_undegraded":"2023-09-30T09:21:28.086050+0000","last_fullsized":"2023-09-30T09:21:28.086050+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090767+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:21:28.090767+0000","last_peered":"2023-09-30T09:21:28.090767+0000","last_clean":"2023-09-30T09:21:28.090767+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:21:28.090767+0000","last_undegraded":"2023-09-30T09:21:28.090767+0000","last_fullsized":"2023-09-30T09:21:28.090767+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089286+0000","last_change":"2023-09-30T09:20:56.406202+0000","last_active":"2023-09-30T09:21:28.089286+0000","last_peered":"2023-09-30T09:21:28.089286+0000","last_clean":"2023-09-30T09:21:28.089286+0000","last_became_active":"2023-09-30T09:20:56.405232+0000","last_became_peered":"2023-09-30T09:20:56.405232+0000","last_unstale":"2023-09-30T09:21:28.089286+0000","last_undegraded":"2023-09-30T09:21:28.089286+0000","last_fullsized":"2023-09-30T09:21:28.089286+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:40:27.385688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081936+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:21:28.081936+0000","last_peered":"2023-09-30T09:21:28.081936+0000","last_clean":"2023-09-30T09:21:28.081936+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:21:28.081936+0000","last_undegraded":"2023-09-30T09:21:28.081936+0000","last_fullsized":"2023-09-30T09:21:28.081936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104691+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:21:29.104691+0000","last_peered":"2023-09-30T09:21:29.104691+0000","last_clean":"2023-09-30T09:21:29.104691+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:21:29.104691+0000","last_undegraded":"2023-09-30T09:21:29.104691+0000","last_fullsized":"2023-09-30T09:21:29.104691+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093267+0000","last_change":"2023-09-30T09:21:29.093267+0000","last_active":"2023-09-30T09:21:29.093267+0000","last_peered":"2023-09-30T09:21:29.093267+0000","last_clean":"2023-09-30T09:21:29.093267+0000","last_became_active":"2023-09-30T09:21:29.092904+0000","last_became_peered":"2023-09-30T09:21:29.092904+0000","last_unstale":"2023-09-30T09:21:29.093267+0000","last_undegraded":"2023-09-30T09:21:29.093267+0000","last_fullsized":"2023-09-30T09:21:29.093267+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:31.930457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083274+0000","last_change":"2023-09-30T09:20:23.567941+0000","last_active":"2023-09-30T09:21:28.083274+0000","last_peered":"2023-09-30T09:21:28.083274+0000","last_clean":"2023-09-30T09:21:28.083274+0000","last_became_active":"2023-09-30T09:20:23.567652+0000","last_became_peered":"2023-09-30T09:20:23.567652+0000","last_unstale":"2023-09-30T09:21:28.083274+0000","last_undegraded":"2023-09-30T09:21:28.083274+0000","last_fullsized":"2023-09-30T09:21:28.083274+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:41:04.133820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104438+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:21:29.104438+0000","last_peered":"2023-09-30T09:21:29.104438+0000","last_clean":"2023-09-30T09:21:29.104438+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:21:29.104438+0000","last_undegraded":"2023-09-30T09:21:29.104438+0000","last_fullsized":"2023-09-30T09:21:29.104438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081779+0000","last_change":"2023-09-30T09:20:23.580109+0000","last_active":"2023-09-30T09:21:28.081779+0000","last_peered":"2023-09-30T09:21:28.081779+0000","last_clean":"2023-09-30T09:21:28.081779+0000","last_became_active":"2023-09-30T09:20:23.579747+0000","last_became_peered":"2023-09-30T09:20:23.579747+0000","last_unstale":"2023-09-30T09:21:28.081779+0000","last_undegraded":"2023-09-30T09:21:28.081779+0000","last_fullsized":"2023-09-30T09:21:28.081779+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:04.192497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086557+0000","last_change":"2023-09-30T09:12:07.944640+0000","last_active":"2023-09-30T09:21:28.086557+0000","last_peered":"2023-09-30T09:21:28.086557+0000","last_clean":"2023-09-30T09:21:28.086557+0000","last_became_active":"2023-09-30T09:12:07.944362+0000","last_became_peered":"2023-09-30T09:12:07.944362+0000","last_unstale":"2023-09-30T09:21:28.086557+0000","last_undegraded":"2023-09-30T09:21:28.086557+0000","last_fullsized":"2023-09-30T09:21:28.086557+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:14.703475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.097658+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:21:29.097658+0000","last_peered":"2023-09-30T09:21:29.097658+0000","last_clean":"2023-09-30T09:21:29.097658+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:21:29.097658+0000","last_undegraded":"2023-09-30T09:21:29.097658+0000","last_fullsized":"2023-09-30T09:21:29.097658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087158+0000","last_change":"2023-09-30T09:20:23.568846+0000","last_active":"2023-09-30T09:21:28.087158+0000","last_peered":"2023-09-30T09:21:28.087158+0000","last_clean":"2023-09-30T09:21:28.087158+0000","last_became_active":"2023-09-30T09:20:23.567910+0000","last_became_peered":"2023-09-30T09:20:23.567910+0000","last_unstale":"2023-09-30T09:21:28.087158+0000","last_undegraded":"2023-09-30T09:21:28.087158+0000","last_fullsized":"2023-09-30T09:21:28.087158+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:48:49.264877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080900+0000","last_change":"2023-09-30T09:12:09.760712+0000","last_active":"2023-09-30T09:21:28.080900+0000","last_peered":"2023-09-30T09:21:28.080900+0000","last_clean":"2023-09-30T09:21:28.080900+0000","last_became_active":"2023-09-30T09:12:09.760435+0000","last_became_peered":"2023-09-30T09:12:09.760435+0000","last_unstale":"2023-09-30T09:21:28.080900+0000","last_undegraded":"2023-09-30T09:21:28.080900+0000","last_fullsized":"2023-09-30T09:21:28.080900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:03.896756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087175+0000","last_change":"2023-09-30T09:19:08.916779+0000","last_active":"2023-09-30T09:21:28.087175+0000","last_peered":"2023-09-30T09:21:28.087175+0000","last_clean":"2023-09-30T09:21:28.087175+0000","last_became_active":"2023-09-30T09:19:08.916333+0000","last_became_peered":"2023-09-30T09:19:08.916333+0000","last_unstale":"2023-09-30T09:21:28.087175+0000","last_undegraded":"2023-09-30T09:21:28.087175+0000","last_fullsized":"2023-09-30T09:21:28.087175+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:25.205240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092140+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:21:29.092140+0000","last_peered":"2023-09-30T09:21:29.092140+0000","last_clean":"2023-09-30T09:21:29.092140+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:21:29.092140+0000","last_undegraded":"2023-09-30T09:21:29.092140+0000","last_fullsized":"2023-09-30T09:21:29.092140+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.084198+0000","last_change":"2023-09-30T09:19:08.913366+0000","last_active":"2023-09-30T09:21:28.084198+0000","last_peered":"2023-09-30T09:21:28.084198+0000","last_clean":"2023-09-30T09:21:28.084198+0000","last_became_active":"2023-09-30T09:19:08.912978+0000","last_became_peered":"2023-09-30T09:19:08.912978+0000","last_unstale":"2023-09-30T09:21:28.084198+0000","last_undegraded":"2023-09-30T09:21:28.084198+0000","last_fullsized":"2023-09-30T09:21:28.084198+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:47:20.546906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096658+0000","last_change":"2023-09-30T09:21:29.094783+0000","last_active":"2023-09-30T09:21:29.096658+0000","last_peered":"2023-09-30T09:21:29.096658+0000","last_clean":"2023-09-30T09:21:29.096658+0000","last_became_active":"2023-09-30T09:21:29.094482+0000","last_became_peered":"2023-09-30T09:21:29.094482+0000","last_unstale":"2023-09-30T09:21:29.096658+0000","last_undegraded":"2023-09-30T09:21:29.096658+0000","last_fullsized":"2023-09-30T09:21:29.096658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:59:14.875660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087317+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:21:28.087317+0000","last_peered":"2023-09-30T09:21:28.087317+0000","last_clean":"2023-09-30T09:21:28.087317+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:21:28.087317+0000","last_undegraded":"2023-09-30T09:21:28.087317+0000","last_fullsized":"2023-09-30T09:21:28.087317+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092932+0000","last_change":"2023-09-30T09:21:29.092932+0000","last_active":"2023-09-30T09:21:29.092932+0000","last_peered":"2023-09-30T09:21:29.092932+0000","last_clean":"2023-09-30T09:21:29.092932+0000","last_became_active":"2023-09-30T09:21:29.092450+0000","last_became_peered":"2023-09-30T09:21:29.092450+0000","last_unstale":"2023-09-30T09:21:29.092932+0000","last_undegraded":"2023-09-30T09:21:29.092932+0000","last_fullsized":"2023-09-30T09:21:29.092932+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:39:06.199633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093775+0000","last_change":"2023-09-30T09:21:29.093775+0000","last_active":"2023-09-30T09:21:29.093775+0000","last_peered":"2023-09-30T09:21:29.093775+0000","last_clean":"2023-09-30T09:21:29.093775+0000","last_became_active":"2023-09-30T09:21:29.093399+0000","last_became_peered":"2023-09-30T09:21:29.093399+0000","last_unstale":"2023-09-30T09:21:29.093775+0000","last_undegraded":"2023-09-30T09:21:29.093775+0000","last_fullsized":"2023-09-30T09:21:29.093775+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:50.192497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092287+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:21:29.092287+0000","last_peered":"2023-09-30T09:21:29.092287+0000","last_clean":"2023-09-30T09:21:29.092287+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:21:29.092287+0000","last_undegraded":"2023-09-30T09:21:29.092287+0000","last_fullsized":"2023-09-30T09:21:29.092287+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085471+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:21:28.085471+0000","last_peered":"2023-09-30T09:21:28.085471+0000","last_clean":"2023-09-30T09:21:28.085471+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:21:28.085471+0000","last_undegraded":"2023-09-30T09:21:28.085471+0000","last_fullsized":"2023-09-30T09:21:28.085471+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090741+0000","last_change":"2023-09-30T09:20:23.574453+0000","last_active":"2023-09-30T09:21:28.090741+0000","last_peered":"2023-09-30T09:21:28.090741+0000","last_clean":"2023-09-30T09:21:28.090741+0000","last_became_active":"2023-09-30T09:20:23.573953+0000","last_became_peered":"2023-09-30T09:20:23.573953+0000","last_unstale":"2023-09-30T09:21:28.090741+0000","last_undegraded":"2023-09-30T09:21:28.090741+0000","last_fullsized":"2023-09-30T09:21:28.090741+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:23:21.634624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089127+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:21:28.089127+0000","last_peered":"2023-09-30T09:21:28.089127+0000","last_clean":"2023-09-30T09:21:28.089127+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:21:28.089127+0000","last_undegraded":"2023-09-30T09:21:28.089127+0000","last_fullsized":"2023-09-30T09:21:28.089127+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081654+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:21:28.081654+0000","last_peered":"2023-09-30T09:21:28.081654+0000","last_clean":"2023-09-30T09:21:28.081654+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:21:28.081654+0000","last_undegraded":"2023-09-30T09:21:28.081654+0000","last_fullsized":"2023-09-30T09:21:28.081654+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092097+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:21:29.092097+0000","last_peered":"2023-09-30T09:21:29.092097+0000","last_clean":"2023-09-30T09:21:29.092097+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:21:29.092097+0000","last_undegraded":"2023-09-30T09:21:29.092097+0000","last_fullsized":"2023-09-30T09:21:29.092097+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104391+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:21:29.104391+0000","last_peered":"2023-09-30T09:21:29.104391+0000","last_clean":"2023-09-30T09:21:29.104391+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:21:29.104391+0000","last_undegraded":"2023-09-30T09:21:29.104391+0000","last_fullsized":"2023-09-30T09:21:29.104391+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080851+0000","last_change":"2023-09-30T09:20:23.573657+0000","last_active":"2023-09-30T09:21:28.080851+0000","last_peered":"2023-09-30T09:21:28.080851+0000","last_clean":"2023-09-30T09:21:28.080851+0000","last_became_active":"2023-09-30T09:20:23.573337+0000","last_became_peered":"2023-09-30T09:20:23.573337+0000","last_unstale":"2023-09-30T09:21:28.080851+0000","last_undegraded":"2023-09-30T09:21:28.080851+0000","last_fullsized":"2023-09-30T09:21:28.080851+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:00.644205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104648+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:21:29.104648+0000","last_peered":"2023-09-30T09:21:29.104648+0000","last_clean":"2023-09-30T09:21:29.104648+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:21:29.104648+0000","last_undegraded":"2023-09-30T09:21:29.104648+0000","last_fullsized":"2023-09-30T09:21:29.104648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.098232+0000","last_change":"2023-09-30T09:21:29.097434+0000","last_active":"2023-09-30T09:21:29.098232+0000","last_peered":"2023-09-30T09:21:29.098232+0000","last_clean":"2023-09-30T09:21:29.098232+0000","last_became_active":"2023-09-30T09:21:29.097116+0000","last_became_peered":"2023-09-30T09:21:29.097116+0000","last_unstale":"2023-09-30T09:21:29.098232+0000","last_undegraded":"2023-09-30T09:21:29.098232+0000","last_fullsized":"2023-09-30T09:21:29.098232+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:41.904331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083111+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:21:28.083111+0000","last_peered":"2023-09-30T09:21:28.083111+0000","last_clean":"2023-09-30T09:21:28.083111+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:21:28.083111+0000","last_undegraded":"2023-09-30T09:21:28.083111+0000","last_fullsized":"2023-09-30T09:21:28.083111+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094532+0000","last_change":"2023-09-30T09:21:29.094532+0000","last_active":"2023-09-30T09:21:29.094532+0000","last_peered":"2023-09-30T09:21:29.094532+0000","last_clean":"2023-09-30T09:21:29.094532+0000","last_became_active":"2023-09-30T09:21:29.093807+0000","last_became_peered":"2023-09-30T09:21:29.093807+0000","last_unstale":"2023-09-30T09:21:29.094532+0000","last_undegraded":"2023-09-30T09:21:29.094532+0000","last_fullsized":"2023-09-30T09:21:29.094532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:39:32.517645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.098125+0000","last_change":"2023-09-30T09:21:29.098125+0000","last_active":"2023-09-30T09:21:29.098125+0000","last_peered":"2023-09-30T09:21:29.098125+0000","last_clean":"2023-09-30T09:21:29.098125+0000","last_became_active":"2023-09-30T09:21:29.097837+0000","last_became_peered":"2023-09-30T09:21:29.097837+0000","last_unstale":"2023-09-30T09:21:29.098125+0000","last_undegraded":"2023-09-30T09:21:29.098125+0000","last_fullsized":"2023-09-30T09:21:29.098125+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:19:36.896416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092203+0000","last_change":"2023-09-30T09:21:29.092203+0000","last_active":"2023-09-30T09:21:29.092203+0000","last_peered":"2023-09-30T09:21:29.092203+0000","last_clean":"2023-09-30T09:21:29.092203+0000","last_became_active":"2023-09-30T09:21:29.091882+0000","last_became_peered":"2023-09-30T09:21:29.091882+0000","last_unstale":"2023-09-30T09:21:29.092203+0000","last_undegraded":"2023-09-30T09:21:29.092203+0000","last_fullsized":"2023-09-30T09:21:29.092203+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:56.866973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093480+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:21:29.093480+0000","last_peered":"2023-09-30T09:21:29.093480+0000","last_clean":"2023-09-30T09:21:29.093480+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:21:29.093480+0000","last_undegraded":"2023-09-30T09:21:29.093480+0000","last_fullsized":"2023-09-30T09:21:29.093480+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085932+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:21:28.085932+0000","last_peered":"2023-09-30T09:21:28.085932+0000","last_clean":"2023-09-30T09:21:28.085932+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:21:28.085932+0000","last_undegraded":"2023-09-30T09:21:28.085932+0000","last_fullsized":"2023-09-30T09:21:28.085932+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234128,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17072,"kb_used_data":5800,"kb_used_omap":0,"kb_used_meta":11264,"kb_avail":93728080,"statfs":{"total":95995035648,"available":95977553920,"internally_reserved":0,"allocated":5939200,"data_stored":5405566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11534336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.60199999999999998,"15min":0.62},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.61199999999999999,"15min":0.64400000000000002},"min":{"1min":0.42599999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.51900000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63500000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:21:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.57699999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.30199999999999999},"max":{"1min":0.68600000000000005,"5min":0.89700000000000002,"15min":1.5429999999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.61199999999999999,"15min":0.58999999999999997},"min":{"1min":0.35799999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.69499999999999995,"5min":0.85899999999999999,"15min":2.359},"last":0.629}]},{"osd":4,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.48399999999999999,"15min":0.46899999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":0.68799999999999994,"5min":0.77800000000000002,"15min":0.878},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.497},"min":{"1min":0.33000000000000002,"5min":0.317,"15min":0.28799999999999998},"max":{"1min":0.73899999999999999,"5min":0.78000000000000003,"15min":1.111},"last":0.44800000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.53500000000000003,"15min":0.51100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":0.754,"5min":0.82499999999999996,"15min":0.86099999999999999},"last":0.498},{"interface":"front","average":{"1min":0.625,"5min":0.58399999999999996,"15min":0.55600000000000005},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.32600000000000001},"max":{"1min":0.81599999999999995,"5min":0.82299999999999995,"15min":0.89400000000000002},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59899999999999998,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34100000000000003},"max":{"1min":0.83899999999999997,"5min":0.85099999999999998,"15min":1.01},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.60799999999999998,"15min":0.55500000000000005},"min":{"1min":0.41599999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.81899999999999995,"15min":1.595},"last":0.65100000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495765,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8224,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":6528,"kb_avail":93736928,"statfs":{"total":95995035648,"available":95986614272,"internally_reserved":0,"allocated":1728512,"data_stored":1210547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6684672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.622,"15min":0.63},"min":{"1min":0.372,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.82899999999999996,"5min":2.673,"15min":2.673},"last":0.75},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.64600000000000002,"15min":0.65300000000000002},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73699999999999999,"5min":3.7450000000000001,"15min":3.7450000000000001},"last":0.64300000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.74199999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.79900000000000004}]},{"osd":3,"last update":"Sat Sep 30 09:20:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.61099999999999999,"15min":0.58399999999999996},"min":{"1min":0.45600000000000002,"5min":0.36899999999999999,"15min":0.29199999999999998},"max":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.58699999999999997,"15min":0.57499999999999996},"min":{"1min":0.441,"5min":0.37,"15min":0.34200000000000003},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":3.6200000000000001},"last":0.69799999999999995}]},{"osd":4,"last update":"Sat Sep 30 09:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.49399999999999999,"15min":0.48699999999999999},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.22700000000000001},"max":{"1min":0.69999999999999996,"5min":0.82599999999999996,"15min":0.83299999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52700000000000002,"15min":0.50800000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.80300000000000005,"5min":0.85299999999999998,"15min":0.92300000000000004},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.50900000000000001,"15min":0.48699999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.24399999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.82099999999999995},"last":0.625},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.57399999999999995,"15min":0.54700000000000004},"min":{"1min":0.39700000000000002,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.95199999999999996},"last":0.66000000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.58599999999999997,"15min":0.57399999999999995},"min":{"1min":0.39900000000000002,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":0.79100000000000004,"5min":0.85599999999999998,"15min":0.877},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.623,"15min":0.60799999999999998},"min":{"1min":0.45000000000000001,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":0.80200000000000005,"5min":0.878,"15min":1.0269999999999999},"last":0.71999999999999997}]}]},{"osd":1,"up_from":99,"seq":425201762329,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32728,"kb_used_data":5252,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93712424,"statfs":{"total":95995035648,"available":95961522176,"internally_reserved":0,"allocated":5378048,"data_stored":4821802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":28108982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.374},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.435}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.52900000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.624}]},{"osd":7,"last update":"Sat Sep 30 09:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.504},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.58799999999999997}]}]},{"osd":0,"up_from":94,"seq":403726925864,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33328,"kb_used_data":5856,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93711824,"statfs":{"total":95995035648,"available":95960907776,"internally_reserved":0,"allocated":5996544,"data_stored":5417259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":28109053},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.54200000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:21:12 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.50700000000000001,"15min":0.51800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.625,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.45700000000000002,"15min":0.46100000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.54900000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.67100000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.68000000000000005,"15min":0.72399999999999998},"min":{"1min":0.41599999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.78200000000000003,"5min":2.363,"15min":2.363},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.72399999999999998,"15min":0.76500000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.399,"5min":2.399,"15min":2.399},"last":0.629}]},{"osd":5,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.76100000000000001,"15min":0.80700000000000005},"min":{"1min":0.36599999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.3610000000000002,"5min":2.3610000000000002,"15min":2.3610000000000002},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.76000000000000001,"15min":0.81200000000000006},"min":{"1min":0.5,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84199999999999997,"5min":2.4260000000000002,"15min":2.4260000000000002},"last":0.78100000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.74399999999999999,"15min":0.79100000000000004},"min":{"1min":0.45200000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93600000000000005,"5min":2.5369999999999999,"15min":2.5369999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.78600000000000003,"15min":0.84599999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89700000000000002,"5min":2.6139999999999999,"15min":2.6139999999999999},"last":0.72399999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.71799999999999997,"15min":0.75900000000000001},"min":{"1min":0.46200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.879,"5min":2.3039999999999998,"15min":2.3039999999999998},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.77100000000000002,"15min":0.83699999999999997},"min":{"1min":0.44,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76500000000000001,"5min":2.492,"15min":2.492},"last":0.68999999999999995}]}]},{"osd":2,"up_from":104,"seq":446676598803,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28612,"kb_used_data":1136,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93716540,"statfs":{"total":95995035648,"available":95965736960,"internally_reserved":0,"allocated":1163264,"data_stored":624404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":28109453},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.42799999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.35999999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.749},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.57099999999999995}]},{"osd":5,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":0.71599999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.139,"5min":1.139,"15min":1.139},"last":0.62}]},{"osd":7,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.72999999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435276,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28532,"kb_used_data":1120,"kb_used_omap":2,"kb_used_meta":27389,"kb_avail":93716620,"statfs":{"total":95995035648,"available":95965818880,"internally_reserved":0,"allocated":1146880,"data_stored":620489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":28046695},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25900000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.379}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.43099999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019038,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12000,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":10880,"kb_avail":93733152,"statfs":{"total":95995035648,"available":95982747648,"internally_reserved":0,"allocated":1138688,"data_stored":620294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11141120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.60399999999999998,"15min":0.63100000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77200000000000002,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.60399999999999998,"15min":0.63300000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65100000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.52900000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59299999999999997,"15min":0.57399999999999995},"min":{"1min":0.41499999999999998,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":0.72099999999999997,"5min":0.81999999999999995,"15min":1.6839999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.63,"5min":0.61399999999999999,"15min":0.58499999999999996},"min":{"1min":0.40600000000000003,"5min":0.38200000000000001,"15min":0.379},"max":{"1min":0.79400000000000004,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.69699999999999995}]},{"osd":5,"last update":"Sat Sep 30 09:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.49299999999999999,"15min":0.46300000000000002},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.27200000000000002},"max":{"1min":0.68500000000000005,"5min":0.74399999999999999,"15min":0.98699999999999999},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54100000000000004,"15min":0.49399999999999999},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":0.70299999999999996,"5min":0.80200000000000005,"15min":0.84099999999999997},"last":0.77100000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54200000000000004,"15min":0.501},"min":{"1min":0.35099999999999998,"5min":0.33100000000000002,"15min":0.30099999999999999},"max":{"1min":0.84399999999999997,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.748},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.59499999999999997,"15min":0.54100000000000004},"min":{"1min":0.47699999999999998,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.78700000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.59399999999999997,"15min":0.56399999999999995},"min":{"1min":0.38100000000000001,"5min":0.375,"15min":0.32200000000000001},"max":{"1min":0.81999999999999995,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.628,"15min":0.57899999999999996},"min":{"1min":0.51600000000000001,"5min":0.36399999999999999,"15min":0.33700000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.85199999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757402,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7384,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":6272,"kb_avail":93737768,"statfs":{"total":95995035648,"available":95987474432,"internally_reserved":0,"allocated":1130496,"data_stored":616276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6422528},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38900000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.501},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.61499999999999999,"15min":0.61299999999999999},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":0.46700000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57899999999999996,"15min":0.57399999999999995},"min":{"1min":0.39000000000000001,"5min":0.34799999999999998,"15min":0.33700000000000002},"max":{"1min":0.77100000000000002,"5min":0.83099999999999996,"15min":2.1110000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.63800000000000001,"15min":0.60599999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.33000000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.53600000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.495,"15min":0.45700000000000002},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":0.63800000000000001,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52500000000000002,"15min":0.502},"min":{"1min":0.371,"5min":0.316,"15min":0.25900000000000001},"max":{"1min":0.66700000000000004,"5min":0.89700000000000002,"15min":0.91000000000000003},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.54900000000000004,"15min":0.501},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.28699999999999998},"max":{"1min":0.78100000000000003,"5min":0.872,"15min":0.872},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63400000000000001,"15min":0.58599999999999997},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.622}]},{"osd":7,"last update":"Sat Sep 30 09:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60699999999999998,"15min":0.58099999999999996},"min":{"1min":0.41099999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.87,"5min":0.88800000000000001,"15min":0.91400000000000003},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":0.57499999999999996},"min":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":0.83799999999999997,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:22:22.686 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-30T09:22:22.686 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-30T09:22:22.687 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-30T09:22:22.687 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-30T09:22:23.052 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:23.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:23 smithi019 ceph-mon[216815]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:23.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:23 smithi019 ceph-mon[224043]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:23 smithi167 ceph-mon[192880]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:22:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:22:23] "GET /metrics HTTP/1.1" 200 33868 "" "Prometheus/2.43.0" 2023-09-30T09:22:24.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:24 smithi019 ceph-mon[216815]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:24.357 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:24 smithi019 ceph-mon[224043]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:24 smithi167 ceph-mon[192880]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:24.845 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:22:24.845 INFO:teuthology.orchestra.run.smithi019.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-30T09:22:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:25 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1133835518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:22:25.572 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:25 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1133835518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:22:25.573 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:25 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1133835518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-30T09:22:25.592 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-30T09:22:25.592 INFO:teuthology.run_tasks:Running task print... 2023-09-30T09:22:25.603 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-30T09:22:25.604 INFO:teuthology.run_tasks:Running task parallel... 2023-09-30T09:22:25.613 INFO:teuthology.task.parallel:starting parallel... 2023-09-30T09:22:25.613 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-30T09:22:25.613 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-30T09:22:25.615 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-30T09:22:25.615 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '1d10b71792f3be8887a7631e69851ac2df3585af'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '1d10b71792f3be8887a7631e69851ac2df3585af'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '1d10b71792f3be8887a7631e69851ac2df3585af'}} 2023-09-30T09:22:25.615 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-30T09:22:25.615 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-30T09:22:25.615 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-30T09:22:25.615 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:22:25.980 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:26 smithi167 ceph-mon[192880]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:26 smithi019 ceph-mon[216815]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:26 smithi019 ceph-mon[224043]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:27.824 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:22:27.826 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:22:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:28 smithi167 ceph-mon[192880]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:28 smithi019 ceph-mon[224043]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:28 smithi019 ceph-mon[216815]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:28.614 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":215,"stamp":"2023-09-30T09:22:26.955859+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790183,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6480,"num_read_kb":14131,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":167880,"kb_used_data":23068,"kb_used_omap":19,"kb_used_meta":144684,"kb_avail":749793336,"statfs":{"total":767960285184,"available":767788376064,"internally_reserved":0,"allocated":23621632,"data_stored":19336637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":148156839},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003117"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085795+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:21:28.085795+0000","last_peered":"2023-09-30T09:21:28.085795+0000","last_clean":"2023-09-30T09:21:28.085795+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:21:28.085795+0000","last_undegraded":"2023-09-30T09:21:28.085795+0000","last_fullsized":"2023-09-30T09:21:28.085795+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082738+0000","last_change":"2023-09-30T09:20:56.404258+0000","last_active":"2023-09-30T09:21:28.082738+0000","last_peered":"2023-09-30T09:21:28.082738+0000","last_clean":"2023-09-30T09:21:28.082738+0000","last_became_active":"2023-09-30T09:20:56.403848+0000","last_became_peered":"2023-09-30T09:20:56.403848+0000","last_unstale":"2023-09-30T09:21:28.082738+0000","last_undegraded":"2023-09-30T09:21:28.082738+0000","last_fullsized":"2023-09-30T09:21:28.082738+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:01.175436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093237+0000","last_change":"2023-09-30T09:21:29.093237+0000","last_active":"2023-09-30T09:21:29.093237+0000","last_peered":"2023-09-30T09:21:29.093237+0000","last_clean":"2023-09-30T09:21:29.093237+0000","last_became_active":"2023-09-30T09:21:29.092638+0000","last_became_peered":"2023-09-30T09:21:29.092638+0000","last_unstale":"2023-09-30T09:21:29.093237+0000","last_undegraded":"2023-09-30T09:21:29.093237+0000","last_fullsized":"2023-09-30T09:21:29.093237+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:11:54.139992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104648+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:21:29.104648+0000","last_peered":"2023-09-30T09:21:29.104648+0000","last_clean":"2023-09-30T09:21:29.104648+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:21:29.104648+0000","last_undegraded":"2023-09-30T09:21:29.104648+0000","last_fullsized":"2023-09-30T09:21:29.104648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.097503+0000","last_change":"2023-09-30T09:21:29.097503+0000","last_active":"2023-09-30T09:21:29.097503+0000","last_peered":"2023-09-30T09:21:29.097503+0000","last_clean":"2023-09-30T09:21:29.097503+0000","last_became_active":"2023-09-30T09:21:29.097151+0000","last_became_peered":"2023-09-30T09:21:29.097151+0000","last_unstale":"2023-09-30T09:21:29.097503+0000","last_undegraded":"2023-09-30T09:21:29.097503+0000","last_fullsized":"2023-09-30T09:21:29.097503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:55.978501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081849+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:21:28.081849+0000","last_peered":"2023-09-30T09:21:28.081849+0000","last_clean":"2023-09-30T09:21:28.081849+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:21:28.081849+0000","last_undegraded":"2023-09-30T09:21:28.081849+0000","last_fullsized":"2023-09-30T09:21:28.081849+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096090+0000","last_change":"2023-09-30T09:21:29.096090+0000","last_active":"2023-09-30T09:21:29.096090+0000","last_peered":"2023-09-30T09:21:29.096090+0000","last_clean":"2023-09-30T09:21:29.096090+0000","last_became_active":"2023-09-30T09:21:29.095664+0000","last_became_peered":"2023-09-30T09:21:29.095664+0000","last_unstale":"2023-09-30T09:21:29.096090+0000","last_undegraded":"2023-09-30T09:21:29.096090+0000","last_fullsized":"2023-09-30T09:21:29.096090+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:58.213312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082945+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:21:28.082945+0000","last_peered":"2023-09-30T09:21:28.082945+0000","last_clean":"2023-09-30T09:21:28.082945+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:21:28.082945+0000","last_undegraded":"2023-09-30T09:21:28.082945+0000","last_fullsized":"2023-09-30T09:21:28.082945+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096256+0000","last_change":"2023-09-30T09:21:29.096256+0000","last_active":"2023-09-30T09:21:29.096256+0000","last_peered":"2023-09-30T09:21:29.096256+0000","last_clean":"2023-09-30T09:21:29.096256+0000","last_became_active":"2023-09-30T09:21:29.095930+0000","last_became_peered":"2023-09-30T09:21:29.095930+0000","last_unstale":"2023-09-30T09:21:29.096256+0000","last_undegraded":"2023-09-30T09:21:29.096256+0000","last_fullsized":"2023-09-30T09:21:29.096256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:25.270756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.082734+0000","last_change":"2023-09-30T09:19:08.915504+0000","last_active":"2023-09-30T09:21:29.082734+0000","last_peered":"2023-09-30T09:21:29.082734+0000","last_clean":"2023-09-30T09:21:29.082734+0000","last_became_active":"2023-09-30T09:19:08.915150+0000","last_became_peered":"2023-09-30T09:19:08.915150+0000","last_unstale":"2023-09-30T09:21:29.082734+0000","last_undegraded":"2023-09-30T09:21:29.082734+0000","last_fullsized":"2023-09-30T09:21:29.082734+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:56:15.982749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089369+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:21:28.089369+0000","last_peered":"2023-09-30T09:21:28.089369+0000","last_clean":"2023-09-30T09:21:28.089369+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:21:28.089369+0000","last_undegraded":"2023-09-30T09:21:28.089369+0000","last_fullsized":"2023-09-30T09:21:28.089369+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082758+0000","last_change":"2023-09-30T09:20:23.568788+0000","last_active":"2023-09-30T09:21:28.082758+0000","last_peered":"2023-09-30T09:21:28.082758+0000","last_clean":"2023-09-30T09:21:28.082758+0000","last_became_active":"2023-09-30T09:20:23.568454+0000","last_became_peered":"2023-09-30T09:20:23.568454+0000","last_unstale":"2023-09-30T09:21:28.082758+0000","last_undegraded":"2023-09-30T09:21:28.082758+0000","last_fullsized":"2023-09-30T09:21:28.082758+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:47:26.911269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":363,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083198+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:21:28.083198+0000","last_peered":"2023-09-30T09:21:28.083198+0000","last_clean":"2023-09-30T09:21:28.083198+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:21:28.083198+0000","last_undegraded":"2023-09-30T09:21:28.083198+0000","last_fullsized":"2023-09-30T09:21:28.083198+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094736+0000","last_change":"2023-09-30T09:21:29.094736+0000","last_active":"2023-09-30T09:21:29.094736+0000","last_peered":"2023-09-30T09:21:29.094736+0000","last_clean":"2023-09-30T09:21:29.094736+0000","last_became_active":"2023-09-30T09:21:29.094220+0000","last_became_peered":"2023-09-30T09:21:29.094220+0000","last_unstale":"2023-09-30T09:21:29.094736+0000","last_undegraded":"2023-09-30T09:21:29.094736+0000","last_fullsized":"2023-09-30T09:21:29.094736+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:48:58.067703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1055,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.084527+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:21:28.084527+0000","last_peered":"2023-09-30T09:21:28.084527+0000","last_clean":"2023-09-30T09:21:28.084527+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:21:28.084527+0000","last_undegraded":"2023-09-30T09:21:28.084527+0000","last_fullsized":"2023-09-30T09:21:28.084527+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092954+0000","last_change":"2023-09-30T09:21:29.092954+0000","last_active":"2023-09-30T09:21:29.092954+0000","last_peered":"2023-09-30T09:21:29.092954+0000","last_clean":"2023-09-30T09:21:29.092954+0000","last_became_active":"2023-09-30T09:21:29.092563+0000","last_became_peered":"2023-09-30T09:21:29.092563+0000","last_unstale":"2023-09-30T09:21:29.092954+0000","last_undegraded":"2023-09-30T09:21:29.092954+0000","last_fullsized":"2023-09-30T09:21:29.092954+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:54.133382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082795+0000","last_change":"2023-09-30T09:19:08.911812+0000","last_active":"2023-09-30T09:21:28.082795+0000","last_peered":"2023-09-30T09:21:28.082795+0000","last_clean":"2023-09-30T09:21:28.082795+0000","last_became_active":"2023-09-30T09:19:08.911477+0000","last_became_peered":"2023-09-30T09:19:08.911477+0000","last_unstale":"2023-09-30T09:21:28.082795+0000","last_undegraded":"2023-09-30T09:21:28.082795+0000","last_fullsized":"2023-09-30T09:21:28.082795+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:04.525132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.085255+0000","last_change":"2023-09-30T09:20:56.402640+0000","last_active":"2023-09-30T09:21:29.085255+0000","last_peered":"2023-09-30T09:21:29.085255+0000","last_clean":"2023-09-30T09:21:29.085255+0000","last_became_active":"2023-09-30T09:20:56.402324+0000","last_became_peered":"2023-09-30T09:20:56.402324+0000","last_unstale":"2023-09-30T09:21:29.085255+0000","last_undegraded":"2023-09-30T09:21:29.085255+0000","last_fullsized":"2023-09-30T09:21:29.085255+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:27:18.825926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":650,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086457+0000","last_change":"2023-09-30T09:19:08.914400+0000","last_active":"2023-09-30T09:21:28.086457+0000","last_peered":"2023-09-30T09:21:28.086457+0000","last_clean":"2023-09-30T09:21:28.086457+0000","last_became_active":"2023-09-30T09:19:08.913855+0000","last_became_peered":"2023-09-30T09:19:08.913855+0000","last_unstale":"2023-09-30T09:21:28.086457+0000","last_undegraded":"2023-09-30T09:21:28.086457+0000","last_fullsized":"2023-09-30T09:21:28.086457+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:27.706105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":64,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082854+0000","last_change":"2023-09-30T09:20:56.397596+0000","last_active":"2023-09-30T09:21:28.082854+0000","last_peered":"2023-09-30T09:21:28.082854+0000","last_clean":"2023-09-30T09:21:28.082854+0000","last_became_active":"2023-09-30T09:20:56.397283+0000","last_became_peered":"2023-09-30T09:20:56.397283+0000","last_unstale":"2023-09-30T09:21:28.082854+0000","last_undegraded":"2023-09-30T09:21:28.082854+0000","last_fullsized":"2023-09-30T09:21:28.082854+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:28.695903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094162+0000","last_change":"2023-09-30T09:21:29.094162+0000","last_active":"2023-09-30T09:21:29.094162+0000","last_peered":"2023-09-30T09:21:29.094162+0000","last_clean":"2023-09-30T09:21:29.094162+0000","last_became_active":"2023-09-30T09:21:29.093839+0000","last_became_peered":"2023-09-30T09:21:29.093839+0000","last_unstale":"2023-09-30T09:21:29.094162+0000","last_undegraded":"2023-09-30T09:21:29.094162+0000","last_fullsized":"2023-09-30T09:21:29.094162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:38:47.314823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":590,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086978+0000","last_change":"2023-09-30T09:20:23.569228+0000","last_active":"2023-09-30T09:21:28.086978+0000","last_peered":"2023-09-30T09:21:28.086978+0000","last_clean":"2023-09-30T09:21:28.086978+0000","last_became_active":"2023-09-30T09:20:23.568481+0000","last_became_peered":"2023-09-30T09:20:23.568481+0000","last_unstale":"2023-09-30T09:21:28.086978+0000","last_undegraded":"2023-09-30T09:21:28.086978+0000","last_fullsized":"2023-09-30T09:21:28.086978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:31:06.051903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082973+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:21:28.082973+0000","last_peered":"2023-09-30T09:21:28.082973+0000","last_clean":"2023-09-30T09:21:28.082973+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:21:28.082973+0000","last_undegraded":"2023-09-30T09:21:28.082973+0000","last_fullsized":"2023-09-30T09:21:28.082973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104912+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:21:29.104912+0000","last_peered":"2023-09-30T09:21:29.104912+0000","last_clean":"2023-09-30T09:21:29.104912+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:21:29.104912+0000","last_undegraded":"2023-09-30T09:21:29.104912+0000","last_fullsized":"2023-09-30T09:21:29.104912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":451,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086908+0000","last_change":"2023-09-30T09:19:08.915371+0000","last_active":"2023-09-30T09:21:28.086908+0000","last_peered":"2023-09-30T09:21:28.086908+0000","last_clean":"2023-09-30T09:21:28.086908+0000","last_became_active":"2023-09-30T09:19:08.914936+0000","last_became_peered":"2023-09-30T09:19:08.914936+0000","last_unstale":"2023-09-30T09:21:28.086908+0000","last_undegraded":"2023-09-30T09:21:28.086908+0000","last_fullsized":"2023-09-30T09:21:28.086908+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:41.291423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":517,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080969+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:21:28.080969+0000","last_peered":"2023-09-30T09:21:28.080969+0000","last_clean":"2023-09-30T09:21:28.080969+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:21:28.080969+0000","last_undegraded":"2023-09-30T09:21:28.080969+0000","last_fullsized":"2023-09-30T09:21:28.080969+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085377+0000","last_change":"2023-09-30T09:20:23.574542+0000","last_active":"2023-09-30T09:21:28.085377+0000","last_peered":"2023-09-30T09:21:28.085377+0000","last_clean":"2023-09-30T09:21:28.085377+0000","last_became_active":"2023-09-30T09:20:23.574196+0000","last_became_peered":"2023-09-30T09:20:23.574196+0000","last_unstale":"2023-09-30T09:21:28.085377+0000","last_undegraded":"2023-09-30T09:21:28.085377+0000","last_fullsized":"2023-09-30T09:21:28.085377+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:24:08.605395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.087220+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:21:29.087220+0000","last_peered":"2023-09-30T09:21:29.087220+0000","last_clean":"2023-09-30T09:21:29.087220+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:21:29.087220+0000","last_undegraded":"2023-09-30T09:21:29.087220+0000","last_fullsized":"2023-09-30T09:21:29.087220+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":413,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094131+0000","last_change":"2023-09-30T09:21:29.094131+0000","last_active":"2023-09-30T09:21:29.094131+0000","last_peered":"2023-09-30T09:21:29.094131+0000","last_clean":"2023-09-30T09:21:29.094131+0000","last_became_active":"2023-09-30T09:21:29.093405+0000","last_became_peered":"2023-09-30T09:21:29.093405+0000","last_unstale":"2023-09-30T09:21:29.094131+0000","last_undegraded":"2023-09-30T09:21:29.094131+0000","last_fullsized":"2023-09-30T09:21:29.094131+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:14.014784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092892+0000","last_change":"2023-09-30T09:21:29.092892+0000","last_active":"2023-09-30T09:21:29.092892+0000","last_peered":"2023-09-30T09:21:29.092892+0000","last_clean":"2023-09-30T09:21:29.092892+0000","last_became_active":"2023-09-30T09:21:29.092476+0000","last_became_peered":"2023-09-30T09:21:29.092476+0000","last_unstale":"2023-09-30T09:21:29.092892+0000","last_undegraded":"2023-09-30T09:21:29.092892+0000","last_fullsized":"2023-09-30T09:21:29.092892+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:38.113124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090919+0000","last_change":"2023-09-30T09:19:08.918284+0000","last_active":"2023-09-30T09:21:28.090919+0000","last_peered":"2023-09-30T09:21:28.090919+0000","last_clean":"2023-09-30T09:21:28.090919+0000","last_became_active":"2023-09-30T09:19:08.917861+0000","last_became_peered":"2023-09-30T09:19:08.917861+0000","last_unstale":"2023-09-30T09:21:28.090919+0000","last_undegraded":"2023-09-30T09:21:28.090919+0000","last_fullsized":"2023-09-30T09:21:28.090919+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:51.339869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":331,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081772+0000","last_change":"2023-09-30T09:20:23.574312+0000","last_active":"2023-09-30T09:21:28.081772+0000","last_peered":"2023-09-30T09:21:28.081772+0000","last_clean":"2023-09-30T09:21:28.081772+0000","last_became_active":"2023-09-30T09:20:23.573886+0000","last_became_peered":"2023-09-30T09:20:23.573886+0000","last_unstale":"2023-09-30T09:21:28.081772+0000","last_undegraded":"2023-09-30T09:21:28.081772+0000","last_fullsized":"2023-09-30T09:21:28.081772+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:45:27.269020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":219,"num_read_kb":2265,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.085074+0000","last_change":"2023-09-30T09:20:23.572662+0000","last_active":"2023-09-30T09:21:29.085074+0000","last_peered":"2023-09-30T09:21:29.085074+0000","last_clean":"2023-09-30T09:21:29.085074+0000","last_became_active":"2023-09-30T09:20:23.572374+0000","last_became_peered":"2023-09-30T09:20:23.572374+0000","last_unstale":"2023-09-30T09:21:29.085074+0000","last_undegraded":"2023-09-30T09:21:29.085074+0000","last_fullsized":"2023-09-30T09:21:29.085074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:11.881192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092844+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:21:29.092844+0000","last_peered":"2023-09-30T09:21:29.092844+0000","last_clean":"2023-09-30T09:21:29.092844+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:21:29.092844+0000","last_undegraded":"2023-09-30T09:21:29.092844+0000","last_fullsized":"2023-09-30T09:21:29.092844+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":396,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.085336+0000","last_change":"2023-09-30T09:20:23.576133+0000","last_active":"2023-09-30T09:21:29.085336+0000","last_peered":"2023-09-30T09:21:29.085336+0000","last_clean":"2023-09-30T09:21:29.085336+0000","last_became_active":"2023-09-30T09:20:23.575571+0000","last_became_peered":"2023-09-30T09:20:23.575571+0000","last_unstale":"2023-09-30T09:21:29.085336+0000","last_undegraded":"2023-09-30T09:21:29.085336+0000","last_fullsized":"2023-09-30T09:21:29.085336+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:14:08.450502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092042+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:21:29.092042+0000","last_peered":"2023-09-30T09:21:29.092042+0000","last_clean":"2023-09-30T09:21:29.092042+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:21:29.092042+0000","last_undegraded":"2023-09-30T09:21:29.092042+0000","last_fullsized":"2023-09-30T09:21:29.092042+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.089363+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:21:29.089363+0000","last_peered":"2023-09-30T09:21:29.089363+0000","last_clean":"2023-09-30T09:21:29.089363+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:21:29.089363+0000","last_undegraded":"2023-09-30T09:21:29.089363+0000","last_fullsized":"2023-09-30T09:21:29.089363+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086050+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:21:28.086050+0000","last_peered":"2023-09-30T09:21:28.086050+0000","last_clean":"2023-09-30T09:21:28.086050+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:21:28.086050+0000","last_undegraded":"2023-09-30T09:21:28.086050+0000","last_fullsized":"2023-09-30T09:21:28.086050+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090767+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:21:28.090767+0000","last_peered":"2023-09-30T09:21:28.090767+0000","last_clean":"2023-09-30T09:21:28.090767+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:21:28.090767+0000","last_undegraded":"2023-09-30T09:21:28.090767+0000","last_fullsized":"2023-09-30T09:21:28.090767+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089286+0000","last_change":"2023-09-30T09:20:56.406202+0000","last_active":"2023-09-30T09:21:28.089286+0000","last_peered":"2023-09-30T09:21:28.089286+0000","last_clean":"2023-09-30T09:21:28.089286+0000","last_became_active":"2023-09-30T09:20:56.405232+0000","last_became_peered":"2023-09-30T09:20:56.405232+0000","last_unstale":"2023-09-30T09:21:28.089286+0000","last_undegraded":"2023-09-30T09:21:28.089286+0000","last_fullsized":"2023-09-30T09:21:28.089286+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:40:27.385688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081936+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:21:28.081936+0000","last_peered":"2023-09-30T09:21:28.081936+0000","last_clean":"2023-09-30T09:21:28.081936+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:21:28.081936+0000","last_undegraded":"2023-09-30T09:21:28.081936+0000","last_fullsized":"2023-09-30T09:21:28.081936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104691+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:21:29.104691+0000","last_peered":"2023-09-30T09:21:29.104691+0000","last_clean":"2023-09-30T09:21:29.104691+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:21:29.104691+0000","last_undegraded":"2023-09-30T09:21:29.104691+0000","last_fullsized":"2023-09-30T09:21:29.104691+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093267+0000","last_change":"2023-09-30T09:21:29.093267+0000","last_active":"2023-09-30T09:21:29.093267+0000","last_peered":"2023-09-30T09:21:29.093267+0000","last_clean":"2023-09-30T09:21:29.093267+0000","last_became_active":"2023-09-30T09:21:29.092904+0000","last_became_peered":"2023-09-30T09:21:29.092904+0000","last_unstale":"2023-09-30T09:21:29.093267+0000","last_undegraded":"2023-09-30T09:21:29.093267+0000","last_fullsized":"2023-09-30T09:21:29.093267+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:31.930457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083274+0000","last_change":"2023-09-30T09:20:23.567941+0000","last_active":"2023-09-30T09:21:28.083274+0000","last_peered":"2023-09-30T09:21:28.083274+0000","last_clean":"2023-09-30T09:21:28.083274+0000","last_became_active":"2023-09-30T09:20:23.567652+0000","last_became_peered":"2023-09-30T09:20:23.567652+0000","last_unstale":"2023-09-30T09:21:28.083274+0000","last_undegraded":"2023-09-30T09:21:28.083274+0000","last_fullsized":"2023-09-30T09:21:28.083274+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:41:04.133820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104438+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:21:29.104438+0000","last_peered":"2023-09-30T09:21:29.104438+0000","last_clean":"2023-09-30T09:21:29.104438+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:21:29.104438+0000","last_undegraded":"2023-09-30T09:21:29.104438+0000","last_fullsized":"2023-09-30T09:21:29.104438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081779+0000","last_change":"2023-09-30T09:20:23.580109+0000","last_active":"2023-09-30T09:21:28.081779+0000","last_peered":"2023-09-30T09:21:28.081779+0000","last_clean":"2023-09-30T09:21:28.081779+0000","last_became_active":"2023-09-30T09:20:23.579747+0000","last_became_peered":"2023-09-30T09:20:23.579747+0000","last_unstale":"2023-09-30T09:21:28.081779+0000","last_undegraded":"2023-09-30T09:21:28.081779+0000","last_fullsized":"2023-09-30T09:21:28.081779+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:04.192497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086557+0000","last_change":"2023-09-30T09:12:07.944640+0000","last_active":"2023-09-30T09:21:28.086557+0000","last_peered":"2023-09-30T09:21:28.086557+0000","last_clean":"2023-09-30T09:21:28.086557+0000","last_became_active":"2023-09-30T09:12:07.944362+0000","last_became_peered":"2023-09-30T09:12:07.944362+0000","last_unstale":"2023-09-30T09:21:28.086557+0000","last_undegraded":"2023-09-30T09:21:28.086557+0000","last_fullsized":"2023-09-30T09:21:28.086557+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:14.703475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.097658+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:21:29.097658+0000","last_peered":"2023-09-30T09:21:29.097658+0000","last_clean":"2023-09-30T09:21:29.097658+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:21:29.097658+0000","last_undegraded":"2023-09-30T09:21:29.097658+0000","last_fullsized":"2023-09-30T09:21:29.097658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087158+0000","last_change":"2023-09-30T09:20:23.568846+0000","last_active":"2023-09-30T09:21:28.087158+0000","last_peered":"2023-09-30T09:21:28.087158+0000","last_clean":"2023-09-30T09:21:28.087158+0000","last_became_active":"2023-09-30T09:20:23.567910+0000","last_became_peered":"2023-09-30T09:20:23.567910+0000","last_unstale":"2023-09-30T09:21:28.087158+0000","last_undegraded":"2023-09-30T09:21:28.087158+0000","last_fullsized":"2023-09-30T09:21:28.087158+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:48:49.264877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080900+0000","last_change":"2023-09-30T09:12:09.760712+0000","last_active":"2023-09-30T09:21:28.080900+0000","last_peered":"2023-09-30T09:21:28.080900+0000","last_clean":"2023-09-30T09:21:28.080900+0000","last_became_active":"2023-09-30T09:12:09.760435+0000","last_became_peered":"2023-09-30T09:12:09.760435+0000","last_unstale":"2023-09-30T09:21:28.080900+0000","last_undegraded":"2023-09-30T09:21:28.080900+0000","last_fullsized":"2023-09-30T09:21:28.080900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:03.896756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087175+0000","last_change":"2023-09-30T09:19:08.916779+0000","last_active":"2023-09-30T09:21:28.087175+0000","last_peered":"2023-09-30T09:21:28.087175+0000","last_clean":"2023-09-30T09:21:28.087175+0000","last_became_active":"2023-09-30T09:19:08.916333+0000","last_became_peered":"2023-09-30T09:19:08.916333+0000","last_unstale":"2023-09-30T09:21:28.087175+0000","last_undegraded":"2023-09-30T09:21:28.087175+0000","last_fullsized":"2023-09-30T09:21:28.087175+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:25.205240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092140+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:21:29.092140+0000","last_peered":"2023-09-30T09:21:29.092140+0000","last_clean":"2023-09-30T09:21:29.092140+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:21:29.092140+0000","last_undegraded":"2023-09-30T09:21:29.092140+0000","last_fullsized":"2023-09-30T09:21:29.092140+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.084198+0000","last_change":"2023-09-30T09:19:08.913366+0000","last_active":"2023-09-30T09:21:28.084198+0000","last_peered":"2023-09-30T09:21:28.084198+0000","last_clean":"2023-09-30T09:21:28.084198+0000","last_became_active":"2023-09-30T09:19:08.912978+0000","last_became_peered":"2023-09-30T09:19:08.912978+0000","last_unstale":"2023-09-30T09:21:28.084198+0000","last_undegraded":"2023-09-30T09:21:28.084198+0000","last_fullsized":"2023-09-30T09:21:28.084198+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:47:20.546906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096658+0000","last_change":"2023-09-30T09:21:29.094783+0000","last_active":"2023-09-30T09:21:29.096658+0000","last_peered":"2023-09-30T09:21:29.096658+0000","last_clean":"2023-09-30T09:21:29.096658+0000","last_became_active":"2023-09-30T09:21:29.094482+0000","last_became_peered":"2023-09-30T09:21:29.094482+0000","last_unstale":"2023-09-30T09:21:29.096658+0000","last_undegraded":"2023-09-30T09:21:29.096658+0000","last_fullsized":"2023-09-30T09:21:29.096658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:59:14.875660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087317+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:21:28.087317+0000","last_peered":"2023-09-30T09:21:28.087317+0000","last_clean":"2023-09-30T09:21:28.087317+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:21:28.087317+0000","last_undegraded":"2023-09-30T09:21:28.087317+0000","last_fullsized":"2023-09-30T09:21:28.087317+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092932+0000","last_change":"2023-09-30T09:21:29.092932+0000","last_active":"2023-09-30T09:21:29.092932+0000","last_peered":"2023-09-30T09:21:29.092932+0000","last_clean":"2023-09-30T09:21:29.092932+0000","last_became_active":"2023-09-30T09:21:29.092450+0000","last_became_peered":"2023-09-30T09:21:29.092450+0000","last_unstale":"2023-09-30T09:21:29.092932+0000","last_undegraded":"2023-09-30T09:21:29.092932+0000","last_fullsized":"2023-09-30T09:21:29.092932+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:39:06.199633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093775+0000","last_change":"2023-09-30T09:21:29.093775+0000","last_active":"2023-09-30T09:21:29.093775+0000","last_peered":"2023-09-30T09:21:29.093775+0000","last_clean":"2023-09-30T09:21:29.093775+0000","last_became_active":"2023-09-30T09:21:29.093399+0000","last_became_peered":"2023-09-30T09:21:29.093399+0000","last_unstale":"2023-09-30T09:21:29.093775+0000","last_undegraded":"2023-09-30T09:21:29.093775+0000","last_fullsized":"2023-09-30T09:21:29.093775+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:50.192497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092287+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:21:29.092287+0000","last_peered":"2023-09-30T09:21:29.092287+0000","last_clean":"2023-09-30T09:21:29.092287+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:21:29.092287+0000","last_undegraded":"2023-09-30T09:21:29.092287+0000","last_fullsized":"2023-09-30T09:21:29.092287+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085471+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:21:28.085471+0000","last_peered":"2023-09-30T09:21:28.085471+0000","last_clean":"2023-09-30T09:21:28.085471+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:21:28.085471+0000","last_undegraded":"2023-09-30T09:21:28.085471+0000","last_fullsized":"2023-09-30T09:21:28.085471+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090741+0000","last_change":"2023-09-30T09:20:23.574453+0000","last_active":"2023-09-30T09:21:28.090741+0000","last_peered":"2023-09-30T09:21:28.090741+0000","last_clean":"2023-09-30T09:21:28.090741+0000","last_became_active":"2023-09-30T09:20:23.573953+0000","last_became_peered":"2023-09-30T09:20:23.573953+0000","last_unstale":"2023-09-30T09:21:28.090741+0000","last_undegraded":"2023-09-30T09:21:28.090741+0000","last_fullsized":"2023-09-30T09:21:28.090741+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:23:21.634624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089127+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:21:28.089127+0000","last_peered":"2023-09-30T09:21:28.089127+0000","last_clean":"2023-09-30T09:21:28.089127+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:21:28.089127+0000","last_undegraded":"2023-09-30T09:21:28.089127+0000","last_fullsized":"2023-09-30T09:21:28.089127+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081654+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:21:28.081654+0000","last_peered":"2023-09-30T09:21:28.081654+0000","last_clean":"2023-09-30T09:21:28.081654+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:21:28.081654+0000","last_undegraded":"2023-09-30T09:21:28.081654+0000","last_fullsized":"2023-09-30T09:21:28.081654+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092097+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:21:29.092097+0000","last_peered":"2023-09-30T09:21:29.092097+0000","last_clean":"2023-09-30T09:21:29.092097+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:21:29.092097+0000","last_undegraded":"2023-09-30T09:21:29.092097+0000","last_fullsized":"2023-09-30T09:21:29.092097+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104391+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:21:29.104391+0000","last_peered":"2023-09-30T09:21:29.104391+0000","last_clean":"2023-09-30T09:21:29.104391+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:21:29.104391+0000","last_undegraded":"2023-09-30T09:21:29.104391+0000","last_fullsized":"2023-09-30T09:21:29.104391+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080851+0000","last_change":"2023-09-30T09:20:23.573657+0000","last_active":"2023-09-30T09:21:28.080851+0000","last_peered":"2023-09-30T09:21:28.080851+0000","last_clean":"2023-09-30T09:21:28.080851+0000","last_became_active":"2023-09-30T09:20:23.573337+0000","last_became_peered":"2023-09-30T09:20:23.573337+0000","last_unstale":"2023-09-30T09:21:28.080851+0000","last_undegraded":"2023-09-30T09:21:28.080851+0000","last_fullsized":"2023-09-30T09:21:28.080851+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:00.644205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104648+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:21:29.104648+0000","last_peered":"2023-09-30T09:21:29.104648+0000","last_clean":"2023-09-30T09:21:29.104648+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:21:29.104648+0000","last_undegraded":"2023-09-30T09:21:29.104648+0000","last_fullsized":"2023-09-30T09:21:29.104648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.098232+0000","last_change":"2023-09-30T09:21:29.097434+0000","last_active":"2023-09-30T09:21:29.098232+0000","last_peered":"2023-09-30T09:21:29.098232+0000","last_clean":"2023-09-30T09:21:29.098232+0000","last_became_active":"2023-09-30T09:21:29.097116+0000","last_became_peered":"2023-09-30T09:21:29.097116+0000","last_unstale":"2023-09-30T09:21:29.098232+0000","last_undegraded":"2023-09-30T09:21:29.098232+0000","last_fullsized":"2023-09-30T09:21:29.098232+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:41.904331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083111+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:21:28.083111+0000","last_peered":"2023-09-30T09:21:28.083111+0000","last_clean":"2023-09-30T09:21:28.083111+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:21:28.083111+0000","last_undegraded":"2023-09-30T09:21:28.083111+0000","last_fullsized":"2023-09-30T09:21:28.083111+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094532+0000","last_change":"2023-09-30T09:21:29.094532+0000","last_active":"2023-09-30T09:21:29.094532+0000","last_peered":"2023-09-30T09:21:29.094532+0000","last_clean":"2023-09-30T09:21:29.094532+0000","last_became_active":"2023-09-30T09:21:29.093807+0000","last_became_peered":"2023-09-30T09:21:29.093807+0000","last_unstale":"2023-09-30T09:21:29.094532+0000","last_undegraded":"2023-09-30T09:21:29.094532+0000","last_fullsized":"2023-09-30T09:21:29.094532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:39:32.517645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.098125+0000","last_change":"2023-09-30T09:21:29.098125+0000","last_active":"2023-09-30T09:21:29.098125+0000","last_peered":"2023-09-30T09:21:29.098125+0000","last_clean":"2023-09-30T09:21:29.098125+0000","last_became_active":"2023-09-30T09:21:29.097837+0000","last_became_peered":"2023-09-30T09:21:29.097837+0000","last_unstale":"2023-09-30T09:21:29.098125+0000","last_undegraded":"2023-09-30T09:21:29.098125+0000","last_fullsized":"2023-09-30T09:21:29.098125+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:19:36.896416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092203+0000","last_change":"2023-09-30T09:21:29.092203+0000","last_active":"2023-09-30T09:21:29.092203+0000","last_peered":"2023-09-30T09:21:29.092203+0000","last_clean":"2023-09-30T09:21:29.092203+0000","last_became_active":"2023-09-30T09:21:29.091882+0000","last_became_peered":"2023-09-30T09:21:29.091882+0000","last_unstale":"2023-09-30T09:21:29.092203+0000","last_undegraded":"2023-09-30T09:21:29.092203+0000","last_fullsized":"2023-09-30T09:21:29.092203+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:56.866973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093480+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:21:29.093480+0000","last_peered":"2023-09-30T09:21:29.093480+0000","last_clean":"2023-09-30T09:21:29.093480+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:21:29.093480+0000","last_undegraded":"2023-09-30T09:21:29.093480+0000","last_fullsized":"2023-09-30T09:21:29.093480+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085932+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:21:28.085932+0000","last_peered":"2023-09-30T09:21:28.085932+0000","last_clean":"2023-09-30T09:21:28.085932+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:21:28.085932+0000","last_undegraded":"2023-09-30T09:21:28.085932+0000","last_fullsized":"2023-09-30T09:21:28.085932+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234129,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17072,"kb_used_data":5800,"kb_used_omap":0,"kb_used_meta":11264,"kb_avail":93728080,"statfs":{"total":95995035648,"available":95977553920,"internally_reserved":0,"allocated":5939200,"data_stored":5405566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11534336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.60199999999999998,"15min":0.62},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.61199999999999999,"15min":0.64400000000000002},"min":{"1min":0.42599999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.53200000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:21:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.72299999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.30199999999999999},"max":{"1min":0.68600000000000005,"5min":0.89700000000000002,"15min":1.5429999999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.61199999999999999,"15min":0.58999999999999997},"min":{"1min":0.35799999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.69499999999999995,"5min":0.85899999999999999,"15min":2.359},"last":0.67600000000000005}]},{"osd":4,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.48399999999999999,"15min":0.46899999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":0.68799999999999994,"5min":0.77800000000000002,"15min":0.878},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.497},"min":{"1min":0.33000000000000002,"5min":0.317,"15min":0.28799999999999998},"max":{"1min":0.73899999999999999,"5min":0.78000000000000003,"15min":1.111},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.53500000000000003,"15min":0.51100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":0.754,"5min":0.82499999999999996,"15min":0.86099999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.58399999999999996,"15min":0.55600000000000005},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.32600000000000001},"max":{"1min":0.81599999999999995,"5min":0.82299999999999995,"15min":0.89400000000000002},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59899999999999998,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34100000000000003},"max":{"1min":0.83899999999999997,"5min":0.85099999999999998,"15min":1.01},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.60799999999999998,"15min":0.55500000000000005},"min":{"1min":0.41599999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.81899999999999995,"15min":1.595},"last":0.56699999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495766,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8224,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":6528,"kb_avail":93736928,"statfs":{"total":95995035648,"available":95986614272,"internally_reserved":0,"allocated":1728512,"data_stored":1210547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6684672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.622,"15min":0.63},"min":{"1min":0.372,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.82899999999999996,"5min":2.673,"15min":2.673},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.64600000000000002,"15min":0.65300000000000002},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73699999999999999,"5min":3.7450000000000001,"15min":3.7450000000000001},"last":0.51000000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.55500000000000005}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:20:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.61099999999999999,"15min":0.58399999999999996},"min":{"1min":0.45600000000000002,"5min":0.36899999999999999,"15min":0.29199999999999998},"max":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.58699999999999997,"15min":0.57499999999999996},"min":{"1min":0.441,"5min":0.37,"15min":0.34200000000000003},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":3.6200000000000001},"last":0.46600000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.49399999999999999,"15min":0.48699999999999999},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.22700000000000001},"max":{"1min":0.69999999999999996,"5min":0.82599999999999996,"15min":0.83299999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52700000000000002,"15min":0.50800000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.80300000000000005,"5min":0.85299999999999998,"15min":0.92300000000000004},"last":0.65600000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.50900000000000001,"15min":0.48699999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.24399999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.82099999999999995},"last":0.433},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.57399999999999995,"15min":0.54700000000000004},"min":{"1min":0.39700000000000002,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.95199999999999996},"last":0.67800000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.58599999999999997,"15min":0.57399999999999995},"min":{"1min":0.39900000000000002,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":0.79100000000000004,"5min":0.85599999999999998,"15min":0.877},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.623,"15min":0.60799999999999998},"min":{"1min":0.45000000000000001,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":0.80200000000000005,"5min":0.878,"15min":1.0269999999999999},"last":0.63200000000000001}]}]},{"osd":1,"up_from":99,"seq":425201762331,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32728,"kb_used_data":5252,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93712424,"statfs":{"total":95995035648,"available":95961522176,"internally_reserved":0,"allocated":5378048,"data_stored":4821802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":28108982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.36499999999999999,"15min":0.36899999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.313,"5min":0.38800000000000001,"15min":0.40000000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.499,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.252}]},{"osd":2,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.30099999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.63800000000000001,"15min":0.64200000000000002},"min":{"1min":0.44400000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.88500000000000001,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.66900000000000004,"15min":0.67700000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82399999999999995,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.39500000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.91600000000000004,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.63,"5min":0.64400000000000002,"15min":0.64600000000000002},"min":{"1min":0.48099999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.84099999999999997,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.64300000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82999999999999996,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.67000000000000004,"15min":0.68000000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.80500000000000005,"5min":1.038,"15min":1.038},"last":0.49099999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.66300000000000003,"15min":0.67400000000000004},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.87,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.68500000000000005,"15min":0.69099999999999995},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.89600000000000002,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.45900000000000002}]}]},{"osd":0,"up_from":94,"seq":403726925866,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33328,"kb_used_data":5856,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93711824,"statfs":{"total":95995035648,"available":95960907776,"internally_reserved":0,"allocated":5996544,"data_stored":5417259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":28109053},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:21:12 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.50700000000000001,"15min":0.51800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.625,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.45700000000000002,"15min":0.46100000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.54900000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.36299999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.68000000000000005,"15min":0.72399999999999998},"min":{"1min":0.41599999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.78200000000000003,"5min":2.363,"15min":2.363},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.72399999999999998,"15min":0.76500000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.399,"5min":2.399,"15min":2.399},"last":0.505}]},{"osd":5,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.76100000000000001,"15min":0.80700000000000005},"min":{"1min":0.36599999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.3610000000000002,"5min":2.3610000000000002,"15min":2.3610000000000002},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.76000000000000001,"15min":0.81200000000000006},"min":{"1min":0.5,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84199999999999997,"5min":2.4260000000000002,"15min":2.4260000000000002},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.74399999999999999,"15min":0.79100000000000004},"min":{"1min":0.45200000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93600000000000005,"5min":2.5369999999999999,"15min":2.5369999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.629,"5min":0.78600000000000003,"15min":0.84599999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89700000000000002,"5min":2.6139999999999999,"15min":2.6139999999999999},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.71799999999999997,"15min":0.75900000000000001},"min":{"1min":0.46200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.879,"5min":2.3039999999999998,"15min":2.3039999999999998},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.77100000000000002,"15min":0.83699999999999997},"min":{"1min":0.44,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76500000000000001,"5min":2.492,"15min":2.492},"last":0.52400000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598804,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28612,"kb_used_data":1136,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93716540,"statfs":{"total":95995035648,"available":95965736960,"internally_reserved":0,"allocated":1163264,"data_stored":624404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":28109453},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.434}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.498}]},{"osd":5,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.139,"5min":1.139,"15min":1.139},"last":0.622}]},{"osd":7,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.55200000000000005}]}]},{"osd":3,"up_from":109,"seq":468151435277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28532,"kb_used_data":1120,"kb_used_omap":2,"kb_used_meta":27389,"kb_avail":93716620,"statfs":{"total":95995035648,"available":95965818880,"internally_reserved":0,"allocated":1146880,"data_stored":620489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":28046695},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33400000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.372}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40600000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48999999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54500000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019039,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12000,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":10880,"kb_avail":93733152,"statfs":{"total":95995035648,"available":95982747648,"internally_reserved":0,"allocated":1138688,"data_stored":620294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11141120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.60399999999999998,"15min":0.63100000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77200000000000002,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.60399999999999998,"15min":0.63300000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65100000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.51500000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.58999999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.65000000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59299999999999997,"15min":0.57399999999999995},"min":{"1min":0.41499999999999998,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":0.72099999999999997,"5min":0.81999999999999995,"15min":1.6839999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.61399999999999999,"15min":0.58499999999999996},"min":{"1min":0.40600000000000003,"5min":0.38200000000000001,"15min":0.379},"max":{"1min":0.79400000000000004,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.60799999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.49299999999999999,"15min":0.46300000000000002},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.27200000000000002},"max":{"1min":0.68500000000000005,"5min":0.74399999999999999,"15min":0.98699999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54100000000000004,"15min":0.49399999999999999},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":0.70299999999999996,"5min":0.80200000000000005,"15min":0.84099999999999997},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54200000000000004,"15min":0.501},"min":{"1min":0.35099999999999998,"5min":0.33100000000000002,"15min":0.30099999999999999},"max":{"1min":0.84399999999999997,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.497},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.59499999999999997,"15min":0.54100000000000004},"min":{"1min":0.47699999999999998,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.63200000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.59399999999999997,"15min":0.56399999999999995},"min":{"1min":0.38100000000000001,"5min":0.375,"15min":0.32200000000000001},"max":{"1min":0.81999999999999995,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.628,"15min":0.57899999999999996},"min":{"1min":0.51600000000000001,"5min":0.36399999999999999,"15min":0.33700000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.67900000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757403,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7384,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":6272,"kb_avail":93737768,"statfs":{"total":95995035648,"available":95987474432,"internally_reserved":0,"allocated":1130496,"data_stored":616276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6422528},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38900000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.61499999999999999,"15min":0.61299999999999999},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":0.55500000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57899999999999996,"15min":0.57399999999999995},"min":{"1min":0.39000000000000001,"5min":0.34799999999999998,"15min":0.33700000000000002},"max":{"1min":0.77100000000000002,"5min":0.83099999999999996,"15min":2.1110000000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.63800000000000001,"15min":0.60599999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.33000000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.629}]},{"osd":4,"last update":"Sat Sep 30 09:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.495,"15min":0.45700000000000002},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":0.63800000000000001,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52500000000000002,"15min":0.502},"min":{"1min":0.371,"5min":0.316,"15min":0.25900000000000001},"max":{"1min":0.66700000000000004,"5min":0.89700000000000002,"15min":0.91000000000000003},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.54900000000000004,"15min":0.501},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.28699999999999998},"max":{"1min":0.78100000000000003,"5min":0.872,"15min":0.872},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63400000000000001,"15min":0.58599999999999997},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.65400000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60699999999999998,"15min":0.58099999999999996},"min":{"1min":0.41099999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.87,"5min":0.88800000000000001,"15min":0.91400000000000003},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":0.57499999999999996},"min":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":0.83799999999999997,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.57299999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:22:28.621 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:22:28.986 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:29.451 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:29 smithi019 ceph-mon[216815]: from='client.44525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:29.452 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:29 smithi019 ceph-mon[224043]: from='client.44525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:29 smithi167 ceph-mon[192880]: from='client.44525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:30.364 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:30 smithi019 ceph-mon[216815]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:30.365 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:30 smithi019 ceph-mon[224043]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:30 smithi167 ceph-mon[192880]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:30.779 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:22:30.781 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:22:31.528 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":216,"stamp":"2023-09-30T09:22:28.956394+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790183,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6480,"num_read_kb":14131,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":167880,"kb_used_data":23068,"kb_used_omap":19,"kb_used_meta":144684,"kb_avail":749793336,"statfs":{"total":767960285184,"available":767788376064,"internally_reserved":0,"allocated":23621632,"data_stored":19336637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":148156839},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003132"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085795+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:21:28.085795+0000","last_peered":"2023-09-30T09:21:28.085795+0000","last_clean":"2023-09-30T09:21:28.085795+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:21:28.085795+0000","last_undegraded":"2023-09-30T09:21:28.085795+0000","last_fullsized":"2023-09-30T09:21:28.085795+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082738+0000","last_change":"2023-09-30T09:20:56.404258+0000","last_active":"2023-09-30T09:21:28.082738+0000","last_peered":"2023-09-30T09:21:28.082738+0000","last_clean":"2023-09-30T09:21:28.082738+0000","last_became_active":"2023-09-30T09:20:56.403848+0000","last_became_peered":"2023-09-30T09:20:56.403848+0000","last_unstale":"2023-09-30T09:21:28.082738+0000","last_undegraded":"2023-09-30T09:21:28.082738+0000","last_fullsized":"2023-09-30T09:21:28.082738+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:01.175436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093237+0000","last_change":"2023-09-30T09:21:29.093237+0000","last_active":"2023-09-30T09:21:29.093237+0000","last_peered":"2023-09-30T09:21:29.093237+0000","last_clean":"2023-09-30T09:21:29.093237+0000","last_became_active":"2023-09-30T09:21:29.092638+0000","last_became_peered":"2023-09-30T09:21:29.092638+0000","last_unstale":"2023-09-30T09:21:29.093237+0000","last_undegraded":"2023-09-30T09:21:29.093237+0000","last_fullsized":"2023-09-30T09:21:29.093237+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:11:54.139992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104648+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:21:29.104648+0000","last_peered":"2023-09-30T09:21:29.104648+0000","last_clean":"2023-09-30T09:21:29.104648+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:21:29.104648+0000","last_undegraded":"2023-09-30T09:21:29.104648+0000","last_fullsized":"2023-09-30T09:21:29.104648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.097503+0000","last_change":"2023-09-30T09:21:29.097503+0000","last_active":"2023-09-30T09:21:29.097503+0000","last_peered":"2023-09-30T09:21:29.097503+0000","last_clean":"2023-09-30T09:21:29.097503+0000","last_became_active":"2023-09-30T09:21:29.097151+0000","last_became_peered":"2023-09-30T09:21:29.097151+0000","last_unstale":"2023-09-30T09:21:29.097503+0000","last_undegraded":"2023-09-30T09:21:29.097503+0000","last_fullsized":"2023-09-30T09:21:29.097503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:55.978501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081849+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:21:28.081849+0000","last_peered":"2023-09-30T09:21:28.081849+0000","last_clean":"2023-09-30T09:21:28.081849+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:21:28.081849+0000","last_undegraded":"2023-09-30T09:21:28.081849+0000","last_fullsized":"2023-09-30T09:21:28.081849+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096090+0000","last_change":"2023-09-30T09:21:29.096090+0000","last_active":"2023-09-30T09:21:29.096090+0000","last_peered":"2023-09-30T09:21:29.096090+0000","last_clean":"2023-09-30T09:21:29.096090+0000","last_became_active":"2023-09-30T09:21:29.095664+0000","last_became_peered":"2023-09-30T09:21:29.095664+0000","last_unstale":"2023-09-30T09:21:29.096090+0000","last_undegraded":"2023-09-30T09:21:29.096090+0000","last_fullsized":"2023-09-30T09:21:29.096090+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:58.213312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082945+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:21:28.082945+0000","last_peered":"2023-09-30T09:21:28.082945+0000","last_clean":"2023-09-30T09:21:28.082945+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:21:28.082945+0000","last_undegraded":"2023-09-30T09:21:28.082945+0000","last_fullsized":"2023-09-30T09:21:28.082945+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096256+0000","last_change":"2023-09-30T09:21:29.096256+0000","last_active":"2023-09-30T09:21:29.096256+0000","last_peered":"2023-09-30T09:21:29.096256+0000","last_clean":"2023-09-30T09:21:29.096256+0000","last_became_active":"2023-09-30T09:21:29.095930+0000","last_became_peered":"2023-09-30T09:21:29.095930+0000","last_unstale":"2023-09-30T09:21:29.096256+0000","last_undegraded":"2023-09-30T09:21:29.096256+0000","last_fullsized":"2023-09-30T09:21:29.096256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:25.270756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.082734+0000","last_change":"2023-09-30T09:19:08.915504+0000","last_active":"2023-09-30T09:21:29.082734+0000","last_peered":"2023-09-30T09:21:29.082734+0000","last_clean":"2023-09-30T09:21:29.082734+0000","last_became_active":"2023-09-30T09:19:08.915150+0000","last_became_peered":"2023-09-30T09:19:08.915150+0000","last_unstale":"2023-09-30T09:21:29.082734+0000","last_undegraded":"2023-09-30T09:21:29.082734+0000","last_fullsized":"2023-09-30T09:21:29.082734+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:56:15.982749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089369+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:21:28.089369+0000","last_peered":"2023-09-30T09:21:28.089369+0000","last_clean":"2023-09-30T09:21:28.089369+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:21:28.089369+0000","last_undegraded":"2023-09-30T09:21:28.089369+0000","last_fullsized":"2023-09-30T09:21:28.089369+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082758+0000","last_change":"2023-09-30T09:20:23.568788+0000","last_active":"2023-09-30T09:21:28.082758+0000","last_peered":"2023-09-30T09:21:28.082758+0000","last_clean":"2023-09-30T09:21:28.082758+0000","last_became_active":"2023-09-30T09:20:23.568454+0000","last_became_peered":"2023-09-30T09:20:23.568454+0000","last_unstale":"2023-09-30T09:21:28.082758+0000","last_undegraded":"2023-09-30T09:21:28.082758+0000","last_fullsized":"2023-09-30T09:21:28.082758+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:47:26.911269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":363,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083198+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:21:28.083198+0000","last_peered":"2023-09-30T09:21:28.083198+0000","last_clean":"2023-09-30T09:21:28.083198+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:21:28.083198+0000","last_undegraded":"2023-09-30T09:21:28.083198+0000","last_fullsized":"2023-09-30T09:21:28.083198+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094736+0000","last_change":"2023-09-30T09:21:29.094736+0000","last_active":"2023-09-30T09:21:29.094736+0000","last_peered":"2023-09-30T09:21:29.094736+0000","last_clean":"2023-09-30T09:21:29.094736+0000","last_became_active":"2023-09-30T09:21:29.094220+0000","last_became_peered":"2023-09-30T09:21:29.094220+0000","last_unstale":"2023-09-30T09:21:29.094736+0000","last_undegraded":"2023-09-30T09:21:29.094736+0000","last_fullsized":"2023-09-30T09:21:29.094736+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:48:58.067703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1055,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.084527+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:21:28.084527+0000","last_peered":"2023-09-30T09:21:28.084527+0000","last_clean":"2023-09-30T09:21:28.084527+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:21:28.084527+0000","last_undegraded":"2023-09-30T09:21:28.084527+0000","last_fullsized":"2023-09-30T09:21:28.084527+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092954+0000","last_change":"2023-09-30T09:21:29.092954+0000","last_active":"2023-09-30T09:21:29.092954+0000","last_peered":"2023-09-30T09:21:29.092954+0000","last_clean":"2023-09-30T09:21:29.092954+0000","last_became_active":"2023-09-30T09:21:29.092563+0000","last_became_peered":"2023-09-30T09:21:29.092563+0000","last_unstale":"2023-09-30T09:21:29.092954+0000","last_undegraded":"2023-09-30T09:21:29.092954+0000","last_fullsized":"2023-09-30T09:21:29.092954+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:54.133382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082795+0000","last_change":"2023-09-30T09:19:08.911812+0000","last_active":"2023-09-30T09:21:28.082795+0000","last_peered":"2023-09-30T09:21:28.082795+0000","last_clean":"2023-09-30T09:21:28.082795+0000","last_became_active":"2023-09-30T09:19:08.911477+0000","last_became_peered":"2023-09-30T09:19:08.911477+0000","last_unstale":"2023-09-30T09:21:28.082795+0000","last_undegraded":"2023-09-30T09:21:28.082795+0000","last_fullsized":"2023-09-30T09:21:28.082795+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:04.525132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.085255+0000","last_change":"2023-09-30T09:20:56.402640+0000","last_active":"2023-09-30T09:21:29.085255+0000","last_peered":"2023-09-30T09:21:29.085255+0000","last_clean":"2023-09-30T09:21:29.085255+0000","last_became_active":"2023-09-30T09:20:56.402324+0000","last_became_peered":"2023-09-30T09:20:56.402324+0000","last_unstale":"2023-09-30T09:21:29.085255+0000","last_undegraded":"2023-09-30T09:21:29.085255+0000","last_fullsized":"2023-09-30T09:21:29.085255+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:27:18.825926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":650,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086457+0000","last_change":"2023-09-30T09:19:08.914400+0000","last_active":"2023-09-30T09:21:28.086457+0000","last_peered":"2023-09-30T09:21:28.086457+0000","last_clean":"2023-09-30T09:21:28.086457+0000","last_became_active":"2023-09-30T09:19:08.913855+0000","last_became_peered":"2023-09-30T09:19:08.913855+0000","last_unstale":"2023-09-30T09:21:28.086457+0000","last_undegraded":"2023-09-30T09:21:28.086457+0000","last_fullsized":"2023-09-30T09:21:28.086457+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:27.706105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":64,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082854+0000","last_change":"2023-09-30T09:20:56.397596+0000","last_active":"2023-09-30T09:21:28.082854+0000","last_peered":"2023-09-30T09:21:28.082854+0000","last_clean":"2023-09-30T09:21:28.082854+0000","last_became_active":"2023-09-30T09:20:56.397283+0000","last_became_peered":"2023-09-30T09:20:56.397283+0000","last_unstale":"2023-09-30T09:21:28.082854+0000","last_undegraded":"2023-09-30T09:21:28.082854+0000","last_fullsized":"2023-09-30T09:21:28.082854+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:28.695903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094162+0000","last_change":"2023-09-30T09:21:29.094162+0000","last_active":"2023-09-30T09:21:29.094162+0000","last_peered":"2023-09-30T09:21:29.094162+0000","last_clean":"2023-09-30T09:21:29.094162+0000","last_became_active":"2023-09-30T09:21:29.093839+0000","last_became_peered":"2023-09-30T09:21:29.093839+0000","last_unstale":"2023-09-30T09:21:29.094162+0000","last_undegraded":"2023-09-30T09:21:29.094162+0000","last_fullsized":"2023-09-30T09:21:29.094162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:38:47.314823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":590,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086978+0000","last_change":"2023-09-30T09:20:23.569228+0000","last_active":"2023-09-30T09:21:28.086978+0000","last_peered":"2023-09-30T09:21:28.086978+0000","last_clean":"2023-09-30T09:21:28.086978+0000","last_became_active":"2023-09-30T09:20:23.568481+0000","last_became_peered":"2023-09-30T09:20:23.568481+0000","last_unstale":"2023-09-30T09:21:28.086978+0000","last_undegraded":"2023-09-30T09:21:28.086978+0000","last_fullsized":"2023-09-30T09:21:28.086978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:31:06.051903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.082973+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:21:28.082973+0000","last_peered":"2023-09-30T09:21:28.082973+0000","last_clean":"2023-09-30T09:21:28.082973+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:21:28.082973+0000","last_undegraded":"2023-09-30T09:21:28.082973+0000","last_fullsized":"2023-09-30T09:21:28.082973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104912+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:21:29.104912+0000","last_peered":"2023-09-30T09:21:29.104912+0000","last_clean":"2023-09-30T09:21:29.104912+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:21:29.104912+0000","last_undegraded":"2023-09-30T09:21:29.104912+0000","last_fullsized":"2023-09-30T09:21:29.104912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":451,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086908+0000","last_change":"2023-09-30T09:19:08.915371+0000","last_active":"2023-09-30T09:21:28.086908+0000","last_peered":"2023-09-30T09:21:28.086908+0000","last_clean":"2023-09-30T09:21:28.086908+0000","last_became_active":"2023-09-30T09:19:08.914936+0000","last_became_peered":"2023-09-30T09:19:08.914936+0000","last_unstale":"2023-09-30T09:21:28.086908+0000","last_undegraded":"2023-09-30T09:21:28.086908+0000","last_fullsized":"2023-09-30T09:21:28.086908+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:41.291423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":517,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080969+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:21:28.080969+0000","last_peered":"2023-09-30T09:21:28.080969+0000","last_clean":"2023-09-30T09:21:28.080969+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:21:28.080969+0000","last_undegraded":"2023-09-30T09:21:28.080969+0000","last_fullsized":"2023-09-30T09:21:28.080969+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085377+0000","last_change":"2023-09-30T09:20:23.574542+0000","last_active":"2023-09-30T09:21:28.085377+0000","last_peered":"2023-09-30T09:21:28.085377+0000","last_clean":"2023-09-30T09:21:28.085377+0000","last_became_active":"2023-09-30T09:20:23.574196+0000","last_became_peered":"2023-09-30T09:20:23.574196+0000","last_unstale":"2023-09-30T09:21:28.085377+0000","last_undegraded":"2023-09-30T09:21:28.085377+0000","last_fullsized":"2023-09-30T09:21:28.085377+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:24:08.605395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.087220+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:21:29.087220+0000","last_peered":"2023-09-30T09:21:29.087220+0000","last_clean":"2023-09-30T09:21:29.087220+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:21:29.087220+0000","last_undegraded":"2023-09-30T09:21:29.087220+0000","last_fullsized":"2023-09-30T09:21:29.087220+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":413,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094131+0000","last_change":"2023-09-30T09:21:29.094131+0000","last_active":"2023-09-30T09:21:29.094131+0000","last_peered":"2023-09-30T09:21:29.094131+0000","last_clean":"2023-09-30T09:21:29.094131+0000","last_became_active":"2023-09-30T09:21:29.093405+0000","last_became_peered":"2023-09-30T09:21:29.093405+0000","last_unstale":"2023-09-30T09:21:29.094131+0000","last_undegraded":"2023-09-30T09:21:29.094131+0000","last_fullsized":"2023-09-30T09:21:29.094131+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:14.014784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092892+0000","last_change":"2023-09-30T09:21:29.092892+0000","last_active":"2023-09-30T09:21:29.092892+0000","last_peered":"2023-09-30T09:21:29.092892+0000","last_clean":"2023-09-30T09:21:29.092892+0000","last_became_active":"2023-09-30T09:21:29.092476+0000","last_became_peered":"2023-09-30T09:21:29.092476+0000","last_unstale":"2023-09-30T09:21:29.092892+0000","last_undegraded":"2023-09-30T09:21:29.092892+0000","last_fullsized":"2023-09-30T09:21:29.092892+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:38.113124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090919+0000","last_change":"2023-09-30T09:19:08.918284+0000","last_active":"2023-09-30T09:21:28.090919+0000","last_peered":"2023-09-30T09:21:28.090919+0000","last_clean":"2023-09-30T09:21:28.090919+0000","last_became_active":"2023-09-30T09:19:08.917861+0000","last_became_peered":"2023-09-30T09:19:08.917861+0000","last_unstale":"2023-09-30T09:21:28.090919+0000","last_undegraded":"2023-09-30T09:21:28.090919+0000","last_fullsized":"2023-09-30T09:21:28.090919+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:51.339869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":331,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081772+0000","last_change":"2023-09-30T09:20:23.574312+0000","last_active":"2023-09-30T09:21:28.081772+0000","last_peered":"2023-09-30T09:21:28.081772+0000","last_clean":"2023-09-30T09:21:28.081772+0000","last_became_active":"2023-09-30T09:20:23.573886+0000","last_became_peered":"2023-09-30T09:20:23.573886+0000","last_unstale":"2023-09-30T09:21:28.081772+0000","last_undegraded":"2023-09-30T09:21:28.081772+0000","last_fullsized":"2023-09-30T09:21:28.081772+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:45:27.269020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":219,"num_read_kb":2265,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.085074+0000","last_change":"2023-09-30T09:20:23.572662+0000","last_active":"2023-09-30T09:21:29.085074+0000","last_peered":"2023-09-30T09:21:29.085074+0000","last_clean":"2023-09-30T09:21:29.085074+0000","last_became_active":"2023-09-30T09:20:23.572374+0000","last_became_peered":"2023-09-30T09:20:23.572374+0000","last_unstale":"2023-09-30T09:21:29.085074+0000","last_undegraded":"2023-09-30T09:21:29.085074+0000","last_fullsized":"2023-09-30T09:21:29.085074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:11.881192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092844+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:21:29.092844+0000","last_peered":"2023-09-30T09:21:29.092844+0000","last_clean":"2023-09-30T09:21:29.092844+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:21:29.092844+0000","last_undegraded":"2023-09-30T09:21:29.092844+0000","last_fullsized":"2023-09-30T09:21:29.092844+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":396,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.085336+0000","last_change":"2023-09-30T09:20:23.576133+0000","last_active":"2023-09-30T09:21:29.085336+0000","last_peered":"2023-09-30T09:21:29.085336+0000","last_clean":"2023-09-30T09:21:29.085336+0000","last_became_active":"2023-09-30T09:20:23.575571+0000","last_became_peered":"2023-09-30T09:20:23.575571+0000","last_unstale":"2023-09-30T09:21:29.085336+0000","last_undegraded":"2023-09-30T09:21:29.085336+0000","last_fullsized":"2023-09-30T09:21:29.085336+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:14:08.450502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092042+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:21:29.092042+0000","last_peered":"2023-09-30T09:21:29.092042+0000","last_clean":"2023-09-30T09:21:29.092042+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:21:29.092042+0000","last_undegraded":"2023-09-30T09:21:29.092042+0000","last_fullsized":"2023-09-30T09:21:29.092042+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.089363+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:21:29.089363+0000","last_peered":"2023-09-30T09:21:29.089363+0000","last_clean":"2023-09-30T09:21:29.089363+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:21:29.089363+0000","last_undegraded":"2023-09-30T09:21:29.089363+0000","last_fullsized":"2023-09-30T09:21:29.089363+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086050+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:21:28.086050+0000","last_peered":"2023-09-30T09:21:28.086050+0000","last_clean":"2023-09-30T09:21:28.086050+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:21:28.086050+0000","last_undegraded":"2023-09-30T09:21:28.086050+0000","last_fullsized":"2023-09-30T09:21:28.086050+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090767+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:21:28.090767+0000","last_peered":"2023-09-30T09:21:28.090767+0000","last_clean":"2023-09-30T09:21:28.090767+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:21:28.090767+0000","last_undegraded":"2023-09-30T09:21:28.090767+0000","last_fullsized":"2023-09-30T09:21:28.090767+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089286+0000","last_change":"2023-09-30T09:20:56.406202+0000","last_active":"2023-09-30T09:21:28.089286+0000","last_peered":"2023-09-30T09:21:28.089286+0000","last_clean":"2023-09-30T09:21:28.089286+0000","last_became_active":"2023-09-30T09:20:56.405232+0000","last_became_peered":"2023-09-30T09:20:56.405232+0000","last_unstale":"2023-09-30T09:21:28.089286+0000","last_undegraded":"2023-09-30T09:21:28.089286+0000","last_fullsized":"2023-09-30T09:21:28.089286+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:40:27.385688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081936+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:21:28.081936+0000","last_peered":"2023-09-30T09:21:28.081936+0000","last_clean":"2023-09-30T09:21:28.081936+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:21:28.081936+0000","last_undegraded":"2023-09-30T09:21:28.081936+0000","last_fullsized":"2023-09-30T09:21:28.081936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104691+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:21:29.104691+0000","last_peered":"2023-09-30T09:21:29.104691+0000","last_clean":"2023-09-30T09:21:29.104691+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:21:29.104691+0000","last_undegraded":"2023-09-30T09:21:29.104691+0000","last_fullsized":"2023-09-30T09:21:29.104691+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093267+0000","last_change":"2023-09-30T09:21:29.093267+0000","last_active":"2023-09-30T09:21:29.093267+0000","last_peered":"2023-09-30T09:21:29.093267+0000","last_clean":"2023-09-30T09:21:29.093267+0000","last_became_active":"2023-09-30T09:21:29.092904+0000","last_became_peered":"2023-09-30T09:21:29.092904+0000","last_unstale":"2023-09-30T09:21:29.093267+0000","last_undegraded":"2023-09-30T09:21:29.093267+0000","last_fullsized":"2023-09-30T09:21:29.093267+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:31.930457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083274+0000","last_change":"2023-09-30T09:20:23.567941+0000","last_active":"2023-09-30T09:21:28.083274+0000","last_peered":"2023-09-30T09:21:28.083274+0000","last_clean":"2023-09-30T09:21:28.083274+0000","last_became_active":"2023-09-30T09:20:23.567652+0000","last_became_peered":"2023-09-30T09:20:23.567652+0000","last_unstale":"2023-09-30T09:21:28.083274+0000","last_undegraded":"2023-09-30T09:21:28.083274+0000","last_fullsized":"2023-09-30T09:21:28.083274+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:41:04.133820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104438+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:21:29.104438+0000","last_peered":"2023-09-30T09:21:29.104438+0000","last_clean":"2023-09-30T09:21:29.104438+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:21:29.104438+0000","last_undegraded":"2023-09-30T09:21:29.104438+0000","last_fullsized":"2023-09-30T09:21:29.104438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081779+0000","last_change":"2023-09-30T09:20:23.580109+0000","last_active":"2023-09-30T09:21:28.081779+0000","last_peered":"2023-09-30T09:21:28.081779+0000","last_clean":"2023-09-30T09:21:28.081779+0000","last_became_active":"2023-09-30T09:20:23.579747+0000","last_became_peered":"2023-09-30T09:20:23.579747+0000","last_unstale":"2023-09-30T09:21:28.081779+0000","last_undegraded":"2023-09-30T09:21:28.081779+0000","last_fullsized":"2023-09-30T09:21:28.081779+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:04.192497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.086557+0000","last_change":"2023-09-30T09:12:07.944640+0000","last_active":"2023-09-30T09:21:28.086557+0000","last_peered":"2023-09-30T09:21:28.086557+0000","last_clean":"2023-09-30T09:21:28.086557+0000","last_became_active":"2023-09-30T09:12:07.944362+0000","last_became_peered":"2023-09-30T09:12:07.944362+0000","last_unstale":"2023-09-30T09:21:28.086557+0000","last_undegraded":"2023-09-30T09:21:28.086557+0000","last_fullsized":"2023-09-30T09:21:28.086557+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:14.703475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.097658+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:21:29.097658+0000","last_peered":"2023-09-30T09:21:29.097658+0000","last_clean":"2023-09-30T09:21:29.097658+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:21:29.097658+0000","last_undegraded":"2023-09-30T09:21:29.097658+0000","last_fullsized":"2023-09-30T09:21:29.097658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087158+0000","last_change":"2023-09-30T09:20:23.568846+0000","last_active":"2023-09-30T09:21:28.087158+0000","last_peered":"2023-09-30T09:21:28.087158+0000","last_clean":"2023-09-30T09:21:28.087158+0000","last_became_active":"2023-09-30T09:20:23.567910+0000","last_became_peered":"2023-09-30T09:20:23.567910+0000","last_unstale":"2023-09-30T09:21:28.087158+0000","last_undegraded":"2023-09-30T09:21:28.087158+0000","last_fullsized":"2023-09-30T09:21:28.087158+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:48:49.264877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080900+0000","last_change":"2023-09-30T09:12:09.760712+0000","last_active":"2023-09-30T09:21:28.080900+0000","last_peered":"2023-09-30T09:21:28.080900+0000","last_clean":"2023-09-30T09:21:28.080900+0000","last_became_active":"2023-09-30T09:12:09.760435+0000","last_became_peered":"2023-09-30T09:12:09.760435+0000","last_unstale":"2023-09-30T09:21:28.080900+0000","last_undegraded":"2023-09-30T09:21:28.080900+0000","last_fullsized":"2023-09-30T09:21:28.080900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:03.896756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087175+0000","last_change":"2023-09-30T09:19:08.916779+0000","last_active":"2023-09-30T09:21:28.087175+0000","last_peered":"2023-09-30T09:21:28.087175+0000","last_clean":"2023-09-30T09:21:28.087175+0000","last_became_active":"2023-09-30T09:19:08.916333+0000","last_became_peered":"2023-09-30T09:19:08.916333+0000","last_unstale":"2023-09-30T09:21:28.087175+0000","last_undegraded":"2023-09-30T09:21:28.087175+0000","last_fullsized":"2023-09-30T09:21:28.087175+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:25.205240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092140+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:21:29.092140+0000","last_peered":"2023-09-30T09:21:29.092140+0000","last_clean":"2023-09-30T09:21:29.092140+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:21:29.092140+0000","last_undegraded":"2023-09-30T09:21:29.092140+0000","last_fullsized":"2023-09-30T09:21:29.092140+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.084198+0000","last_change":"2023-09-30T09:19:08.913366+0000","last_active":"2023-09-30T09:21:28.084198+0000","last_peered":"2023-09-30T09:21:28.084198+0000","last_clean":"2023-09-30T09:21:28.084198+0000","last_became_active":"2023-09-30T09:19:08.912978+0000","last_became_peered":"2023-09-30T09:19:08.912978+0000","last_unstale":"2023-09-30T09:21:28.084198+0000","last_undegraded":"2023-09-30T09:21:28.084198+0000","last_fullsized":"2023-09-30T09:21:28.084198+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:47:20.546906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.096658+0000","last_change":"2023-09-30T09:21:29.094783+0000","last_active":"2023-09-30T09:21:29.096658+0000","last_peered":"2023-09-30T09:21:29.096658+0000","last_clean":"2023-09-30T09:21:29.096658+0000","last_became_active":"2023-09-30T09:21:29.094482+0000","last_became_peered":"2023-09-30T09:21:29.094482+0000","last_unstale":"2023-09-30T09:21:29.096658+0000","last_undegraded":"2023-09-30T09:21:29.096658+0000","last_fullsized":"2023-09-30T09:21:29.096658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:59:14.875660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.087317+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:21:28.087317+0000","last_peered":"2023-09-30T09:21:28.087317+0000","last_clean":"2023-09-30T09:21:28.087317+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:21:28.087317+0000","last_undegraded":"2023-09-30T09:21:28.087317+0000","last_fullsized":"2023-09-30T09:21:28.087317+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092932+0000","last_change":"2023-09-30T09:21:29.092932+0000","last_active":"2023-09-30T09:21:29.092932+0000","last_peered":"2023-09-30T09:21:29.092932+0000","last_clean":"2023-09-30T09:21:29.092932+0000","last_became_active":"2023-09-30T09:21:29.092450+0000","last_became_peered":"2023-09-30T09:21:29.092450+0000","last_unstale":"2023-09-30T09:21:29.092932+0000","last_undegraded":"2023-09-30T09:21:29.092932+0000","last_fullsized":"2023-09-30T09:21:29.092932+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:39:06.199633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093775+0000","last_change":"2023-09-30T09:21:29.093775+0000","last_active":"2023-09-30T09:21:29.093775+0000","last_peered":"2023-09-30T09:21:29.093775+0000","last_clean":"2023-09-30T09:21:29.093775+0000","last_became_active":"2023-09-30T09:21:29.093399+0000","last_became_peered":"2023-09-30T09:21:29.093399+0000","last_unstale":"2023-09-30T09:21:29.093775+0000","last_undegraded":"2023-09-30T09:21:29.093775+0000","last_fullsized":"2023-09-30T09:21:29.093775+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:50.192497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092287+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:21:29.092287+0000","last_peered":"2023-09-30T09:21:29.092287+0000","last_clean":"2023-09-30T09:21:29.092287+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:21:29.092287+0000","last_undegraded":"2023-09-30T09:21:29.092287+0000","last_fullsized":"2023-09-30T09:21:29.092287+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085471+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:21:28.085471+0000","last_peered":"2023-09-30T09:21:28.085471+0000","last_clean":"2023-09-30T09:21:28.085471+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:21:28.085471+0000","last_undegraded":"2023-09-30T09:21:28.085471+0000","last_fullsized":"2023-09-30T09:21:28.085471+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.090741+0000","last_change":"2023-09-30T09:20:23.574453+0000","last_active":"2023-09-30T09:21:28.090741+0000","last_peered":"2023-09-30T09:21:28.090741+0000","last_clean":"2023-09-30T09:21:28.090741+0000","last_became_active":"2023-09-30T09:20:23.573953+0000","last_became_peered":"2023-09-30T09:20:23.573953+0000","last_unstale":"2023-09-30T09:21:28.090741+0000","last_undegraded":"2023-09-30T09:21:28.090741+0000","last_fullsized":"2023-09-30T09:21:28.090741+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:23:21.634624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.089127+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:21:28.089127+0000","last_peered":"2023-09-30T09:21:28.089127+0000","last_clean":"2023-09-30T09:21:28.089127+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:21:28.089127+0000","last_undegraded":"2023-09-30T09:21:28.089127+0000","last_fullsized":"2023-09-30T09:21:28.089127+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.081654+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:21:28.081654+0000","last_peered":"2023-09-30T09:21:28.081654+0000","last_clean":"2023-09-30T09:21:28.081654+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:21:28.081654+0000","last_undegraded":"2023-09-30T09:21:28.081654+0000","last_fullsized":"2023-09-30T09:21:28.081654+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092097+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:21:29.092097+0000","last_peered":"2023-09-30T09:21:29.092097+0000","last_clean":"2023-09-30T09:21:29.092097+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:21:29.092097+0000","last_undegraded":"2023-09-30T09:21:29.092097+0000","last_fullsized":"2023-09-30T09:21:29.092097+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104391+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:21:29.104391+0000","last_peered":"2023-09-30T09:21:29.104391+0000","last_clean":"2023-09-30T09:21:29.104391+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:21:29.104391+0000","last_undegraded":"2023-09-30T09:21:29.104391+0000","last_fullsized":"2023-09-30T09:21:29.104391+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.080851+0000","last_change":"2023-09-30T09:20:23.573657+0000","last_active":"2023-09-30T09:21:28.080851+0000","last_peered":"2023-09-30T09:21:28.080851+0000","last_clean":"2023-09-30T09:21:28.080851+0000","last_became_active":"2023-09-30T09:20:23.573337+0000","last_became_peered":"2023-09-30T09:20:23.573337+0000","last_unstale":"2023-09-30T09:21:28.080851+0000","last_undegraded":"2023-09-30T09:21:28.080851+0000","last_fullsized":"2023-09-30T09:21:28.080851+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:00.644205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.104648+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:21:29.104648+0000","last_peered":"2023-09-30T09:21:29.104648+0000","last_clean":"2023-09-30T09:21:29.104648+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:21:29.104648+0000","last_undegraded":"2023-09-30T09:21:29.104648+0000","last_fullsized":"2023-09-30T09:21:29.104648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.098232+0000","last_change":"2023-09-30T09:21:29.097434+0000","last_active":"2023-09-30T09:21:29.098232+0000","last_peered":"2023-09-30T09:21:29.098232+0000","last_clean":"2023-09-30T09:21:29.098232+0000","last_became_active":"2023-09-30T09:21:29.097116+0000","last_became_peered":"2023-09-30T09:21:29.097116+0000","last_unstale":"2023-09-30T09:21:29.098232+0000","last_undegraded":"2023-09-30T09:21:29.098232+0000","last_fullsized":"2023-09-30T09:21:29.098232+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:41.904331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.083111+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:21:28.083111+0000","last_peered":"2023-09-30T09:21:28.083111+0000","last_clean":"2023-09-30T09:21:28.083111+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:21:28.083111+0000","last_undegraded":"2023-09-30T09:21:28.083111+0000","last_fullsized":"2023-09-30T09:21:28.083111+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.094532+0000","last_change":"2023-09-30T09:21:29.094532+0000","last_active":"2023-09-30T09:21:29.094532+0000","last_peered":"2023-09-30T09:21:29.094532+0000","last_clean":"2023-09-30T09:21:29.094532+0000","last_became_active":"2023-09-30T09:21:29.093807+0000","last_became_peered":"2023-09-30T09:21:29.093807+0000","last_unstale":"2023-09-30T09:21:29.094532+0000","last_undegraded":"2023-09-30T09:21:29.094532+0000","last_fullsized":"2023-09-30T09:21:29.094532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:39:32.517645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.098125+0000","last_change":"2023-09-30T09:21:29.098125+0000","last_active":"2023-09-30T09:21:29.098125+0000","last_peered":"2023-09-30T09:21:29.098125+0000","last_clean":"2023-09-30T09:21:29.098125+0000","last_became_active":"2023-09-30T09:21:29.097837+0000","last_became_peered":"2023-09-30T09:21:29.097837+0000","last_unstale":"2023-09-30T09:21:29.098125+0000","last_undegraded":"2023-09-30T09:21:29.098125+0000","last_fullsized":"2023-09-30T09:21:29.098125+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:19:36.896416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.092203+0000","last_change":"2023-09-30T09:21:29.092203+0000","last_active":"2023-09-30T09:21:29.092203+0000","last_peered":"2023-09-30T09:21:29.092203+0000","last_clean":"2023-09-30T09:21:29.092203+0000","last_became_active":"2023-09-30T09:21:29.091882+0000","last_became_peered":"2023-09-30T09:21:29.091882+0000","last_unstale":"2023-09-30T09:21:29.092203+0000","last_undegraded":"2023-09-30T09:21:29.092203+0000","last_fullsized":"2023-09-30T09:21:29.092203+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:56.866973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-30T09:21:29.093480+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:21:29.093480+0000","last_peered":"2023-09-30T09:21:29.093480+0000","last_clean":"2023-09-30T09:21:29.093480+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:21:29.093480+0000","last_undegraded":"2023-09-30T09:21:29.093480+0000","last_fullsized":"2023-09-30T09:21:29.093480+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-30T09:21:28.085932+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:21:28.085932+0000","last_peered":"2023-09-30T09:21:28.085932+0000","last_clean":"2023-09-30T09:21:28.085932+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:21:28.085932+0000","last_undegraded":"2023-09-30T09:21:28.085932+0000","last_fullsized":"2023-09-30T09:21:28.085932+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234130,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17072,"kb_used_data":5800,"kb_used_omap":0,"kb_used_meta":11264,"kb_avail":93728080,"statfs":{"total":95995035648,"available":95977553920,"internally_reserved":0,"allocated":5939200,"data_stored":5405566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11534336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.60199999999999998,"15min":0.62},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.61199999999999999,"15min":0.64400000000000002},"min":{"1min":0.42599999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.47799999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.996,"5min":0.996,"15min":0.996},"last":0.504},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.622}]},{"osd":2,"last update":"Sat Sep 30 09:21:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.59099999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.30199999999999999},"max":{"1min":0.68600000000000005,"5min":0.89700000000000002,"15min":1.5429999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.61199999999999999,"15min":0.58999999999999997},"min":{"1min":0.35799999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.69499999999999995,"5min":0.85899999999999999,"15min":2.359},"last":0.73099999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.48399999999999999,"15min":0.46899999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":0.68799999999999994,"5min":0.77800000000000002,"15min":0.878},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.497},"min":{"1min":0.33000000000000002,"5min":0.317,"15min":0.28799999999999998},"max":{"1min":0.73899999999999999,"5min":0.78000000000000003,"15min":1.111},"last":0.82699999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.53500000000000003,"15min":0.51100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":0.754,"5min":0.82499999999999996,"15min":0.86099999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.58399999999999996,"15min":0.55600000000000005},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.32600000000000001},"max":{"1min":0.81599999999999995,"5min":0.82299999999999995,"15min":0.89400000000000002},"last":0.81000000000000005}]},{"osd":6,"last update":"Sat Sep 30 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59899999999999998,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34100000000000003},"max":{"1min":0.83899999999999997,"5min":0.85099999999999998,"15min":1.01},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.60799999999999998,"15min":0.55500000000000005},"min":{"1min":0.41599999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.81899999999999995,"15min":1.595},"last":0.79300000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495767,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8224,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":6528,"kb_avail":93736928,"statfs":{"total":95995035648,"available":95986614272,"internally_reserved":0,"allocated":1728512,"data_stored":1210547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6684672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.622,"15min":0.63},"min":{"1min":0.372,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.82899999999999996,"5min":2.673,"15min":2.673},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.64600000000000002,"15min":0.65300000000000002},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73699999999999999,"5min":3.7450000000000001,"15min":3.7450000000000001},"last":0.82099999999999995}]},{"osd":1,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.622,"15min":0.629},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.623,"15min":0.63100000000000001},"min":{"1min":0.40000000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77500000000000002,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.77600000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.67000000000000004}]},{"osd":3,"last update":"Sat Sep 30 09:20:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.61099999999999999,"15min":0.58399999999999996},"min":{"1min":0.45600000000000002,"5min":0.36899999999999999,"15min":0.29199999999999998},"max":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.58699999999999997,"15min":0.57499999999999996},"min":{"1min":0.441,"5min":0.37,"15min":0.34200000000000003},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":3.6200000000000001},"last":0.72199999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.48899999999999999},"min":{"1min":0.32700000000000001,"5min":0.28399999999999997,"15min":0.22700000000000001},"max":{"1min":0.76200000000000001,"5min":0.77500000000000002,"15min":0.83299999999999996},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.54400000000000004,"15min":0.51500000000000001},"min":{"1min":0.34899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.749,"5min":0.85299999999999998,"15min":0.92300000000000004},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51000000000000001,"15min":0.49099999999999999},"min":{"1min":0.37,"5min":0.308,"15min":0.24399999999999999},"max":{"1min":0.68500000000000005,"5min":0.77500000000000002,"15min":0.82099999999999995},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58399999999999996,"15min":0.55400000000000005},"min":{"1min":0.39400000000000002,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":0.78000000000000003,"5min":0.88400000000000001,"15min":0.95199999999999996},"last":0.52600000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.58599999999999997,"15min":0.57399999999999995},"min":{"1min":0.39900000000000002,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":0.79100000000000004,"5min":0.85599999999999998,"15min":0.877},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.623,"15min":0.60799999999999998},"min":{"1min":0.45000000000000001,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":0.80200000000000005,"5min":0.878,"15min":1.0269999999999999},"last":0.58099999999999996}]}]},{"osd":1,"up_from":99,"seq":425201762331,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32728,"kb_used_data":5252,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93712424,"statfs":{"total":95995035648,"available":95961522176,"internally_reserved":0,"allocated":5378048,"data_stored":4821802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":28108982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.36499999999999999,"15min":0.36899999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.313,"5min":0.38800000000000001,"15min":0.40000000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.499,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.252}]},{"osd":2,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.30099999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.63800000000000001,"15min":0.64200000000000002},"min":{"1min":0.44400000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.88500000000000001,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.66900000000000004,"15min":0.67700000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82399999999999995,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.39500000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.91600000000000004,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.63,"5min":0.64400000000000002,"15min":0.64600000000000002},"min":{"1min":0.48099999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.84099999999999997,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.64300000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82999999999999996,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.67000000000000004,"15min":0.68000000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.80500000000000005,"5min":1.038,"15min":1.038},"last":0.49099999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.66300000000000003,"15min":0.67400000000000004},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.87,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.68500000000000005,"15min":0.69099999999999995},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.89600000000000002,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.45900000000000002}]}]},{"osd":0,"up_from":94,"seq":403726925866,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33328,"kb_used_data":5856,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93711824,"statfs":{"total":95995035648,"available":95960907776,"internally_reserved":0,"allocated":5996544,"data_stored":5417259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":28109053},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:21:12 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.50700000000000001,"15min":0.51800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.625,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.45700000000000002,"15min":0.46100000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.54900000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.36299999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.68000000000000005,"15min":0.72399999999999998},"min":{"1min":0.41599999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.78200000000000003,"5min":2.363,"15min":2.363},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.72399999999999998,"15min":0.76500000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.399,"5min":2.399,"15min":2.399},"last":0.505}]},{"osd":5,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.76100000000000001,"15min":0.80700000000000005},"min":{"1min":0.36599999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.3610000000000002,"5min":2.3610000000000002,"15min":2.3610000000000002},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.76000000000000001,"15min":0.81200000000000006},"min":{"1min":0.5,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84199999999999997,"5min":2.4260000000000002,"15min":2.4260000000000002},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.74399999999999999,"15min":0.79100000000000004},"min":{"1min":0.45200000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93600000000000005,"5min":2.5369999999999999,"15min":2.5369999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.629,"5min":0.78600000000000003,"15min":0.84599999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89700000000000002,"5min":2.6139999999999999,"15min":2.6139999999999999},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.71799999999999997,"15min":0.75900000000000001},"min":{"1min":0.46200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.879,"5min":2.3039999999999998,"15min":2.3039999999999998},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.77100000000000002,"15min":0.83699999999999997},"min":{"1min":0.44,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76500000000000001,"5min":2.492,"15min":2.492},"last":0.52400000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598804,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28612,"kb_used_data":1136,"kb_used_omap":5,"kb_used_meta":27450,"kb_avail":93716540,"statfs":{"total":95995035648,"available":95965736960,"internally_reserved":0,"allocated":1163264,"data_stored":624404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":28109453},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.434}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.498}]},{"osd":5,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.139,"5min":1.139,"15min":1.139},"last":0.622}]},{"osd":7,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.55200000000000005}]}]},{"osd":3,"up_from":109,"seq":468151435278,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28532,"kb_used_data":1120,"kb_used_omap":2,"kb_used_meta":27389,"kb_avail":93716620,"statfs":{"total":95995035648,"available":95965818880,"internally_reserved":0,"allocated":1146880,"data_stored":620489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":28046695},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22700000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32200000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53900000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019040,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12000,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":10880,"kb_avail":93733152,"statfs":{"total":95995035648,"available":95982747648,"internally_reserved":0,"allocated":1138688,"data_stored":620294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11141120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.60399999999999998,"15min":0.63100000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77200000000000002,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.60399999999999998,"15min":0.63300000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65100000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.65500000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.70599999999999996,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.63400000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.443}]},{"osd":2,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.504}]},{"osd":3,"last update":"Sat Sep 30 09:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59299999999999997,"15min":0.57399999999999995},"min":{"1min":0.41499999999999998,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":0.72099999999999997,"5min":0.81999999999999995,"15min":1.6839999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.61399999999999999,"15min":0.58499999999999996},"min":{"1min":0.40600000000000003,"5min":0.38200000000000001,"15min":0.379},"max":{"1min":0.79400000000000004,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.49299999999999999,"15min":0.46300000000000002},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.27200000000000002},"max":{"1min":0.68500000000000005,"5min":0.74399999999999999,"15min":0.98699999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54100000000000004,"15min":0.49399999999999999},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":0.70299999999999996,"5min":0.80200000000000005,"15min":0.84099999999999997},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54200000000000004,"15min":0.501},"min":{"1min":0.35099999999999998,"5min":0.33100000000000002,"15min":0.30099999999999999},"max":{"1min":0.84399999999999997,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.59499999999999997,"15min":0.54100000000000004},"min":{"1min":0.47699999999999998,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.68000000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.59399999999999997,"15min":0.56399999999999995},"min":{"1min":0.38100000000000001,"5min":0.375,"15min":0.32200000000000001},"max":{"1min":0.81999999999999995,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.628,"15min":0.57899999999999996},"min":{"1min":0.51600000000000001,"5min":0.36399999999999999,"15min":0.33700000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.625}]}]},{"osd":5,"up_from":38,"seq":163208757404,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7384,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":6272,"kb_avail":93737768,"statfs":{"total":95995035648,"available":95987474432,"internally_reserved":0,"allocated":1130496,"data_stored":616276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6422528},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:22:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38900000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.498},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.61499999999999999,"15min":0.61299999999999999},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":0.69099999999999995}]},{"osd":1,"last update":"Sat Sep 30 09:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56899999999999995,"15min":0.57399999999999995},"min":{"1min":0.36199999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.63500000000000001,"15min":0.64300000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80100000000000005,"5min":1.502,"15min":1.502},"last":0.80100000000000005}]},{"osd":2,"last update":"Sat Sep 30 09:21:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":0.55800000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57899999999999996,"15min":0.57399999999999995},"min":{"1min":0.39000000000000001,"5min":0.34799999999999998,"15min":0.33700000000000002},"max":{"1min":0.77100000000000002,"5min":0.83099999999999996,"15min":2.1110000000000002},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.63800000000000001,"15min":0.60599999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.33000000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.747}]},{"osd":4,"last update":"Sat Sep 30 09:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.495,"15min":0.45700000000000002},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":0.63800000000000001,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52500000000000002,"15min":0.502},"min":{"1min":0.371,"5min":0.316,"15min":0.25900000000000001},"max":{"1min":0.66700000000000004,"5min":0.89700000000000002,"15min":0.91000000000000003},"last":0.63500000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.54900000000000004,"15min":0.501},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.28699999999999998},"max":{"1min":0.78100000000000003,"5min":0.872,"15min":0.872},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63400000000000001,"15min":0.58599999999999997},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.76900000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60699999999999998,"15min":0.58099999999999996},"min":{"1min":0.41099999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.87,"5min":0.88800000000000001,"15min":0.91400000000000003},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":0.57499999999999996},"min":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":0.83799999999999997,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.84999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:22:31.533 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-30T09:22:31.534 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-30T09:22:31.897 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:32.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:32 smithi019 ceph-mon[224043]: from='client.44531 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:32.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:32 smithi019 ceph-mon[224043]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:32 smithi019 ceph-mon[216815]: from='client.44531 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:32 smithi019 ceph-mon[216815]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:32 smithi167 ceph-mon[192880]: from='client.44531 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:22:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:32 smithi167 ceph-mon[192880]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:33.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:33.603 INFO:teuthology.orchestra.run.smithi019.stdout:epoch 110 2023-09-30T09:22:33.603 INFO:teuthology.orchestra.run.smithi019.stdout:fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:created 2023-09-30T09:05:29.547224+0000 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:modified 2023-09-30T09:21:29.074381+0000 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:crush_version 22 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:full_ratio 0.95 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:backfillfull_ratio 0.9 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:nearfull_ratio 0.85 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:require_min_compat_client luminous 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:min_compat_client luminous 2023-09-30T09:22:33.604 INFO:teuthology.orchestra.run.smithi019.stdout:require_osd_release quincy 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:stretch_mode_enabled false 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 85 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:max_osd 8 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591] [v2:172.21.15.19:6804/2904823591,v1:172.21.15.19:6805/2904823591] exists,up 91b26362-fc1a-4b53-a717-f8d48720a017 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404] [v2:172.21.15.19:6812/1402281404,v1:172.21.15.19:6813/1402281404] exists,up 64e9dc26-b495-4d5f-83f7-a38a9466d999 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465] [v2:172.21.15.19:6820/2623064465,v1:172.21.15.19:6821/2623064465] exists,up c5256ec9-304c-4028-8256-970c6bd03bb2 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409] [v2:172.21.15.19:6828/3712654409,v1:172.21.15.19:6829/3712654409] exists,up 1f9fc59d-faab-4c30-938c-f244c7f972d9 2023-09-30T09:22:33.605 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573] [v2:172.21.15.167:6802/3844930573,v1:172.21.15.167:6803/3844930573] exists,up cb4cb89f-3933-48cd-a588-95a2f2bf8d13 2023-09-30T09:22:33.606 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 up in weight 1 up_from 38 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136] [v2:172.21.15.167:6810/2834758136,v1:172.21.15.167:6811/2834758136] exists,up 1bc3ed47-225a-429a-a825-36b3357cf271 2023-09-30T09:22:33.606 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 up in weight 1 up_from 46 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455] [v2:172.21.15.167:6818/1453229455,v1:172.21.15.167:6819/1453229455] exists,up 3a4720ae-d64a-4bda-b7d3-478ec2742819 2023-09-30T09:22:33.606 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 up in weight 1 up_from 54 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506] [v2:172.21.15.167:6826/1838064506,v1:172.21.15.167:6827/1838064506] exists,up 88b4cb7b-99e4-46e9-a9aa-24f886b0174b 2023-09-30T09:22:33.606 INFO:teuthology.orchestra.run.smithi019.stdout:pg_upmap_items 4.4 [1,5] 2023-09-30T09:22:33.606 INFO:teuthology.orchestra.run.smithi019.stdout:pg_upmap_items 4.14 [1,2] 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:pg_upmap_items 4.17 [1,2] 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/4266586334 expires 2023-10-01T09:16:02.836873+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/1275272106 expires 2023-10-01T09:16:02.836873+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6800/4099317090 expires 2023-10-01T09:05:58.980233+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.167:0/625642582 expires 2023-10-01T09:15:21.453516+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/956155 expires 2023-10-01T09:14:41.945226+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6800/1837817935 expires 2023-10-01T09:06:26.045265+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6801/1837817935 expires 2023-10-01T09:06:26.045265+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.167:6835/1123065418 expires 2023-10-01T09:15:21.453516+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/3001919590 expires 2023-10-01T09:06:26.045265+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/861352263 expires 2023-10-01T09:16:02.836873+0000 2023-09-30T09:22:33.607 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6800/605197989 expires 2023-10-01T09:16:02.836873+0000 2023-09-30T09:22:33.608 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.167:6834/1123065418 expires 2023-10-01T09:15:21.453516+0000 2023-09-30T09:22:33.608 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/3324367156 expires 2023-10-01T09:11:06.498249+0000 2023-09-30T09:22:33.608 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.167:0/1856357891 expires 2023-10-01T09:15:21.453516+0000 2023-09-30T09:22:33.608 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/2559536428 expires 2023-10-01T09:06:26.045265+0000 2023-09-30T09:22:33.608 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/3152350528 expires 2023-10-01T09:11:06.498249+0000 2023-09-30T09:22:33.608 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/1638946842 expires 2023-10-01T09:14:41.945226+0000 2023-09-30T09:22:33.608 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.167:0/798457358 expires 2023-10-01T09:15:21.453516+0000 2023-09-30T09:22:33.608 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.167:0/320118286 expires 2023-10-01T09:15:21.453516+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6800/2650629233 expires 2023-10-01T09:14:41.945226+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/238111816 expires 2023-10-01T09:14:41.945226+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/3793098865 expires 2023-10-01T09:05:58.980233+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6800/1814390438 expires 2023-10-01T09:11:06.498249+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/3931489393 expires 2023-10-01T09:14:41.945226+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6801/2650629233 expires 2023-10-01T09:14:41.945226+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/936416645 expires 2023-10-01T09:11:06.498249+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.167:0/2022890892 expires 2023-10-01T09:15:21.453516+0000 2023-09-30T09:22:33.609 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/1679148187 expires 2023-10-01T09:11:06.498249+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/3546350991 expires 2023-10-01T09:05:58.980233+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/309324931 expires 2023-10-01T09:16:02.836873+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/572446726 expires 2023-10-01T09:05:58.980233+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6801/1814390438 expires 2023-10-01T09:11:06.498249+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.167:0/2218856933 expires 2023-10-01T09:15:21.453516+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/2702007278 expires 2023-10-01T09:06:26.045265+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/3289282363 expires 2023-10-01T09:16:02.836873+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6801/4099317090 expires 2023-10-01T09:05:58.980233+0000 2023-09-30T09:22:33.610 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:0/554243229 expires 2023-10-01T09:16:02.836873+0000 2023-09-30T09:22:33.611 INFO:teuthology.orchestra.run.smithi019.stdout:blocklist 172.21.15.19:6801/605197989 expires 2023-10-01T09:16:02.836873+0000 2023-09-30T09:22:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:22:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:22:33] "GET /metrics HTTP/1.1" 200 33858 "" "Prometheus/2.43.0" 2023-09-30T09:22:34.312 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.19:6802/2904823591,v1:172.21.15.19:6803/2904823591] [v2:172.21.15.19:6804/2904823591,v1:172.21.15.19:6805/2904823591] exists,up 91b26362-fc1a-4b53-a717-f8d48720a017', 'osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.19:6810/1402281404,v1:172.21.15.19:6811/1402281404] [v2:172.21.15.19:6812/1402281404,v1:172.21.15.19:6813/1402281404] exists,up 64e9dc26-b495-4d5f-83f7-a38a9466d999', 'osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.19:6818/2623064465,v1:172.21.15.19:6819/2623064465] [v2:172.21.15.19:6820/2623064465,v1:172.21.15.19:6821/2623064465] exists,up c5256ec9-304c-4028-8256-970c6bd03bb2', 'osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.19:6826/3712654409,v1:172.21.15.19:6827/3712654409] [v2:172.21.15.19:6828/3712654409,v1:172.21.15.19:6829/3712654409] exists,up 1f9fc59d-faab-4c30-938c-f244c7f972d9', 'osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.167:6800/3844930573,v1:172.21.15.167:6801/3844930573] [v2:172.21.15.167:6802/3844930573,v1:172.21.15.167:6803/3844930573] exists,up cb4cb89f-3933-48cd-a588-95a2f2bf8d13', 'osd.5 up in weight 1 up_from 38 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.167:6808/2834758136,v1:172.21.15.167:6809/2834758136] [v2:172.21.15.167:6810/2834758136,v1:172.21.15.167:6811/2834758136] exists,up 1bc3ed47-225a-429a-a825-36b3357cf271', 'osd.6 up in weight 1 up_from 46 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.167:6816/1453229455,v1:172.21.15.167:6817/1453229455] [v2:172.21.15.167:6818/1453229455,v1:172.21.15.167:6819/1453229455] exists,up 3a4720ae-d64a-4bda-b7d3-478ec2742819', 'osd.7 up in weight 1 up_from 54 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.167:6824/1838064506,v1:172.21.15.167:6825/1838064506] [v2:172.21.15.167:6826/1838064506,v1:172.21.15.167:6827/1838064506] exists,up 88b4cb7b-99e4-46e9-a9aa-24f886b0174b'] 2023-09-30T09:22:34.313 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-30T09:22:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:34 smithi167 ceph-mon[192880]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/38518212' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:22:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:34 smithi019 ceph-mon[216815]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/38518212' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:22:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:34 smithi019 ceph-mon[224043]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/38518212' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:22:34.688 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:36.269 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:22:36.269 INFO:teuthology.orchestra.run.smithi019.stdout: "name": "mon.a", 2023-09-30T09:22:36.269 INFO:teuthology.orchestra.run.smithi019.stdout: "cluster": "ceph", 2023-09-30T09:22:36.269 INFO:teuthology.orchestra.run.smithi019.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-30T09:22:36.269 INFO:teuthology.orchestra.run.smithi019.stdout: "admin_socket_mode": "", 2023-09-30T09:22:36.269 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-30T09:22:36.269 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_client_required": "cephx, none", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_cluster_required": "cephx", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_debug": "true", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_service_required": "cephx", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "auth_supported": "", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_aio": "true", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_aio_poll_ms": "250", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_aio_reap_max": "16", 2023-09-30T09:22:36.270 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_async_discard": "false", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_block_size": "4096", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_debug_aio": "false", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_debug_inflight_ios": "false", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_enable_discard": "false", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_flock_retry": "3", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_inject_crash": "0", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_ioring": "false", 2023-09-30T09:22:36.271 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_ioring_hipri": "false", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bdev_type": "", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_alloc_size": "1048576", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_allocator": "hybrid", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_buffered_io": "true", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_check_for_zeros": "false", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-30T09:22:36.272 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_compact_log_sync": "false", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_max_log_runway": "4194304", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_max_prefetch": "1048576", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_min_flush_size": "524288", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_min_log_runway": "1048576", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_replay_recovery": "false", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluefs_sync_write": "false", 2023-09-30T09:22:36.273 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_allocation_from_file": "true", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_allocator": "hybrid", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-30T09:22:36.274 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_create": "true", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_db_create": "false", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_db_path": "", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_db_size": "0", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_path": "", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_preallocate_file": "false", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_size": "107374182400", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_wal_create": "false", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_wal_path": "", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_block_wal_size": "100663296", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_bluefs": "true", 2023-09-30T09:22:36.275 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_autotune": "true", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-30T09:22:36.276 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_size": "0", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cache_type": "2q", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_clone_cow": "true", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-30T09:22:36.277 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_mode": "none", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_csum_type": "crc32c", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_freelist": "false", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-30T09:22:36.278 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_small_allocations": "0", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-30T09:22:36.279 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_default_buffered_read": "true", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_default_buffered_write": "false", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_elastic_shared_blobs": "true", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fail_eio": "false", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-30T09:22:36.280 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_on_mount": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_on_umount": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-30T09:22:36.281 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_ignore_data_csum": "false", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_log_op_age": "5.000000", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_max_alloc_size": "0", 2023-09-30T09:22:36.282 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_max_blob_size": "0", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_max_deferred_txc": "32", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_min_alloc_size": "0", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_nid_prealloc": "1024", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-30T09:22:36.283 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_retry_disk_reads": "3", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_rocksdb_cf": "true", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_spdk_mem": "512", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-30T09:22:36.284 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_tracing": "false", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-30T09:22:36.285 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "bluestore_zero_block_detection": "false", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-30T09:22:36.286 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-30T09:22:36.287 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-30T09:22:36.287 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-30T09:22:36.289 INFO:teuthology.orchestra.run.smithi019.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-30T09:22:36.289 INFO:teuthology.orchestra.run.smithi019.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-30T09:22:36.289 INFO:teuthology.orchestra.run.smithi019.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-30T09:22:36.289 INFO:teuthology.orchestra.run.smithi019.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-30T09:22:36.289 INFO:teuthology.orchestra.run.smithi019.stdout: "cephx_cluster_require_signatures": "false", 2023-09-30T09:22:36.289 INFO:teuthology.orchestra.run.smithi019.stdout: "cephx_cluster_require_version": "2", 2023-09-30T09:22:36.289 INFO:teuthology.orchestra.run.smithi019.stdout: "cephx_require_signatures": "false", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "cephx_require_version": "2", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "cephx_service_require_signatures": "false", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "cephx_service_require_version": "2", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "cephx_sign_messages": "true", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "chdir": "", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client_acl_type": "", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client_asio_thread_count": "2", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client_cache_mid": "0.750000", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client_cache_size": "16384", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client_caps_release_delay": "5", 2023-09-30T09:22:36.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client_check_pool_perm": "true", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_debug_force_sync_read": "false", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_debug_getattr_caps": "false", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_debug_inject_tick_delay": "0", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_die_on_failed_remount": "false", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_dirsize_rbytes": "true", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_force_lazyio": "false", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_fs": "", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_inject_release_failure": "false", 2023-09-30T09:22:36.291 INFO:teuthology.orchestra.run.smithi019.stdout: "client_max_inline_size": "4096", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_mds_namespace": "", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_metadata": "", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_mount_gid": "-1", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_mount_timeout": "300", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_mount_uid": "-1", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_mountpoint": "/", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_notify_timeout": "10", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_oc": "true", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_oc_max_dirty": "104857600", 2023-09-30T09:22:36.292 INFO:teuthology.orchestra.run.smithi019.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_oc_max_objects": "1000", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_oc_size": "209715200", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_oc_target_dirty": "8388608", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_permissions": "true", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_quota": "true", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_quota_df": "true", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_readahead_max_bytes": "0", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_readahead_max_periods": "4", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_readahead_min": "131072", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_reconnect_stale": "false", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_shutdown_timeout": "30", 2023-09-30T09:22:36.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_snapdir": ".snap", 2023-09-30T09:22:36.294 INFO:teuthology.orchestra.run.smithi019.stdout: "client_tick_interval": "1", 2023-09-30T09:22:36.294 INFO:teuthology.orchestra.run.smithi019.stdout: "client_trace": "", 2023-09-30T09:22:36.294 INFO:teuthology.orchestra.run.smithi019.stdout: "client_try_dentry_invalidate": "false", 2023-09-30T09:22:36.294 INFO:teuthology.orchestra.run.smithi019.stdout: "client_use_faked_inos": "false", 2023-09-30T09:22:36.294 INFO:teuthology.orchestra.run.smithi019.stdout: "client_use_random_mds": "false", 2023-09-30T09:22:36.294 INFO:teuthology.orchestra.run.smithi019.stdout: "clog_to_graylog": "false", 2023-09-30T09:22:36.294 INFO:teuthology.orchestra.run.smithi019.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-30T09:22:36.294 INFO:teuthology.orchestra.run.smithi019.stdout: "clog_to_graylog_port": "12201", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "clog_to_monitors": "default=true", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "clog_to_syslog": "false", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "clog_to_syslog_level": "info", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "cluster_addr": "-", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "cluster_network": "", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "cluster_network_interface": "", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "compressor_zlib_isal": "false", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "compressor_zlib_level": "5", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "compressor_zlib_winsize": "-15", 2023-09-30T09:22:36.295 INFO:teuthology.orchestra.run.smithi019.stdout: "compressor_zstd_level": "1", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:842fd637775ede364d8d63f7b11a1a5940cb8e14653cebf5ae1ff346603854c0", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crimson_alien_op_num_threads": "6", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crimson_seastar_smp": "1", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crush_location": "", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crush_location_hook": "", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "crush_location_hook_timeout": "10", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "daemonize": "false", 2023-09-30T09:22:36.296 INFO:teuthology.orchestra.run.smithi019.stdout: "daos_pool": "tank", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_alienstore": "0/5", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_allow_any_pool_priority": "false", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_asok": "1/5", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_asok_assert_abort": "false", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_asserts_on_shutdown": "true", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_auth": "1/5", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_bdev": "1/3", 2023-09-30T09:22:36.297 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_bluefs": "1/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_bluestore": "1/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_buffer": "0/1", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_ceph_exporter": "1/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_cephfs_mirror": "0/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_cephsqlite": "0/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_client": "0/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_compressor": "1/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_context": "0/1", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_crush": "1/1", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_crypto": "1/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_cyanstore": "0/5", 2023-09-30T09:22:36.298 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_deliberately_leak_memory": "false", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_disable_randomized_ping": "false", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_dpdk": "1/5", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_eventtrace": "1/5", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_filer": "0/1", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_filestore": "1/3", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_finisher": "1/1", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_fuse": "1/5", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_heartbeat_testing_span": "0", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_heartbeatmap": "1/5", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_javaclient": "1/5", 2023-09-30T09:22:36.299 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_journal": "1/3", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_journaler": "0/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_kstore": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_lockdep": "0/1", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mclock": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mds": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mds_balancer": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mds_locker": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mds_log": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mds_log_expire": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mds_migrator": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_memstore": "1/5", 2023-09-30T09:22:36.300 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mgr": "2/5", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mgrc": "1/5", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_mon": "20/20", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_monc": "0/10", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_ms": "1/1", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_none": "0/5", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_objclass": "0/5", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_objectcacher": "0/5", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_objecter": "0/1", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_optracker": "0/5", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_osd": "1/5", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_paxos": "20/20", 2023-09-30T09:22:36.301 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_perfcounter": "1/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_prioritycache": "1/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rados": "0/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rbd": "0/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rbd_mirror": "0/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rbd_pwl": "0/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rbd_replay": "0/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_refs": "0/0", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_reserver": "1/1", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rgw": "1/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rgw_access": "1/5", 2023-09-30T09:22:36.302 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rgw_datacache": "1/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rgw_dbstore": "1/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rgw_flight": "1/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rgw_sync": "1/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_rocksdb": "4/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore": "0/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_backref": "0/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_cache": "0/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_cleaner": "0/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_device": "0/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_epm": "0/5", 2023-09-30T09:22:36.303 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_journal": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_lba": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_odata": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_omap": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_onode": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_t": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_seastore_tm": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_striper": "0/1", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_test": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_throttle": "1/1", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_timer": "0/1", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_tp": "0/5", 2023-09-30T09:22:36.304 INFO:teuthology.orchestra.run.smithi019.stdout: "debug_trace": "1/5", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "defer_client_eviction_on_laggy_osds": "true", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "device_failure_prediction_mode": "none", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "err_to_graylog": "false", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "err_to_journald": "false", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "err_to_stderr": "true", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "err_to_syslog": "false", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "event_tracing": "false", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "exporter_addr": "0.0.0.0", 2023-09-30T09:22:36.305 INFO:teuthology.orchestra.run.smithi019.stdout: "exporter_http_port": "9926", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "exporter_prio_limit": "5", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "exporter_sort_metrics": "true", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "exporter_stats_period": "5", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "fake_statfs_for_testing": "0", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "fatal_signal_handlers": "true", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "filer_max_purge_ops": "10", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "filer_max_truncate_ops": "128", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_apply_finisher_threads": "1", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_blackhole": "false", 2023-09-30T09:22:36.306 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_btrfs_clone_range": "true", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_btrfs_snap": "true", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_caller_concurrency": "10", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_collect_device_partition_information": "true", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_commit_timeout": "600.000000", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_debug_inject_read_err": "false", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_debug_omap_check": "false", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_debug_verify_split": "false", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_dump_file": "", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-30T09:22:36.307 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_fadvise": "true", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_fail_eio": "true", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_fd_cache_shards": "16", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_fd_cache_size": "128", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_fiemap": "false", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_fiemap_threshold": "4096", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_inject_stall": "0", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_journal_parallel": "false", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_journal_trailing": "false", 2023-09-30T09:22:36.308 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_journal_writeahead": "false", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_kill_at": "0", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_inline_xattrs": "0", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-30T09:22:36.309 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_xattr_value_size": "0", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_merge_threshold": "-10", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_odsync_write": "false", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_omap_backend": "rocksdb", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_omap_backend_path": "", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-30T09:22:36.310 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_op_thread_timeout": "60", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_op_threads": "2", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_punch_hole": "false", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-30T09:22:36.311 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_queue_max_ops": "50", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_seek_data_hole": "false", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_sloppy_crc": "false", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_splice": "false", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_split_multiple": "2", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_split_rand_factor": "20", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_update_to": "1000", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-30T09:22:36.312 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_enable": "true", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_xfs_extsize": "false", 2023-09-30T09:22:36.313 INFO:teuthology.orchestra.run.smithi019.stdout: "filestore_zfs_snap": "false", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fio_dir": "/tmp/fio", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fsid": "608495ac-5f70-11ee-8db4-212e2dc638e7", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_allow_other": "true", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_atomic_o_trunc": "true", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_big_writes": "true", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_debug": "false", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_default_permissions": "false", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_disable_pagecache": "false", 2023-09-30T09:22:36.314 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_max_write": "0", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_multithreaded": "true", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_require_active_mds": "true", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_set_user_groups": "true", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_splice_move": "true", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_splice_read": "true", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_splice_write": "true", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "fuse_use_invalidate_cb": "true", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "gss_target_name": "ceph", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "heartbeat_file": "", 2023-09-30T09:22:36.315 INFO:teuthology.orchestra.run.smithi019.stdout: "heartbeat_inject_failure": "0", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "heartbeat_interval": "5", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "host": "smithi019", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-30T09:22:36.316 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "inject_early_sigterm": "false", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "jaeger_agent_port": "6799", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "jaeger_tracing_enable": "false", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_aio": "true", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_align_min_size": "65536", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_block_align": "true", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_block_size": "4096", 2023-09-30T09:22:36.317 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_dio": "true", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_discard": "false", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_force_aio": "false", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_ignore_corruption": "false", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_max_write_bytes": "10485760", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_max_write_entries": "100", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_replay_from": "0", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-30T09:22:36.318 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_write_header_frequency": "0", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "journal_zero_on_create": "false", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "journaler_prefetch_periods": "10", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "journaler_prezero_periods": "5", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "journaler_write_head_interval": "15", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "key": "", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "keyfile": "", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_backend": "rocksdb", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_default_stripe_size": "65536", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_fsck_on_mount": "false", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_max_bytes": "67108864", 2023-09-30T09:22:36.319 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_max_ops": "512", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_nid_prealloc": "1024", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_onode_map_size": "1024", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_sync_submit_transaction": "false", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "kstore_sync_transaction": "false", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "librados_thread_count": "2", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "lockdep": "false", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "lockdep_force_backtrace": "false", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "log_coarse_timestamps": "true", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "log_flush_on_exit": "false", 2023-09-30T09:22:36.320 INFO:teuthology.orchestra.run.smithi019.stdout: "log_graylog_host": "127.0.0.1", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_graylog_port": "12201", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_max_new": "1000", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_max_recent": "10000", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_stderr_prefix": "", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_stop_at_utilization": "0.970000", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_to_file": "true", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_to_graylog": "false", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_to_journald": "false", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_to_stderr": "false", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "log_to_syslog": "false", 2023-09-30T09:22:36.321 INFO:teuthology.orchestra.run.smithi019.stdout: "max_rotating_auth_attempts": "10", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_action_on_write_error": "1", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_alternate_name_max": "8192", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_asio_thread_count": "2", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_export_pin": "true", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_fragment_dirs": "true", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_fragment_interval": "5", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-30T09:22:36.322 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_interval": "10", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_max": "-1", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_max_until": "-1", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_merge_size": "50", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_midchunk": "0.300000", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_min_start": "0.200000", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_minchunk": "0.001000", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_mode": "0", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_need_max": "1.200000", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_need_min": "0.800000", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-30T09:22:36.323 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_split_bits": "3", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_split_size": "10000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_target_decay": "10.000000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_beacon_grace": "15.000000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_beacon_interval": "4.000000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cache_mid": "0.700000", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cache_release_free_interval": "10", 2023-09-30T09:22:36.324 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cache_reservation": "0.050000", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cache_trim_interval": "1", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cache_trim_threshold": "262144", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_client_prealloc_inos": "1000", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_connect_bootstrapping": "false", 2023-09-30T09:22:36.325 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_damage_table_max_entries": "10000", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_debug_auth_pins": "false", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_debug_frag": "false", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_debug_scatterstat": "false", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_debug_subtrees": "false", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_decay_halflife": "5.000000", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_default_dir_hash": "2", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_defer_session_stale": "true", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_deny_all_reconnect": "false", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dir_keys_per_op": "16384", 2023-09-30T09:22:36.326 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dir_max_commit_size": "10", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dir_max_entries": "0", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dir_prefetch": "true", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dump_cache_on_map": "false", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_early_reply": "true", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_enable_op_tracker": "true", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_enforce_unique_name": "true", 2023-09-30T09:22:36.327 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_export_ephemeral_random": "true", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_health_summarize_threshold": "10", 2023-09-30T09:22:36.328 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_inject_health_dummy": "false", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_inject_migrator_session_race": "false", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_join_fs": "", 2023-09-30T09:22:36.329 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_journal_format": "1", 2023-09-30T09:22:36.330 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_create_at": "0", 2023-09-30T09:22:36.330 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_export_at": "0", 2023-09-30T09:22:36.330 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_import_at": "0", 2023-09-30T09:22:36.330 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_journal_at": "0", 2023-09-30T09:22:36.330 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_journal_expire_at": "0", 2023-09-30T09:22:36.330 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_journal_replay_at": "0", 2023-09-30T09:22:36.330 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_link_at": "0", 2023-09-30T09:22:36.330 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_mdstable_at": "0", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_openc_at": "0", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_rename_at": "0", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_shutdown_at": "0", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_event_large_threshold": "524288", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_events_per_segment": "1024", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_major_segment_event_ratio": "12", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_max_events": "-1", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_max_segments": "128", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_pause": "false", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_segment_size": "0", 2023-09-30T09:22:36.331 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_skip_unbounded_events": "false", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_log_warn_factor": "2.000000", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_caps_per_client": "1048576", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_completed_flushes": "100000", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_completed_requests": "100000", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_export_size": "20971520", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_file_recover": "32", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_purge_files": "64", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_purge_ops": "8192", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-30T09:22:36.332 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_max_snaps_per_dir": "100", 2023-09-30T09:22:36.333 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_metrics_update_interval": "2", 2023-09-30T09:22:36.333 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_min_caps_per_client": "100", 2023-09-30T09:22:36.333 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_min_caps_working_set": "10000", 2023-09-30T09:22:36.333 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-30T09:22:36.333 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_numa_node": "-1", 2023-09-30T09:22:36.333 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-30T09:22:36.333 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_op_complaint_time": "30.000000", 2023-09-30T09:22:36.333 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_op_history_duration": "600", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_op_history_size": "20", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_op_history_slow_op_size": "20", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_op_history_slow_op_threshold": "10", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_op_log_threshold": "5", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_ping_grace": "15", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_ping_interval": "5", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-30T09:22:36.334 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_recall_max_caps": "30000", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_recall_warning_threshold": "262144", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_replay_interval": "1.000000", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_root_ino_gid": "0", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_root_ino_uid": "0", 2023-09-30T09:22:36.335 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_session_metadata_threshold": "16777216", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_shutdown_check": "0", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_skip_ino": "0", 2023-09-30T09:22:36.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_snap_max_uid": "4294967294", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_snap_min_uid": "0", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_snap_rstat": "false", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_standby_replay_damaged": "false", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_symlink_recovery": "true", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_thrash_exports": "0", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_thrash_fragments": "0", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_tick_interval": "5.000000", 2023-09-30T09:22:36.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_valgrind_exit": "false", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_verify_backtrace": "1", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_verify_scatter": "false", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "mds_wipe_sessions": "false", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "mempool_debug": "false", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "memstore_device_bytes": "1073741824", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "memstore_page_set": "false", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "memstore_page_size": "65536", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_client_bytes": "134217728", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_client_messages": "512", 2023-09-30T09:22:36.338 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_disabled_modules": "", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_max_pg_num_change": "128", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_mds_bytes": "134217728", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_mds_messages": "128", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-30T09:22:36.339 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_mon_bytes": "134217728", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_mon_messages": "128", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_osd_bytes": "536870912", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_osd_messages": "8192", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_pool": "true", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_standby_modules": "true", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_stats_period": "5", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_stats_threshold": "5", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_tick_period": "2", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-30T09:22:36.340 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_allow_pool_delete": "true", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_allow_pool_size_one": "false", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_auth_validate_all_caps": "true", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_bytes": "104857600", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_directed_command_retry": "2", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-30T09:22:36.341 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_hunt_parallel": "3", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_log_interval": "1.000000", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_ping_interval": "10.000000", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_client_target_rank": "-1", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_file_level": "debug", 2023-09-30T09:22:36.342 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_file": "false", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_journald": "true", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-30T09:22:36.343 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_compact_on_bootstrap": "false", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_compact_on_start": "false", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_compact_on_trim": "true", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_cpu_threads": "4", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_crush_min_required_version": "hammer", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_daemon_bytes": "419430400", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-30T09:22:36.344 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_data_avail_crit": "5", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_data_avail_warn": "5", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_data_size_warn": "16106127360", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_dump_json": "false", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_dump_transactions": "false", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_extra_checks": "false", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_no_require_quincy": "false", 2023-09-30T09:22:36.345 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_no_require_reef": "false", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_down_mkfs_grace": "60", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_election_default_strategy": "1", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_election_timeout": "5.000000", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_elector_ping_divisor": "2", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_enable_op_tracker": "true", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_fake_pool_delete": "false", 2023-09-30T09:22:36.346 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_force_quorum_join": "false", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_globalid_prealloc": "10000", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_health_detail_to_clog": "true", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_health_log_update_period": "5", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_health_max_detail": "50", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_health_to_clog": "true", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_health_to_clog_interval": "600", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_host": "[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_host_override": "", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_initial_members": "", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-30T09:22:36.347 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_lease": "5.000000", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_log_full_interval": "50", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_log_max": "10000", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_log_max_summary": "50", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_max_log_epochs": "500", 2023-09-30T09:22:36.348 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_max_osd": "10000", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_max_pg_per_osd": "10000", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_max_pool_pg_num": "65536", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mds_force_trim_to": "0", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mds_skip_sanity": "true", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_memory_autotune": "true", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_memory_target": "2147483648", 2023-09-30T09:22:36.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mgr_beacon_grace": "30", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mgr_digest_period": "5", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mgr_inactive_grace": "60", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_min_osdmap_epochs": "500", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_op_complaint_time": "30", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_op_history_duration": "600", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_op_history_size": "20", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_op_history_slow_op_size": "20", 2023-09-30T09:22:36.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_op_log_threshold": "5", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_auto_mark_in": "false", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_cache_size": "500", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-30T09:22:36.351 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_down_out_interval": "600", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_force_trim_to": "0", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-30T09:22:36.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_min_down_reporters": "2", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_report_timeout": "900", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-30T09:22:36.353 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_pg_stuck_threshold": "60", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-30T09:22:36.354 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_probe_timeout": "2.000000", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_reweight_max_change": "0.050000", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_reweight_max_osds": "4", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_scrub_interval": "86400", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_scrub_max_keys": "100", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_scrub_timeout": "300", 2023-09-30T09:22:36.355 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_session_timeout": "300", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_smart_report_timeout": "5", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_stat_smooth_intervals": "6", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_stretch_pool_min_size": "2", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_stretch_pool_size": "4", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_sync_debug": "false", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_sync_provider_kill_at": "0", 2023-09-30T09:22:36.356 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_sync_requester_kill_at": "0", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_sync_timeout": "60.000000", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_target_pg_per_osd": "100", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_tick_interval": "5", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_timecheck_interval": "300.000000", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_use_min_delay_socket": "false", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_older_version_delay": "604800", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-30T09:22:36.357 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_misplaced": "false", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_older_version": "true", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-30T09:22:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "monmap": "", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-30T09:22:36.359 INFO:teuthology.orchestra.run.smithi019.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "motr_tracing_enabled": "false", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_op_threads": "3", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_cm": "false", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_device_name": "", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_dscp": "96", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_local_gid": "", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_port_num": "1", 2023-09-30T09:22:36.360 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_sl": "3", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_support_srq": "true", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_rdma_type": "ib", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_async_reap_threshold": "5", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_before_connect": "false", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_ipv4": "true", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_ipv6": "false", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_msgr1": "true", 2023-09-30T09:22:36.361 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_msgr2": "true", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_port_max": "7568", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_port_min": "6800", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_retry_count": "3", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_bind_retry_delay": "5", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_blackhole_client": "false", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_blackhole_mds": "false", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_blackhole_mgr": "false", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_blackhole_mon": "false", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_blackhole_osd": "false", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_client_mode": "crc secure", 2023-09-30T09:22:36.362 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_cluster_mode": "crc secure", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_cluster_type": "", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_compress_secure": "false", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_connection_idle_timeout": "900", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_connection_ready_timeout": "10", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_crc_data": "true", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_crc_header": "true", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_die_on_bad_msg": "false", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_die_on_bug": "true", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_die_on_old_message": "true", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_die_on_skipped_message": "false", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-30T09:22:36.363 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_coremask": "0xF", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_enable_tso": "true", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_hugepages": "", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_lro": "true", 2023-09-30T09:22:36.364 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_memory_channel": "4", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_pmd": "", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_port_id": "0", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_dump_on_send": "false", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_initial_backoff": "0.200000", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_inject_delay_max": "1.000000", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_inject_delay_type": "", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-30T09:22:36.365 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_inject_network_congestion": "0", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_inject_socket_failures": "0", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_learn_addr_from_peer": "true", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_max_accept_failures": "4", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_max_backoff": "15.000000", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_mon_client_mode": "secure crc", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_mon_service_mode": "secure crc", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_osd_compress_min_size": "1024", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_osd_compress_mode": "none", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-30T09:22:36.366 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_pq_min_cost": "65536", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_public_type": "", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_service_mode": "crc secure", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_tcp_listen_backlog": "512", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_tcp_nodelay": "true", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_tcp_rcvbuf": "0", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "ms_type": "async+posix", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "no_config_file": "false", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "objecter_completion_locks_per_session": "32", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-30T09:22:36.367 INFO:teuthology.orchestra.run.smithi019.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "objecter_inflight_ops": "1024", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "objecter_tick_interval": "5.000000", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "objecter_timeout": "10.000000", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "objectstore_blackhole": "false", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "openssl_engine_opts": "", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_agent_delay_time": "5.000000", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_agent_hist_halflife": "1000", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_agent_max_low_ops": "2", 2023-09-30T09:22:36.368 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_agent_max_ops": "4", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_agent_slop": "0.020000", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_asio_thread_count": "2", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_async_recovery_min_cost": "100", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_backfill_scan_max": "512", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_backfill_scan_min": "64", 2023-09-30T09:22:36.369 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_backoff_on_degraded": "false", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_backoff_on_peering": "false", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_backoff_on_unfound": "true", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_beacon_report_interval": "300", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_bench_duration": "30", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_bench_max_block_size": "67108864", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_blkin_trace_all": "false", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-30T09:22:36.370 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_check_for_log_corruption": "false", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_class_update_on_start": "true", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_client_message_cap": "256", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_client_message_size_cap": "524288000", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_client_op_priority": "63", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_client_watch_timeout": "30", 2023-09-30T09:22:36.371 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_command_max_records": "256", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_compact_on_start": "false", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_crush_update_on_start": "true", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_crush_update_weight_set": "true", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-30T09:22:36.372 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_feed_pullee": "-1", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_misdirected_ops": "false", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_no_acting_change": "false", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_no_purge_strays": "false", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_op_order": "false", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-30T09:22:36.373 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_shutdown": "false", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_verify_snaps": "false", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_deep_scrub_keys": "1024", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-30T09:22:36.374 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_deep_scrub_stride": "524288", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_default_notify_timeout": "30", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_delete_sleep": "0.000000", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_discard_disconnected_ops": "true", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_enable_op_tracker": "true", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-30T09:22:36.375 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_fast_info": "true", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_fast_shutdown": "true", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_function_tracing": "false", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_heartbeat_grace": "20", 2023-09-30T09:22:36.376 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_heartbeat_interval": "6", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_heartbeat_min_peers": "10", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_heartbeat_min_size": "2000", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_heartbeat_stale": "600", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_hit_set_max_size": "100000", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_hit_set_min_size": "1000", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-30T09:22:36.377 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_journal_size": "5120", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_kill_backfill_at": "0", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_map_cache_size": "50", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_map_dedup": "true", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_map_message_max": "40", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_map_share_max_epochs": "40", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_attr_name_len": "100", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_attr_size": "0", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_backfills": "1", 2023-09-30T09:22:36.378 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_markdown_count": "5", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_markdown_period": "600", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_object_name_len": "2048", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_object_namespace_len": "256", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_object_size": "134217728", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_pg_blocked_by": "16", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_pg_log_entries": "10000", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-30T09:22:36.379 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_pgls": "1024", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_push_cost": "8388608", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_push_objects": "10", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_scrubs": "3", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_trimming_pgs": "2", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_write_op_reply_len": "64", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_max_write_size": "90", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-30T09:22:36.380 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_profile": "balanced", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-30T09:22:36.381 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_memory_base": "805306368", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_memory_cache_min": "134217728", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_memory_target": "4294967296", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_memory_target_autotune": "false", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_min_pg_log_entries": "250", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_min_recovery_priority": "0", 2023-09-30T09:22:36.382 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mon_report_interval": "5", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_num_cache_shards": "32", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_num_op_tracker_shard": "32", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_numa_auto_affinity": "true", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_numa_node": "-1", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_numa_prefer_iface": "true", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_objecter_finishers": "1", 2023-09-30T09:22:36.383 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_objectstore": "bluestore", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_objectstore_fuse": "false", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_objectstore_tracing": "false", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_complaint_time": "30.000000", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_history_duration": "600", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_history_size": "20", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_history_slow_op_size": "20", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_log_threshold": "5", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_num_shards": "0", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_num_shards_hdd": "5", 2023-09-30T09:22:36.384 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_num_shards_ssd": "8", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_pq_min_cost": "65536", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_queue_cut_off": "high", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_op_thread_timeout": "15", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_open_classes_on_start": "true", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_os_flags": "0", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_peering_op_priority": "255", 2023-09-30T09:22:36.385 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_delete_cost": "1048576", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_delete_priority": "5", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_log_trim_max": "10000", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_log_trim_min": "100", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-30T09:22:36.386 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_crimson": "false", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-30T09:22:36.387 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_flags": "0", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_min_size": "0", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_pg_num": "32", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_pgp_num": "0", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_size": "3", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_default_type": "replicated", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_push_per_object_cost": "1000", 2023-09-30T09:22:36.388 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recover_clone_overlap": "true", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_cost": "20971520", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_max_active": "0", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_max_single_start": "1", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_op_priority": "3", 2023-09-30T09:22:36.389 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_priority": "5", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_sleep": "0.000000", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_repair_during_recovery": "false", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_requested_scrub_priority": "120", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-30T09:22:36.390 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_auto_repair": "false", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_begin_hour": "0", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_begin_week_day": "0", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_chunk_max": "25", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_chunk_min": "5", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_cost": "52428800", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_during_recovery": "false", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_end_hour": "0", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_end_week_day": "0", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_event_cost": "4096", 2023-09-30T09:22:36.391 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_invalid_stats": "true", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_max_preemptions": "5", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_priority": "5", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_sleep": "0.000000", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-30T09:22:36.392 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_skip_data_digest": "false", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_smart_report_timeout": "5", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_snap_trim_cost": "1048576", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_snap_trim_priority": "5", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-30T09:22:36.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_target_transaction_size": "30", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-30T09:22:36.394 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_tracing": "false", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_use_stale_snap": "false", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "osdc_blkin_trace_all": "false", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_kill_at": "0", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_max_join_drift": "10", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_min": "500", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_min_wait": "0.050000", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_propose_interval": "1.000000", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_service_trim_max": "500", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_service_trim_min": "250", 2023-09-30T09:22:36.395 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_stash_full_interval": "25", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_trim_max": "500", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "paxos_trim_min": "250", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "perf": "true", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "pid_file": "", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "public_addr": "-", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "public_addrv": "", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "public_bind_addr": "-", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "public_network": "172.21.0.0/20", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "public_network_interface": "", 2023-09-30T09:22:36.396 INFO:teuthology.orchestra.run.smithi019.stdout: "qat_compressor_enabled": "false", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "qat_compressor_session_max_number": "256", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rados_mon_op_timeout": "0", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rados_osd_op_timeout": "0", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rados_tracing": "false", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_atime_update_interval": "60", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_balance_parent_reads": "false", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_balance_snap_reads": "false", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_blkin_trace_all": "false", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-30T09:22:36.397 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache": "true", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache_policy": "writearound", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache_size": "33554432", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_clone_copy_on_read": "false", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_compression_hint": "none", 2023-09-30T09:22:36.398 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_concurrent_management_ops": "10", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_clone_format": "auto", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_data_pool": "", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_features": "61", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_format": "2", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_map_options": "", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_order": "22", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_pool": "rbd", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_stripe_count": "0", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_default_stripe_unit": "0", 2023-09-30T09:22:36.399 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_enable_alloc_hint": "true", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_io_scheduler": "simple", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-30T09:22:36.400 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_order": "24", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_pool": "", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_journal_splay_width": "4", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_localize_parent_reads": "false", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_localize_snap_reads": "false", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-30T09:22:36.401 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-30T09:22:36.402 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-30T09:22:36.403 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_mtime_update_interval": "60", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_non_blocking_aio": "true", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_op_thread_timeout": "60", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_op_threads": "1", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_parent_cache_enabled": "false", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_plugins": "", 2023-09-30T09:22:36.404 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_bps_burst": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_bps_limit": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_exclude_ops": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_iops_burst": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_iops_limit": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-30T09:22:36.405 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_read_from_replica_policy": "default", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-30T09:22:36.406 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_skip_partial_discard": "true", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_tracing": "false", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_validate_names": "true", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rbd_validate_pool": "true", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "restapi_base_url": "", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "restapi_log_level": "", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_acl_grants_max_num": "100", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_admin_entry": "admin", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_backend_store": "rados", 2023-09-30T09:22:36.407 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_barbican_url": "", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_beast_enable_async": "true", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_cache_enabled": "true", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_cache_expiry_interval": "900", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_cache_lru_size": "10000", 2023-09-30T09:22:36.408 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_config_store": "rados", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_content_length_compat": "false", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_copy_obj_progress": "true", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_cors_rules_max_num": "100", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_cross_domain_policy": "", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_kmip_addr": "", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_kmip_password": "", 2023-09-30T09:22:36.409 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_kmip_username": "", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_require_ssl": "true", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-30T09:22:36.410 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_addr": "", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_auth": "token", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_namespace": "", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_prefix": "", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-30T09:22:36.411 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_token_file": "", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_curl_buffersize": "524288", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_curl_low_speed_time": "300", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-30T09:22:36.412 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d4n_host": "127.0.0.1", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_d4n_port": "6379", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_data_log_changes_size": "1000", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_data_log_num_shards": "128", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_data_log_window": "30", 2023-09-30T09:22:36.413 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-30T09:22:36.414 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-30T09:22:36.415 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-30T09:22:36.415 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-30T09:22:36.415 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-30T09:22:36.415 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-30T09:22:36.415 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-30T09:22:36.415 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-30T09:22:36.415 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-30T09:22:36.415 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dns_name": "", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dns_s3website_name": "", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_dynamic_resharding": "true", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_enable_gc_threads": "true", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_enable_lc_threads": "true", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_enable_ops_log": "false", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_enable_quota_threads": "true", 2023-09-30T09:22:36.416 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_enable_static_website": "false", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_enable_usage_log": "false", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_enforce_swift_acls": "true", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_exit_timeout_secs": "120", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_expose_bucket": "false", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_extended_http_attrs": "", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_filter": "none", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_frontends": "beast port=7480", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_max_deferred": "50", 2023-09-30T09:22:36.417 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_max_objs": "32", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_gc_processor_period": "3600", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_init_timeout": "300", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-30T09:22:36.418 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_accepted_reader_roles": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_admin_domain": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_admin_password": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_admin_password_path": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_admin_project": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_admin_tenant": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_admin_token": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_admin_token_path": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_admin_user": "", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_api_version": "2", 2023-09-30T09:22:36.419 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_barbican_domain": "", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_barbican_password": "", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_barbican_project": "", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_barbican_user": "", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_url": "", 2023-09-30T09:22:36.420 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lc_debug_interval": "-1", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lc_lock_max_time": "90", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lc_max_objs": "32", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lc_max_rules": "1000", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lc_max_worker": "3", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lc_thread_delay": "0", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ldap_dnattr": "uid", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ldap_searchfilter": "", 2023-09-30T09:22:36.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_log_http_headers": "", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_log_object_name_utc": "false", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_lua_max_memory_per_state": "128000", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-30T09:22:36.422 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_attr_name_len": "0", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_attr_size": "0", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_chunk_size": "4194304", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_listing_results": "1000", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_notify_retries": "10", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-30T09:22:36.423 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_put_param_size": "1048576", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_put_size": "5368709120", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_max_slo_entries": "1000", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_md_log_max_shards": "64", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_mp_lock_max_time": "600", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-30T09:22:36.424 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_max_gc": "300", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-30T09:22:36.425 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_num_async_rados_threads": "32", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_num_control_oids": "8", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_numa_node": "-1", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_objexp_chunk_size": "100", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_objexp_gc_interval": "600", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-30T09:22:36.426 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_op_thread_timeout": "600", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_op_tracing": "false", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_opa_token": "", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_opa_url": "", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_opa_verify_ssl": "true", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ops_log_rados": "false", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_ops_log_socket_path": "", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-30T09:22:36.427 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_period_push_interval": "2.000000", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_posix_cache_lanes": "3", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_posix_cache_lmdb_count": "3", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_posix_cache_max_buckets": "100", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_posix_cache_partitions": "3", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_print_continue": "true", 2023-09-30T09:22:36.428 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_rados_tracing": "false", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_realm": "", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_realm_id": "", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_region": "", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-30T09:22:36.429 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_request_uri": "", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_reshard_batch_size": "64", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_reshard_max_aio": "128", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_reshard_num_logs": "16", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_reshard_thread_interval": "600", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_resolve_cname": "false", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_run_sync_thread": "true", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-30T09:22:36.430 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_scheduler_type": "throttler", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_script_uri": "", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_service_provider_name": "", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-30T09:22:36.431 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sts_client_id": "", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sts_client_secret": "", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sts_entry": "sts", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sts_key": "sts", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sts_min_session_duration": "900", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sts_token_introspection_url": "", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_account_in_url": "false", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_auth_entry": "auth", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_auth_url": "", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_custom_header": "", 2023-09-30T09:22:36.432 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_need_stats": "true", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_tenant_name": "", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_token_expiration": "86400", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_url": "", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_url_prefix": "swift", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_lease_period": "120", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-30T09:22:36.433 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_thread_pool_size": "512", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_topic_persistency_max_retries": "0", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_topic_persistency_sleep_duration": "0", 2023-09-30T09:22:36.434 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_topic_persistency_time_to_live": "0", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_torrent_comment": "", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_torrent_createby": "", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_torrent_encoding": "", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_torrent_flag": "false", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_torrent_max_size": "5368709120", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_torrent_origin": "", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_torrent_tracker": "", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_trust_forwarded_https": "false", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-30T09:22:36.435 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_usage_max_shards": "32", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_usage_max_user_shards": "1", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_use_opa_authz": "false", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_max_buckets": "1000", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_policies_max_num": "100", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-30T09:22:36.436 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_user_unique_email": "true", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_verify_ssl": "true", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_zone": "", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_zone_id": "", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_zonegroup": "", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_zonegroup_id": "", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgwlc_auto_session_clear": "true", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_block_size": "4096", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-30T09:22:36.437 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cache_size": "536870912", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-30T09:22:36.438 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_index_type": "binary_search", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_partition_filters": "false", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_perf": "false", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "run_dir": "/var/run/ceph", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_block_create": "true", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_cache_lru_size": "67108864", 2023-09-30T09:22:36.439 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_default_max_object_size": "16777216", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_device_size": "53687091200", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_journal_batch_capacity": "16", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_main_device_type": "SSD", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-30T09:22:36.440 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "seastore_segment_size": "67108864", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "set_keepcaps": "false", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "setgroup": "ceph", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "setuser": "ceph", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "setuser_match_path": "", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "thp": "false", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "threadpool_default_timeout": "60", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout: "throttler_perf_counter": "true" 2023-09-30T09:22:36.441 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:22:36.442 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:36 smithi019 ceph-mon[216815]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:36.442 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:36 smithi019 ceph-mon[224043]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:36 smithi167 ceph-mon[192880]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:36.987 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-30T09:22:37.353 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:38 smithi167 ceph-mon[192880]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:38 smithi019 ceph-mon[216815]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:38.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:38 smithi019 ceph-mon[224043]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:38.827 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a: {} 2023-09-30T09:22:38.838 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b: {} 2023-09-30T09:22:38.848 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c: {} 2023-09-30T09:22:38.849 INFO:teuthology.orchestra.run.smithi019.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-30T09:22:38.849 INFO:teuthology.orchestra.run.smithi019.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-30T09:22:38.849 INFO:teuthology.orchestra.run.smithi019.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-30T09:22:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4001782760' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:22:39.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4001782760' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:22:39.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4001782760' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:22:39.562 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-30T09:22:39.562 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-30T09:22:39.563 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-30T09:22:39.565 INFO:tasks.radosbench:Beginning radosbench... 2023-09-30T09:22:39.565 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-30T09:22:39.565 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-30T09:22:39.565 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-30T09:22:39.566 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:22:39.566 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:22:39.566 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:22:39.566 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd out 0 2023-09-30T09:22:39.566 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-30T09:22:39.567 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:22:39.567 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-30T09:22:39.567 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:22:39.567 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-30T09:22:39.568 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:22:39.568 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-30T09:22:39.568 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:22:39.620 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:39.669 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:22:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:39.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:22:39.775 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:22:39.818 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:39.888 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:22:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:39.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:22:39.994 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:22:40.030 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:40.030 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:40.093 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:22:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:40.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:22:40.142 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:40.198 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:40.226 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:40.252 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:40.387 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:40 smithi019 ceph-mon[216815]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:40.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:40 smithi019 ceph-mon[224043]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:40.387 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:40.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:40.492 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:40.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:40 smithi167 ceph-mon[192880]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:40.526 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:40.575 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:40.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:40.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:40.586 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:40.682 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:22:40.710 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:40.755 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:22:40.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:40.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:22:40.862 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:40.888 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:40.931 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:40.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:40.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:41.037 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:22:41.067 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:41.468 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:22:41.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:41.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:22:41.572 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:22:41.599 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:41.641 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:22:41.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:41.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:22:41.748 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:41.773 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:41.814 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:41.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:41.921 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:22:41.951 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:42 smithi167 ceph-mon[192880]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:42.584 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:22:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:42 smithi019 ceph-mon[216815]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:42 smithi019 ceph-mon[224043]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:42.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:42.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:22:42.690 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:22:42.720 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:42.766 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:22:42.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:42.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:22:42.871 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:22:42.883 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:42.900 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:42.943 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:22:42.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:42.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:22:42.979 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:22:42.998 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:22:43.018 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:22:43.037 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:22:43.051 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:43.063 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:22:43.077 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:43.081 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:22:43.103 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:22:43.120 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:43.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:43.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:43.125 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:22:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:22:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:22:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:22:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:22:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:22:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:22:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:22:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:22:43.225 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:22:43.255 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:43.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1709080316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:22:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1709080316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:22:43.478 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:22:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:43.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:22:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1709080316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:22:43.583 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:22:43.612 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:43.656 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:22:43.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:43.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:22:43.761 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:22:43.790 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:44.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:22:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:22:43] "GET /metrics HTTP/1.1" 200 33858 "" "Prometheus/2.43.0" 2023-09-30T09:22:44.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:44 smithi019 ceph-mon[216815]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:44.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:44 smithi019 ceph-mon[224043]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:44 smithi167 ceph-mon[192880]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:44.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:44 smithi019 conmon[216791]: 2023-09-30T09:22:44.621+0000 7faf52971700 -1 mon.a@0(leader).osd e110 definitely_dead 0 2023-09-30T09:22:45.091 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:22:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:45.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:22:45.195 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:22:45.234 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:45.241 INFO:teuthology.orchestra.run.smithi019.stderr:marked out osd.0. 2023-09-30T09:22:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:45 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1899099921' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:22:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:45 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:22:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:45 smithi019 ceph-mon[216815]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-30T09:22:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:45 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1899099921' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:22:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:45 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:22:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:45 smithi019 ceph-mon[224043]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-30T09:22:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:45 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1899099921' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:22:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:45 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:22:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:45 smithi167 ceph-mon[192880]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-30T09:22:45.570 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:22:45.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:45.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:22:45.674 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:22:45.703 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:45.970 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:22:45.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:45.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:22:46.073 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:46.101 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:46.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:46.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-30T09:22:46.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: osdmap e111: 8 total, 8 up, 7 in 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:22:46.246 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:46.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:46.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:46.277 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:22:46.352 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:46.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:46.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-30T09:22:46.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: osdmap e111: 8 total, 8 up, 7 in 2023-09-30T09:22:46.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.378 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.378 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.378 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.378 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:22:46.379 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:46.427 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:46.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:46.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:46.532 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:46.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-30T09:22:46.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-30T09:22:46.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: osdmap e111: 8 total, 8 up, 7 in 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: failed to encode map e111 with expected crc 2023-09-30T09:22:46.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:22:46.569 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:46.675 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:46.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:46.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:46.779 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:22:46.811 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:46.949 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:22:46.950 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:22:47.169 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:22:47.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:47.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:22:47.274 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:47.301 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:47.344 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:47.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:47.453 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:22:47.481 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:47.558 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_0' created 2023-09-30T09:22:47.576 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:22:47.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: osdmap e112: 8 total, 8 up, 7 in 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2735524021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: osdmap e112: 8 total, 8 up, 7 in 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: failed to encode map e112 with expected crc 2023-09-30T09:22:47.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2735524021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:22:47.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:47 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:22:47.934 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:22:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:47.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:22:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: osdmap e112: 8 total, 8 up, 7 in 2023-09-30T09:22:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: failed to encode map e112 with expected crc 2023-09-30T09:22:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2735524021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:22:48.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:47 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:22:48.038 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:22:48.067 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:48.686 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:22:48.721 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:22:48.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:48.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:22:48.806 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-30T09:22:48.826 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 131 MiB used, 626 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:22:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4016826489' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-30T09:22:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:22:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:22:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4016826489' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-30T09:22:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: osdmap e113: 8 total, 8 up, 7 in 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[224043]: osdmap e114: 8 total, 8 up, 7 in 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 131 MiB used, 626 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4016826489' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-30T09:22:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4016826489' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: osdmap e113: 8 total, 8 up, 7 in 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:48.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:48 smithi019 ceph-mon[216815]: osdmap e114: 8 total, 8 up, 7 in 2023-09-30T09:22:48.849 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 131 MiB used, 626 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4016826489' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4016826489' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: osdmap e113: 8 total, 8 up, 7 in 2023-09-30T09:22:48.850 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: failed to encode map e113 with expected crc 2023-09-30T09:22:48.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:48.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:22:48.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:48 smithi167 ceph-mon[192880]: osdmap e114: 8 total, 8 up, 7 in 2023-09-30T09:22:48.852 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:48.857 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:48.907 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:48.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:49.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:49.039 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:49.544 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:49.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:49.649 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:22:49.679 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:49.727 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:22:49.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:49.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:22:49.830 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: osdmap e115: 8 total, 8 up, 7 in 2023-09-30T09:22:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: osdmap e115: 8 total, 8 up, 7 in 2023-09-30T09:22:49.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:49.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:49.856 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.856 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.856 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: failed to encode map e114 with expected crc 2023-09-30T09:22:49.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: osdmap e115: 8 total, 8 up, 7 in 2023-09-30T09:22:49.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:49.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:49.858 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:49.901 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:49.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:49.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:50.006 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:50.033 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:50.666 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:50.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:50.667 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:50.667 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[224043]: pgmap v230: 89 pgs: 3 peering, 8 unknown, 78 active+clean; 4.6 MiB data, 132 MiB used, 626 GiB / 626 GiB avail; 1.1 MiB/s, 10 objects/s recovering 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[224043]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[224043]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[224043]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[224043]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[224043]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[224043]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[216815]: pgmap v230: 89 pgs: 3 peering, 8 unknown, 78 active+clean; 4.6 MiB data, 132 MiB used, 626 GiB / 626 GiB avail; 1.1 MiB/s, 10 objects/s recovering 2023-09-30T09:22:50.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[216815]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[216815]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[216815]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[216815]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[216815]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:50 smithi019 ceph-mon[216815]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:22:50.726 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:22:50.768 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:22:50.799 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:50.846 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:22:50.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:50 smithi167 ceph-mon[192880]: pgmap v230: 89 pgs: 3 peering, 8 unknown, 78 active+clean; 4.6 MiB data, 132 MiB used, 626 GiB / 626 GiB avail; 1.1 MiB/s, 10 objects/s recovering 2023-09-30T09:22:50.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:50 smithi167 ceph-mon[192880]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:50 smithi167 ceph-mon[192880]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:50 smithi167 ceph-mon[192880]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:50 smithi167 ceph-mon[192880]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.851 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:50 smithi167 ceph-mon[192880]: failed to encode map e115 with expected crc 2023-09-30T09:22:50.852 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:50 smithi167 ceph-mon[192880]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:22:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:50.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:22:50.931 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:22:50.951 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:22:50.953 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:22:50.978 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:22:50.986 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:51.003 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:22:51.025 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:22:51.034 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:22:51.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:51.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:22:51.048 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:22:51.069 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:22:51.088 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:22:51.089 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:22:51.089 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:22:51.089 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:22:51.090 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:22:51.090 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:22:51.090 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:22:51.090 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:22:51.090 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:22:51.139 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:22:51.169 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:51.212 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:22:51.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:51.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:22:51.317 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:22:51.323 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:22:51.323 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:22:51.323 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-30T09:22:51.323 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:22:51.324 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:22:51.324 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-30T09:22:51.344 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:51.387 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:22:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:51.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:22:51.494 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:22:51.522 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:51.685 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:22:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2741469357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:22:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2741469357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:22:51.881 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2741469357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:22:51.996 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:22:51.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:51.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:22:52.099 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:52.130 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:52.377 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:52.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:52.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:52.481 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:52.509 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:52.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:52 smithi019 ceph-mon[216815]: pgmap v232: 89 pgs: 3 peering, 3 unknown, 83 active+clean; 4.6 MiB data, 132 MiB used, 626 GiB / 626 GiB avail; 986 KiB/s, 11 objects/s recovering 2023-09-30T09:22:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:52 smithi019 ceph-mon[224043]: pgmap v232: 89 pgs: 3 peering, 3 unknown, 83 active+clean; 4.6 MiB data, 132 MiB used, 626 GiB / 626 GiB avail; 986 KiB/s, 11 objects/s recovering 2023-09-30T09:22:52.881 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:52.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:52.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:52.888 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:52 smithi167 ceph-mon[192880]: pgmap v232: 89 pgs: 3 peering, 3 unknown, 83 active+clean; 4.6 MiB data, 132 MiB used, 626 GiB / 626 GiB avail; 986 KiB/s, 11 objects/s recovering 2023-09-30T09:22:52.985 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:22:53.014 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:53.056 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:22:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:53.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:22:53.160 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:22:53.188 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:53.348 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:22:53.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:53.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:22:53.394 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:22:53.394 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:22:53.394 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:22:53.399 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:53.453 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:53.484 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:53.765 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:53.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:53.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:53.868 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:22:53.899 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:54.054 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:22:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:22:53] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-30T09:22:54.392 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:22:54.592 INFO:teuthology.orchestra.run.smithi019.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-30T09:22:54.617 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:54 smithi019 ceph-mon[216815]: pgmap v233: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 779 KiB/s, 7 objects/s recovering 2023-09-30T09:22:54.617 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:54 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/518258638' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:22:54.617 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:54 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:22:54.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:54 smithi019 ceph-mon[224043]: pgmap v233: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 779 KiB/s, 7 objects/s recovering 2023-09-30T09:22:54.618 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:54 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/518258638' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:22:54.618 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:54 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:22:54.764 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:22:54.764 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:22:54.764 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:22:54.764 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:22:54.764 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:22:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:54 smithi167 ceph-mon[192880]: pgmap v233: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 779 KiB/s, 7 objects/s recovering 2023-09-30T09:22:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:54 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/518258638' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:22:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:54 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:22:55.160 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:22:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:55.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:22:55.263 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:22:55.299 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:55.350 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:22:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:55.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:22:55.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:55.485 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:55.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:22:55.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[216815]: osdmap e116: 8 total, 8 up, 7 in 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[216815]: failed to encode map e116 with expected crc 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[216815]: failed to encode map e116 with expected crc 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[216815]: failed to encode map e116 with expected crc 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[216815]: pgmap v235: 89 pgs: 89 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 680 KiB/s, 6 objects/s recovering 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[224043]: osdmap e116: 8 total, 8 up, 7 in 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[224043]: failed to encode map e116 with expected crc 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[224043]: failed to encode map e116 with expected crc 2023-09-30T09:22:55.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[224043]: failed to encode map e116 with expected crc 2023-09-30T09:22:55.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:55 smithi019 ceph-mon[224043]: pgmap v235: 89 pgs: 89 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 680 KiB/s, 6 objects/s recovering 2023-09-30T09:22:55.933 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:22:55.968 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:55.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:55.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:56.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:55 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:22:56.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:55 smithi167 ceph-mon[192880]: osdmap e116: 8 total, 8 up, 7 in 2023-09-30T09:22:56.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:55 smithi167 ceph-mon[192880]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:55 smithi167 ceph-mon[192880]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:55 smithi167 ceph-mon[192880]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:55 smithi167 ceph-mon[192880]: pgmap v235: 89 pgs: 89 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 680 KiB/s, 6 objects/s recovering 2023-09-30T09:22:56.072 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:22:56.104 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:56.155 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:22:56.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:56.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:22:56.261 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:22:56.290 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:56.589 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:22:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:56.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:22:56.693 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:22:56.723 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:56.775 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:22:56.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:56.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:22:56.880 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:22:56.910 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:56 smithi167 ceph-mon[192880]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.910 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:56 smithi167 ceph-mon[192880]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.911 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:56 smithi167 ceph-mon[192880]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:22:56.912 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:56.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:56 smithi019 ceph-mon[216815]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:56 smithi019 ceph-mon[216815]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.924 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:56 smithi019 ceph-mon[216815]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:22:56.924 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:56 smithi019 ceph-mon[224043]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.924 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:56 smithi019 ceph-mon[224043]: failed to encode map e116 with expected crc 2023-09-30T09:22:56.924 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:56 smithi019 ceph-mon[224043]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:22:56.961 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:22:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:56.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:22:56.990 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:57.043 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:22:57.068 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:22:57.096 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:57.142 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:22:57.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:57.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:22:57.250 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:22:57.277 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:57.319 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:22:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:57.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:22:57.425 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:22:57.457 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:57.566 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:22:57.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:57.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:22:57.598 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:22:57.671 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:22:57.701 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: failed to encode map e116 with expected crc 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: failed to encode map e116 with expected crc 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3348578859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: pgmap v236: 89 pgs: 89 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: failed to encode map e116 with expected crc 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: failed to encode map e116 with expected crc 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3348578859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: pgmap v236: 89 pgs: 89 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:22:57.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:57.955 DEBUG:teuthology.orchestra.run.smithi019:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-30T09:22:57.958 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-30T09:22:57.958 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-30T09:22:58.007 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:hints = 1 2023-09-30T09:22:58.007 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-30T09:22:58.007 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object prefix: benchmark_data_smithi019_287194 2023-09-30T09:22:58.008 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:22:58.008 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 0 0 0 0 0 0 - 0 2023-09-30T09:22:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: failed to encode map e116 with expected crc 2023-09-30T09:22:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: failed to encode map e116 with expected crc 2023-09-30T09:22:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3348578859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:22:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:22:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: pgmap v236: 89 pgs: 89 active+clean; 4.6 MiB data, 137 MiB used, 626 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:22:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:22:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:22:58.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:22:58.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:22:58.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:22:58.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[224043]: failed to encode map e117 with expected crc 2023-09-30T09:22:58.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[224043]: osdmap e117: 8 total, 8 up, 7 in 2023-09-30T09:22:58.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[224043]: failed to encode map e117 with expected crc 2023-09-30T09:22:58.860 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:22:58.861 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:22:58.861 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[216815]: failed to encode map e117 with expected crc 2023-09-30T09:22:58.861 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[216815]: osdmap e117: 8 total, 8 up, 7 in 2023-09-30T09:22:58.861 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:58 smithi019 ceph-mon[216815]: failed to encode map e117 with expected crc 2023-09-30T09:22:59.008 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 1 16 3498 3482 217.607 217.625 0.00332079 0.00457833 2023-09-30T09:22:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:58 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:22:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:58 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:22:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:58 smithi167 ceph-mon[192880]: failed to encode map e117 with expected crc 2023-09-30T09:22:59.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:58 smithi167 ceph-mon[192880]: osdmap e117: 8 total, 8 up, 7 in 2023-09-30T09:22:59.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:58 smithi167 ceph-mon[192880]: failed to encode map e117 with expected crc 2023-09-30T09:22:59.062 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:22:59.062 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:22:59.062 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-09-30T09:22:59.063 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd in 0 2023-09-30T09:22:59.790 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:22:59.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:22:59.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:22:59.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:59 smithi019 ceph-mon[216815]: failed to encode map e117 with expected crc 2023-09-30T09:22:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:59 smithi019 ceph-mon[216815]: failed to encode map e117 with expected crc 2023-09-30T09:22:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:22:59 smithi019 ceph-mon[216815]: pgmap v238: 89 pgs: 89 active+clean; 101 MiB data, 308 MiB used, 626 GiB / 626 GiB avail; 12 MiB/s wr, 193 op/s 2023-09-30T09:22:59.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:59 smithi019 ceph-mon[224043]: failed to encode map e117 with expected crc 2023-09-30T09:22:59.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:59 smithi019 ceph-mon[224043]: failed to encode map e117 with expected crc 2023-09-30T09:22:59.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:22:59 smithi019 ceph-mon[224043]: pgmap v238: 89 pgs: 89 active+clean; 101 MiB data, 308 MiB used, 626 GiB / 626 GiB avail; 12 MiB/s wr, 193 op/s 2023-09-30T09:22:59.894 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:22:59.946 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:00.008 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 2 16 6929 6913 215.94 214.438 0.0063792 0.00462139 2023-09-30T09:23:00.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:59 smithi167 ceph-mon[192880]: failed to encode map e117 with expected crc 2023-09-30T09:23:00.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:59 smithi167 ceph-mon[192880]: failed to encode map e117 with expected crc 2023-09-30T09:23:00.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:22:59 smithi167 ceph-mon[192880]: pgmap v238: 89 pgs: 89 active+clean; 101 MiB data, 308 MiB used, 626 GiB / 626 GiB avail; 12 MiB/s wr, 193 op/s 2023-09-30T09:23:00.158 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:00.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:00.263 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:00.313 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:00.417 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:00.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:00.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:00.522 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:00.534 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:23:00.566 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:00.570 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:00.577 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:23:00.620 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:23:00.656 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:00.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:00.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:00.663 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:23:00.690 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:23:00.690 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:23:00.700 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:23:00.740 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:23:00.761 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:00.784 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:23:00.818 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:00.824 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:23:00.839 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:23:00.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:23:00.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:23:00.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:23:00.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:23:00.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:23:00.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:23:00.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:23:01.009 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 3 16 10005 9989 208.039 192.25 0.00368076 0.00480022 2023-09-30T09:23:01.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1240477296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1240477296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1240477296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:01.910 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:23:01.956 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:01 smithi019 ceph-mon[216815]: pgmap v239: 89 pgs: 89 active+clean; 196 MiB data, 360 MiB used, 625 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s 2023-09-30T09:23:01.956 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:01 smithi019 ceph-mon[224043]: pgmap v239: 89 pgs: 89 active+clean; 196 MiB data, 360 MiB used, 625 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s 2023-09-30T09:23:02.009 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 4 16 13261 13245 206.889 203.5 0.00550668 0.004826 2023-09-30T09:23:02.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:01 smithi167 ceph-mon[192880]: pgmap v239: 89 pgs: 89 active+clean; 196 MiB data, 360 MiB used, 625 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s 2023-09-30T09:23:03.009 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 5 16 16224 16208 202.534 185.188 0.00457799 0.00493051 2023-09-30T09:23:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-09-30T09:23:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-09-30T09:23:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-09-30T09:23:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-09-30T09:23:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-09-30T09:23:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-09-30T09:23:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:03.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:23:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:23:03] "GET /metrics HTTP/1.1" 200 36277 "" "Prometheus/2.43.0" 2023-09-30T09:23:03.921 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:03.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:04.000 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:23:04.009 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 6 16 18294 18278 190.341 129.375 0.00482725 0.00524786 2023-09-30T09:23:04.025 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:04.093 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:04.197 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:04.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:04.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:04.210 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-09-30T09:23:04.210 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-09-30T09:23:04.210 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[216815]: failed to encode map e118 with expected crc 2023-09-30T09:23:04.210 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[216815]: osdmap e118: 8 total, 8 up, 7 in 2023-09-30T09:23:04.210 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[216815]: failed to encode map e118 with expected crc 2023-09-30T09:23:04.210 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[216815]: pgmap v241: 89 pgs: 89 active+clean; 338 MiB data, 783 MiB used, 625 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-30T09:23:04.210 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:04.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-09-30T09:23:04.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-09-30T09:23:04.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[224043]: failed to encode map e118 with expected crc 2023-09-30T09:23:04.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[224043]: osdmap e118: 8 total, 8 up, 7 in 2023-09-30T09:23:04.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[224043]: failed to encode map e118 with expected crc 2023-09-30T09:23:04.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[224043]: pgmap v241: 89 pgs: 89 active+clean; 338 MiB data, 783 MiB used, 625 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-30T09:23:04.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-09-30T09:23:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-09-30T09:23:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:03 smithi167 ceph-mon[192880]: failed to encode map e118 with expected crc 2023-09-30T09:23:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:03 smithi167 ceph-mon[192880]: osdmap e118: 8 total, 8 up, 7 in 2023-09-30T09:23:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:03 smithi167 ceph-mon[192880]: failed to encode map e118 with expected crc 2023-09-30T09:23:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:03 smithi167 ceph-mon[192880]: pgmap v241: 89 pgs: 89 active+clean; 338 MiB data, 783 MiB used, 625 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-30T09:23:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:04.306 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:04.356 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:05.004 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:05.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:05.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:05.010 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 7 16 19992 19976 178.292 106.125 0.00625334 0.00560388 2023-09-30T09:23:05.109 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:05.144 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:05.183 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:05.257 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:04 smithi167 ceph-mon[192880]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:04 smithi167 ceph-mon[192880]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:04 smithi167 ceph-mon[192880]: osdmap e119: 8 total, 8 up, 7 in 2023-09-30T09:23:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:04 smithi167 ceph-mon[192880]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:04 smithi167 ceph-mon[192880]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:04 smithi167 ceph-mon[192880]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:04 smithi167 ceph-mon[192880]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:04 smithi167 ceph-mon[192880]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.290 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:05.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[224043]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[224043]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[224043]: osdmap e119: 8 total, 8 up, 7 in 2023-09-30T09:23:05.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[224043]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[224043]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[224043]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[224043]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[224043]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[216815]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[216815]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[216815]: osdmap e119: 8 total, 8 up, 7 in 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[216815]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[216815]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.294 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[216815]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.294 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[216815]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.294 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:04 smithi019 ceph-mon[216815]: failed to encode map e119 with expected crc 2023-09-30T09:23:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:05.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:05.395 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:05.466 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:05.602 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:05.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:05.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:05.716 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:05.770 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:05.868 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:23:06.010 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 8 16 22114 22098 172.582 132.625 0.00918021 0.00578862 2023-09-30T09:23:06.160 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:06.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:06.269 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:05 smithi167 ceph-mon[192880]: pgmap v243: 89 pgs: 2 peering, 87 active+clean; 1.1 GiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-09-30T09:23:06.334 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:06.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:05 smithi019 ceph-mon[224043]: pgmap v243: 89 pgs: 2 peering, 87 active+clean; 1.1 GiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-09-30T09:23:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:05 smithi019 ceph-mon[216815]: pgmap v243: 89 pgs: 2 peering, 87 active+clean; 1.1 GiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-09-30T09:23:06.445 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:06.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:06.554 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:06.611 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:06.758 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:06.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:06 smithi019 conmon[216791]: 2023-09-30T09:23:06.551+0000 7faf52971700 -1 mon.a@0(leader).osd e119 definitely_dead 0 2023-09-30T09:23:06.869 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:23:06.922 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:07.013 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 9 16 24137 24121 167.404 126.438 0.0136143 0.00596536 2023-09-30T09:23:07.016 INFO:teuthology.orchestra.run.smithi019.stderr:marked in osd.0. 2023-09-30T09:23:07.036 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:23:07.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:07.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:23:07.143 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:07.166 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3477764284' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:23:07.166 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:06 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:23:07.169 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:07.217 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:07.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:07.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:07.321 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3477764284' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:23:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:06 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:23:07.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3477764284' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:23:07.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:06 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:23:07.352 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:07.914 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:07.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:07.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:08.017 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:08.018 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 10 16 24151 24135 150.698 0.875 0.00344042 0.00597068 2023-09-30T09:23:08.051 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:08.095 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:08.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:08.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:08.202 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:08.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: pgmap v244: 89 pgs: 2 peering, 87 active+clean; 1.1 GiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-30T09:23:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-30T09:23:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: osdmap e120: 8 total, 8 up, 8 in 2023-09-30T09:23:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[224043]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: pgmap v244: 89 pgs: 2 peering, 87 active+clean; 1.1 GiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: osdmap e120: 8 total, 8 up, 8 in 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:07 smithi019 ceph-mon[216815]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.231 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:07 smithi167 ceph-mon[192880]: pgmap v244: 89 pgs: 2 peering, 87 active+clean; 1.1 GiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:08 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:08 smithi167 ceph-mon[192880]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:08 smithi167 ceph-mon[192880]: osdmap e120: 8 total, 8 up, 8 in 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:08 smithi167 ceph-mon[192880]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:08 smithi167 ceph-mon[192880]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:08 smithi167 ceph-mon[192880]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:08 smithi167 ceph-mon[192880]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:08 smithi167 ceph-mon[192880]: failed to encode map e120 with expected crc 2023-09-30T09:23:08.578 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:08.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:08.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:08.587 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-09-30T09:23:08.683 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:08.714 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:09.018 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 11 16 24151 24135 136.995 0 - 0.00597068 2023-09-30T09:23:09.032 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:09.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:09.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:09.047 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:09.148 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:09.173 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:23:09.173 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:23:09.178 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:09.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: osdmap e121: 8 total, 8 up, 8 in 2023-09-30T09:23:09.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[216815]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: osdmap e121: 8 total, 8 up, 8 in 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:09 smithi019 ceph-mon[224043]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.244 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:09.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: osdmap e121: 8 total, 8 up, 8 in 2023-09-30T09:23:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:09 smithi167 ceph-mon[192880]: failed to encode map e121 with expected crc 2023-09-30T09:23:09.351 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:09.396 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:09.464 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:09.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:09.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:09.572 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:09.660 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:10.018 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 12 16 25385 25369 132.01 38.5625 0.00633186 0.00756802 2023-09-30T09:23:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: pgmap v247: 89 pgs: 7 remapped+peering, 5 peering, 77 active+clean; 1.4 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 179 MiB/s wr, 2.86k op/s; 3 B/s, 0 objects/s recovering 2023-09-30T09:23:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: osdmap e122: 8 total, 8 up, 8 in 2023-09-30T09:23:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[216815]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: pgmap v247: 89 pgs: 7 remapped+peering, 5 peering, 77 active+clean; 1.4 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 179 MiB/s wr, 2.86k op/s; 3 B/s, 0 objects/s recovering 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: osdmap e122: 8 total, 8 up, 8 in 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:10 smithi019 ceph-mon[224043]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.106 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:23:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: pgmap v247: 89 pgs: 7 remapped+peering, 5 peering, 77 active+clean; 1.4 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 179 MiB/s wr, 2.86k op/s; 3 B/s, 0 objects/s recovering 2023-09-30T09:23:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: osdmap e122: 8 total, 8 up, 8 in 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:10 smithi167 ceph-mon[192880]: failed to encode map e122 with expected crc 2023-09-30T09:23:11.019 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 13 16 26864 26848 128.967 92.4375 0.00166846 0.00774022 2023-09-30T09:23:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/182414498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:23:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/182414498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:23:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[216815]: failed to encode map e123 with expected crc 2023-09-30T09:23:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[216815]: osdmap e123: 8 total, 8 up, 8 in 2023-09-30T09:23:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[216815]: failed to encode map e123 with expected crc 2023-09-30T09:23:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[216815]: failed to encode map e123 with expected crc 2023-09-30T09:23:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[216815]: failed to encode map e123 with expected crc 2023-09-30T09:23:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/182414498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:23:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/182414498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:23:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[224043]: failed to encode map e123 with expected crc 2023-09-30T09:23:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[224043]: osdmap e123: 8 total, 8 up, 8 in 2023-09-30T09:23:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[224043]: failed to encode map e123 with expected crc 2023-09-30T09:23:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[224043]: failed to encode map e123 with expected crc 2023-09-30T09:23:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:11 smithi019 ceph-mon[224043]: failed to encode map e123 with expected crc 2023-09-30T09:23:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/182414498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:23:12.037 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:11 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:12.038 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/182414498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:23:12.038 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:11 smithi167 ceph-mon[192880]: failed to encode map e123 with expected crc 2023-09-30T09:23:12.038 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:11 smithi167 ceph-mon[192880]: osdmap e123: 8 total, 8 up, 8 in 2023-09-30T09:23:12.038 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:11 smithi167 ceph-mon[192880]: failed to encode map e123 with expected crc 2023-09-30T09:23:12.038 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:11 smithi167 ceph-mon[192880]: failed to encode map e123 with expected crc 2023-09-30T09:23:12.039 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:11 smithi167 ceph-mon[192880]: failed to encode map e123 with expected crc 2023-09-30T09:23:12.040 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:23:12.122 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 14 16 28058 28042 125.085 74.625 0.0115143 0.00798889 2023-09-30T09:23:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[216815]: failed to encode map e123 with expected crc 2023-09-30T09:23:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[216815]: pgmap v250: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 7 remapped+peering, 5 peering, 74 active+clean; 1.5 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 3040/72525 objects degraded (4.192%); 5 B/s, 3 objects/s recovering 2023-09-30T09:23:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 3040/72525 objects degraded (4.192%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[216815]: osdmap e124: 8 total, 8 up, 8 in 2023-09-30T09:23:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[216815]: failed to encode map e124 with expected crc 2023-09-30T09:23:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[224043]: failed to encode map e123 with expected crc 2023-09-30T09:23:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[224043]: pgmap v250: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 7 remapped+peering, 5 peering, 74 active+clean; 1.5 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 3040/72525 objects degraded (4.192%); 5 B/s, 3 objects/s recovering 2023-09-30T09:23:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 3040/72525 objects degraded (4.192%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[224043]: osdmap e124: 8 total, 8 up, 8 in 2023-09-30T09:23:12.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:12 smithi019 ceph-mon[224043]: failed to encode map e124 with expected crc 2023-09-30T09:23:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:12 smithi167 ceph-mon[192880]: failed to encode map e123 with expected crc 2023-09-30T09:23:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:12 smithi167 ceph-mon[192880]: pgmap v250: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 7 remapped+peering, 5 peering, 74 active+clean; 1.5 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 3040/72525 objects degraded (4.192%); 5 B/s, 3 objects/s recovering 2023-09-30T09:23:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:12 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 3040/72525 objects degraded (4.192%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:12 smithi167 ceph-mon[192880]: osdmap e124: 8 total, 8 up, 8 in 2023-09-30T09:23:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:12 smithi167 ceph-mon[192880]: failed to encode map e124 with expected crc 2023-09-30T09:23:12.560 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:12.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:12.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:12.674 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:12.749 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:12.886 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:12.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:12.991 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:13.019 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 15 16 29365 29349 122.193 81.6875 0.0208281 0.00817487 2023-09-30T09:23:13.041 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:13.134 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:13.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:13.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:13.223 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:13.246 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:13.284 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:13 smithi019 ceph-mon[224043]: failed to encode map e124 with expected crc 2023-09-30T09:23:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:13 smithi019 ceph-mon[224043]: failed to encode map e124 with expected crc 2023-09-30T09:23:13.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:13 smithi019 ceph-mon[216815]: failed to encode map e124 with expected crc 2023-09-30T09:23:13.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:13 smithi019 ceph-mon[216815]: failed to encode map e124 with expected crc 2023-09-30T09:23:13.350 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:13.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:13.355 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:13 smithi167 ceph-mon[192880]: failed to encode map e124 with expected crc 2023-09-30T09:23:13.355 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:13 smithi167 ceph-mon[192880]: failed to encode map e124 with expected crc 2023-09-30T09:23:13.403 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:23:13.440 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:23:13.456 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:13.487 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:23:13.500 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:13.528 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:23:13.564 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:13.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:13.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:13.569 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:23:13.588 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:23:13.588 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:23:13.588 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:23:13.588 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.871726 2023-09-30T09:23:13.588 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.8717261637091928 2023-09-30T09:23:13.605 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:23:13.651 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:23:13.670 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:13.681 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:23:13.705 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:23:13.705 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:23:13.705 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:23:13.705 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:23:13.705 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:23:13.705 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:23:13.705 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:23:13.705 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:23:13.711 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:13.761 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:13.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:13.864 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:13.903 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:13.972 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:13.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:13.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:14.019 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 16 16 30735 30719 119.908 85.625 0.0128387 0.00833314 2023-09-30T09:23:14.080 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:14.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:23:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:23:13] "GET /metrics HTTP/1.1" 200 36277 "" "Prometheus/2.43.0" 2023-09-30T09:23:14.156 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:14 smithi167 ceph-mon[192880]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 7 remapped+peering, 77 active+clean; 1.5 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 4495/74661 objects degraded (6.021%); 1311/74661 objects misplaced (1.756%); 2.5 MiB/s, 45 objects/s recovering 2023-09-30T09:23:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3598641844' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:14.574 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:23:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:14 smithi019 ceph-mon[224043]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 7 remapped+peering, 77 active+clean; 1.5 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 4495/74661 objects degraded (6.021%); 1311/74661 objects misplaced (1.756%); 2.5 MiB/s, 45 objects/s recovering 2023-09-30T09:23:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3598641844' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:14.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:14 smithi019 ceph-mon[216815]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 7 remapped+peering, 77 active+clean; 1.5 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 4495/74661 objects degraded (6.021%); 1311/74661 objects misplaced (1.756%); 2.5 MiB/s, 45 objects/s recovering 2023-09-30T09:23:14.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3598641844' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:15.019 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 17 16 32210 32194 118.278 92.1875 0.00183451 0.00843832 2023-09-30T09:23:16.019 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 18 16 33642 33626 116.68 89.5 0.0162802 0.00856429 2023-09-30T09:23:16.170 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:16.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:16.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:16.284 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:16.328 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:16.395 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:16.401 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:16 smithi167 ceph-mon[192880]: pgmap v253: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10502/90882 objects degraded (11.556%); 1311/90882 objects misplaced (1.443%); 2.1 MiB/s, 39 objects/s recovering 2023-09-30T09:23:16.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:16.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:16.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:16 smithi019 ceph-mon[224043]: pgmap v253: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10502/90882 objects degraded (11.556%); 1311/90882 objects misplaced (1.443%); 2.1 MiB/s, 39 objects/s recovering 2023-09-30T09:23:16.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:16 smithi019 ceph-mon[216815]: pgmap v253: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10502/90882 objects degraded (11.556%); 1311/90882 objects misplaced (1.443%); 2.1 MiB/s, 39 objects/s recovering 2023-09-30T09:23:16.503 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:16.542 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:16.605 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:16.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:16.648 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:16.712 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:16.772 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:16.799 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:17.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 19 16 35064 35048 115.208 88.875 0.0105725 0.00867426 2023-09-30T09:23:17.147 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:17.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:17.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:17.256 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:17.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:17 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 10502/93621 objects degraded (11.218%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:17.310 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:17 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 10502/93621 objects degraded (11.218%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:17.342 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:17.504 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:17.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:17.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:17.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:17 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 10502/93621 objects degraded (11.218%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:17.612 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:17.692 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:17.924 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:23:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:23:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:23:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:23:17.925 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:23:18.021 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:23:18.021318+0000 min lat: 0.00105726 max lat: 2.21224 avg lat: 0.00880838 2023-09-30T09:23:18.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:23:18.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 20 16 36348 36332 113.461 80.25 0.00797241 0.00880838 2023-09-30T09:23:18.473 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:23:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:18 smithi167 ceph-mon[192880]: pgmap v254: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10502/93621 objects degraded (11.218%); 829/93621 objects misplaced (0.885%); 5.4 MiB/s, 90 objects/s recovering 2023-09-30T09:23:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:18.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:18 smithi019 ceph-mon[216815]: pgmap v254: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10502/93621 objects degraded (11.218%); 829/93621 objects misplaced (0.885%); 5.4 MiB/s, 90 objects/s recovering 2023-09-30T09:23:18.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:18.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:18.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:18.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:18 smithi019 ceph-mon[224043]: pgmap v254: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10502/93621 objects degraded (11.218%); 829/93621 objects misplaced (0.885%); 5.4 MiB/s, 90 objects/s recovering 2023-09-30T09:23:18.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:18.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:18.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:23:19.021 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 21 16 37528 37512 111.57 73.75 0.0056276 0.00895736 2023-09-30T09:23:20.021 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 22 16 38953 38937 110.547 89.0625 0.0128406 0.00903956 2023-09-30T09:23:20.167 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:20.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:20.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:20.271 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:20.322 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:20.409 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:20.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:20.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:20.437 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:20 smithi019 ceph-mon[216815]: pgmap v255: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 10502/104988 objects degraded (10.003%); 829/104988 objects misplaced (0.790%); 4.8 MiB/s, 79 objects/s recovering 2023-09-30T09:23:20.439 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:20 smithi019 ceph-mon[224043]: pgmap v255: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 10502/104988 objects degraded (10.003%); 829/104988 objects misplaced (0.790%); 4.8 MiB/s, 79 objects/s recovering 2023-09-30T09:23:20.520 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:20.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:20 smithi167 ceph-mon[192880]: pgmap v255: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 10502/104988 objects degraded (10.003%); 829/104988 objects misplaced (0.790%); 4.8 MiB/s, 79 objects/s recovering 2023-09-30T09:23:20.545 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:20.587 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:20.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:20.692 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:20.761 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:20.983 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:20.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:21.009 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:23:21.021 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 23 16 40076 40060 108.793 70.1875 0.00929244 0.00918728 2023-09-30T09:23:21.089 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:21.129 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:21.198 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:21.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:21.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:21.303 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:21.433 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:21.531 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:21.612 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:21.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:21.724 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:21.827 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:21.977 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:21.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:21.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:22.021 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 24 16 41168 41152 107.104 68.25 0.00821324 0.00933124 2023-09-30T09:23:22.081 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:22.161 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:22.168 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:22.235 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:23:22.249 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.5 primary-affinity to 0.871726 (8571292) 2023-09-30T09:23:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:22 smithi167 ceph-mon[192880]: pgmap v256: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10502/111195 objects degraded (9.445%); 829/111195 objects misplaced (0.746%); 4.3 MiB/s, 70 objects/s recovering 2023-09-30T09:23:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3274194821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:22 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2568691436' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8717261637091928}]: dispatch 2023-09-30T09:23:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:22 smithi019 ceph-mon[216815]: pgmap v256: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10502/111195 objects degraded (9.445%); 829/111195 objects misplaced (0.746%); 4.3 MiB/s, 70 objects/s recovering 2023-09-30T09:23:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3274194821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:22 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2568691436' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8717261637091928}]: dispatch 2023-09-30T09:23:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:22 smithi019 ceph-mon[224043]: pgmap v256: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10502/111195 objects degraded (9.445%); 829/111195 objects misplaced (0.746%); 4.3 MiB/s, 70 objects/s recovering 2023-09-30T09:23:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3274194821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:22 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2568691436' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8717261637091928}]: dispatch 2023-09-30T09:23:23.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 25 16 42516 42500 106.19 84.25 0.00264597 0.00939091 2023-09-30T09:23:23.453 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:23.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:23:23.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:23:23.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2568691436' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8717261637091928}]': finished 2023-09-30T09:23:23.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: osdmap e125: 8 total, 8 up, 8 in 2023-09-30T09:23:23.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 10502/111195 objects degraded (9.445%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:23.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2568691436' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8717261637091928}]': finished 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: osdmap e125: 8 total, 8 up, 8 in 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 10502/111195 objects degraded (9.445%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:23:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:23:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2568691436' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8717261637091928}]': finished 2023-09-30T09:23:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: osdmap e125: 8 total, 8 up, 8 in 2023-09-30T09:23:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: failed to encode map e125 with expected crc 2023-09-30T09:23:23.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 10502/111195 objects degraded (9.445%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:23.561 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:23.621 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:23.809 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:23:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:23:23] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.43.0" 2023-09-30T09:23:23.834 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:23.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:23.941 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:23.967 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:24.009 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:24.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:24.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:24.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 26 16 43703 43687 104.958 74.1875 0.00591881 0.0095225 2023-09-30T09:23:24.119 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:24.151 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:24.192 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:24.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:24.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:24.297 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:23:24.342 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:24.414 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:23:24.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:24.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:23:24.521 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 10502/114618 objects degraded (9.163%); 375/114618 objects misplaced (0.327%); 5.9 MiB/s, 95 objects/s recovering 2023-09-30T09:23:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: osdmap e126: 8 total, 8 up, 8 in 2023-09-30T09:23:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:24 smithi167 ceph-mon[192880]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 10502/114618 objects degraded (9.163%); 375/114618 objects misplaced (0.327%); 5.9 MiB/s, 95 objects/s recovering 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: osdmap e126: 8 total, 8 up, 8 in 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[216815]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 10502/114618 objects degraded (9.163%); 375/114618 objects misplaced (0.327%); 5.9 MiB/s, 95 objects/s recovering 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: osdmap e126: 8 total, 8 up, 8 in 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:24 smithi019 ceph-mon[224043]: failed to encode map e126 with expected crc 2023-09-30T09:23:24.588 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:25.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 27 15 44599 44584 103.148 56.0625 0.00440997 0.00968997 2023-09-30T09:23:25.322 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:25.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:25.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:25.430 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:25.479 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:25.549 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:25.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:25.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:25.657 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:25.717 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:25.812 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:25.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:25.920 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:26.006 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:26.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 28 16 46008 45992 102.607 88 0.00329561 0.00973637 2023-09-30T09:23:26.233 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:26 smithi167 ceph-mon[192880]: pgmap v260: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 10502/129714 objects degraded (8.096%); 375/129714 objects misplaced (0.289%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:23:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:26 smithi019 ceph-mon[224043]: pgmap v260: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 10502/129714 objects degraded (8.096%); 375/129714 objects misplaced (0.289%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:23:26.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:26 smithi019 ceph-mon[216815]: pgmap v260: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 10502/129714 objects degraded (8.096%); 375/129714 objects misplaced (0.289%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:23:26.578 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:26.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:26.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:26.682 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:26.708 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:26.751 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:26.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:26.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:26.838 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:23:26.856 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:26.859 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:23:26.880 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:23:26.888 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:26.905 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:23:26.928 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:23:26.941 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:26.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:26.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:26.957 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:23:26.978 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:23:26.999 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:23:27.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:23:27.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:23:27.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:23:27.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:23:27.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:23:27.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:23:27.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:23:27.002 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:23:27.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 29 16 46391 46375 99.8955 23.9375 0.00303911 0.00978837 2023-09-30T09:23:27.045 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:27.077 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:27.175 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:23:27.175 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:23:27.203 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:23:27.285 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:27.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:27.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:27.407 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:27.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[216815]: osdmap e127: 8 total, 8 up, 8 in 2023-09-30T09:23:27.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[216815]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[216815]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[216815]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[216815]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[216815]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2924108105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:27.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[224043]: osdmap e127: 8 total, 8 up, 8 in 2023-09-30T09:23:27.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[224043]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[224043]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[224043]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[224043]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[224043]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2924108105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:27.472 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:27.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:27 smithi167 ceph-mon[192880]: osdmap e127: 8 total, 8 up, 8 in 2023-09-30T09:23:27.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:27 smithi167 ceph-mon[192880]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:27 smithi167 ceph-mon[192880]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:27 smithi167 ceph-mon[192880]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:27 smithi167 ceph-mon[192880]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:27 smithi167 ceph-mon[192880]: failed to encode map e127 with expected crc 2023-09-30T09:23:27.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2924108105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:28.022 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 30 16 47366 47350 98.5973 60.9375 0.00809606 0.0101371 2023-09-30T09:23:28.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[216815]: pgmap v262: 89 pgs: 1 creating+peering, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 10502/131676 objects degraded (7.976%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[216815]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[216815]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[216815]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[216815]: osdmap e128: 8 total, 8 up, 8 in 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[216815]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[216815]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 10502/131676 objects degraded (7.976%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[224043]: pgmap v262: 89 pgs: 1 creating+peering, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 10502/131676 objects degraded (7.976%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:23:28.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[224043]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[224043]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[224043]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[224043]: osdmap e128: 8 total, 8 up, 8 in 2023-09-30T09:23:28.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[224043]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[224043]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 10502/131676 objects degraded (7.976%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:28.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:28 smithi167 ceph-mon[192880]: pgmap v262: 89 pgs: 1 creating+peering, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 10502/131676 objects degraded (7.976%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:23:28.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:28 smithi167 ceph-mon[192880]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:28 smithi167 ceph-mon[192880]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:28 smithi167 ceph-mon[192880]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:28 smithi167 ceph-mon[192880]: osdmap e128: 8 total, 8 up, 8 in 2023-09-30T09:23:28.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:28 smithi167 ceph-mon[192880]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:28 smithi167 ceph-mon[192880]: failed to encode map e128 with expected crc 2023-09-30T09:23:28.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 10502/131676 objects degraded (7.976%), 9 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:28.648 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:28.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:28.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:28.743 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:23:28.753 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:28.864 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:29.000 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:29.025 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 31 16 48587 48571 97.8716 76.3125 0.0167335 0.0102106 2023-09-30T09:23:29.071 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:29.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:29.187 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:29.250 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:29.330 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:29.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:29.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:29.437 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:29.491 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:29.594 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:29.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:29.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:29.701 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:29.750 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:29.848 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:29.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:29.955 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:30.022 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:30.024 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 32 16 49902 49886 97.3814 82.1875 0.0116054 0.0102636 2023-09-30T09:23:30.117 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:30.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:30.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:30.227 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:30.253 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:30.288 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:23:30.289 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:23:30.289 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-30T09:23:30.289 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:23:30.289 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:23:30.289 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-30T09:23:30.295 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:30.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:30.400 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:30.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:30 smithi019 ceph-mon[216815]: pgmap v264: 89 pgs: 1 creating+peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 9019/139959 objects degraded (6.444%); 1034/139959 objects misplaced (0.739%); 8.7 MiB/s, 137 objects/s recovering 2023-09-30T09:23:30.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:30 smithi019 ceph-mon[224043]: pgmap v264: 89 pgs: 1 creating+peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 9019/139959 objects degraded (6.444%); 1034/139959 objects misplaced (0.739%); 8.7 MiB/s, 137 objects/s recovering 2023-09-30T09:23:30.451 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:30.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:30 smithi167 ceph-mon[192880]: pgmap v264: 89 pgs: 1 creating+peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 9019/139959 objects degraded (6.444%); 1034/139959 objects misplaced (0.739%); 8.7 MiB/s, 137 objects/s recovering 2023-09-30T09:23:30.702 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:23:30.725 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:30.731 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:30.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:30.841 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:23:30.849 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:23:30.900 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:31.025 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 33 16 50897 50881 96.3152 62.1875 0.0213596 0.0103771 2023-09-30T09:23:31.029 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:23:31.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:31.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:23:31.141 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:31.228 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:31.410 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:31.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:31.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:31.515 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:23:31.575 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:31.687 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:23:31.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:31.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:23:31.791 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:31.842 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:31.924 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:31.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:31.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:32.025 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 34 16 52129 52113 95.7472 77 0.00595512 0.0104363 2023-09-30T09:23:32.031 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:32.095 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:32.168 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:32.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:32.272 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:32.281 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:23:32.300 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:32 smithi019 ceph-mon[216815]: pgmap v265: 89 pgs: 1 creating+peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 9019/144882 objects degraded (6.225%); 1034/144882 objects misplaced (0.714%); 6.7 MiB/s, 106 objects/s recovering 2023-09-30T09:23:32.300 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1406016778' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:23:32.301 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:32 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:23:32.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:32 smithi019 ceph-mon[224043]: pgmap v265: 89 pgs: 1 creating+peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 9019/144882 objects degraded (6.225%); 1034/144882 objects misplaced (0.714%); 6.7 MiB/s, 106 objects/s recovering 2023-09-30T09:23:32.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1406016778' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:23:32.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:32 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:23:32.303 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:32.365 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:32.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:32.469 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:32.530 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:32.631 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:32.636 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:32 smithi167 ceph-mon[192880]: pgmap v265: 89 pgs: 1 creating+peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 9019/144882 objects degraded (6.225%); 1034/144882 objects misplaced (0.714%); 6.7 MiB/s, 106 objects/s recovering 2023-09-30T09:23:32.637 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1406016778' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:23:32.637 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:32 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:23:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:32.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:32.738 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:34.153 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:34.153 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:23:34.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: osdmap e129: 8 total, 8 up, 8 in 2023-09-30T09:23:34.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:34.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9019/144882 objects degraded (6.225%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:34.163 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 35 16 53152 53136 94.8385 63.9375 0.00285075 0.0105279 2023-09-30T09:23:34.163 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:34.163 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:23:34.163 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.163 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: osdmap e129: 8 total, 8 up, 8 in 2023-09-30T09:23:34.163 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.163 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9019/144882 objects degraded (6.225%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: osdmap e129: 8 total, 8 up, 8 in 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: failed to encode map e129 with expected crc 2023-09-30T09:23:34.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:34.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9019/144882 objects degraded (6.225%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:34.165 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:34.166 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 36 16 54502 54486 94.5381 84.375 0.0144351 0.010572 2023-09-30T09:23:34.167 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:23:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:23:33] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.43.0" 2023-09-30T09:23:34.211 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:23:34.233 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:23:34.308 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:34.497 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:34 smithi019 ceph-mon[224043]: pgmap v267: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 9019/147921 objects degraded (6.097%); 1034/147921 objects misplaced (0.699%); 7.7 MiB/s, 122 objects/s recovering 2023-09-30T09:23:34.497 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:34 smithi019 ceph-mon[216815]: pgmap v267: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 9019/147921 objects degraded (6.097%); 1034/147921 objects misplaced (0.699%); 7.7 MiB/s, 122 objects/s recovering 2023-09-30T09:23:34.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:34 smithi167 ceph-mon[192880]: pgmap v267: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 9019/147921 objects degraded (6.097%); 1034/147921 objects misplaced (0.699%); 7.7 MiB/s, 122 objects/s recovering 2023-09-30T09:23:35.031 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 37 16 55759 55743 94.1017 78.5625 0.00840194 0.0106218 2023-09-30T09:23:36.033 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 38 16 56935 56919 93.5597 73.5 0.00513832 0.0106802 2023-09-30T09:23:36.167 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:23:36.228 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:23:36.233 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:36.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:36.341 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:23:36.389 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:36.466 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:23:36.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:36.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:23:36.573 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:36 smithi019 ceph-mon[216815]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 9019/162240 objects degraded (5.559%); 553/162240 objects misplaced (0.341%); 7.3 MiB/s, 116 objects/s recovering 2023-09-30T09:23:36.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:36 smithi019 ceph-mon[224043]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 9019/162240 objects degraded (5.559%); 553/162240 objects misplaced (0.341%); 7.3 MiB/s, 116 objects/s recovering 2023-09-30T09:23:36.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:36 smithi167 ceph-mon[192880]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 9019/162240 objects degraded (5.559%); 553/162240 objects misplaced (0.341%); 7.3 MiB/s, 116 objects/s recovering 2023-09-30T09:23:36.628 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:36.706 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:36.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:36.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:36.813 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:36.901 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:37.031 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 39 16 57969 57953 92.818 64.625 0.0264798 0.0107688 2023-09-30T09:23:37.069 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:23:37.111 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:23:37.112 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:37.155 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:23:37.160 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:37.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:37.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:37.195 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:23:37.200 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:37.232 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:23:37.269 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:37.278 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:23:37.317 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:23:37.342 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:37.359 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:23:37.368 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:23:37.368 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:23:37.368 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:23:37.369 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:23:37.369 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:23:37.369 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:23:37.369 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:23:37.369 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:23:37.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2826357628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:37.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2826357628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:37.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2826357628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:37.908 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:23:37.908 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "" 2023-09-30T09:23:37.908 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:23:38.031 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:23:38.031126+0000 min lat: 0.00105726 max lat: 2.21224 avg lat: 0.0108329 2023-09-30T09:23:38.031 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:23:38.031 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 40 16 59047 59031 92.1821 67.375 0.0658652 0.0108329 2023-09-30T09:23:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:38 smithi019 ceph-mon[216815]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 9019/164481 objects degraded (5.483%); 553/164481 objects misplaced (0.336%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:38.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9019/164481 objects degraded (5.483%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:38.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:38 smithi019 ceph-mon[224043]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 9019/164481 objects degraded (5.483%); 553/164481 objects misplaced (0.336%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:38.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9019/164481 objects degraded (5.483%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:38.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:38 smithi167 ceph-mon[192880]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 9019/164481 objects degraded (5.483%); 553/164481 objects misplaced (0.336%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:38.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9019/164481 objects degraded (5.483%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:39.032 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 41 16 59911 59895 91.2502 54 0.078277 0.0109443 2023-09-30T09:23:39.466 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:39.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:39.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:39.574 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:39.624 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:39.733 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:39.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:39.840 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:39.902 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:40.032 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 42 16 61142 61126 90.9094 76.9375 0.0107167 0.0109949 2023-09-30T09:23:40.118 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:40.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:40.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:40.225 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:40.270 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:40 smithi019 ceph-mon[224043]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 9019/173781 objects degraded (5.190%); 79/173781 objects misplaced (0.045%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:40.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:40 smithi019 ceph-mon[216815]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 9019/173781 objects degraded (5.190%); 79/173781 objects misplaced (0.045%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:40.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:40 smithi167 ceph-mon[192880]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 9019/173781 objects degraded (5.190%); 79/173781 objects misplaced (0.045%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:40.824 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:40.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:40.927 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:40.960 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:41.032 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 43 16 61753 61737 89.6838 38.1875 0.00251134 0.0110367 2023-09-30T09:23:41.063 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:41.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:41.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:41.167 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:41.200 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:41.247 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:23:41.351 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:41.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:41.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:41.459 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:41.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:41 smithi019 ceph-mon[216815]: osdmap e130: 8 total, 8 up, 8 in 2023-09-30T09:23:41.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:41 smithi019 ceph-mon[216815]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:41 smithi019 ceph-mon[216815]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:41 smithi019 ceph-mon[216815]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.470 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:41 smithi019 ceph-mon[224043]: osdmap e130: 8 total, 8 up, 8 in 2023-09-30T09:23:41.470 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:41 smithi019 ceph-mon[224043]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.470 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:41 smithi019 ceph-mon[224043]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.470 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:41 smithi019 ceph-mon[224043]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.569 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:41.685 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:41.793 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:41 smithi167 ceph-mon[192880]: osdmap e130: 8 total, 8 up, 8 in 2023-09-30T09:23:41.794 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:41 smithi167 ceph-mon[192880]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.794 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:41 smithi167 ceph-mon[192880]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.794 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:41 smithi167 ceph-mon[192880]: failed to encode map e130 with expected crc 2023-09-30T09:23:41.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:41.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:41.795 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:23:41.795 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:23:41.795 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:23:41.795 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:23:41.795 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:23:41.896 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:41.942 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:42.032 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 44 16 62421 62405 88.5948 41.75 0.0172681 0.0112815 2023-09-30T09:23:42.060 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:42.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:42.167 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:42.231 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:42.344 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:42.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:42.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:42.395 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:23:42.452 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:42.483 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:42.572 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:42.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:42.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:42.680 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:23:42.737 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:42 smithi167 ceph-mon[192880]: pgmap v272: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 9019/179019 objects degraded (5.038%); 79/179019 objects misplaced (0.044%); 6.9 MiB/s, 109 objects/s recovering 2023-09-30T09:23:42.738 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:42 smithi167 ceph-mon[192880]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.738 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:42 smithi167 ceph-mon[192880]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.738 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:42 smithi167 ceph-mon[192880]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.738 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:42 smithi167 ceph-mon[192880]: osdmap e131: 8 total, 8 up, 8 in 2023-09-30T09:23:42.738 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:42 smithi167 ceph-mon[192880]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.738 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:42 smithi167 ceph-mon[192880]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.738 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/391282322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:42.743 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[224043]: pgmap v272: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 9019/179019 objects degraded (5.038%); 79/179019 objects misplaced (0.044%); 6.9 MiB/s, 109 objects/s recovering 2023-09-30T09:23:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[224043]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[224043]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[224043]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[224043]: osdmap e131: 8 total, 8 up, 8 in 2023-09-30T09:23:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[224043]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[224043]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/391282322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[216815]: pgmap v272: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 9019/179019 objects degraded (5.038%); 79/179019 objects misplaced (0.044%); 6.9 MiB/s, 109 objects/s recovering 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[216815]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[216815]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[216815]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[216815]: osdmap e131: 8 total, 8 up, 8 in 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[216815]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[216815]: failed to encode map e131 with expected crc 2023-09-30T09:23:42.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/391282322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:23:42.842 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:23:42.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:42.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:23:42.951 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:43.032 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:43.032 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 45 16 63572 63556 88.2247 71.9375 0.0245916 0.0113282 2023-09-30T09:23:43.497 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:43.524 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:23:43.621 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:43.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:43.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:43.625 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:43.625 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/391282322' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:23:43.625 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: osdmap e132: 8 total, 8 up, 8 in 2023-09-30T09:23:43.626 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.626 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.626 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.626 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.626 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.626 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9019/179019 objects degraded (5.038%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:43.629 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:43.629 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/391282322' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:23:43.629 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: osdmap e132: 8 total, 8 up, 8 in 2023-09-30T09:23:43.629 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.629 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.630 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.630 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.630 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.630 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9019/179019 objects degraded (5.038%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:43.726 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:43.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:43.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/391282322' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:23:43.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: osdmap e132: 8 total, 8 up, 8 in 2023-09-30T09:23:43.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: failed to encode map e132 with expected crc 2023-09-30T09:23:43.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9019/179019 objects degraded (5.038%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:43.774 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:43.845 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:43.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:43.949 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:23:43.974 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:23:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:23:43] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.43.0" 2023-09-30T09:23:44.002 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:44.032 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 46 16 64446 64430 87.4945 54.625 0.0142253 0.0114236 2023-09-30T09:23:44.098 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:23:44.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:44.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:23:44.105 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:23:44.105 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:23:44.105 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-30T09:23:44.106 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-30T09:23:44.203 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:44.245 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:44.307 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:44.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:44.375 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:44 smithi167 ceph-mon[192880]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 9019/181491 objects degraded (4.969%); 79/181491 objects misplaced (0.044%); 4.9 MiB/s, 78 objects/s recovering 2023-09-30T09:23:44.419 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:44.489 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:44.600 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:44.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:44.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:44.704 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:44.729 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:44.771 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:44.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:44.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:44 smithi019 ceph-mon[216815]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 9019/181491 objects degraded (4.969%); 79/181491 objects misplaced (0.044%); 4.9 MiB/s, 78 objects/s recovering 2023-09-30T09:23:44.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:44 smithi019 ceph-mon[224043]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 9019/181491 objects degraded (4.969%); 79/181491 objects misplaced (0.044%); 4.9 MiB/s, 78 objects/s recovering 2023-09-30T09:23:44.875 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:44.926 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:45.032 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 47 16 65301 65285 86.7701 53.4375 0.00957623 0.0115196 2023-09-30T09:23:45.235 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:45.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:45.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:45.310 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:45.343 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:45.393 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:45.470 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:45.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:45.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:45.501 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:45.575 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:45.643 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:45.723 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:45.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:45.798 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:23:45.834 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:45.966 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:46.033 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 48 16 66373 66357 86.3578 67 0.00857551 0.0115734 2023-09-30T09:23:46.071 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:23:46.112 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:23:46.114 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:46.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:46.146 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:23:46.182 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:23:46.222 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:23:46.223 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:46.254 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:23:46.268 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:46.305 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:23:46.342 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:23:46.348 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:23:46.348 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:23:46.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:23:46.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:23:46.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:23:46.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:23:46.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:23:46.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:23:46.355 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:46.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:46.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:46.462 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:46 smithi167 ceph-mon[192880]: pgmap v276: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 7550/192423 objects degraded (3.924%); 1023/192423 objects misplaced (0.532%); 5.5 MiB/s, 87 objects/s recovering 2023-09-30T09:23:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3258610715' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:46.538 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:46.580 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:46.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:46.685 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:46.752 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:46 smithi019 ceph-mon[216815]: pgmap v276: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 7550/192423 objects degraded (3.924%); 1023/192423 objects misplaced (0.532%); 5.5 MiB/s, 87 objects/s recovering 2023-09-30T09:23:46.752 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3258610715' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:46.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:46 smithi019 ceph-mon[224043]: pgmap v276: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 7550/192423 objects degraded (3.924%); 1023/192423 objects misplaced (0.532%); 5.5 MiB/s, 87 objects/s recovering 2023-09-30T09:23:46.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3258610715' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:46.754 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:47.032 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 49 16 67383 67367 85.8837 63.125 0.00248884 0.0116293 2023-09-30T09:23:47.637 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:47.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:47.748 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:47.824 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:48.036 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 50 16 68305 68289 85.3121 57.625 0.012526 0.0117155 2023-09-30T09:23:48.299 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:48.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:48.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:48.404 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:23:48.462 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:48.561 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:23:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:48.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:23:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:48 smithi019 ceph-mon[216815]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s; 7550/193377 objects degraded (3.904%); 1023/193377 objects misplaced (0.529%); 4.9 MiB/s, 78 objects/s recovering 2023-09-30T09:23:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7550/193377 objects degraded (3.904%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:48 smithi019 ceph-mon[224043]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s; 7550/193377 objects degraded (3.904%); 1023/193377 objects misplaced (0.529%); 4.9 MiB/s, 78 objects/s recovering 2023-09-30T09:23:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7550/193377 objects degraded (3.904%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:48.666 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:48.717 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:48 smithi167 ceph-mon[192880]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s; 7550/193377 objects degraded (3.904%); 1023/193377 objects misplaced (0.529%); 4.9 MiB/s, 78 objects/s recovering 2023-09-30T09:23:48.717 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:23:48.717 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7550/193377 objects degraded (3.904%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:48.721 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:48.832 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:48.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:48.939 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:49.009 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:49.037 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 51 16 69425 69409 85.0118 70 0.0106427 0.0117583 2023-09-30T09:23:49.108 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:49.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:49.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:49.232 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:49.272 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:49.358 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:49.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:49.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:49.467 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:49.475 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:49.475 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:23:49.475 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:23:49.504 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:49.583 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:49.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:49.691 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:49.768 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:50.037 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 52 16 70481 70465 84.6463 66 0.0113425 0.011808 2023-09-30T09:23:50.295 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:50.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:50.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:50.384 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:23:50.403 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:50.450 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:50.566 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:50.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:50.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:50.675 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:50.723 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:50 smithi019 ceph-mon[224043]: pgmap v278: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 7550/202602 objects degraded (3.727%); 627/202602 objects misplaced (0.309%); 7.2 MiB/s, 115 objects/s recovering 2023-09-30T09:23:50.723 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:50 smithi019 ceph-mon[216815]: pgmap v278: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 7550/202602 objects degraded (3.727%); 627/202602 objects misplaced (0.309%); 7.2 MiB/s, 115 objects/s recovering 2023-09-30T09:23:50.725 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:50 smithi167 ceph-mon[192880]: pgmap v278: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 7550/202602 objects degraded (3.727%); 627/202602 objects misplaced (0.309%); 7.2 MiB/s, 115 objects/s recovering 2023-09-30T09:23:50.725 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:50.814 INFO:teuthology.orchestra.run.smithi019.stdout:moved 0 / 267 (0%) 2023-09-30T09:23:50.814 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:23:50.814 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 4.02919 -> 4.02919 (expected baseline 5.40399) 2023-09-30T09:23:50.814 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.2 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-09-30T09:23:50.814 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.4 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-30T09:23:50.815 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:23:50.815 INFO:teuthology.orchestra.run.smithi019.stdout:oload 1000 2023-09-30T09:23:50.815 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 1 2023-09-30T09:23:50.815 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:23:50.815 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0182 2023-09-30T09:23:50.815 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.1824 2023-09-30T09:23:50.817 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:50.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:50.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:50.823 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:23:50.924 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:51.000 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:51.041 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 53 16 71377 71361 84.0994 56 0.0159511 0.0118842 2023-09-30T09:23:51.153 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:23:51.218 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:51.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:51.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:51.334 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:51.392 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:51.474 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:51.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:51.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:51.581 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:51.640 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:51.732 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:51.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:51.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:51.838 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:51.901 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:51.991 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:52.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:52.041 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 54 16 72318 72302 83.6312 58.8125 0.00514378 0.0119502 2023-09-30T09:23:52.106 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:52.135 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:52.540 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:52.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:52.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:52.552 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-30T09:23:52.553 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-30T09:23:52.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:52 smithi019 ceph-mon[216815]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:23:52.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:52 smithi019 ceph-mon[216815]: pgmap v279: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 7550/207174 objects degraded (3.644%); 627/207174 objects misplaced (0.303%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:52 smithi019 ceph-mon[224043]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:23:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:52 smithi019 ceph-mon[224043]: pgmap v279: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 7550/207174 objects degraded (3.644%); 627/207174 objects misplaced (0.303%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:52.650 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:52.694 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:52 smithi167 ceph-mon[192880]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:23:52.694 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:52 smithi167 ceph-mon[192880]: pgmap v279: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 7550/207174 objects degraded (3.644%); 627/207174 objects misplaced (0.303%); 6.0 MiB/s, 95 objects/s recovering 2023-09-30T09:23:52.700 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:52.768 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:52.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:52.878 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:23:52.915 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:52.966 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:23:52.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:52.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:23:53.000 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:53.015 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:53.041 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 55 16 73196 73180 83.1086 54.875 0.00964001 0.0120271 2023-09-30T09:23:53.077 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:23:53.187 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:53.383 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:23:53.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:53.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:23:53.489 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:53.586 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7550/207174 objects degraded (3.644%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:53.802 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:53.810 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7550/207174 objects degraded (3.644%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:53.810 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:23:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:23:53] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.43.0" 2023-09-30T09:23:53.810 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7550/207174 objects degraded (3.644%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:53.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:53.913 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:23:53.986 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:54.041 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 56 16 74156 74140 82.696 60 0.0164199 0.0120874 2023-09-30T09:23:54.090 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:23:54.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:54.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:23:54.199 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:54.247 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:54.495 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:23:54.514 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:23:54.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:54 smithi019 ceph-mon[224043]: pgmap v280: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 7550/209985 objects degraded (3.595%); 627/209985 objects misplaced (0.299%); 5.4 MiB/s, 86 objects/s recovering 2023-09-30T09:23:54.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:54 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3410796271' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:23:54.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:54 smithi019 ceph-mon[216815]: pgmap v280: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 7550/209985 objects degraded (3.595%); 627/209985 objects misplaced (0.299%); 5.4 MiB/s, 86 objects/s recovering 2023-09-30T09:23:54.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:54 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3410796271' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:23:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:54 smithi167 ceph-mon[192880]: pgmap v280: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 7550/209985 objects degraded (3.595%); 627/209985 objects misplaced (0.299%); 5.4 MiB/s, 86 objects/s recovering 2023-09-30T09:23:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:54 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3410796271' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:23:55.044 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 57 16 74959 74943 82.1238 50.1875 0.0033683 0.0121583 2023-09-30T09:23:55.579 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:55.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:55.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:55.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:55 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:55.795 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3410796271' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:23:55.795 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:55 smithi167 ceph-mon[192880]: osdmap e133: 8 total, 8 up, 8 in 2023-09-30T09:23:55.795 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:55 smithi167 ceph-mon[192880]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.795 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:55 smithi167 ceph-mon[192880]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.795 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:55 smithi167 ceph-mon[192880]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.795 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:55 smithi167 ceph-mon[192880]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.830 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:23:55.868 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:55.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3410796271' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:23:55.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[224043]: osdmap e133: 8 total, 8 up, 8 in 2023-09-30T09:23:55.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[224043]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[224043]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[224043]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.869 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[224043]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:23:55.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3410796271' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:23:55.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[216815]: osdmap e133: 8 total, 8 up, 8 in 2023-09-30T09:23:55.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[216815]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[216815]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[216815]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:55 smithi019 ceph-mon[216815]: failed to encode map e133 with expected crc 2023-09-30T09:23:55.913 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:56.043 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 58 16 75855 75839 81.6735 56 0.0038839 0.0121797 2023-09-30T09:23:56.531 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:23:56.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:56.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:23:56.641 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:56.701 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 7550/221394 objects degraded (3.410%); 117/221394 objects misplaced (0.053%); 5.7 MiB/s, 90 objects/s recovering 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: osdmap e134: 8 total, 8 up, 8 in 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:56 smithi167 ceph-mon[192880]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 7550/221394 objects degraded (3.410%); 117/221394 objects misplaced (0.053%); 5.7 MiB/s, 90 objects/s recovering 2023-09-30T09:23:56.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: osdmap e134: 8 total, 8 up, 8 in 2023-09-30T09:23:56.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.780 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.780 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.780 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[216815]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.780 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 7550/221394 objects degraded (3.410%); 117/221394 objects misplaced (0.053%); 5.7 MiB/s, 90 objects/s recovering 2023-09-30T09:23:56.781 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.781 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.781 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: osdmap e134: 8 total, 8 up, 8 in 2023-09-30T09:23:56.781 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.781 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.781 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.781 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.781 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:56 smithi019 ceph-mon[224043]: failed to encode map e134 with expected crc 2023-09-30T09:23:56.973 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:56.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:56.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:57.046 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 59 16 76445 76429 80.9141 36.875 0.0126031 0.0123537 2023-09-30T09:23:57.082 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:23:57.178 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:57.248 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:23:57.248 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:23:57.306 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:23:57.344 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:23:57.403 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:23:57.445 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:23:57.481 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:23:57.517 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:23:57.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[216815]: osdmap e135: 8 total, 8 up, 8 in 2023-09-30T09:23:57.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[216815]: failed to encode map e135 with expected crc 2023-09-30T09:23:57.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[216815]: failed to encode map e135 with expected crc 2023-09-30T09:23:57.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[216815]: failed to encode map e135 with expected crc 2023-09-30T09:23:57.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4161877629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:57.550 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:23:57.597 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:23:57.624 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:23:57.624 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:23:57.624 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:23:57.624 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:23:57.624 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:23:57.624 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:23:57.624 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:23:57.624 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:23:57.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[224043]: osdmap e135: 8 total, 8 up, 8 in 2023-09-30T09:23:57.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[224043]: failed to encode map e135 with expected crc 2023-09-30T09:23:57.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[224043]: failed to encode map e135 with expected crc 2023-09-30T09:23:57.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[224043]: failed to encode map e135 with expected crc 2023-09-30T09:23:57.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:57 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4161877629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:57 smithi167 ceph-mon[192880]: osdmap e135: 8 total, 8 up, 8 in 2023-09-30T09:23:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:57 smithi167 ceph-mon[192880]: failed to encode map e135 with expected crc 2023-09-30T09:23:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:57 smithi167 ceph-mon[192880]: failed to encode map e135 with expected crc 2023-09-30T09:23:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:57 smithi167 ceph-mon[192880]: failed to encode map e135 with expected crc 2023-09-30T09:23:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:57 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4161877629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:23:58.043 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:23:58.043492+0000 min lat: 0.00105726 max lat: 2.21224 avg lat: 0.0124155 2023-09-30T09:23:58.043 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:23:58.043 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 60 16 77325 77309 80.4823 55 0.00990937 0.0124155 2023-09-30T09:23:58.482 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:23:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:58.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:23:58.552 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:23:58.587 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:23:58.647 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:58.745 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:23:58.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:58.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:23:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:58 smithi019 ceph-mon[216815]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 7550/222261 objects degraded (3.397%); 117/222261 objects misplaced (0.053%); 5.3 MiB/s, 84 objects/s recovering 2023-09-30T09:23:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:23:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:23:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7550/222261 objects degraded (3.397%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:58 smithi019 ceph-mon[224043]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 7550/222261 objects degraded (3.397%); 117/222261 objects misplaced (0.053%); 5.3 MiB/s, 84 objects/s recovering 2023-09-30T09:23:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:23:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:23:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7550/222261 objects degraded (3.397%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:58.858 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:58.925 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:58 smithi167 ceph-mon[192880]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 7550/222261 objects degraded (3.397%); 117/222261 objects misplaced (0.053%); 5.3 MiB/s, 84 objects/s recovering 2023-09-30T09:23:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:23:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:23:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7550/222261 objects degraded (3.397%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:23:59.043 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 61 16 78201 78185 80.0605 54.75 0.0141505 0.0124855 2023-09-30T09:23:59.285 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:59.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:59.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:59.393 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:59.488 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:23:59.585 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:23:59.669 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:23:59.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:23:59.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:23:59.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:23:59.901 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:00.020 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:00.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:00.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:00.044 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 62 16 79052 79036 79.6272 53.1875 0.0250035 0.0125515 2023-09-30T09:24:00.129 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:00.220 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:00.244 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:00.492 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:00.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:00.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:00.602 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:00.644 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:00.707 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:00.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:00.780 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:00 smithi019 ceph-mon[216815]: pgmap v286: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 7550/229977 objects degraded (3.283%); 6.5 MiB/s, 104 objects/s recovering 2023-09-30T09:24:00.780 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:00 smithi019 ceph-mon[224043]: pgmap v286: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 7550/229977 objects degraded (3.283%); 6.5 MiB/s, 104 objects/s recovering 2023-09-30T09:24:00.816 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:00.863 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:00 smithi167 ceph-mon[192880]: pgmap v286: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 7550/229977 objects degraded (3.283%); 6.5 MiB/s, 104 objects/s recovering 2023-09-30T09:24:00.864 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:00.991 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:01.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:01.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:01.043 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:01.044 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 63 16 79895 79879 79.1997 52.6875 0.0279329 0.0126203 2023-09-30T09:24:01.106 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:01.177 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:01.273 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:01.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:01.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:01.380 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:01.466 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:01.517 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:24:01.599 INFO:teuthology.orchestra.run.smithi019.stdout:moved 21 / 267 (7.86517%) 2023-09-30T09:24:01.599 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:24:01.599 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 4.02919 -> 5.36045 (expected baseline 5.40399) 2023-09-30T09:24:01.599 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.2 with 26 -> 30 pgs (0.779026 -> 0.898876 * mean) 2023-09-30T09:24:01.599 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.4 with 41 -> 33 pgs (1.22846 -> 0.988764 * mean) 2023-09-30T09:24:01.600 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:24:01.600 INFO:teuthology.orchestra.run.smithi019.stdout:oload 110 2023-09-30T09:24:01.600 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 1 2023-09-30T09:24:01.600 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:24:01.600 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0207 2023-09-30T09:24:01.600 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0228 2023-09-30T09:24:01.600 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 weight 1.0000 -> 0.6540 2023-09-30T09:24:01.600 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 weight 1.0000 -> 0.7426 2023-09-30T09:24:01.601 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:01.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:01.709 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:01.740 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:01.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:01 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'48667','5':'42860'}"}]: dispatch 2023-09-30T09:24:01.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:01 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'48667','5':'42860'}"}]: dispatch 2023-09-30T09:24:01.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:01 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'48667','5':'42860'}"}]: dispatch 2023-09-30T09:24:02.044 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 64 16 80465 80449 78.5192 35.625 0.00445095 0.0126475 2023-09-30T09:24:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: from='client.44663 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 639 op/s; 6012/233892 objects degraded (2.570%); 1236/233892 objects misplaced (0.528%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:24:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'48667','5':'42860'}"}]': finished 2023-09-30T09:24:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: osdmap e136: 8 total, 8 up, 8 in 2023-09-30T09:24:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:03.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:02 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:03.044 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 65 16 80465 80449 77.3117 0 - 0.0126475 2023-09-30T09:24:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: from='client.44663 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 639 op/s; 6012/233892 objects degraded (2.570%); 1236/233892 objects misplaced (0.528%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:24:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'48667','5':'42860'}"}]': finished 2023-09-30T09:24:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: osdmap e136: 8 total, 8 up, 8 in 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: from='client.44663 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 639 op/s; 6012/233892 objects degraded (2.570%); 1236/233892 objects misplaced (0.528%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'48667','5':'42860'}"}]': finished 2023-09-30T09:24:03.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: osdmap e136: 8 total, 8 up, 8 in 2023-09-30T09:24:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: failed to encode map e136 with expected crc 2023-09-30T09:24:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:02 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:03.186 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:03.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:03.290 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:03.327 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:03.388 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:03.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:03.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:03.492 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:03.526 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:03.782 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:03.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:03.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: osdmap e137: 8 total, 8 up, 8 in 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.794 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.795 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:03.795 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: pgmap v290: 89 pgs: 2 unknown, 7 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 6012/237054 objects degraded (2.536%); 1236/237054 objects misplaced (0.521%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:24:03.795 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6012/233892 objects degraded (2.570%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:03.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: osdmap e137: 8 total, 8 up, 8 in 2023-09-30T09:24:03.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:03.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: pgmap v290: 89 pgs: 2 unknown, 7 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 6012/237054 objects degraded (2.536%); 1236/237054 objects misplaced (0.521%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:24:03.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6012/233892 objects degraded (2.570%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:03.838 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-30T09:24:03.839 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-30T09:24:03.894 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:03.920 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: osdmap e137: 8 total, 8 up, 8 in 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: failed to encode map e137 with expected crc 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:03.921 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: pgmap v290: 89 pgs: 2 unknown, 7 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 6012/237054 objects degraded (2.536%); 1236/237054 objects misplaced (0.521%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:24:03.922 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6012/233892 objects degraded (2.570%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:03.925 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:03.977 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:03.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:03.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:04.047 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 66 16 80496 80480 76.1666 0.96875 0.0111942 0.0129054 2023-09-30T09:24:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:24:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:24:03] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2023-09-30T09:24:04.085 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:04.111 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:04.157 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:04.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:04.262 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:04.289 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:04.333 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:04.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:04.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:04.440 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:04.477 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:04.626 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[216815]: failed to encode map e138 with expected crc 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[216815]: 5.7 starting backfill to osd.4 from (0'0,119'1481] MIN to 135'4973 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[216815]: osdmap e138: 8 total, 8 up, 8 in 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[216815]: failed to encode map e138 with expected crc 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[216815]: failed to encode map e138 with expected crc 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[216815]: failed to encode map e138 with expected crc 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3093339340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:04.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:24:04.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[224043]: failed to encode map e138 with expected crc 2023-09-30T09:24:04.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[224043]: 5.7 starting backfill to osd.4 from (0'0,119'1481] MIN to 135'4973 2023-09-30T09:24:04.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[224043]: osdmap e138: 8 total, 8 up, 8 in 2023-09-30T09:24:04.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[224043]: failed to encode map e138 with expected crc 2023-09-30T09:24:04.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[224043]: failed to encode map e138 with expected crc 2023-09-30T09:24:04.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[224043]: failed to encode map e138 with expected crc 2023-09-30T09:24:04.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3093339340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:05.003 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:05.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:05.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:04 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:24:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:04 smithi167 ceph-mon[192880]: failed to encode map e138 with expected crc 2023-09-30T09:24:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:04 smithi167 ceph-mon[192880]: 5.7 starting backfill to osd.4 from (0'0,119'1481] MIN to 135'4973 2023-09-30T09:24:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:04 smithi167 ceph-mon[192880]: osdmap e138: 8 total, 8 up, 8 in 2023-09-30T09:24:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:04 smithi167 ceph-mon[192880]: failed to encode map e138 with expected crc 2023-09-30T09:24:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:04 smithi167 ceph-mon[192880]: failed to encode map e138 with expected crc 2023-09-30T09:24:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:04 smithi167 ceph-mon[192880]: failed to encode map e138 with expected crc 2023-09-30T09:24:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3093339340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:05.047 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 67 16 80509 80493 75.0424 0.8125 0.0112832 0.0129057 2023-09-30T09:24:05.109 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:05.150 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:05.887 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:05.889 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:24:05.889 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3093339340' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:05.889 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[216815]: failed to encode map e139 with expected crc 2023-09-30T09:24:05.889 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[216815]: osdmap e139: 8 total, 8 up, 8 in 2023-09-30T09:24:05.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[216815]: failed to encode map e139 with expected crc 2023-09-30T09:24:05.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[216815]: pgmap v293: 89 pgs: 2 unknown, 1 activating+undersized+degraded+remapped, 7 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 18914/241026 objects degraded (7.847%); 1351/241026 objects misplaced (0.561%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:24:05.890 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:24:05.890 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3093339340' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:05.890 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[224043]: failed to encode map e139 with expected crc 2023-09-30T09:24:05.890 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[224043]: osdmap e139: 8 total, 8 up, 8 in 2023-09-30T09:24:05.890 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[224043]: failed to encode map e139 with expected crc 2023-09-30T09:24:05.890 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:05 smithi019 ceph-mon[224043]: pgmap v293: 89 pgs: 2 unknown, 1 activating+undersized+degraded+remapped, 7 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 18914/241026 objects degraded (7.847%); 1351/241026 objects misplaced (0.561%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:24:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:05.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:05.992 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:05 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:24:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3093339340' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:05 smithi167 ceph-mon[192880]: failed to encode map e139 with expected crc 2023-09-30T09:24:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:05 smithi167 ceph-mon[192880]: osdmap e139: 8 total, 8 up, 8 in 2023-09-30T09:24:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:05 smithi167 ceph-mon[192880]: failed to encode map e139 with expected crc 2023-09-30T09:24:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:05 smithi167 ceph-mon[192880]: pgmap v293: 89 pgs: 2 unknown, 1 activating+undersized+degraded+remapped, 7 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 18914/241026 objects degraded (7.847%); 1351/241026 objects misplaced (0.561%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:24:06.038 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:06.048 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 68 16 80509 80493 73.9394 0 - 0.0129057 2023-09-30T09:24:06.114 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:06.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:06.185 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:06.220 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:06.268 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:06.357 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:06.411 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:06.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:06.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:06.447 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:24:06.447 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:24:06.447 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:24:06.448 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:24:06.448 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:24:06.516 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:06.562 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:06.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:06 smithi019 ceph-mon[216815]: failed to encode map e139 with expected crc 2023-09-30T09:24:06.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:06 smithi019 ceph-mon[216815]: failed to encode map e139 with expected crc 2023-09-30T09:24:06.809 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:06 smithi019 ceph-mon[224043]: failed to encode map e139 with expected crc 2023-09-30T09:24:06.809 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:06 smithi019 ceph-mon[224043]: failed to encode map e139 with expected crc 2023-09-30T09:24:07.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:06 smithi167 ceph-mon[192880]: failed to encode map e139 with expected crc 2023-09-30T09:24:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:06 smithi167 ceph-mon[192880]: failed to encode map e139 with expected crc 2023-09-30T09:24:07.047 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 69 16 80509 80493 72.8683 0 - 0.0129057 2023-09-30T09:24:07.999 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:08.001 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:07 smithi019 ceph-mon[224043]: pgmap v294: 89 pgs: 75 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 peering, 1 activating+undersized+degraded+remapped, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 28 MiB/s wr, 442 op/s; 18914/241071 objects degraded (7.846%); 6333/241071 objects misplaced (2.627%); 6.4 MiB/s, 104 objects/s recovering 2023-09-30T09:24:08.002 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:07 smithi019 ceph-mon[216815]: pgmap v294: 89 pgs: 75 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 peering, 1 activating+undersized+degraded+remapped, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 28 MiB/s wr, 442 op/s; 18914/241071 objects degraded (7.846%); 6333/241071 objects misplaced (2.627%); 6.4 MiB/s, 104 objects/s recovering 2023-09-30T09:24:08.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:08.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:08.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:07 smithi167 ceph-mon[192880]: pgmap v294: 89 pgs: 75 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 peering, 1 activating+undersized+degraded+remapped, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 28 MiB/s wr, 442 op/s; 18914/241071 objects degraded (7.846%); 6333/241071 objects misplaced (2.627%); 6.4 MiB/s, 104 objects/s recovering 2023-09-30T09:24:08.024 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:24:08.047 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 70 16 80509 80493 71.8278 0 - 0.0129057 2023-09-30T09:24:08.104 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:08.132 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:08.177 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:08.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:08.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:08.284 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:08.325 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:08.663 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:24:08.780 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:08.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:08.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:08.886 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:08.958 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18914/241071 objects degraded (7.846%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:08.958 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18914/241071 objects degraded (7.846%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:08.962 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18914/241071 objects degraded (7.846%), 7 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:09.047 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 71 16 80509 80493 70.8167 0 - 0.0129057 2023-09-30T09:24:09.539 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:09.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:09.643 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:09.670 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:09.715 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:09.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:09.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:09.821 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:09 smithi019 ceph-mon[216815]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 activating+undersized+degraded+remapped, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 13 MiB/s wr, 210 op/s; 18914/241107 objects degraded (7.845%); 5423/241107 objects misplaced (2.249%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:24:09.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:09 smithi019 ceph-mon[224043]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 activating+undersized+degraded+remapped, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 13 MiB/s wr, 210 op/s; 18914/241107 objects degraded (7.845%); 5423/241107 objects misplaced (2.249%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:24:09.847 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:09.892 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:09.896 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:09 smithi167 ceph-mon[192880]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 activating+undersized+degraded+remapped, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 13 MiB/s wr, 210 op/s; 18914/241107 objects degraded (7.845%); 5423/241107 objects misplaced (2.249%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:24:09.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:09.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:09.999 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:10.046 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:10.047 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 72 16 80509 80493 69.8336 0 - 0.0129057 2023-09-30T09:24:10.141 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:10.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:10.254 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:10.259 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:10.286 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:10.332 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:10.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:10.437 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:10.463 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:10.507 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:10.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:10.613 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:10.640 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:10.685 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:10.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:10.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:10.706 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:10.790 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:10.816 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:10.859 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:10.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:10.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:10.963 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:10.999 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:10 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-30T09:24:10.999 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/242050753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:10.999 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:10 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-30T09:24:10.999 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/242050753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:11.003 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:10 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-30T09:24:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/242050753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:11.048 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 73 16 80561 80545 68.9219 0.541667 0.00362171 0.0136505 2023-09-30T09:24:11.417 INFO:teuthology.orchestra.run.smithi019.stdout:moved 0 / 267 (0%) 2023-09-30T09:24:11.417 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:24:11.417 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 5.36045 -> 5.36045 (expected baseline 5.40399) 2023-09-30T09:24:11.418 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.5 with 21 -> 21 pgs (0.629214 -> 0.629214 * mean) 2023-09-30T09:24:11.418 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.0 with 39 -> 39 pgs (1.16854 -> 1.16854 * mean) 2023-09-30T09:24:11.418 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:24:11.418 INFO:teuthology.orchestra.run.smithi019.stdout:oload 1000 2023-09-30T09:24:11.418 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 1 2023-09-30T09:24:11.418 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:24:11.418 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0217 2023-09-30T09:24:11.418 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.2175 2023-09-30T09:24:11.422 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:24:11.585 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:11.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:11.690 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:11.716 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:11.759 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:11.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:11.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:11.864 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:11.896 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:11 smithi019 ceph-mon[224043]: pgmap v296: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s; 18914/241107 objects degraded (7.845%); 5423/241107 objects misplaced (2.249%); 11 MiB/s, 184 objects/s recovering 2023-09-30T09:24:11.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:11 smithi019 ceph-mon[224043]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:11.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:11 smithi019 ceph-mon[216815]: pgmap v296: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s; 18914/241107 objects degraded (7.845%); 5423/241107 objects misplaced (2.249%); 11 MiB/s, 184 objects/s recovering 2023-09-30T09:24:11.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:11 smithi019 ceph-mon[216815]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:11.902 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:12.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:11 smithi167 ceph-mon[192880]: pgmap v296: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s; 18914/241107 objects degraded (7.845%); 5423/241107 objects misplaced (2.249%); 11 MiB/s, 184 objects/s recovering 2023-09-30T09:24:12.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:11 smithi167 ceph-mon[192880]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:12.048 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 74 16 80561 80545 67.9909 0 - 0.0136505 2023-09-30T09:24:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:12 smithi167 ceph-mon[192880]: osdmap e140: 8 total, 8 up, 8 in 2023-09-30T09:24:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:12 smithi167 ceph-mon[192880]: failed to encode map e140 with expected crc 2023-09-30T09:24:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:12 smithi167 ceph-mon[192880]: failed to encode map e140 with expected crc 2023-09-30T09:24:13.048 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 75 16 80561 80545 67.0848 0 - 0.0136505 2023-09-30T09:24:13.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:12 smithi019 ceph-mon[216815]: osdmap e140: 8 total, 8 up, 8 in 2023-09-30T09:24:13.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:12 smithi019 ceph-mon[216815]: failed to encode map e140 with expected crc 2023-09-30T09:24:13.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:12 smithi019 ceph-mon[216815]: failed to encode map e140 with expected crc 2023-09-30T09:24:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:12 smithi019 ceph-mon[224043]: osdmap e140: 8 total, 8 up, 8 in 2023-09-30T09:24:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:12 smithi019 ceph-mon[224043]: failed to encode map e140 with expected crc 2023-09-30T09:24:13.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:12 smithi019 ceph-mon[224043]: failed to encode map e140 with expected crc 2023-09-30T09:24:13.554 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:13.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:13.661 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:13.669 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[224043]: failed to encode map e141 with expected crc 2023-09-30T09:24:13.670 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[224043]: osdmap e141: 8 total, 8 up, 8 in 2023-09-30T09:24:13.670 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[224043]: failed to encode map e141 with expected crc 2023-09-30T09:24:13.670 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[224043]: failed to encode map e141 with expected crc 2023-09-30T09:24:13.670 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[224043]: pgmap v299: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 288 KiB/s wr, 4 op/s; 17459/241134 objects degraded (7.240%); 6769/241134 objects misplaced (2.807%); 8.0 MiB/s, 127 objects/s recovering 2023-09-30T09:24:13.670 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18914/241107 objects degraded (7.845%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:13.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[216815]: failed to encode map e141 with expected crc 2023-09-30T09:24:13.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[216815]: osdmap e141: 8 total, 8 up, 8 in 2023-09-30T09:24:13.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[216815]: failed to encode map e141 with expected crc 2023-09-30T09:24:13.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[216815]: failed to encode map e141 with expected crc 2023-09-30T09:24:13.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[216815]: pgmap v299: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 288 KiB/s wr, 4 op/s; 17459/241134 objects degraded (7.240%); 6769/241134 objects misplaced (2.807%); 8.0 MiB/s, 127 objects/s recovering 2023-09-30T09:24:13.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18914/241107 objects degraded (7.845%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:13.711 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:13.769 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:13.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:13.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:13.876 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:13.911 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:13 smithi167 ceph-mon[192880]: failed to encode map e141 with expected crc 2023-09-30T09:24:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:13 smithi167 ceph-mon[192880]: osdmap e141: 8 total, 8 up, 8 in 2023-09-30T09:24:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:13 smithi167 ceph-mon[192880]: failed to encode map e141 with expected crc 2023-09-30T09:24:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:13 smithi167 ceph-mon[192880]: failed to encode map e141 with expected crc 2023-09-30T09:24:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:13 smithi167 ceph-mon[192880]: pgmap v299: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 77 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 288 KiB/s wr, 4 op/s; 17459/241134 objects degraded (7.240%); 6769/241134 objects misplaced (2.807%); 8.0 MiB/s, 127 objects/s recovering 2023-09-30T09:24:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18914/241107 objects degraded (7.845%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:14.048 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 76 16 80561 80545 66.2025 0 - 0.0136505 2023-09-30T09:24:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:24:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:24:13] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2023-09-30T09:24:14.976 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:14 smithi019 ceph-mon[216815]: failed to encode map e141 with expected crc 2023-09-30T09:24:14.977 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:14 smithi019 ceph-mon[224043]: failed to encode map e141 with expected crc 2023-09-30T09:24:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:14 smithi167 ceph-mon[192880]: failed to encode map e141 with expected crc 2023-09-30T09:24:15.048 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 77 16 80561 80545 65.343 0 - 0.0136505 2023-09-30T09:24:15.788 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:15.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:15.892 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:15.927 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:15.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:15 smithi019 ceph-mon[216815]: pgmap v300: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 384 KiB/s wr, 5 op/s; 17459/241215 objects degraded (7.238%); 6328/241215 objects misplaced (2.623%); 11 MiB/s, 182 objects/s recovering 2023-09-30T09:24:15.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:15 smithi019 ceph-mon[224043]: pgmap v300: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 384 KiB/s wr, 5 op/s; 17459/241215 objects degraded (7.238%); 6328/241215 objects misplaced (2.623%); 11 MiB/s, 182 objects/s recovering 2023-09-30T09:24:16.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:15 smithi167 ceph-mon[192880]: pgmap v300: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 384 KiB/s wr, 5 op/s; 17459/241215 objects degraded (7.238%); 6328/241215 objects misplaced (2.623%); 11 MiB/s, 182 objects/s recovering 2023-09-30T09:24:16.026 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-30T09:24:16.026 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-30T09:24:16.048 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 78 16 80561 80545 64.5057 0 - 0.0136505 2023-09-30T09:24:16.324 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:16.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:16.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:16.430 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:1226 5.0 5:0deac523:::benchmark_data_smithi019_287194_object1225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:22:58.306896+0000", 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 78.507317389999997, 2023-09-30T09:24:16.983 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14286222500000001, 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 1226 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:16.984 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.306896+0000", 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.306896+0000", 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.306898+0000", 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3740000000000001e-06 2023-09-30T09:24:16.985 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.306921+0000", 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2949e-05 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.306939+0000", 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8281999999999999e-05 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.986 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.306948+0000", 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1870000000000003e-06 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.307000+0000", 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1391000000000003e-05 2023-09-30T09:24:16.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.307106+0000", 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000106718 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.307285+0000", 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000178276 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.307784+0000", 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00049945000000000002 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.308774+0000", 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0009903729999999999 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:16.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.449656+0000", 2023-09-30T09:24:16.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14088136700000001 2023-09-30T09:24:16.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.993 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:16.993 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.449720+0000", 2023-09-30T09:24:16.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4032e-05 2023-09-30T09:24:16.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.449759+0000", 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8825999999999997e-05 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:1237 5.0 5:043cd26c:::benchmark_data_smithi019_287194_object1236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:22:58.310552+0000", 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 78.503661453999996, 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14024456900000001, 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 1237 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:16.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310552+0000", 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310552+0000", 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:16.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310553+0000", 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1310000000000001e-06 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310575+0000", 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1829000000000001e-05 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310590+0000", 2023-09-30T09:24:16.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5126999999999999e-05 2023-09-30T09:24:17.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310598+0000", 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0839999999999992e-06 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310658+0000", 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9188999999999999e-05 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310780+0000", 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012268500000000001 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.310920+0000", 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013992200000000001 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.311516+0000", 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00059627200000000001 2023-09-30T09:24:17.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.312587+0000", 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001070784 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.450715+0000", 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.138127577 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.450764+0000", 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8820999999999999e-05 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.450797+0000", 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3148000000000002e-05 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.038 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:1258 5.0 5:09ef51ab:::benchmark_data_smithi019_287194_object1257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:22:58.318320+0000", 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 78.495893687000006, 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.132619603, 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.039 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 1258 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318320+0000", 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318320+0000", 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318321+0000", 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0920000000000001e-06 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318340+0000", 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9318e-05 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318354+0000", 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.33e-05 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318360+0000", 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8139999999999997e-06 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318411+0000", 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1755000000000002e-05 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318500+0000", 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.852e-05 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318595+0000", 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5308000000000003e-05 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.318904+0000", 2023-09-30T09:24:17.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00030878600000000002 2023-09-30T09:24:17.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.319332+0000", 2023-09-30T09:24:17.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00042773900000000001 2023-09-30T09:24:17.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.450850+0000", 2023-09-30T09:24:17.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.13151818500000001 2023-09-30T09:24:17.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.450904+0000", 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3944999999999999e-05 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.450940+0000", 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5840999999999998e-05 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.046 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.047 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:1266 5.0 5:010df711:::benchmark_data_smithi019_287194_object1265:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-30T09:24:17.047 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:22:58.326546+0000", 2023-09-30T09:24:17.047 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 78.487668068000005, 2023-09-30T09:24:17.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.12453995499999999, 2023-09-30T09:24:17.047 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.047 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.048 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.048 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.048 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.048 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 1266 2023-09-30T09:24:17.048 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.048 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.048 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.048 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326546+0000", 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326546+0000", 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326547+0000", 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5799999999999998e-07 2023-09-30T09:24:17.049 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326564+0000", 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7221e-05 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326575+0000", 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1644999999999999e-05 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326581+0000", 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.755e-06 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326634+0000", 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2553000000000003e-05 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326721+0000", 2023-09-30T09:24:17.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7181000000000004e-05 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.326809+0000", 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8299000000000003e-05 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.327159+0000", 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035012599999999997 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.327504+0000", 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00034487199999999997 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.450982+0000", 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.123477925 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.451051+0000", 2023-09-30T09:24:17.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8899000000000006e-05 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:22:58.451086+0000", 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4620999999999999e-05 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.44600.0:32729 5.1 e124/113 5:868395c1:::benchmark_data_smithi019_287194_object32728:head v 124'2110, mlcod=124'2108)", 2023-09-30T09:24:17.054 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:15.323267+0000", 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 61.490946590999997, 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.106677173, 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.323267+0000", 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.323267+0000", 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.323268+0000", 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1600000000000001e-07 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.323352+0000", 2023-09-30T09:24:17.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4124999999999998e-05 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.323361+0000", 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1689999999999998e-06 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.323368+0000", 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6000000000000003e-06 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.057 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.323460+0000", 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2311000000000004e-05 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.323480+0000", 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0247000000000001e-05 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.058 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.429902+0000", 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.10642159900000001 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.429944+0000", 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2506000000000003e-05 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.44600.0:32733 5.a e124/113 5:5048b6ac:::benchmark_data_smithi019_287194_object32732:head v 124'2040, mlcod=124'2038)", 2023-09-30T09:24:17.059 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:15.324558+0000", 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 61.489655251000002, 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.105467503, 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.324558+0000", 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.324558+0000", 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.324559+0000", 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9800000000000003e-07 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.324585+0000", 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5284000000000001e-05 2023-09-30T09:24:17.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426230+0000", 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.101645164 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426244+0000", 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.429e-05 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426306+0000", 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1966999999999998e-05 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426330+0000", 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3660999999999999e-05 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.430002+0000", 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036728540000000001 2023-09-30T09:24:17.063 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.430026+0000", 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3485e-05 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:32761 5.0 5:0f364751:::benchmark_data_smithi019_287194_object32760:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:15.338361+0000", 2023-09-30T09:24:17.064 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 61.475852779, 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.093211262000000003, 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 32761 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.065 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.338361+0000", 2023-09-30T09:24:17.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.338361+0000", 2023-09-30T09:24:17.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.338363+0000", 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3879999999999998e-06 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.338390+0000", 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6452999999999999e-05 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426240+0000", 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.087850710999999998 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426254+0000", 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3611999999999999e-05 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426318+0000", 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4325999999999994e-05 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426479+0000", 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016013 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.426596+0000", 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000117892 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.430016+0000", 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003419582 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.430074+0000", 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7967999999999998e-05 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.431459+0000", 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0013854990000000001 2023-09-30T09:24:17.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.431527+0000", 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7197000000000001e-05 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:15.431572+0000", 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5503999999999997e-05 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.108 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:36123 5.0 5:0ab50927:::benchmark_data_smithi019_287194_object36122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:17.790374+0000", 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 59.023840086, 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.083857583999999999, 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.109 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36123 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790374+0000", 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790374+0000", 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790376+0000", 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.198e-06 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790401+0000", 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4689999999999999e-05 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790418+0000", 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7357999999999999e-05 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790427+0000", 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4490000000000005e-06 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790506+0000", 2023-09-30T09:24:17.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8224000000000002e-05 2023-09-30T09:24:17.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790646+0000", 2023-09-30T09:24:17.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014052800000000001 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790779+0000", 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013261499999999999 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794106+0000", 2023-09-30T09:24:17.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033271680000000001 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794804+0000", 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00069853500000000002 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.874104+0000", 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.079300082999999993 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.874172+0000", 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7104e-05 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.874231+0000", 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9632000000000001e-05 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:36127 5.0 5:0eab8398:::benchmark_data_smithi019_287194_object36126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:17.790467+0000", 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 59.023746639000002, 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.085052759000000006, 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36127 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.117 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790467+0000", 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790467+0000", 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.118 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790468+0000", 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1200000000000003e-07 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790495+0000", 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7693999999999999e-05 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790504+0000", 2023-09-30T09:24:17.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6829999999999997e-06 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790510+0000", 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0630000000000002e-06 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.791979+0000", 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0014694459999999999 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.792063+0000", 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3894999999999996e-05 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.792197+0000", 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000133413 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794815+0000", 2023-09-30T09:24:17.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026185570000000001 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794854+0000", 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8518e-05 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875429+0000", 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.080574969999999996 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875482+0000", 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3156999999999999e-05 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875520+0000", 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7851000000000001e-05 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.123 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:36128 5.0 5:0744adc4:::benchmark_data_smithi019_287194_object36127:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:17.790525+0000", 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 59.023689140000002, 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.085181194000000002, 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36128 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.124 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790525+0000", 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790525+0000", 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790525+0000", 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3600000000000004e-07 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790545+0000", 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9544000000000001e-05 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790551+0000", 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8210000000000004e-06 2023-09-30T09:24:17.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.790557+0000", 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.259e-06 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.793001+0000", 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024438810000000002 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.127 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.793129+0000", 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000128202 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.793238+0000", 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000108723 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794887+0000", 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001649108 2023-09-30T09:24:17.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794895+0000", 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9379999999999992e-06 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875568+0000", 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.080673024999999995 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875652+0000", 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4347999999999999e-05 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875706+0000", 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3808999999999999e-05 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:36133 5.0 5:0a3e2efc:::benchmark_data_smithi019_287194_object36132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:17.792427+0000", 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 59.021786368999997, 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.083437786, 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36133 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.792427+0000", 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.792427+0000", 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.792429+0000", 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1999999999999999e-06 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.792447+0000", 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8464000000000001e-05 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.792457+0000", 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6320000000000003e-06 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.792462+0000", 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.614e-06 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794140+0000", 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00167818 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794242+0000", 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000101427 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794319+0000", 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7560999999999998e-05 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.794972+0000", 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00065256400000000001 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.795313+0000", 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00034063600000000001 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875756+0000", 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.080443287000000002 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875822+0000", 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5976999999999995e-05 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:17.875865+0000", 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3244000000000001e-05 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:37258 5.0 5:05f832e5:::benchmark_data_smithi019_287194_object37257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.137 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:18.699176+0000", 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 58.115037583000003, 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.179164408, 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 37258 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.138 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699176+0000", 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699176+0000", 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699178+0000", 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6199999999999999e-06 2023-09-30T09:24:17.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699209+0000", 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1272999999999997e-05 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699237+0000", 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8031999999999999e-05 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699246+0000", 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.541e-06 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699506+0000", 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00026085300000000002 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699654+0000", 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000147133 2023-09-30T09:24:17.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699809+0000", 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015585600000000001 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.704237+0000", 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004427149 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.707663+0000", 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034260509999999998 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.878236+0000", 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.17057333499999999 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.878298+0000", 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1576999999999997e-05 2023-09-30T09:24:17.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.878341+0000", 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2988000000000002e-05 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:37259 5.0 5:0c7a126e:::benchmark_data_smithi019_287194_object37258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:18.699273+0000", 2023-09-30T09:24:17.144 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 58.114940988999997, 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.179183015, 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 37259 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699273+0000", 2023-09-30T09:24:17.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699273+0000", 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699273+0000", 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7700000000000004e-07 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699297+0000", 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3632e-05 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699309+0000", 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2471e-05 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.699315+0000", 2023-09-30T09:24:17.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9050000000000002e-06 2023-09-30T09:24:17.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.701257+0000", 2023-09-30T09:24:17.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019416539999999999 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.701398+0000", 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000141326 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.701534+0000", 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013523800000000001 2023-09-30T09:24:17.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.705513+0000", 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0039798469999999999 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.707700+0000", 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021870079999999998 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.878377+0000", 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.17067686400000001 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.878425+0000", 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7954999999999997e-05 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.878456+0000", 2023-09-30T09:24:17.191 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0537999999999998e-05 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:37267 5.0 5:0a37dc6a:::benchmark_data_smithi019_287194_object37266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:18.709342+0000", 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 58.104871348000003, 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.17140628299999999, 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.192 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 37267 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709342+0000", 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.193 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709342+0000", 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709345+0000", 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8540000000000001e-06 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709374+0000", 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8714999999999999e-05 2023-09-30T09:24:17.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709396+0000", 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2305000000000001e-05 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709409+0000", 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2652e-05 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709494+0000", 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5348999999999999e-05 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709674+0000", 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000180197 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.709849+0000", 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017468200000000001 2023-09-30T09:24:17.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.711453+0000", 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001604126 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.713261+0000", 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001807304 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.880643+0000", 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.167382212 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.880708+0000", 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4800000000000003e-05 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.880749+0000", 2023-09-30T09:24:17.198 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1087000000000002e-05 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:37285 5.0 5:0f83f1ab:::benchmark_data_smithi019_287194_object37284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:18.723138+0000", 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 58.091075541999999, 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.16116872900000001, 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.199 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 37285 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723138+0000", 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.200 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723138+0000", 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723140+0000", 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4959999999999999e-06 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723160+0000", 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0245e-05 2023-09-30T09:24:17.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723175+0000", 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4958e-05 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723205+0000", 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9762999999999999e-05 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723283+0000", 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8296999999999999e-05 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723450+0000", 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016666200000000001 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.723623+0000", 2023-09-30T09:24:17.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017347200000000001 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.724792+0000", 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011686769999999999 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.725090+0000", 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00029848400000000001 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.884209+0000", 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15911843000000001 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.884267+0000", 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8387000000000002e-05 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:18.884307+0000", 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9857999999999997e-05 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:39531 5.0 5:09fbb5bc:::benchmark_data_smithi019_287194_object39530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:20.454667+0000", 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 56.359547181000003, 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.174717452, 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.206 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 39531 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.454667+0000", 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.454667+0000", 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.454668+0000", 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.641e-06 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.454695+0000", 2023-09-30T09:24:17.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6418999999999999e-05 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.454708+0000", 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3268000000000001e-05 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.454716+0000", 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8229999999999995e-06 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.454777+0000", 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0878999999999998e-05 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.454926+0000", 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000149328 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.455069+0000", 2023-09-30T09:24:17.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014286600000000001 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.456617+0000", 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015479630000000001 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.462013+0000", 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0053957969999999999 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.629257+0000", 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.167244119 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.629333+0000", 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6851e-05 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.629384+0000", 2023-09-30T09:24:17.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0498000000000003e-05 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:39540 5.0 5:0dc841da:::benchmark_data_smithi019_287194_object39539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:20.457298+0000", 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 56.356916013999999, 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.172230525, 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.213 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 39540 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457298+0000", 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457298+0000", 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457299+0000", 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4309999999999999e-06 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457326+0000", 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7080000000000002e-05 2023-09-30T09:24:17.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457339+0000", 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2748000000000001e-05 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457348+0000", 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1640000000000007e-06 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.241 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457411+0000", 2023-09-30T09:24:17.241 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2565999999999997e-05 2023-09-30T09:24:17.241 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457551+0000", 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014003299999999999 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.457696+0000", 2023-09-30T09:24:17.242 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000145103 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.459236+0000", 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015405620000000001 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.462055+0000", 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028190099999999998 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.243 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.629428+0000", 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.16737264399999999 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.629486+0000", 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7868999999999998e-05 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:20.629528+0000", 2023-09-30T09:24:17.244 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2314999999999997e-05 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.44600.0:42957 5.a e126/113 5:5952d1c4:::benchmark_data_smithi019_287194_object42956:head v 126'2665, mlcod=126'2663)", 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:23.360205+0000", 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 53.454008449, 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.081067272999999995, 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.245 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.360205+0000", 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.360205+0000", 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.246 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.360206+0000", 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.54e-07 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.360230+0000", 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4340000000000001e-05 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.434295+0000", 2023-09-30T09:24:17.247 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.074065133000000005 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.434311+0000", 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5138e-05 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.434366+0000", 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4994999999999998e-05 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.248 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.434387+0000", 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1753e-05 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.441209+0000", 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0068214970000000002 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:23.441273+0000", 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3663000000000004e-05 2023-09-30T09:24:17.249 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:63741 5.0 5:0492f242:::benchmark_data_smithi019_287194_object63740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:43.237509+0000", 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 33.576704859000003, 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15293327200000001, 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.250 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 63741 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237509+0000", 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.251 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237509+0000", 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237511+0000", 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2120000000000002e-06 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237538+0000", 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6444999999999998e-05 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.252 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237558+0000", 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0482000000000001e-05 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237569+0000", 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0722e-05 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.253 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237639+0000", 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0601999999999999e-05 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237767+0000", 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012802700000000001 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.237881+0000", 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113919 2023-09-30T09:24:17.254 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.239808+0000", 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001927015 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.242662+0000", 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028539849999999999 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.255 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.390327+0000", 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14766483399999999 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.390401+0000", 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3654999999999999e-05 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:43.390442+0000", 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1374000000000001e-05 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.256 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.257 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.257 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.257 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.257 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:77350 5.0 5:038ab2b4:::benchmark_data_smithi019_287194_object77349:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:24:17.257 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:23:58.128848+0000", 2023-09-30T09:24:17.257 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 18.685365851, 2023-09-30T09:24:17.257 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.085467314000000003, 2023-09-30T09:24:17.268 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:24:17.268 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 77350 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.128848+0000", 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.128848+0000", 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.128849+0000", 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.238e-06 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:24:17.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.128872+0000", 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2832000000000001e-05 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.128884+0000", 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2361000000000001e-05 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.128893+0000", 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3110000000000006e-06 2023-09-30T09:24:17.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.128955+0000", 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2076999999999995e-05 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.129085+0000", 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013055900000000001 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,4", 2023-09-30T09:24:17.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.129246+0000", 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016092100000000001 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.132115+0000", 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028693350000000002 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.134750+0000", 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026342409999999998 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.214214+0000", 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.079464060000000003 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.214275+0000", 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1011999999999998e-05 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:24:17.274 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:23:58.214315+0000", 2023-09-30T09:24:17.275 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0367000000000002e-05 2023-09-30T09:24:17.275 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.275 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.275 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.275 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:24:17.275 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:24:17.275 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:24:17.275 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:17.276 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 79 16 80561 80545 63.6895 0 - 0.0136505 2023-09-30T09:24:17.303 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:17.348 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:17.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:17.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:17.492 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:17.547 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:24:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:17 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:17 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3256700005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:24:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:17 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:24:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:17 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:24:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:17 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:17.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3256700005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3256700005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:24:17.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:24:17.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:17 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:24:18.048 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:24:18.048550+0000 min lat: 0.00105726 max lat: 8.52161 avg lat: 0.0136505 2023-09-30T09:24:18.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:24:18.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 80 16 80561 80545 62.8937 0 - 0.0136505 2023-09-30T09:24:18.354 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:24:18.383 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:24:18.414 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:24:18.438 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:24:18.467 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:24:18.496 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:24:18.530 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:24:18.562 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:24:18.567 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:24:18.567 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:24:18.567 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:24:18.567 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:24:18.567 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:24:18.567 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:24:18.567 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:24:18.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:24:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:18 smithi167 ceph-mon[192880]: pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 288 KiB/s wr, 4 op/s; 17459/241215 objects degraded (7.238%); 5409/241215 objects misplaced (2.242%); 11 MiB/s, 183 objects/s recovering 2023-09-30T09:24:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:18 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3256700005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:24:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:18 smithi167 ceph-mon[192880]: failed to encode map e142 with expected crc 2023-09-30T09:24:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:18 smithi167 ceph-mon[192880]: osdmap e142: 8 total, 8 up, 8 in 2023-09-30T09:24:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 17459/241215 objects degraded (7.238%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[216815]: pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 288 KiB/s wr, 4 op/s; 17459/241215 objects degraded (7.238%); 5409/241215 objects misplaced (2.242%); 11 MiB/s, 183 objects/s recovering 2023-09-30T09:24:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3256700005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:24:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[216815]: failed to encode map e142 with expected crc 2023-09-30T09:24:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[216815]: osdmap e142: 8 total, 8 up, 8 in 2023-09-30T09:24:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 17459/241215 objects degraded (7.238%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[224043]: pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 288 KiB/s wr, 4 op/s; 17459/241215 objects degraded (7.238%); 5409/241215 objects misplaced (2.242%); 11 MiB/s, 183 objects/s recovering 2023-09-30T09:24:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3256700005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:24:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[224043]: failed to encode map e142 with expected crc 2023-09-30T09:24:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[224043]: osdmap e142: 8 total, 8 up, 8 in 2023-09-30T09:24:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 17459/241215 objects degraded (7.238%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:19.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 81 16 80721 80705 62.2409 1.25 0.00738102 0.0155838 2023-09-30T09:24:20.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 82 16 80945 80929 61.6529 14 0.00842721 0.0159229 2023-09-30T09:24:20.301 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:20.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:20.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:20.345 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:24:20.410 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:20.455 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[216815]: pgmap v303: 89 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 79 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2.1 MiB/s wr, 32 op/s; 24533/241839 objects degraded (10.144%); 7434/241839 objects misplaced (3.074%); 25 MiB/s, 388 objects/s recovering 2023-09-30T09:24:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[216815]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[216815]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[216815]: osdmap e143: 8 total, 8 up, 8 in 2023-09-30T09:24:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[216815]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[216815]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[216815]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[224043]: pgmap v303: 89 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 79 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2.1 MiB/s wr, 32 op/s; 24533/241839 objects degraded (10.144%); 7434/241839 objects misplaced (3.074%); 25 MiB/s, 388 objects/s recovering 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[224043]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[224043]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[224043]: osdmap e143: 8 total, 8 up, 8 in 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[224043]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[224043]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:20 smithi019 ceph-mon[224043]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.706 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:20.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:20.731 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:20 smithi167 ceph-mon[192880]: pgmap v303: 89 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 79 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2.1 MiB/s wr, 32 op/s; 24533/241839 objects degraded (10.144%); 7434/241839 objects misplaced (3.074%); 25 MiB/s, 388 objects/s recovering 2023-09-30T09:24:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:20 smithi167 ceph-mon[192880]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:20 smithi167 ceph-mon[192880]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:20 smithi167 ceph-mon[192880]: osdmap e143: 8 total, 8 up, 8 in 2023-09-30T09:24:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:20 smithi167 ceph-mon[192880]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:20 smithi167 ceph-mon[192880]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:20 smithi167 ceph-mon[192880]: failed to encode map e143 with expected crc 2023-09-30T09:24:20.817 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:20.847 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:20.925 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:20.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:20.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:21.028 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:21.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 83 16 81205 81189 61.106 16.25 0.00985207 0.0160736 2023-09-30T09:24:21.080 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:21.228 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:21.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:21.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:21.335 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:21.372 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:21.444 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:21.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:21.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:21.549 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:21.579 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:21.624 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:21.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:21.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:21.727 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: osdmap e144: 8 total, 8 up, 8 in 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[224043]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: osdmap e144: 8 total, 8 up, 8 in 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:21 smithi019 ceph-mon[216815]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.758 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:21.826 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:21.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:21.932 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:21.958 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.958 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.958 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.958 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.958 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.959 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: osdmap e144: 8 total, 8 up, 8 in 2023-09-30T09:24:21.959 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.959 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.959 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.959 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:21 smithi167 ceph-mon[192880]: failed to encode map e144 with expected crc 2023-09-30T09:24:21.961 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:22.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 84 16 81396 81380 60.5209 11.9375 0.00706418 0.0161923 2023-09-30T09:24:22.196 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:22.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:22.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:22.301 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:22.338 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:22.415 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:22.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:22.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:22.523 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:22.554 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:22.608 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:22.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:22.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:22.668 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:24:22.681 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:24:22.681 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:24:22.704 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:22 smithi167 ceph-mon[192880]: pgmap v306: 89 pgs: 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 79 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 3.3 MiB/s wr, 52 op/s; 23653/242157 objects degraded (9.768%); 8198/242157 objects misplaced (3.385%); 27 MiB/s, 422 objects/s recovering 2023-09-30T09:24:22.709 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:22 smithi019 ceph-mon[216815]: pgmap v306: 89 pgs: 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 79 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 3.3 MiB/s wr, 52 op/s; 23653/242157 objects degraded (9.768%); 8198/242157 objects misplaced (3.385%); 27 MiB/s, 422 objects/s recovering 2023-09-30T09:24:22.709 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:22 smithi019 ceph-mon[224043]: pgmap v306: 89 pgs: 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 79 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 3.3 MiB/s wr, 52 op/s; 23653/242157 objects degraded (9.768%); 8198/242157 objects misplaced (3.385%); 27 MiB/s, 422 objects/s recovering 2023-09-30T09:24:22.713 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:22.722 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:24:22.740 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:22.893 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:22.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:22.996 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:23.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 85 16 81576 81560 59.9415 11.25 0.0128554 0.0163216 2023-09-30T09:24:23.067 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:23.234 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:23.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:23.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:23.341 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:23.413 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:23.511 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:24:23.600 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:23.611 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:23.619 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23653/242157 objects degraded (9.768%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:23.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:23.620 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23653/242157 objects degraded (9.768%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:23.667 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:23.721 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:23.765 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:23.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23653/242157 objects degraded (9.768%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:24:23.868 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:24:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:24:23] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.43.0" 2023-09-30T09:24:23.881 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:24.004 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:24.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:24.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:24.017 INFO:teuthology.orchestra.run.smithi019.stdout:moved 0 / 267 (0%) 2023-09-30T09:24:24.017 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:24:24.017 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 5.36045 -> 5.36045 (expected baseline 5.40399) 2023-09-30T09:24:24.017 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.5 with 21 -> 21 pgs (0.629214 -> 0.629214 * mean) 2023-09-30T09:24:24.017 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.0 with 39 -> 39 pgs (1.16854 -> 1.16854 * mean) 2023-09-30T09:24:24.017 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:24:24.017 INFO:teuthology.orchestra.run.smithi019.stdout:oload 1000 2023-09-30T09:24:24.018 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 3 2023-09-30T09:24:24.018 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:24:24.018 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0221 2023-09-30T09:24:24.018 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.2205 2023-09-30T09:24:24.028 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:24:24.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 86 16 81755 81739 59.3747 11.1875 0.00681892 0.0164677 2023-09-30T09:24:24.110 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:24.158 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:24 smithi167 ceph-mon[192880]: pgmap v307: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 4.8 MiB/s wr, 77 op/s; 23653/242604 objects degraded (9.750%); 7418/242604 objects misplaced (3.058%); 26 MiB/s, 406 objects/s recovering 2023-09-30T09:24:24.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:24 smithi019 ceph-mon[216815]: pgmap v307: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 4.8 MiB/s wr, 77 op/s; 23653/242604 objects degraded (9.750%); 7418/242604 objects misplaced (3.058%); 26 MiB/s, 406 objects/s recovering 2023-09-30T09:24:24.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:24 smithi019 ceph-mon[224043]: pgmap v307: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 4.8 MiB/s wr, 77 op/s; 23653/242604 objects degraded (9.750%); 7418/242604 objects misplaced (3.058%); 26 MiB/s, 406 objects/s recovering 2023-09-30T09:24:25.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 87 16 81962 81946 58.8411 12.9375 0.00334472 0.0166408 2023-09-30T09:24:25.550 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:24:25.550 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:24:25.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[224043]: from='client.34782 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:25.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[224043]: osdmap e145: 8 total, 8 up, 8 in 2023-09-30T09:24:25.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[224043]: failed to encode map e145 with expected crc 2023-09-30T09:24:25.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[224043]: failed to encode map e145 with expected crc 2023-09-30T09:24:25.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[224043]: failed to encode map e145 with expected crc 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[224043]: failed to encode map e145 with expected crc 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[224043]: osdmap e146: 8 total, 8 up, 8 in 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[216815]: from='client.34782 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[216815]: osdmap e145: 8 total, 8 up, 8 in 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[216815]: failed to encode map e145 with expected crc 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[216815]: failed to encode map e145 with expected crc 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[216815]: failed to encode map e145 with expected crc 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[216815]: failed to encode map e145 with expected crc 2023-09-30T09:24:25.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:25 smithi019 ceph-mon[216815]: osdmap e146: 8 total, 8 up, 8 in 2023-09-30T09:24:26.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:25 smithi167 ceph-mon[192880]: from='client.34782 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:26.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:25 smithi167 ceph-mon[192880]: osdmap e145: 8 total, 8 up, 8 in 2023-09-30T09:24:26.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:25 smithi167 ceph-mon[192880]: failed to encode map e145 with expected crc 2023-09-30T09:24:26.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:25 smithi167 ceph-mon[192880]: failed to encode map e145 with expected crc 2023-09-30T09:24:26.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:25 smithi167 ceph-mon[192880]: failed to encode map e145 with expected crc 2023-09-30T09:24:26.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:25 smithi167 ceph-mon[192880]: failed to encode map e145 with expected crc 2023-09-30T09:24:26.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:25 smithi167 ceph-mon[192880]: osdmap e146: 8 total, 8 up, 8 in 2023-09-30T09:24:26.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 88 16 82225 82209 58.3595 16.4375 0.0115481 0.0168033 2023-09-30T09:24:26.264 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:26.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:26.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:26.300 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:24:26.372 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:26.408 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:26.462 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:26.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:26.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:26.568 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:26.583 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[216815]: pgmap v309: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 178 op/s; 20617/245061 objects degraded (8.413%); 9670/245061 objects misplaced (3.946%); 18 MiB/s, 286 objects/s recovering 2023-09-30T09:24:26.583 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[216815]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[216815]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:26 smithi167 ceph-mon[192880]: pgmap v309: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 178 op/s; 20617/245061 objects degraded (8.413%); 9670/245061 objects misplaced (3.946%); 18 MiB/s, 286 objects/s recovering 2023-09-30T09:24:26.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:26 smithi167 ceph-mon[192880]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:26 smithi167 ceph-mon[192880]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:26 smithi167 ceph-mon[192880]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:26 smithi167 ceph-mon[192880]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.604 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:26.659 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:26.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:26.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:26.709 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:26.765 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:26.808 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[216815]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[216815]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[224043]: pgmap v309: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 178 op/s; 20617/245061 objects degraded (8.413%); 9670/245061 objects misplaced (3.946%); 18 MiB/s, 286 objects/s recovering 2023-09-30T09:24:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[224043]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[224043]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[224043]: failed to encode map e146 with expected crc 2023-09-30T09:24:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:26 smithi019 ceph-mon[224043]: failed to encode map e146 with expected crc 2023-09-30T09:24:27.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 89 16 82540 82524 57.9251 19.6875 0.0048968 0.0168981 2023-09-30T09:24:28.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 90 13 82775 82762 57.4469 14.875 0.00498923 0.0170278 2023-09-30T09:24:28.184 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-30T09:24:28.184 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-30T09:24:28.197 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:28.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:28.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:28.301 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:28.341 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:28.350 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:24:28.350 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:24:28.426 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:28.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:28.530 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:28.570 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:28.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:28 smithi019 ceph-mon[216815]: pgmap v311: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 170 op/s; 20617/245472 objects degraded (8.399%); 8929/245472 objects misplaced (3.637%); 22 MiB/s, 347 objects/s recovering 2023-09-30T09:24:28.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20617/245472 objects degraded (8.399%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:24:28.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:28 smithi019 ceph-mon[224043]: pgmap v311: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 170 op/s; 20617/245472 objects degraded (8.399%); 8929/245472 objects misplaced (3.637%); 22 MiB/s, 347 objects/s recovering 2023-09-30T09:24:28.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20617/245472 objects degraded (8.399%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:24:28.686 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:28.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:28.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:28.790 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:28.818 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:28 smithi167 ceph-mon[192880]: pgmap v311: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 170 op/s; 20617/245472 objects degraded (8.399%); 8929/245472 objects misplaced (3.637%); 22 MiB/s, 347 objects/s recovering 2023-09-30T09:24:28.818 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20617/245472 objects degraded (8.399%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:24:28.821 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:28.888 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:24:28.919 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:24:28.948 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:24:28.952 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:28.976 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:24:28.992 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:28.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:28.999 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:24:29.025 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:24:29.045 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:24:29.049 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total time run: 90.7654 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total writes made: 82775 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Write size: 65536 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object size: 65536 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Bandwidth (MB/sec): 56.9979 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Bandwidth: 49.7532 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max bandwidth (MB/sec): 217.625 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min bandwidth (MB/sec): 0 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average IOPS: 911 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev IOPS: 796.066 2023-09-30T09:24:29.050 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max IOPS: 3482 2023-09-30T09:24:29.051 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min IOPS: 0 2023-09-30T09:24:29.051 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average Latency(s): 0.0174524 2023-09-30T09:24:29.051 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Latency(s): 0.21012 2023-09-30T09:24:29.051 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max latency(s): 27.166 2023-09-30T09:24:29.051 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min latency(s): 0.00105726 2023-09-30T09:24:29.052 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:24:29.064 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Cleaning up (deleting benchmark objects) 2023-09-30T09:24:29.089 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:24:29.096 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:29.107 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:24:29.107 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:24:29.107 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:24:29.107 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:24:29.107 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:24:29.107 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:24:29.108 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:24:29.108 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:24:29.128 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:29.188 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:29.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:29.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:29.293 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:29.324 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:29.377 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:29.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:29.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:29.473 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:29.486 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:29.552 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:29.633 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:24:29.697 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:29 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1284058770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:29.698 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:29 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/339099112' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:29.698 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:29 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:29.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1284058770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:29.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/339099112' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:29.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:29 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:29.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:29 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1284058770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:29 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/339099112' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:29.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:29 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:29.996 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:29.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:29.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:30.100 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:30.130 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:30.264 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:30.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:30.368 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:30.394 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:30.438 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:30.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:30.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:30.541 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:30.575 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:30 smithi167 ceph-mon[192880]: pgmap v312: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 13 MiB/s wr, 215 op/s; 20617/247332 objects degraded (8.336%); 8929/247332 objects misplaced (3.610%); 18 MiB/s, 281 objects/s recovering 2023-09-30T09:24:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:30 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:30 smithi167 ceph-mon[192880]: failed to encode map e147 with expected crc 2023-09-30T09:24:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:30 smithi167 ceph-mon[192880]: osdmap e147: 8 total, 8 up, 8 in 2023-09-30T09:24:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:30 smithi167 ceph-mon[192880]: failed to encode map e147 with expected crc 2023-09-30T09:24:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:30 smithi167 ceph-mon[192880]: failed to encode map e147 with expected crc 2023-09-30T09:24:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[224043]: pgmap v312: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 13 MiB/s wr, 215 op/s; 20617/247332 objects degraded (8.336%); 8929/247332 objects misplaced (3.610%); 18 MiB/s, 281 objects/s recovering 2023-09-30T09:24:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[224043]: failed to encode map e147 with expected crc 2023-09-30T09:24:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[224043]: osdmap e147: 8 total, 8 up, 8 in 2023-09-30T09:24:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[224043]: failed to encode map e147 with expected crc 2023-09-30T09:24:30.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[224043]: failed to encode map e147 with expected crc 2023-09-30T09:24:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[216815]: pgmap v312: 89 pgs: 1 active+remapped+backfilling, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 13 MiB/s wr, 215 op/s; 20617/247332 objects degraded (8.336%); 8929/247332 objects misplaced (3.610%); 18 MiB/s, 281 objects/s recovering 2023-09-30T09:24:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[216815]: failed to encode map e147 with expected crc 2023-09-30T09:24:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[216815]: osdmap e147: 8 total, 8 up, 8 in 2023-09-30T09:24:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[216815]: failed to encode map e147 with expected crc 2023-09-30T09:24:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:30 smithi019 ceph-mon[216815]: failed to encode map e147 with expected crc 2023-09-30T09:24:31.267 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:24:31.267 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:24:31.267 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:24:31.267 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:24:31.956 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:24:32.236 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:32.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:32.327 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:32.339 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:32.373 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:32.916 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:32 smithi167 ceph-mon[192880]: pgmap v314: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 158 B/s rd, 10 MiB/s wr, 173 op/s; 20605/248031 objects degraded (8.307%); 8929/248031 objects misplaced (3.600%); 6.4 MiB/s, 102 objects/s recovering 2023-09-30T09:24:33.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:32 smithi019 ceph-mon[216815]: pgmap v314: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 158 B/s rd, 10 MiB/s wr, 173 op/s; 20605/248031 objects degraded (8.307%); 8929/248031 objects misplaced (3.600%); 6.4 MiB/s, 102 objects/s recovering 2023-09-30T09:24:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:32 smithi019 ceph-mon[224043]: pgmap v314: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 158 B/s rd, 10 MiB/s wr, 173 op/s; 20605/248031 objects degraded (8.307%); 8929/248031 objects misplaced (3.600%); 6.4 MiB/s, 102 objects/s recovering 2023-09-30T09:24:33.159 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:33.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:33.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:33.264 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:33.304 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:33.362 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:33.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:33.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:33.473 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:33.509 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:33.622 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:33.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:33.727 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:33.743 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:33.743 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20605/248031 objects degraded (8.307%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:33.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:33.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20605/248031 objects degraded (8.307%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:33.757 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:33.757 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20605/248031 objects degraded (8.307%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:33.761 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:33.820 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:33.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:33.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:33.924 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:33.956 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:33.982 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:24:33.982 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:24:33.983 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:24:33.983 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:24:33.983 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:24:34.005 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:34.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:34.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:34.067 INFO:teuthology.orchestra.run.smithi019.stdout:moved 0 / 267 (0%) 2023-09-30T09:24:34.067 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 5.36045 -> 5.36045 (expected baseline 5.40399) 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.5 with 21 -> 21 pgs (0.629214 -> 0.629214 * mean) 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.0 with 39 -> 39 pgs (1.16854 -> 1.16854 * mean) 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout:oload 1000 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 3 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0229 2023-09-30T09:24:34.068 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.2293 2023-09-30T09:24:34.075 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:24:34.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:24:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:24:33] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-09-30T09:24:34.109 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:34.148 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:34.197 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:34.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:34.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:34.301 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:34.326 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:34.368 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:34.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:34.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:34.471 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:34.506 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:34.560 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:34.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:34.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:34.664 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:34.670 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:24:34.692 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:34.741 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:34.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:34.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:34.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:34 smithi019 ceph-mon[216815]: pgmap v315: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 8.7 MiB/s wr, 158 op/s; 20605/247941 objects degraded (8.310%); 7875/247941 objects misplaced (3.176%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:24:34.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:34 smithi019 ceph-mon[216815]: from='client.34797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:34.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:34 smithi019 ceph-mon[224043]: pgmap v315: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 8.7 MiB/s wr, 158 op/s; 20605/247941 objects degraded (8.310%); 7875/247941 objects misplaced (3.176%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:24:34.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:34 smithi019 ceph-mon[224043]: from='client.34797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:34.846 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:34.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:34 smithi167 ceph-mon[192880]: pgmap v315: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 8.7 MiB/s wr, 158 op/s; 20605/247941 objects degraded (8.310%); 7875/247941 objects misplaced (3.176%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:24:34.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:34 smithi167 ceph-mon[192880]: from='client.34797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:24:34.873 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:34.921 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:34.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:35.026 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:35.059 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:35.076 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:35.121 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:35.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:35.178 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:24:35.225 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:35.241 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:24:35.258 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:35.341 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:35.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:35.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:35.447 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:35.484 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:35.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:35 smithi019 ceph-mon[224043]: pgmap v316: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 B/s rd, 7.4 MiB/s wr, 197 op/s; 20564/246144 objects degraded (8.354%); 7829/246144 objects misplaced (3.181%); 10 MiB/s, 159 objects/s recovering 2023-09-30T09:24:35.617 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:35 smithi019 ceph-mon[216815]: pgmap v316: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 B/s rd, 7.4 MiB/s wr, 197 op/s; 20564/246144 objects degraded (8.354%); 7829/246144 objects misplaced (3.181%); 10 MiB/s, 159 objects/s recovering 2023-09-30T09:24:35.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:35 smithi167 ceph-mon[192880]: pgmap v316: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 B/s rd, 7.4 MiB/s wr, 197 op/s; 20564/246144 objects degraded (8.354%); 7829/246144 objects misplaced (3.181%); 10 MiB/s, 159 objects/s recovering 2023-09-30T09:24:35.721 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:35.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:35.826 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:35.855 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:35.898 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:35.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:35.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:35.940 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:35.964 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:36.001 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:36.074 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:36.197 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:36.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:36.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:36.302 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:36.330 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:36.375 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:36.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:36.479 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:36.514 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:36.671 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:46003 5.7 5:e9f5404d:::benchmark_data_smithi019_287194_object46002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:26.010034+0000", 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 70.658442500999996, 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21684166099999999, 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.672 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 46003 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.010034+0000", 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.010034+0000", 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.010033+0000", 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.010087+0000", 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4681e-05 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.010101+0000", 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3326000000000001e-05 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.010106+0000", 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0440000000000003e-06 2023-09-30T09:24:36.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.010921+0000", 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00081512200000000005 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.011017+0000", 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6193000000000005e-05 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.011135+0000", 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000117933 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.013466+0000", 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002331283 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.017190+0000", 2023-09-30T09:24:36.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037235049999999998 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.225911+0000", 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.208721568 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.225954+0000", 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2503999999999998e-05 2023-09-30T09:24:36.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:26.226875+0000", 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00092156499999999995 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61518 5.7 5:e18009d3:::benchmark_data_smithi019_287194_object61517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.345480+0000", 2023-09-30T09:24:36.679 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.322995857999999, 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0257004590000001, 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61518 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.680 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345480+0000", 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345480+0000", 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345479+0000", 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345523+0000", 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4019e-05 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345541+0000", 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7693000000000002e-05 2023-09-30T09:24:36.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345548+0000", 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5120000000000002e-06 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345603+0000", 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4543000000000003e-05 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345722+0000", 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011874700000000001 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345851+0000", 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012963800000000001 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.347660+0000", 2023-09-30T09:24:36.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018084659999999999 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.348395+0000", 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00073505900000000004 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.353323+0000", 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049284050000000003 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.353336+0000", 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3338e-05 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371124+0000", 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.017787105 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.686 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371181+0000", 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7352000000000002e-05 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61530 5.7 5:ea633b77:::benchmark_data_smithi019_287194_object61529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.512460+0000", 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.156016237000003, 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.877191371, 2023-09-30T09:24:36.687 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61530 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512460+0000", 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.688 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512460+0000", 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512459+0000", 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512489+0000", 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9590999999999998e-05 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512497+0000", 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4209999999999995e-06 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512502+0000", 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0540000000000002e-06 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512572+0000", 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9250000000000003e-05 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512581+0000", 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6949999999999995e-06 2023-09-30T09:24:36.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371197+0000", 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.85861520499999999 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371276+0000", 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9523000000000004e-05 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371425+0000", 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014907200000000001 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.377793+0000", 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0063680660000000004 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389534+0000", 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011741122 2023-09-30T09:24:36.693 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389564+0000", 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9206000000000001e-05 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389615+0000", 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1591999999999997e-05 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.694 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389651+0000", 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6310999999999998e-05 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61551 5.7 5:eed4ef95:::benchmark_data_smithi019_287194_object61550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.536037+0000", 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.132439599000001, 2023-09-30T09:24:36.695 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.85396402599999999, 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61551 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536037+0000", 2023-09-30T09:24:36.696 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536037+0000", 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536035+0000", 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536072+0000", 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6541999999999997e-05 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536088+0000", 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5914999999999999e-05 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536094+0000", 2023-09-30T09:24:36.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3520000000000003e-06 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536160+0000", 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5940999999999998e-05 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536171+0000", 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1406999999999999e-05 2023-09-30T09:24:36.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.372958+0000", 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83678686899999999 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.373042+0000", 2023-09-30T09:24:36.701 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4040000000000002e-05 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.373136+0000", 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3954000000000002e-05 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.377800+0000", 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004663365 2023-09-30T09:24:36.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389684+0000", 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011883906 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389944+0000", 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00026025100000000003 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389975+0000", 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1619999999999999e-05 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.390001+0000", 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5270999999999999e-05 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.704 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61563 5.7 5:e210ad0e:::benchmark_data_smithi019_287194_object61562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.542990+0000", 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.125485824000002, 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.84709957000000002, 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61563 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.705 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.542990+0000", 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.542990+0000", 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.706 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.542989+0000", 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543032+0000", 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3235000000000002e-05 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543046+0000", 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4044e-05 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543053+0000", 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5930000000000002e-06 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543438+0000", 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00038565099999999998 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543448+0000", 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5570000000000001e-06 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.374254+0000", 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83080556800000005 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.374332+0000", 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8102000000000005e-05 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.374433+0000", 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101196 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379043+0000", 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046102929999999997 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389723+0000", 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010679615999999999 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.390039+0000", 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031628500000000003 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.390068+0000", 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9300000000000001e-05 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.390090+0000", 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1701000000000001e-05 2023-09-30T09:24:36.712 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61568 5.7 5:e495c4d5:::benchmark_data_smithi019_287194_object61567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.545646+0000", 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.122829971000002, 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.85827185699999997, 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.713 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61568 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545646+0000", 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545646+0000", 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545646+0000", 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545675+0000", 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9413000000000001e-05 2023-09-30T09:24:36.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545681+0000", 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0109999999999997e-06 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545685+0000", 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6059999999999999e-06 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545750+0000", 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5037999999999996e-05 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545759+0000", 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2690000000000005e-06 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.375557+0000", 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82979780400000003 2023-09-30T09:24:36.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.375662+0000", 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105048 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.375765+0000", 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000102701 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379053+0000", 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032880639999999998 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389742+0000", 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010688939 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.719 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403825+0000", 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014083469 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403880+0000", 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5074000000000002e-05 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403918+0000", 2023-09-30T09:24:36.720 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7787999999999998e-05 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61624 5.7 5:ea2cafff:::benchmark_data_smithi019_287194_object61623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.592059+0000", 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.076417616000001, 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81197014700000003, 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.721 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61624 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592059+0000", 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592059+0000", 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592057+0000", 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592149+0000", 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2001000000000002e-05 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592164+0000", 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4605e-05 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592171+0000", 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9630000000000001e-06 2023-09-30T09:24:36.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.595232+0000", 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030614380000000001 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.595243+0000", 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0117e-05 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.376729+0000", 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.78148660800000003 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.376805+0000", 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5838000000000001e-05 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.376892+0000", 2023-09-30T09:24:36.726 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7280000000000005e-05 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.381389+0000", 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044968070000000002 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389842+0000", 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0084531889999999998 2023-09-30T09:24:36.727 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403951+0000", 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014109116 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403994+0000", 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2341e-05 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.728 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404029+0000", 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5113000000000001e-05 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61627 5.7 5:e02a1d6c:::benchmark_data_smithi019_287194_object61626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.596894+0000", 2023-09-30T09:24:36.729 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.071581924, 2023-09-30T09:24:36.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.80724527599999996, 2023-09-30T09:24:36.730 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.730 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.730 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.730 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.730 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.730 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61627 2023-09-30T09:24:36.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596894+0000", 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596894+0000", 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596893+0000", 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596950+0000", 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6727000000000001e-05 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596961+0000", 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1207e-05 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596968+0000", 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2500000000000003e-06 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597025+0000", 2023-09-30T09:24:36.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7632e-05 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597034+0000", 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8570000000000001e-06 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.377818+0000", 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.78078413999999996 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.377890+0000", 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.169e-05 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.378021+0000", 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013104300000000001 2023-09-30T09:24:36.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383651+0000", 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0056301830000000004 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389885+0000", 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062335450000000001 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404061+0000", 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014175956 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404105+0000", 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4431000000000002e-05 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404140+0000", 2023-09-30T09:24:36.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4637000000000001e-05 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61628 5.7 5:ee236a29:::benchmark_data_smithi019_287194_object61627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.596983+0000", 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.071493709999999, 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.80726483400000004, 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.738 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61628 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596983+0000", 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596983+0000", 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596982+0000", 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.740 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597013+0000", 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1186000000000003e-05 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597028+0000", 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5129e-05 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597034+0000", 2023-09-30T09:24:36.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.891e-06 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597075+0000", 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0327000000000002e-05 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597083+0000", 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5290000000000008e-06 2023-09-30T09:24:36.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379093+0000", 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.78200961700000005 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379208+0000", 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011480300000000001 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379300+0000", 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2001000000000002e-05 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383663+0000", 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043638419999999997 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389914+0000", 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062502749999999996 2023-09-30T09:24:36.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404173+0000", 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014258966 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404213+0000", 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0880999999999997e-05 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404247+0000", 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3939000000000002e-05 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61650 5.7 5:e92d62dc:::benchmark_data_smithi019_287194_object61649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.615308+0000", 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.053167971999997, 2023-09-30T09:24:36.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.78907863300000003, 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61650 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615308+0000", 2023-09-30T09:24:36.747 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615308+0000", 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615306+0000", 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615343+0000", 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6816000000000001e-05 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615361+0000", 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8280000000000001e-05 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615368+0000", 2023-09-30T09:24:36.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9739999999999996e-06 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615441+0000", 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3008000000000003e-05 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615451+0000", 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6490000000000003e-06 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.380228+0000", 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.76477674600000001 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.380321+0000", 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2794000000000006e-05 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.380444+0000", 2023-09-30T09:24:36.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000123688 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.385970+0000", 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055259710000000002 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.393531+0000", 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075608689999999996 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.752 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404295+0000", 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010763477 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404350+0000", 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5593000000000001e-05 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404387+0000", 2023-09-30T09:24:36.753 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6807000000000002e-05 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61667 5.7 5:e26b6ee9:::benchmark_data_smithi019_287194_object61666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.630906+0000", 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.037570551999998, 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.773589215, 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.754 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61667 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630906+0000", 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630906+0000", 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630905+0000", 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630952+0000", 2023-09-30T09:24:36.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7675e-05 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630965+0000", 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3206e-05 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630971+0000", 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1000000000000003e-06 2023-09-30T09:24:36.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.631033+0000", 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2259000000000001e-05 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.631043+0000", 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6919999999999997e-06 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.381434+0000", 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.75039142599999997 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.381539+0000", 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010470499999999999 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.381630+0000", 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1679000000000001e-05 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.385976+0000", 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043454490000000004 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.760 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.393576+0000", 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075999409999999998 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404433+0000", 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010857232 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404473+0000", 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9950000000000002e-05 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404495+0000", 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2050999999999999e-05 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61696 5.7 5:e6913336:::benchmark_data_smithi019_287194_object61695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.658118+0000", 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 56.010358531000001, 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.74646133999999997, 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61696 2023-09-30T09:24:36.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658118+0000", 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658118+0000", 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658116+0000", 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658170+0000", 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3786000000000003e-05 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658180+0000", 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0378e-05 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658193+0000", 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2658e-05 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658254+0000", 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1359999999999995e-05 2023-09-30T09:24:36.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658263+0000", 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8990000000000007e-06 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.382547+0000", 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.724284072 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.382670+0000", 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012278699999999999 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.382762+0000", 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1383000000000002e-05 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389213+0000", 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064515149999999997 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.396897+0000", 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076843409999999999 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404518+0000", 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007620679 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404548+0000", 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9479999999999999e-05 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404579+0000", 2023-09-30T09:24:36.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1501000000000002e-05 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61735 5.7 5:e5c7b539:::benchmark_data_smithi019_287194_object61734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.698538+0000", 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 55.969938304000003, 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.70766760500000003, 2023-09-30T09:24:36.771 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61735 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698538+0000", 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698538+0000", 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698537+0000", 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698572+0000", 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4150000000000003e-05 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698580+0000", 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3729999999999993e-06 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698585+0000", 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.13e-06 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698681+0000", 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6032000000000005e-05 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698691+0000", 2023-09-30T09:24:36.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7850000000000003e-06 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383703+0000", 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.68501200699999998 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383793+0000", 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0301000000000002e-05 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383885+0000", 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2262000000000005e-05 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389222+0000", 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0053361379999999998 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.396940+0000", 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077180720000000003 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406079+0000", 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091388609999999999 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406151+0000", 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2158999999999998e-05 2023-09-30T09:24:36.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406206+0000", 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4895999999999998e-05 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61739 5.7 5:efc4b11e:::benchmark_data_smithi019_287194_object61738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.701702+0000", 2023-09-30T09:24:36.779 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 55.966774196000003, 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.70468040600000004, 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61739 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.780 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701702+0000", 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701702+0000", 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701701+0000", 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:24:36.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701760+0000", 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8833999999999999e-05 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701772+0000", 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1620999999999999e-05 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701777+0000", 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2429999999999996e-06 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701822+0000", 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4644999999999997e-05 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701829+0000", 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4200000000000001e-06 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.384884+0000", 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.68305470300000004 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.384993+0000", 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010960500000000001 2023-09-30T09:24:36.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.385082+0000", 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8086999999999999e-05 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389325+0000", 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042431379999999996 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.402848+0000", 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013523373 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406276+0000", 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034283489999999998 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406345+0000", 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8824000000000004e-05 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406382+0000", 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7313e-05 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61741 5.7 5:e7992001:::benchmark_data_smithi019_287194_object61740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.701920+0000", 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 55.966556773000001, 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.712059569, 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61741 2023-09-30T09:24:36.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701920+0000", 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701920+0000", 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701919+0000", 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701991+0000", 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2441999999999995e-05 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701999+0000", 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7209999999999994e-06 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.702006+0000", 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7259999999999996e-06 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.702038+0000", 2023-09-30T09:24:36.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2450999999999998e-05 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.702044+0000", 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3680000000000001e-06 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.386014+0000", 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.68397022399999996 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.386094+0000", 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9943000000000004e-05 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.386196+0000", 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101778 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389331+0000", 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031357960000000002 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.402875+0000", 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013543914000000001 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.413867+0000", 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010991618999999999 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.413936+0000", 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8697999999999999e-05 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.413979+0000", 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3498999999999998e-05 2023-09-30T09:24:36.795 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61746 5.7 5:e394340c:::benchmark_data_smithi019_287194_object61745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.711658+0000", 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 55.956817991000001, 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.70245204800000005, 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.796 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61746 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711658+0000", 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711658+0000", 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711657+0000", 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711735+0000", 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7892000000000006e-05 2023-09-30T09:24:36.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711751+0000", 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6310000000000001e-05 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711758+0000", 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6769999999999998e-06 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711843+0000", 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5019999999999996e-05 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711856+0000", 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2869e-05 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.387094+0000", 2023-09-30T09:24:36.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.67523834800000004 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.387201+0000", 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010642899999999999 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.387290+0000", 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9501999999999998e-05 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389336+0000", 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002045561 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.402893+0000", 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01355757 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414022+0000", 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011128895999999999 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414063+0000", 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0883000000000001e-05 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414110+0000", 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7240999999999997e-05 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.803 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61753 5.7 5:e75af397:::benchmark_data_smithi019_287194_object61752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.728877+0000", 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 55.939599725999997, 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.68544973399999998, 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.804 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61753 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728877+0000", 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728877+0000", 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728875+0000", 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728933+0000", 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7878999999999999e-05 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728949+0000", 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5931e-05 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728956+0000", 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9099999999999999e-06 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.729011+0000", 2023-09-30T09:24:36.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5245999999999999e-05 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.729021+0000", 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7010000000000008e-06 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.388165+0000", 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.65914413800000005 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.388241+0000", 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6187999999999996e-05 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.388331+0000", 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0103000000000001e-05 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389689+0000", 2023-09-30T09:24:36.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013576490000000001 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.402907+0000", 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013218647 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414240+0000", 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011332493000000001 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414285+0000", 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5207000000000003e-05 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414326+0000", 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1149000000000002e-05 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:80441 5.3 5:c3f79ee6:::benchmark_data_smithi019_287194_object80440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:24:01.607700+0000", 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 35.060776277000002, 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0053129009999999, 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.812 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 80441 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607700+0000", 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607700+0000", 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607699+0000", 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607729+0000", 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0666000000000001e-05 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607743+0000", 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3380999999999999e-05 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607752+0000", 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0180000000000007e-06 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607817+0000", 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5216999999999996e-05 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607989+0000", 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000171857 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,6", 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608168+0000", 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017956800000000001 2023-09-30T09:24:36.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.615044+0000", 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006875613 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.615057+0000", 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3441000000000001e-05 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.612993+0000", 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99793548200000004 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613013+0000", 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0143e-05 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:80443 5.3 5.414b81e3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:24:01.608633+0000", 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 35.059843657999998, 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.004423759, 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 80443 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608633+0000", 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608633+0000", 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608624+0000", 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999914 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608673+0000", 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9038e-05 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608686+0000", 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2653000000000001e-05 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608693+0000", 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9299999999999997e-06 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.615099+0000", 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006406747 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.615109+0000", 2023-09-30T09:24:36.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7010000000000008e-06 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613038+0000", 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99792904500000001 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613056+0000", 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8216999999999999e-05 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.823 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:80448 5.3 5.44d7ee03 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:24:01.627487+0000", 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 35.040989721999999, 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98560211399999997, 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:24:36.824 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 80448 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627487+0000", 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627487+0000", 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627485+0000", 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627550+0000", 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4855000000000002e-05 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:24:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627566+0000", 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6478999999999999e-05 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627574+0000", 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7209999999999994e-06 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627639+0000", 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5351999999999994e-05 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627650+0000", 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0311e-05 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613079+0000", 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98542918400000001 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613089+0000", 2023-09-30T09:24:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8390000000000002e-06 2023-09-30T09:24:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:24:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:24:36.829 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:24:36.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:36.830 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:36.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:36.931 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:36.969 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:37.075 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:37.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:37.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:37.178 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:37.208 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:37.442 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:24:37.763 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:38.187 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:38.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:38.288 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:38.319 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:38 smithi019 ceph-mon[216815]: pgmap v317: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 181 op/s; 20564/245913 objects degraded (8.362%); 6349/245913 objects misplaced (2.582%); 12 MiB/s, 189 objects/s recovering 2023-09-30T09:24:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:38 smithi019 ceph-mon[224043]: pgmap v317: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 181 op/s; 20564/245913 objects degraded (8.362%); 6349/245913 objects misplaced (2.582%); 12 MiB/s, 189 objects/s recovering 2023-09-30T09:24:38.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:38 smithi167 ceph-mon[192880]: pgmap v317: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 181 op/s; 20564/245913 objects degraded (8.362%); 6349/245913 objects misplaced (2.582%); 12 MiB/s, 189 objects/s recovering 2023-09-30T09:24:38.466 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:38.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:38.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:38.569 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:38.595 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:38.638 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:38.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:38.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:38.741 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:38.778 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:38.863 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:38.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:38.971 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:39.001 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:39.062 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:39.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:39.168 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:39.198 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:39 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20564/245913 objects degraded (8.362%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:39 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20564/245913 objects degraded (8.362%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:39 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20564/245913 objects degraded (8.362%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:39.339 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:24:39.343 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:39.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:39.364 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:24:39.393 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:24:39.420 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:24:39.444 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:24:39.447 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:39.468 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:24:39.477 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:39.498 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:24:39.528 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:39.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:39.535 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:24:39.539 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:24:39.539 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:24:39.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:24:39.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:24:39.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:24:39.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:24:39.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:24:39.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:24:39.577 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:24:39.577 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:24:39.632 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:39.663 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:39.764 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:39.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:39.868 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:39.895 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:39.939 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:39.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:39.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:40.043 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:40.076 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:40.132 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:40.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:40.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:40.144 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:24:40.236 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:40.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:40 smithi019 ceph-mon[216815]: pgmap v318: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 102 B/s rd, 2.2 MiB/s wr, 152 op/s; 20564/244932 objects degraded (8.396%); 6308/244932 objects misplaced (2.575%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:24:40.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3048405629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:40.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2328703539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:24:40.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:40 smithi019 ceph-mon[224043]: pgmap v318: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 102 B/s rd, 2.2 MiB/s wr, 152 op/s; 20564/244932 objects degraded (8.396%); 6308/244932 objects misplaced (2.575%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:24:40.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3048405629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:40.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2328703539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:24:40.266 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:40 smithi167 ceph-mon[192880]: pgmap v318: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 102 B/s rd, 2.2 MiB/s wr, 152 op/s; 20564/244932 objects degraded (8.396%); 6308/244932 objects misplaced (2.575%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:24:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3048405629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2328703539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:24:40.287 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:24:40.316 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:40.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:40.423 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:40.452 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:40.645 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:40.758 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:40.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:40.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:40.817 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:24:40.817 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-30T09:24:40.862 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:40.889 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:40.938 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:40.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:40.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:41.042 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:41.071 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:41.306 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:41.321 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:41.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2328703539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:24:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[216815]: osdmap e148: 8 total, 8 up, 8 in 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[216815]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[216815]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[216815]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[216815]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2328703539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[224043]: osdmap e148: 8 total, 8 up, 8 in 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[224043]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[224043]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[224043]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:41 smithi019 ceph-mon[224043]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.424 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:41.456 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:41 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2328703539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:24:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:41 smithi167 ceph-mon[192880]: osdmap e148: 8 total, 8 up, 8 in 2023-09-30T09:24:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:41 smithi167 ceph-mon[192880]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:41 smithi167 ceph-mon[192880]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:41 smithi167 ceph-mon[192880]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:41 smithi167 ceph-mon[192880]: failed to encode map e148 with expected crc 2023-09-30T09:24:41.626 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:41.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:41.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:41.733 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:41.768 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:41.968 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:41.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:42.074 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:42.102 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:42.147 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:42.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:42.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:42.251 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:42.255 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:24:42.255 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:24:42.283 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:42.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[224043]: pgmap v320: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 397 KiB/s wr, 133 op/s; 20527/244401 objects degraded (8.399%); 6308/244401 objects misplaced (2.581%); 12 MiB/s, 196 objects/s recovering 2023-09-30T09:24:42.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[224043]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[224043]: osdmap e149: 8 total, 8 up, 8 in 2023-09-30T09:24:42.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[224043]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[224043]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[224043]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[216815]: pgmap v320: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 397 KiB/s wr, 133 op/s; 20527/244401 objects degraded (8.399%); 6308/244401 objects misplaced (2.581%); 12 MiB/s, 196 objects/s recovering 2023-09-30T09:24:42.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[216815]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[216815]: osdmap e149: 8 total, 8 up, 8 in 2023-09-30T09:24:42.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[216815]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[216815]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.462 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:42 smithi019 ceph-mon[216815]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.465 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:42 smithi167 ceph-mon[192880]: pgmap v320: 89 pgs: 1 active+remapped+backfilling, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 397 KiB/s wr, 133 op/s; 20527/244401 objects degraded (8.399%); 6308/244401 objects misplaced (2.581%); 12 MiB/s, 196 objects/s recovering 2023-09-30T09:24:42.465 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:42 smithi167 ceph-mon[192880]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.465 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:42 smithi167 ceph-mon[192880]: osdmap e149: 8 total, 8 up, 8 in 2023-09-30T09:24:42.465 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:42 smithi167 ceph-mon[192880]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.466 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:42 smithi167 ceph-mon[192880]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.466 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:42 smithi167 ceph-mon[192880]: failed to encode map e149 with expected crc 2023-09-30T09:24:42.486 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:42.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:42.594 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:42.626 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:42.823 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:42.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:42.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:42.927 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:42.945 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:24:42.967 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:42.981 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:24:43.029 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:43.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:43.135 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:43.160 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20527/244401 objects degraded (8.399%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:43.170 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:43.216 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:43.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:43.223 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:24:43.315 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:43.322 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:43.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20527/244401 objects degraded (8.399%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20527/244401 objects degraded (8.399%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:43.351 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:43.440 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:43.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:43.450 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:43.544 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:43.572 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:43.620 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:43.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:43.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:43.723 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:43.750 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:43.895 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:43.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:43.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:43.999 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:44.027 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:44.028 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:24:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:24:43] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-09-30T09:24:44.073 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:44.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:44.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:44.177 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:44.180 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.1 to 1 (10000) 2023-09-30T09:24:44.223 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:44.281 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:44.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:44 smithi019 ceph-mon[224043]: pgmap v322: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 20527/244014 objects degraded (8.412%); 4954/244014 objects misplaced (2.030%); 16 MiB/s, 261 objects/s recovering 2023-09-30T09:24:44.286 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2424658079' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:24:44.286 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:44 smithi019 ceph-mon[216815]: pgmap v322: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 20527/244014 objects degraded (8.412%); 4954/244014 objects misplaced (2.030%); 16 MiB/s, 261 objects/s recovering 2023-09-30T09:24:44.286 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2424658079' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:24:44.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:44.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:44.388 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:44.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:44 smithi167 ceph-mon[192880]: pgmap v322: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 20527/244014 objects degraded (8.412%); 4954/244014 objects misplaced (2.030%); 16 MiB/s, 261 objects/s recovering 2023-09-30T09:24:44.413 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2424658079' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:24:44.420 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:44.481 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:44.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:44.591 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:44.633 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:44.940 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:24:44.940 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:24:45.222 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:45.233 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:45.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:45.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:45.249 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-30T09:24:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:45 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2424658079' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:24:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:45 smithi167 ceph-mon[192880]: osdmap e150: 8 total, 8 up, 8 in 2023-09-30T09:24:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:45 smithi167 ceph-mon[192880]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:45 smithi167 ceph-mon[192880]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:45 smithi167 ceph-mon[192880]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.320 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2424658079' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[216815]: osdmap e150: 8 total, 8 up, 8 in 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[216815]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[216815]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[216815]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2424658079' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[224043]: osdmap e150: 8 total, 8 up, 8 in 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[224043]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[224043]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.321 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:45 smithi019 ceph-mon[224043]: failed to encode map e150 with expected crc 2023-09-30T09:24:45.339 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:45.370 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:45.418 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:45.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:45.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:45.521 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:45.547 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:45.614 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:45.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:45.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:45.671 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:24:45.716 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:24:45.718 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:45.737 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:24:45.744 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:45.757 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:24:45.782 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:24:45.788 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:45.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:45.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:45.815 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:24:45.839 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:24:45.874 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:24:45.894 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:24:45.894 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:45.895 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:24:45.895 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:24:45.895 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:24:45.896 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:24:45.896 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:24:45.896 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:24:45.896 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:24:45.896 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:24:45.921 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:45.969 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:45.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:46.042 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:46.075 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:46.109 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:46.165 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:46.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:46.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:46.270 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:46.299 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:46.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:46 smithi167 ceph-mon[192880]: pgmap v324: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 20484/242100 objects degraded (8.461%); 4915/242100 objects misplaced (2.030%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:24:46.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1174085608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:46.502 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:46 smithi019 ceph-mon[216815]: pgmap v324: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 20484/242100 objects degraded (8.461%); 4915/242100 objects misplaced (2.030%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:24:46.502 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1174085608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:46.503 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:46 smithi019 ceph-mon[224043]: pgmap v324: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 20484/242100 objects degraded (8.461%); 4915/242100 objects misplaced (2.030%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:24:46.503 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1174085608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:46.919 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:46.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:47.024 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:47.053 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:47.070 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:47.160 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:47.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:47.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:47.263 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:47.312 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:47.422 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:47.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:47.528 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:47.561 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:47.690 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:24:47.691 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:24:47.691 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:24:47.691 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:24:47.770 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:47.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:47.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:47.875 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:47.907 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:47.960 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:47.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:47.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:48.064 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:48.093 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:48.136 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:48.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:48.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:48.202 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:24:48.239 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:48.279 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:48.433 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:48 smithi167 ceph-mon[192880]: pgmap v325: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 20484/241800 objects degraded (8.471%); 4915/241800 objects misplaced (2.033%); 9.6 MiB/s, 153 objects/s recovering 2023-09-30T09:24:48.433 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3048623133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:48.433 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:48.433 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:48.433 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20484/241800 objects degraded (8.471%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:48.442 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[216815]: pgmap v325: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 20484/241800 objects degraded (8.471%); 4915/241800 objects misplaced (2.033%); 9.6 MiB/s, 153 objects/s recovering 2023-09-30T09:24:48.442 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3048623133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:48.442 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:48.442 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:48.442 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20484/241800 objects degraded (8.471%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:48.443 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[224043]: pgmap v325: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 20484/241800 objects degraded (8.471%); 4915/241800 objects misplaced (2.033%); 9.6 MiB/s, 153 objects/s recovering 2023-09-30T09:24:48.443 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3048623133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:48.443 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:24:48.443 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:24:48.443 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20484/241800 objects degraded (8.471%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:48.446 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:24:48.534 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:48.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:48.640 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:48.670 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:48.797 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:48.985 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:48.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:48.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:49.006 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:24:49.088 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:49.117 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:49.265 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:49.267 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:49 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:24:49.267 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:49.267 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:49 smithi167 ceph-mon[192880]: osdmap e151: 8 total, 8 up, 8 in 2023-09-30T09:24:49.267 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:49 smithi167 ceph-mon[192880]: failed to encode map e151 with expected crc 2023-09-30T09:24:49.268 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:49 smithi167 ceph-mon[192880]: failed to encode map e151 with expected crc 2023-09-30T09:24:49.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:49.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:24:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[224043]: osdmap e151: 8 total, 8 up, 8 in 2023-09-30T09:24:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[224043]: failed to encode map e151 with expected crc 2023-09-30T09:24:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[224043]: failed to encode map e151 with expected crc 2023-09-30T09:24:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:24:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:24:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[216815]: osdmap e151: 8 total, 8 up, 8 in 2023-09-30T09:24:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[216815]: failed to encode map e151 with expected crc 2023-09-30T09:24:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:49 smithi019 ceph-mon[216815]: failed to encode map e151 with expected crc 2023-09-30T09:24:49.369 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:49.410 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:49.469 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:49.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:49.521 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:49.573 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:49.605 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:49.710 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:49.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:49.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:49.814 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:49.849 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:49.895 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:49.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:49.999 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:50.029 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:50.078 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:50.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:50.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:50.182 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:50.195 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.2 to 1 (10000) 2023-09-30T09:24:50.213 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:50.309 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:50.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:50.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:50 smithi019 ceph-mon[216815]: pgmap v327: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 20484/240651 objects degraded (8.512%); 4871/240651 objects misplaced (2.024%); 8.7 MiB/s, 139 objects/s recovering 2023-09-30T09:24:50.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1080366268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:24:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:50 smithi019 ceph-mon[224043]: pgmap v327: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 20484/240651 objects degraded (8.512%); 4871/240651 objects misplaced (2.024%); 8.7 MiB/s, 139 objects/s recovering 2023-09-30T09:24:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1080366268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:24:50.384 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:24:50.384 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:24:50.384 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:24:50.384 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:24:50.385 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:24:50.414 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:50.443 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:50.495 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:50.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:50 smithi167 ceph-mon[192880]: pgmap v327: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 20484/240651 objects degraded (8.512%); 4871/240651 objects misplaced (2.024%); 8.7 MiB/s, 139 objects/s recovering 2023-09-30T09:24:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1080366268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:24:50.599 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:50.627 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:50.675 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:50.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:50.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:50.779 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:50.810 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:51.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:24:51.046 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:24:51.064 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:51.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:51.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:51.175 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:51.201 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:51.252 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:51.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:51.328 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1080366268' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:24:51.329 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:51 smithi019 ceph-mon[224043]: osdmap e152: 8 total, 8 up, 8 in 2023-09-30T09:24:51.329 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:51 smithi019 ceph-mon[224043]: failed to encode map e152 with expected crc 2023-09-30T09:24:51.329 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:51 smithi019 ceph-mon[224043]: failed to encode map e152 with expected crc 2023-09-30T09:24:51.329 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1080366268' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:24:51.329 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:51 smithi019 ceph-mon[216815]: osdmap e152: 8 total, 8 up, 8 in 2023-09-30T09:24:51.329 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:51 smithi019 ceph-mon[216815]: failed to encode map e152 with expected crc 2023-09-30T09:24:51.329 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:51 smithi019 ceph-mon[216815]: failed to encode map e152 with expected crc 2023-09-30T09:24:51.357 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-30T09:24:51.357 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:51.383 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1080366268' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:24:51.383 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:51 smithi167 ceph-mon[192880]: osdmap e152: 8 total, 8 up, 8 in 2023-09-30T09:24:51.383 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:51 smithi167 ceph-mon[192880]: failed to encode map e152 with expected crc 2023-09-30T09:24:51.383 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:51 smithi167 ceph-mon[192880]: failed to encode map e152 with expected crc 2023-09-30T09:24:51.386 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:51.427 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:24:51.433 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:51.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:51.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:51.542 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:51.569 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:51.664 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:51.666 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:51.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:51.773 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:51.787 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:24:51.801 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:51.812 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:24:51.842 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:24:51.869 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:24:51.894 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:24:51.897 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:51.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:51.916 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:24:51.941 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:24:51.965 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:24:51.967 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:24:51.967 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:24:51.967 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:24:51.967 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:24:51.967 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:24:51.967 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:24:51.967 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:24:51.967 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:24:52.001 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:52.035 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:52.079 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:52.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:52.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:52.184 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:52.210 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:52.253 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:52.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:52.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:52.358 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:52.397 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:52 smithi167 ceph-mon[192880]: pgmap v329: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 20444/240021 objects degraded (8.518%); 4871/240021 objects misplaced (2.029%); 1.1 MiB/s, 18 objects/s recovering 2023-09-30T09:24:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2626110118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:52 smithi019 ceph-mon[216815]: pgmap v329: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 20444/240021 objects degraded (8.518%); 4871/240021 objects misplaced (2.029%); 1.1 MiB/s, 18 objects/s recovering 2023-09-30T09:24:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2626110118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:52 smithi019 ceph-mon[224043]: pgmap v329: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 20444/240021 objects degraded (8.518%); 4871/240021 objects misplaced (2.029%); 1.1 MiB/s, 18 objects/s recovering 2023-09-30T09:24:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2626110118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:24:52.740 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:52.751 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:52.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:52.857 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:52.887 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:52.986 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:52.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:52.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:53.040 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:24:53.040 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:24:53.089 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:53.117 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:53.421 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:53.424 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20444/240021 objects degraded (8.518%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:53.424 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20444/240021 objects degraded (8.518%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:53.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20444/240021 objects degraded (8.518%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:53.526 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:53.567 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:53.652 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:53.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:53.727 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:24:53.760 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:53.787 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:53.833 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:53.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:53.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:53.941 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:53.967 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:54.009 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:54.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:54.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:24:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:24:53] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-09-30T09:24:54.114 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:54.149 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:54.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:54 smithi167 ceph-mon[192880]: pgmap v330: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 20444/239679 objects degraded (8.530%); 4871/239679 objects misplaced (2.032%); 648 KiB/s, 10 objects/s recovering 2023-09-30T09:24:54.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:54 smithi019 ceph-mon[216815]: pgmap v330: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 20444/239679 objects degraded (8.530%); 4871/239679 objects misplaced (2.032%); 648 KiB/s, 10 objects/s recovering 2023-09-30T09:24:54.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:54 smithi019 ceph-mon[224043]: pgmap v330: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 20444/239679 objects degraded (8.530%); 4871/239679 objects misplaced (2.032%); 648 KiB/s, 10 objects/s recovering 2023-09-30T09:24:54.621 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:54.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:54.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:54.724 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:24:54.763 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:54.822 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:24:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:54.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:24:54.867 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:54.932 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:54.958 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:55.002 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:55.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:55.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:55.107 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:55.146 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:55.230 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:24:55.260 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:55.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:55.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:55.296 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:24:55.300 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1548201899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:24:55.300 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1548201899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:24:55.366 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:55.399 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:55.475 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:55.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1548201899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:24:55.580 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:55.607 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:55.653 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:55.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:55.758 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:55.785 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:55.830 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:55.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:55.934 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:55.969 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:56.071 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:56.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:56.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:56.174 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:24:56.205 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:56.234 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.3 to 1 (10000) 2023-09-30T09:24:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: pgmap v331: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 20409/238020 objects degraded (8.574%); 4829/238020 objects misplaced (2.029%); 1.2 MiB/s, 19 objects/s recovering 2023-09-30T09:24:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1548201899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:24:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: osdmap e153: 8 total, 8 up, 8 in 2023-09-30T09:24:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2218153525' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:24:56.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:56 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:24:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: pgmap v331: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 20409/238020 objects degraded (8.574%); 4829/238020 objects misplaced (2.029%); 1.2 MiB/s, 19 objects/s recovering 2023-09-30T09:24:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1548201899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:24:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: osdmap e153: 8 total, 8 up, 8 in 2023-09-30T09:24:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2218153525' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: pgmap v331: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 20409/238020 objects degraded (8.574%); 4829/238020 objects misplaced (2.029%); 1.2 MiB/s, 19 objects/s recovering 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1548201899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: osdmap e153: 8 total, 8 up, 8 in 2023-09-30T09:24:56.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: failed to encode map e153 with expected crc 2023-09-30T09:24:56.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2218153525' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:24:56.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:56 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:24:56.810 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:24:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:56.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:24:56.914 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:24:56.940 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:56.984 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:24:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:56.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:24:57.090 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:24:57.127 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:57.337 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:24:57.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:57.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:24:57.390 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:57 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:24:57.390 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:57 smithi019 ceph-mon[216815]: failed to encode map e154 with expected crc 2023-09-30T09:24:57.390 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:57 smithi019 ceph-mon[216815]: osdmap e154: 8 total, 8 up, 8 in 2023-09-30T09:24:57.390 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:57 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:24:57.390 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:57 smithi019 ceph-mon[224043]: failed to encode map e154 with expected crc 2023-09-30T09:24:57.390 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:57 smithi019 ceph-mon[224043]: osdmap e154: 8 total, 8 up, 8 in 2023-09-30T09:24:57.391 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-30T09:24:57.442 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:57.469 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:57.513 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:57.518 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:57 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:24:57.519 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:57 smithi167 ceph-mon[192880]: failed to encode map e154 with expected crc 2023-09-30T09:24:57.519 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:57 smithi167 ceph-mon[192880]: osdmap e154: 8 total, 8 up, 8 in 2023-09-30T09:24:57.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:57.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:57.606 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:57.620 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:57.660 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:57.728 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:24:57.728 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:24:58.008 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:58.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:58.111 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:24:58.139 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:58.183 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:24:58.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:58.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:24:58.279 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:58 smithi019 ceph-mon[224043]: pgmap v334: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 20409/237825 objects degraded (8.582%); 4829/237825 objects misplaced (2.030%); 730 KiB/s, 11 objects/s recovering 2023-09-30T09:24:58.279 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:24:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20409/237825 objects degraded (8.582%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:58.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:58 smithi019 ceph-mon[216815]: pgmap v334: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 20409/237825 objects degraded (8.582%); 4829/237825 objects misplaced (2.030%); 730 KiB/s, 11 objects/s recovering 2023-09-30T09:24:58.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:24:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20409/237825 objects degraded (8.582%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:58.292 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:58.330 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:58 smithi167 ceph-mon[192880]: pgmap v334: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 20409/237825 objects degraded (8.582%); 4829/237825 objects misplaced (2.030%); 730 KiB/s, 11 objects/s recovering 2023-09-30T09:24:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:24:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20409/237825 objects degraded (8.582%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:24:58.624 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:24:58.637 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:58.773 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:58.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:58.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:24:58.878 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:24:59.421 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:59.476 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:24:59.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:59.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:24:59.566 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:24:59.580 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:59.606 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:59.650 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:24:59.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:59.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:24:59.760 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:24:59.786 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:24:59.830 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:24:59.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:24:59.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:24:59.939 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:24:59.965 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:00.007 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:00.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:00.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:00.113 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:00.139 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:00.181 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:00.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:00.287 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:00.295 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:00.332 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:00.393 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:00.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:00.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:00.494 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:00 smithi019 ceph-mon[216815]: pgmap v335: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 20409/236793 objects degraded (8.619%); 4788/236793 objects misplaced (2.022%); 960 KiB/s, 14 objects/s recovering 2023-09-30T09:25:00.494 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:00 smithi019 ceph-mon[224043]: pgmap v335: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 20409/236793 objects degraded (8.619%); 4788/236793 objects misplaced (2.022%); 960 KiB/s, 14 objects/s recovering 2023-09-30T09:25:00.498 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:00.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:00 smithi167 ceph-mon[192880]: pgmap v335: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 20409/236793 objects degraded (8.619%); 4788/236793 objects misplaced (2.022%); 960 KiB/s, 14 objects/s recovering 2023-09-30T09:25:00.524 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:00.567 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:00.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:00.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:00.672 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:00.716 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:00.738 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:25:00.778 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:25:00.818 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:25:00.846 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:25:00.871 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:25:00.894 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:25:00.918 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:25:00.930 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:00.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:00.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:00.946 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:25:00.948 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:25:00.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:25:00.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:25:00.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:25:00.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:25:00.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:25:00.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:25:00.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:25:01.046 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:25:01.072 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:01.114 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:25:01.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:01.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:25:01.223 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:01.259 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:01.496 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/867315217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:01.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/867315217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:01.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/867315217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:02.265 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:02.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:02.371 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:02.398 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:02.443 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:02.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:02.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:02 smithi167 ceph-mon[192880]: pgmap v336: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 20379/236289 objects degraded (8.625%); 4788/236289 objects misplaced (2.026%); 1.2 MiB/s, 18 objects/s recovering 2023-09-30T09:25:02.552 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:02.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:02 smithi019 ceph-mon[224043]: pgmap v336: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 20379/236289 objects degraded (8.625%); 4788/236289 objects misplaced (2.026%); 1.2 MiB/s, 18 objects/s recovering 2023-09-30T09:25:02.572 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:02 smithi019 ceph-mon[216815]: pgmap v336: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 20379/236289 objects degraded (8.625%); 4788/236289 objects misplaced (2.026%); 1.2 MiB/s, 18 objects/s recovering 2023-09-30T09:25:02.578 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:02.621 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:02.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:02.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:02.726 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:02.752 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:02.794 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:02.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:02.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:02.898 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:02.927 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:02.971 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:02.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:03.078 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:03.120 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:03.294 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.4 to 1 (10000) 2023-09-30T09:25:03.458 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:25:03.458 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:25:03.458 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:25:03.458 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:25:03.459 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:25:03.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1415426901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:25:03.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:25:03.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:03.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20379/236289 objects degraded (8.625%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:03.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1415426901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:25:03.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:25:03.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:03.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20379/236289 objects degraded (8.625%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:03.508 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:03.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:03.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1415426901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:25:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:25:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20379/236289 objects degraded (8.625%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:03.615 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:03.650 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:25:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:25:03] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-09-30T09:25:04.271 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:25:04.333 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:04.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:04.446 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: pgmap v337: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 83 op/s; 20379/236007 objects degraded (8.635%); 4788/236007 objects misplaced (2.029%); 576 KiB/s, 8 objects/s recovering 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: osdmap e155: 8 total, 8 up, 8 in 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:04 smithi167 ceph-mon[192880]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: pgmap v337: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 83 op/s; 20379/236007 objects degraded (8.635%); 4788/236007 objects misplaced (2.029%); 576 KiB/s, 8 objects/s recovering 2023-09-30T09:25:04.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:25:04.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: osdmap e155: 8 total, 8 up, 8 in 2023-09-30T09:25:04.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[216815]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: pgmap v337: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 83 op/s; 20379/236007 objects degraded (8.635%); 4788/236007 objects misplaced (2.029%); 576 KiB/s, 8 objects/s recovering 2023-09-30T09:25:04.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:25:04.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: osdmap e155: 8 total, 8 up, 8 in 2023-09-30T09:25:04.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.535 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:04 smithi019 ceph-mon[224043]: failed to encode map e155 with expected crc 2023-09-30T09:25:04.535 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:04.938 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-30T09:25:05.078 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:05.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:05.183 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:05.236 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:25:05.264 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:05.336 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:05.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:05.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:05.441 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:05.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: osdmap e156: 8 total, 8 up, 8 in 2023-09-30T09:25:05.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[216815]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.484 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:05.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: osdmap e156: 8 total, 8 up, 8 in 2023-09-30T09:25:05.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:05 smithi019 ceph-mon[224043]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.606 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:05.667 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:05.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:05.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: osdmap e156: 8 total, 8 up, 8 in 2023-09-30T09:25:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:05 smithi167 ceph-mon[192880]: failed to encode map e156 with expected crc 2023-09-30T09:25:05.774 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:05.800 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:05.829 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:05.844 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:05.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:05.950 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:05.995 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:06.122 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:06.127 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:06.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:06.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:06.233 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:25:06.259 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:06.304 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:25:06.307 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:25:06.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:06.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:25:06.409 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:06.443 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:06 smithi167 ceph-mon[192880]: pgmap v340: 89 pgs: 2 remapped+peering, 5 peering, 78 active+clean, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 5458/234276 objects degraded (2.330%); 9490/234276 objects misplaced (4.051%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:25:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4229769030' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:06.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:06 smithi019 ceph-mon[216815]: pgmap v340: 89 pgs: 2 remapped+peering, 5 peering, 78 active+clean, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 5458/234276 objects degraded (2.330%); 9490/234276 objects misplaced (4.051%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:25:06.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4229769030' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:06 smithi019 ceph-mon[224043]: pgmap v340: 89 pgs: 2 remapped+peering, 5 peering, 78 active+clean, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 5458/234276 objects degraded (2.330%); 9490/234276 objects misplaced (4.051%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:25:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4229769030' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:07.548 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:07.548 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4229769030' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:07.548 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[216815]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[216815]: osdmap e157: 8 total, 8 up, 8 in 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[216815]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[216815]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4229769030' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[224043]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[224043]: osdmap e157: 8 total, 8 up, 8 in 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[224043]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.549 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:07 smithi019 ceph-mon[224043]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:07 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:07 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4229769030' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:07 smithi167 ceph-mon[192880]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:07 smithi167 ceph-mon[192880]: osdmap e157: 8 total, 8 up, 8 in 2023-09-30T09:25:07.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:07 smithi167 ceph-mon[192880]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:07 smithi167 ceph-mon[192880]: failed to encode map e157 with expected crc 2023-09-30T09:25:07.942 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:07.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:07.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:08.046 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:08.074 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:08.118 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:08.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:08.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:08.225 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:08.265 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:08.374 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:25:08 smithi167 conmon[172457]: logger=sqlstore.transactions t=2023-09-30T09:25:08.318059752Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-30T09:25:08.374 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:08 smithi167 ceph-mon[192880]: pgmap v342: 89 pgs: 2 remapped+peering, 5 peering, 78 active+clean, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 5458/234000 objects degraded (2.332%); 9490/234000 objects misplaced (4.056%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:25:08.375 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5458/234000 objects degraded (2.332%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:25:08.381 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:08.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:08.487 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:08.514 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:08.557 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:08.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:08 smithi019 ceph-mon[224043]: pgmap v342: 89 pgs: 2 remapped+peering, 5 peering, 78 active+clean, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 5458/234000 objects degraded (2.332%); 9490/234000 objects misplaced (4.056%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:25:08.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5458/234000 objects degraded (2.332%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:25:08.560 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:08 smithi019 ceph-mon[216815]: pgmap v342: 89 pgs: 2 remapped+peering, 5 peering, 78 active+clean, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 5458/234000 objects degraded (2.332%); 9490/234000 objects misplaced (4.056%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:25:08.560 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5458/234000 objects degraded (2.332%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:25:08.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:08.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:08.664 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:08.689 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:25:08 smithi167 conmon[172457]: logger=cleanup t=2023-09-30T09:25:08.374028957Z level=info msg="Completed cleanup jobs" duration=66.266432ms 2023-09-30T09:25:08.691 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:08.734 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:08.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:08.842 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:08.867 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:08.910 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:08.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:08.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:09.015 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:09.043 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:09.084 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:09.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:09.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:09.188 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:09.224 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:10.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:10 smithi019 ceph-mon[216815]: pgmap v343: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 6342/232752 objects degraded (2.725%); 9447/232752 objects misplaced (4.059%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:25:10.353 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:25:10.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:10 smithi019 ceph-mon[224043]: pgmap v343: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 6342/232752 objects degraded (2.725%); 9447/232752 objects misplaced (4.059%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:25:10.581 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:10.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:10.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:10.685 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:10.708 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:10 smithi167 ceph-mon[192880]: pgmap v343: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 6342/232752 objects degraded (2.725%); 9447/232752 objects misplaced (4.059%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:25:10.711 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:10.757 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:10.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:10.862 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:10.904 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:11.026 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:25:11.032 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:11.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:11.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:11.056 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:25:11.084 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:25:11.121 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:25:11.139 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:11.151 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:25:11.168 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:11.173 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:25:11.185 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:11.197 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:25:11.211 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:11.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:11.239 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:25:11.240 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:25:11.240 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:25:11.240 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:25:11.240 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:25:11.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:25:11.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:25:11.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:25:11.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:25:11.318 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:11.321 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:25:11.321 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:25:11.345 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:11.390 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:11.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:11.496 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:11.520 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2749159175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:11.523 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:11.567 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:11.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:11.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:11.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2749159175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2749159175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:11.673 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:11.700 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:11.746 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:11.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:11.852 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:11.878 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:11.919 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:11.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:11.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:12.026 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:12.053 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:12.103 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:12.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:12.209 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:12.235 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:12.277 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:12.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:12.331 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:12 smithi019 ceph-mon[216815]: pgmap v344: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 6301/232047 objects degraded (2.715%); 9162/232047 objects misplaced (3.948%); 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:25:12.382 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:12.422 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:12 smithi167 ceph-mon[192880]: pgmap v344: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 6301/232047 objects degraded (2.715%); 9162/232047 objects misplaced (3.948%); 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:25:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:12 smithi019 ceph-mon[224043]: pgmap v344: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 6301/232047 objects degraded (2.715%); 9162/232047 objects misplaced (3.948%); 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:25:12.789 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:12.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:12.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:12.894 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:12.925 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:13.347 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.5 to 1 (10000) 2023-09-30T09:25:13.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4257960390' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:25:13.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:13 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:25:13.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6301/232047 objects degraded (2.715%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:25:13.754 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4257960390' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:25:13.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:13 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:25:13.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6301/232047 objects degraded (2.715%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:25:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4257960390' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:25:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:13 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:25:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6301/232047 objects degraded (2.715%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:25:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:25:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:25:13] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-09-30T09:25:14.438 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:14.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:14.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:14.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: pgmap v345: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 6301/231693 objects degraded (2.720%); 9162/231693 objects misplaced (3.954%); 3.4 MiB/s, 55 objects/s recovering 2023-09-30T09:25:14.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:25:14.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: osdmap e158: 8 total, 8 up, 8 in 2023-09-30T09:25:14.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[216815]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: pgmap v345: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 6301/231693 objects degraded (2.720%); 9162/231693 objects misplaced (3.954%); 3.4 MiB/s, 55 objects/s recovering 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: osdmap e158: 8 total, 8 up, 8 in 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:14 smithi019 ceph-mon[224043]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.531 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:25:14.542 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:14.569 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:14.614 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:14.617 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: pgmap v345: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 6301/231693 objects degraded (2.720%); 9162/231693 objects misplaced (3.954%); 3.4 MiB/s, 55 objects/s recovering 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: osdmap e158: 8 total, 8 up, 8 in 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.618 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:14 smithi167 ceph-mon[192880]: failed to encode map e158 with expected crc 2023-09-30T09:25:14.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:14.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:14.720 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:14.753 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:15.429 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-30T09:25:15.559 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:15.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:15.663 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: osdmap e159: 8 total, 8 up, 8 in 2023-09-30T09:25:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[224043]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: osdmap e159: 8 total, 8 up, 8 in 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.691 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:15 smithi019 ceph-mon[216815]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.693 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:15.768 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: osdmap e159: 8 total, 8 up, 8 in 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:15 smithi167 ceph-mon[192880]: failed to encode map e159 with expected crc 2023-09-30T09:25:15.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:15.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:15.798 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:15.876 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:15.906 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:15.979 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:15.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:15.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:16.047 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:16.084 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:16.113 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:16.157 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:16.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:16.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:16.263 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:16.268 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:25:16.293 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:16.363 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:16 smithi167 ceph-mon[192880]: pgmap v348: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 6 peering, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 3054/230940 objects degraded (1.322%); 6.4 MiB/s, 137 objects/s recovering 2023-09-30T09:25:16.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:16 smithi167 ceph-mon[192880]: osdmap e160: 8 total, 8 up, 8 in 2023-09-30T09:25:16.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:16 smithi167 ceph-mon[192880]: 5.7 starting backfill to osd.5 from (0'0,147'5218] MIN to 157'5499 2023-09-30T09:25:16.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:16 smithi167 ceph-mon[192880]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:16 smithi167 ceph-mon[192880]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:16 smithi167 ceph-mon[192880]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:16 smithi167 ceph-mon[192880]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.500 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:16.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:16.551 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[216815]: pgmap v348: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 6 peering, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 3054/230940 objects degraded (1.322%); 6.4 MiB/s, 137 objects/s recovering 2023-09-30T09:25:16.552 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[216815]: osdmap e160: 8 total, 8 up, 8 in 2023-09-30T09:25:16.552 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[216815]: 5.7 starting backfill to osd.5 from (0'0,147'5218] MIN to 157'5499 2023-09-30T09:25:16.552 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[216815]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.552 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[216815]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.552 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[216815]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.552 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[216815]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.552 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[224043]: pgmap v348: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 6 peering, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 3054/230940 objects degraded (1.322%); 6.4 MiB/s, 137 objects/s recovering 2023-09-30T09:25:16.552 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[224043]: osdmap e160: 8 total, 8 up, 8 in 2023-09-30T09:25:16.553 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[224043]: 5.7 starting backfill to osd.5 from (0'0,147'5218] MIN to 157'5499 2023-09-30T09:25:16.553 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[224043]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.553 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[224043]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.553 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[224043]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.553 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:16 smithi019 ceph-mon[224043]: failed to encode map e160 with expected crc 2023-09-30T09:25:16.604 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:16.635 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:16.835 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:16.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:16.939 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:16.966 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:17.008 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:17.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:17.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:17.113 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:25:17.139 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:17.181 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:25:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:17.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:25:17.286 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:17.313 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:17.374 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:25:17.375 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:17.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:17.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:17.481 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:17.540 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:17.707 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: osdmap e161: 8 total, 8 up, 8 in 2023-09-30T09:25:17.707 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.707 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.707 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2953208532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[224043]: osdmap e162: 8 total, 8 up, 8 in 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: osdmap e161: 8 total, 8 up, 8 in 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.708 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.709 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2953208532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:25:17.709 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:25:17.709 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:25:17.709 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:17.709 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:25:17.709 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:17 smithi019 ceph-mon[216815]: osdmap e162: 8 total, 8 up, 8 in 2023-09-30T09:25:17.759 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:17.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: osdmap e161: 8 total, 8 up, 8 in 2023-09-30T09:25:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: failed to encode map e161 with expected crc 2023-09-30T09:25:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2953208532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:25:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:25:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:25:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:25:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:17 smithi167 ceph-mon[192880]: osdmap e162: 8 total, 8 up, 8 in 2023-09-30T09:25:17.863 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:17.900 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:18.126 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:18.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:18.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:18.231 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:18.266 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:18.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[216815]: pgmap v351: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 6 peering, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 3054/230874 objects degraded (1.323%); 7.1 MiB/s, 183 objects/s recovering 2023-09-30T09:25:18.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[216815]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[216815]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[216815]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[216815]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:18.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 3054/230874 objects degraded (1.323%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:18.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[224043]: pgmap v351: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 6 peering, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 3054/230874 objects degraded (1.323%); 7.1 MiB/s, 183 objects/s recovering 2023-09-30T09:25:18.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[224043]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[224043]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[224043]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[224043]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:18.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 3054/230874 objects degraded (1.323%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:18.687 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:18.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:18.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:18 smithi167 ceph-mon[192880]: pgmap v351: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 6 peering, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 3054/230874 objects degraded (1.323%); 7.1 MiB/s, 183 objects/s recovering 2023-09-30T09:25:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:18 smithi167 ceph-mon[192880]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:18 smithi167 ceph-mon[192880]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:18 smithi167 ceph-mon[192880]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:18 smithi167 ceph-mon[192880]: failed to encode map e162 with expected crc 2023-09-30T09:25:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 3054/230874 objects degraded (1.323%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:18.790 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:18.822 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:18.876 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:18.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:18.987 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:19.038 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:19.195 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:19.212 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:19.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:19.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:19.317 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:19.365 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:19.498 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:19.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:19.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:19.563 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:25:19.563 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:25:19.602 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:19.656 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:19.786 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:19.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:19.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:19.889 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:25:19.922 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:19.982 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:25:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:20.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:25:20.391 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.6 to 1 (10000) 2023-09-30T09:25:20.477 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:20.511 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:20.549 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:25:20.573 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:20.679 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:20 smithi019 ceph-mon[224043]: pgmap v353: 89 pgs: 1 active+recovering+undersized+remapped, 83 active+clean, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 4170/228747 objects degraded (1.823%); 4693/228747 objects misplaced (2.052%); 8.1 MiB/s, 253 objects/s recovering 2023-09-30T09:25:20.679 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2678782262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:25:20.680 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:20 smithi019 ceph-mon[216815]: pgmap v353: 89 pgs: 1 active+recovering+undersized+remapped, 83 active+clean, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 4170/228747 objects degraded (1.823%); 4693/228747 objects misplaced (2.052%); 8.1 MiB/s, 253 objects/s recovering 2023-09-30T09:25:20.680 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2678782262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:25:20.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:20.726 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:25:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:20 smithi167 ceph-mon[192880]: pgmap v353: 89 pgs: 1 active+recovering+undersized+remapped, 83 active+clean, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 4170/228747 objects degraded (1.823%); 4693/228747 objects misplaced (2.052%); 8.1 MiB/s, 253 objects/s recovering 2023-09-30T09:25:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2678782262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:25:20.782 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:25:20.811 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:20.885 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:25:20.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:20.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:25:20.991 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:21.020 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2678782262' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:25:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: osdmap e163: 8 total, 8 up, 8 in 2023-09-30T09:25:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:21.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:21 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2678782262' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:25:21.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: osdmap e163: 8 total, 8 up, 8 in 2023-09-30T09:25:21.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:21.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2678782262' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: osdmap e163: 8 total, 8 up, 8 in 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: failed to encode map e163 with expected crc 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:21.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:21 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:21.939 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:21.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:22.043 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:22.087 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:22.146 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-30T09:25:22.179 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:22.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:22.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:22.285 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:22.319 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:22.375 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:22.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:22.482 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:22.513 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:22.553 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:22.553 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:22.563 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:22.570 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:22 smithi167 ceph-mon[192880]: pgmap v355: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 2518/227919 objects degraded (1.105%); 5839/227919 objects misplaced (2.562%); 3.5 MiB/s, 110 objects/s recovering 2023-09-30T09:25:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:22.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:22 smithi019 ceph-mon[216815]: pgmap v355: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 2518/227919 objects degraded (1.105%); 5839/227919 objects misplaced (2.562%); 3.5 MiB/s, 110 objects/s recovering 2023-09-30T09:25:22.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:22 smithi019 ceph-mon[224043]: pgmap v355: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 2518/227919 objects degraded (1.105%); 5839/227919 objects misplaced (2.562%); 3.5 MiB/s, 110 objects/s recovering 2023-09-30T09:25:22.672 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:22.685 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:25:22.715 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:22.719 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:25:22.757 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:25:22.788 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:25:22.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:25:22.852 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:25:22.870 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:22.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:22.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:22.888 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:25:22.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:25:22.928 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:25:22.928 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:25:22.928 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:25:22.928 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:25:22.929 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:25:22.929 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:25:22.929 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:25:22.929 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:25:22.984 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:23.013 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:23.069 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:23.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:23.174 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:23.212 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2070018886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:23.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 2518/227919 objects degraded (1.105%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:23.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2070018886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:23.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 2518/227919 objects degraded (1.105%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:23.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2070018886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:23.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 2518/227919 objects degraded (1.105%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:25:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:25:23] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-30T09:25:24.465 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:24.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:24.570 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:24.597 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:24.657 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:24.662 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:24 smithi167 ceph-mon[192880]: pgmap v356: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 2518/227235 objects degraded (1.108%); 5839/227235 objects misplaced (2.570%); 3.0 MiB/s, 94 objects/s recovering 2023-09-30T09:25:24.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:24.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:24.764 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:24.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:24 smithi019 ceph-mon[216815]: pgmap v356: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 2518/227235 objects degraded (1.108%); 5839/227235 objects misplaced (2.570%); 3.0 MiB/s, 94 objects/s recovering 2023-09-30T09:25:24.779 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:24 smithi019 ceph-mon[224043]: pgmap v356: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 2518/227235 objects degraded (1.108%); 5839/227235 objects misplaced (2.570%); 3.0 MiB/s, 94 objects/s recovering 2023-09-30T09:25:24.797 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:24.857 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:24.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:24.962 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:25.003 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:25.068 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:25.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:25.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:25.173 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:25.203 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:25.252 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:25.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:25.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:25.361 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:25.402 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:25.500 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:25.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:25.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:25.603 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:25.619 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:25.631 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:25.676 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:25.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:25.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:25.786 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:25.819 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:26.012 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:26.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:26.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:26.123 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:26.192 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:26.413 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:26 smithi019 ceph-mon[216815]: pgmap v357: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 2389/221811 objects degraded (1.077%); 5648/221811 objects misplaced (2.546%); 4.0 MiB/s, 101 objects/s recovering 2023-09-30T09:25:26.413 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1957411784' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:26.413 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:26 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:26.430 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:25:26.455 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:25:26.721 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:26.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:26.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:26.734 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:26 smithi019 ceph-mon[224043]: pgmap v357: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 2389/221811 objects degraded (1.077%); 5648/221811 objects misplaced (2.546%); 4.0 MiB/s, 101 objects/s recovering 2023-09-30T09:25:26.734 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1957411784' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:26.734 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:26 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:26 smithi167 ceph-mon[192880]: pgmap v357: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 2389/221811 objects degraded (1.077%); 5648/221811 objects misplaced (2.546%); 4.0 MiB/s, 101 objects/s recovering 2023-09-30T09:25:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1957411784' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:26 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:26.830 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:26.856 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:26.900 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:26.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:27.006 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:27.048 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:27.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:25:27.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:27.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[224043]: osdmap e164: 8 total, 8 up, 8 in 2023-09-30T09:25:27.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[224043]: failed to encode map e164 with expected crc 2023-09-30T09:25:27.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[224043]: failed to encode map e164 with expected crc 2023-09-30T09:25:27.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:25:27.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:27.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[216815]: osdmap e164: 8 total, 8 up, 8 in 2023-09-30T09:25:27.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[216815]: failed to encode map e164 with expected crc 2023-09-30T09:25:27.740 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:27 smithi019 ceph-mon[216815]: failed to encode map e164 with expected crc 2023-09-30T09:25:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:27 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:25:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:27 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:27 smithi167 ceph-mon[192880]: osdmap e164: 8 total, 8 up, 8 in 2023-09-30T09:25:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:27 smithi167 ceph-mon[192880]: failed to encode map e164 with expected crc 2023-09-30T09:25:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:27 smithi167 ceph-mon[192880]: failed to encode map e164 with expected crc 2023-09-30T09:25:27.979 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:25:27.980 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:25:27.980 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:25:27.980 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:25:27.980 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:25:28.311 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:28.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:28.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:28.418 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:28.514 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:28 smithi019 ceph-mon[216815]: pgmap v359: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 2389/221421 objects degraded (1.079%); 5648/221421 objects misplaced (2.551%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:25:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 2389/221421 objects degraded (1.079%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:28 smithi019 ceph-mon[224043]: pgmap v359: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 2389/221421 objects degraded (1.079%); 5648/221421 objects misplaced (2.551%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:25:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 2389/221421 objects degraded (1.079%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:28.644 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:28.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:28.750 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:28.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:28 smithi167 ceph-mon[192880]: pgmap v359: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 2389/221421 objects degraded (1.079%); 5648/221421 objects misplaced (2.551%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:25:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 2389/221421 objects degraded (1.079%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:28.775 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:28.817 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:28.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:28.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:28.923 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:28.951 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:29.006 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:29.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:29.115 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:29.218 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:29.625 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:25:29.697 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:29.761 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:29.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:29.871 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:29.925 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:30.026 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:30.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:30.132 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:30.183 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:30.409 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:25:30.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:30 smithi019 ceph-mon[216815]: pgmap v360: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 2254/217287 objects degraded (1.037%); 5504/217287 objects misplaced (2.533%); 2.4 MiB/s, 37 objects/s recovering 2023-09-30T09:25:30.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3153291387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:25:30.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:25:30.452 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.7 to 1 (10000) 2023-09-30T09:25:30.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:30 smithi167 ceph-mon[192880]: pgmap v360: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 2254/217287 objects degraded (1.037%); 5504/217287 objects misplaced (2.533%); 2.4 MiB/s, 37 objects/s recovering 2023-09-30T09:25:30.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3153291387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:25:30.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:25:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:30 smithi019 ceph-mon[224043]: pgmap v360: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 2254/217287 objects degraded (1.037%); 5504/217287 objects misplaced (2.533%); 2.4 MiB/s, 37 objects/s recovering 2023-09-30T09:25:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3153291387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:25:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:25:30.961 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:30.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:30.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:31.066 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:31.105 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:31.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:25:31.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[216815]: osdmap e165: 8 total, 8 up, 8 in 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[216815]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[216815]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[216815]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[216815]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[216815]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[224043]: osdmap e165: 8 total, 8 up, 8 in 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[224043]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[224043]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.544 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[224043]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.544 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[224043]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.544 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:31 smithi019 ceph-mon[224043]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.631 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:31.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:31.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:31.735 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:25:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:31 smithi167 ceph-mon[192880]: osdmap e165: 8 total, 8 up, 8 in 2023-09-30T09:25:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:31 smithi167 ceph-mon[192880]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:31 smithi167 ceph-mon[192880]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:31 smithi167 ceph-mon[192880]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:31 smithi167 ceph-mon[192880]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:31 smithi167 ceph-mon[192880]: failed to encode map e165 with expected crc 2023-09-30T09:25:31.781 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:32.264 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:32.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:32.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:32.372 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:32.410 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:32.476 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:32.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:32.532 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:32.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:32 smithi019 ceph-mon[224043]: pgmap v362: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 2254/215124 objects degraded (1.048%); 5360/215124 objects misplaced (2.492%); 3.7 MiB/s, 58 objects/s recovering 2023-09-30T09:25:32.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:32.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:32.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:32 smithi019 ceph-mon[216815]: pgmap v362: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 2254/215124 objects degraded (1.048%); 5360/215124 objects misplaced (2.492%); 3.7 MiB/s, 58 objects/s recovering 2023-09-30T09:25:32.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:32.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:32.574 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-30T09:25:32.580 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:32.608 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:32.670 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:32.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:32 smithi167 ceph-mon[192880]: pgmap v362: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 2254/215124 objects degraded (1.048%); 5360/215124 objects misplaced (2.492%); 3.7 MiB/s, 58 objects/s recovering 2023-09-30T09:25:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:32.776 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:32.787 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:32.808 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:32.837 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:25:32.865 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:32.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:32.883 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:25:32.918 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:25:32.969 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:32.975 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:25:33.018 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:25:33.067 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:25:33.090 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:33.124 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:25:33.155 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:25:33.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:25:33.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:25:33.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:25:33.161 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:25:33.161 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:25:33.161 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:25:33.161 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:25:33.161 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:25:33.245 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:33.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:33.352 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:25:33.379 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:33.431 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:25:33.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:33.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:25:33.542 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/421587279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 2254/215124 objects degraded (1.048%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/421587279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 2254/215124 objects degraded (1.048%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:25:33.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:33.609 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:33 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/421587279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 2254/215124 objects degraded (1.048%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:25:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:25:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:25:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:25:33] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.43.0" 2023-09-30T09:25:34.285 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:34.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:34.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:34.388 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:25:34.416 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:34.463 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:25:34.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:34.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:25:34.570 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:34 smithi019 ceph-mon[216815]: pgmap v363: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 2254/214266 objects degraded (1.052%); 5360/214266 objects misplaced (2.502%); 2.2 MiB/s, 35 objects/s recovering 2023-09-30T09:25:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:34 smithi019 ceph-mon[224043]: pgmap v363: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 2254/214266 objects degraded (1.052%); 5360/214266 objects misplaced (2.502%); 2.2 MiB/s, 35 objects/s recovering 2023-09-30T09:25:34.600 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:34.656 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:34.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:34 smithi167 ceph-mon[192880]: pgmap v363: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 2254/214266 objects degraded (1.052%); 5360/214266 objects misplaced (2.502%); 2.2 MiB/s, 35 objects/s recovering 2023-09-30T09:25:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:34.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:34.768 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:34.811 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:34.985 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:34.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:35.091 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:35.133 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:35.352 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:35.995 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:25:35.996 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:25:36.294 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:36.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:36.400 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:36.426 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:36.467 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:36.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:36.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:36.574 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:36 smithi019 ceph-mon[216815]: pgmap v364: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 2108/208206 objects degraded (1.012%); 5087/208206 objects misplaced (2.443%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:25:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:36 smithi019 ceph-mon[224043]: pgmap v364: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 2108/208206 objects degraded (1.012%); 5087/208206 objects misplaced (2.443%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:25:36.610 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:36.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:36 smithi167 ceph-mon[192880]: pgmap v364: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 2108/208206 objects degraded (1.012%); 5087/208206 objects misplaced (2.443%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:25:36.866 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:36.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:36.935 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:25:36.971 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:37.055 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:37.090 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:25:37.257 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:37.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:37.365 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:37.394 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:37.461 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:37.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:37.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:37.566 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:37.593 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:37.634 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:37.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:37.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:37.742 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:25:37.785 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:38.274 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:25:38.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:38.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:25:38.403 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:25:38.446 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:38.488 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:38 smithi019 ceph-mon[216815]: pgmap v365: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 2108/207789 objects degraded (1.014%); 5087/207789 objects misplaced (2.448%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:25:38.488 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 2108/207789 objects degraded (1.014%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:38.511 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:25:38.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:38.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:25:38.528 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:38.532 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:38.620 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:38.653 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:38.720 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:38.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:38.731 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:38 smithi167 ceph-mon[192880]: pgmap v365: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 2108/207789 objects degraded (1.014%); 5087/207789 objects misplaced (2.448%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:25:38.731 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 2108/207789 objects degraded (1.014%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:38.743 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:38 smithi019 ceph-mon[224043]: pgmap v365: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 2108/207789 objects degraded (1.014%); 5087/207789 objects misplaced (2.448%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:25:38.743 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 2108/207789 objects degraded (1.014%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:38.831 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:38.891 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:39.116 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:39.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:39.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:39.241 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:39.282 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:39.340 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:39.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:39.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:39.446 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:39.475 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:39.504 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.0 to 1 (10000) 2023-09-30T09:25:39.511 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:25:39.542 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:39.548 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3326817485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:25:39.548 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:25:39.548 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/848681244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:25:39.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:39.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:39.650 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:39.693 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3326817485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:25:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:25:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/848681244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:25:39.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3326817485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:25:39.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:25:39.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/848681244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:25:40.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:40 smithi167 ceph-mon[192880]: pgmap v366: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 2108/205143 objects degraded (1.028%); 5087/205143 objects misplaced (2.480%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:25:40.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:40 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:40.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:40 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:25:40.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/848681244' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:25:40.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:40 smithi167 ceph-mon[192880]: osdmap e166: 8 total, 8 up, 8 in 2023-09-30T09:25:40.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:40 smithi167 ceph-mon[192880]: failed to encode map e166 with expected crc 2023-09-30T09:25:40.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:40 smithi167 ceph-mon[192880]: failed to encode map e166 with expected crc 2023-09-30T09:25:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[224043]: pgmap v366: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 2108/205143 objects degraded (1.028%); 5087/205143 objects misplaced (2.480%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:25:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:25:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/848681244' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:25:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[224043]: osdmap e166: 8 total, 8 up, 8 in 2023-09-30T09:25:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[224043]: failed to encode map e166 with expected crc 2023-09-30T09:25:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[224043]: failed to encode map e166 with expected crc 2023-09-30T09:25:40.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[216815]: pgmap v366: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 2108/205143 objects degraded (1.028%); 5087/205143 objects misplaced (2.480%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:25:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:25:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/848681244' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:25:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[216815]: osdmap e166: 8 total, 8 up, 8 in 2023-09-30T09:25:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[216815]: failed to encode map e166 with expected crc 2023-09-30T09:25:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:40 smithi019 ceph-mon[216815]: failed to encode map e166 with expected crc 2023-09-30T09:25:41.592 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-30T09:25:43.655 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:25:43.955 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:42 smithi167 ceph-mon[192880]: pgmap v368: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 1943/200982 objects degraded (0.967%); 4795/200982 objects misplaced (2.386%); 3.7 MiB/s, 59 objects/s recovering 2023-09-30T09:25:43.956 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1943/200982 objects degraded (0.967%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:43.956 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/523321647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:43.956 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:43.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:43.957 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:42 smithi019 ceph-mon[216815]: pgmap v368: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 1943/200982 objects degraded (0.967%); 4795/200982 objects misplaced (2.386%); 3.7 MiB/s, 59 objects/s recovering 2023-09-30T09:25:43.957 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:42 smithi019 ceph-mon[224043]: pgmap v368: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 1943/200982 objects degraded (0.967%); 4795/200982 objects misplaced (2.386%); 3.7 MiB/s, 59 objects/s recovering 2023-09-30T09:25:43.958 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:25:43.958 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:25:43.959 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:25:43.959 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:25:43.960 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:25:43.960 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:25:43.960 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:25:43.960 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:25:43.960 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:25:43.960 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:25:43.960 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:25:43.960 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:25:43.961 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:25:43.961 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:25:43.961 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:25:43.961 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:25:43.961 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:25:43.961 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:25:43.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1943/200982 objects degraded (0.967%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:43.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/523321647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:43.962 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1943/200982 objects degraded (0.967%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:43.962 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/523321647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:44.056 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:25:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:25:43] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.43.0" 2023-09-30T09:25:44.196 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:44.235 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:44.296 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:44.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:44.511 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:45.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:44 smithi019 ceph-mon[216815]: pgmap v369: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 1943/199986 objects degraded (0.972%); 4795/199986 objects misplaced (2.398%); 3.7 MiB/s, 59 objects/s recovering 2023-09-30T09:25:45.188 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:44 smithi167 ceph-mon[192880]: pgmap v369: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 1943/199986 objects degraded (0.972%); 4795/199986 objects misplaced (2.398%); 3.7 MiB/s, 59 objects/s recovering 2023-09-30T09:25:45.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:44 smithi019 ceph-mon[224043]: pgmap v369: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 1943/199986 objects degraded (0.972%); 4795/199986 objects misplaced (2.398%); 3.7 MiB/s, 59 objects/s recovering 2023-09-30T09:25:45.189 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:25:45.191 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:45.267 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:45.407 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:45.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:45.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:45.603 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:45.730 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:45.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:45.824 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:45.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:45.925 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:45.961 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:25:45.997 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:46.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:46.118 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:46.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:46.219 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:46.355 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:46.439 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:46.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:46.742 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:46.769 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:46.788 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:46 smithi167 ceph-mon[192880]: pgmap v370: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 1761/191820 objects degraded (0.918%); 4392/191820 objects misplaced (2.290%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:46.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:46 smithi019 ceph-mon[216815]: pgmap v370: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 1761/191820 objects degraded (0.918%); 4392/191820 objects misplaced (2.290%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:46.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:46 smithi019 ceph-mon[224043]: pgmap v370: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 1761/191820 objects degraded (0.918%); 4392/191820 objects misplaced (2.290%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:46.901 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:46.942 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:47.029 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:47.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:47.156 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:47.217 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:25:47.250 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:47.320 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:47.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:47.473 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:47.589 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:47.739 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:25:47.758 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:25:48.076 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:48.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:48.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:48.183 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:48.252 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:48 smithi019 ceph-mon[216815]: pgmap v371: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 1761/191319 objects degraded (0.920%); 4392/191319 objects misplaced (2.296%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1761/191319 objects degraded (0.920%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:48 smithi019 ceph-mon[224043]: pgmap v371: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 1761/191319 objects degraded (0.920%); 4392/191319 objects misplaced (2.296%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1761/191319 objects degraded (0.920%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:49.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:48 smithi167 ceph-mon[192880]: pgmap v371: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 1761/191319 objects degraded (0.920%); 4392/191319 objects misplaced (2.296%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:49.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:25:49.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:25:49.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1761/191319 objects degraded (0.920%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:49.551 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:49 smithi019 ceph-mon[216815]: from='client.44831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:25:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:49 smithi019 ceph-mon[224043]: from='client.44831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:25:50.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:49 smithi167 ceph-mon[192880]: from='client.44831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:25:50.218 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":371,"stamp":"2023-09-30T09:25:47.040616+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4182316991,"num_objects":63773,"num_object_clones":0,"num_object_copies":191319,"num_objects_missing_on_primary":317,"num_objects_missing":317,"num_objects_degraded":1761,"num_objects_misplaced":4392,"num_objects_unfound":0,"num_objects_dirty":63773,"num_whiteouts":0,"num_read":6443,"num_read_kb":13071,"num_write":102953,"num_write_kb":5312742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18316,"num_bytes_recovered":1165364610,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102999,"ondisk_log_size":102999,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":281,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13803572,"kb_used_data":13155888,"kb_used_omap":19,"kb_used_meta":647532,"kb_avail":736157644,"statfs":{"total":767960285184,"available":753825427456,"internally_reserved":0,"allocated":13471629312,"data_stored":13465119192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":663073191},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-359792640,"num_objects":-5490,"num_object_clones":0,"num_object_copies":-16470,"num_objects_missing_on_primary":-347,"num_objects_missing":-347,"num_objects_degraded":-347,"num_objects_misplaced":-695,"num_objects_unfound":0,"num_objects_dirty":-5490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5490,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":725,"num_bytes_recovered":47513600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002994"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":143,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.452000+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:25:30.452000+0000","last_peered":"2023-09-30T09:25:30.452000+0000","last_clean":"2023-09-30T09:25:30.452000+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:25:30.452000+0000","last_undegraded":"2023-09-30T09:25:30.452000+0000","last_fullsized":"2023-09-30T09:25:30.452000+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":129,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438002+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:25:30.438002+0000","last_peered":"2023-09-30T09:25:30.438002+0000","last_clean":"2023-09-30T09:25:30.438002+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:25:30.438002+0000","last_undegraded":"2023-09-30T09:25:30.438002+0000","last_fullsized":"2023-09-30T09:25:30.438002+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":119,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.443996+0000","last_change":"2023-09-30T09:25:04.281698+0000","last_active":"2023-09-30T09:25:30.443996+0000","last_peered":"2023-09-30T09:25:30.443996+0000","last_clean":"2023-09-30T09:25:30.443996+0000","last_became_active":"2023-09-30T09:25:04.281351+0000","last_became_peered":"2023-09-30T09:25:04.281351+0000","last_unstale":"2023-09-30T09:25:30.443996+0000","last_undegraded":"2023-09-30T09:25:30.443996+0000","last_fullsized":"2023-09-30T09:25:30.443996+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:17:40.382369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":57,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.434725+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:25:30.434725+0000","last_peered":"2023-09-30T09:25:30.434725+0000","last_clean":"2023-09-30T09:25:30.434725+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:25:30.434725+0000","last_undegraded":"2023-09-30T09:25:30.434725+0000","last_fullsized":"2023-09-30T09:25:30.434725+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":163,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438920+0000","last_change":"2023-09-30T09:23:08.014781+0000","last_active":"2023-09-30T09:25:30.438920+0000","last_peered":"2023-09-30T09:25:30.438920+0000","last_clean":"2023-09-30T09:25:30.438920+0000","last_became_active":"2023-09-30T09:23:08.014387+0000","last_became_peered":"2023-09-30T09:23:08.014387+0000","last_unstale":"2023-09-30T09:25:30.438920+0000","last_undegraded":"2023-09-30T09:25:30.438920+0000","last_fullsized":"2023-09-30T09:25:30.438920+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:52:13.669946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":125,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451023+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:25:30.451023+0000","last_peered":"2023-09-30T09:25:30.451023+0000","last_clean":"2023-09-30T09:25:30.451023+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:25:30.451023+0000","last_undegraded":"2023-09-30T09:25:30.451023+0000","last_fullsized":"2023-09-30T09:25:30.451023+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":57,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.050414+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:25:39.050414+0000","last_peered":"2023-09-30T09:25:39.050414+0000","last_clean":"2023-09-30T09:25:39.050414+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:25:39.050414+0000","last_undegraded":"2023-09-30T09:25:39.050414+0000","last_fullsized":"2023-09-30T09:25:39.050414+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":129,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.439001+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:25:30.439001+0000","last_peered":"2023-09-30T09:25:30.439001+0000","last_clean":"2023-09-30T09:25:30.439001+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:25:30.439001+0000","last_undegraded":"2023-09-30T09:25:30.439001+0000","last_fullsized":"2023-09-30T09:25:30.439001+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":17,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451302+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:25:30.451302+0000","last_peered":"2023-09-30T09:25:30.451302+0000","last_clean":"2023-09-30T09:25:30.451302+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:25:30.451302+0000","last_undegraded":"2023-09-30T09:25:30.451302+0000","last_fullsized":"2023-09-30T09:25:30.451302+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":80,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438978+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:25:30.438978+0000","last_peered":"2023-09-30T09:25:30.438978+0000","last_clean":"2023-09-30T09:25:30.438978+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:25:30.438978+0000","last_undegraded":"2023-09-30T09:25:30.438978+0000","last_fullsized":"2023-09-30T09:25:30.438978+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":129,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.477067+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:25:30.477067+0000","last_peered":"2023-09-30T09:25:30.477067+0000","last_clean":"2023-09-30T09:25:30.477067+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:25:30.477067+0000","last_undegraded":"2023-09-30T09:25:30.477067+0000","last_fullsized":"2023-09-30T09:25:30.477067+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":104,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.444318+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:25:30.444318+0000","last_peered":"2023-09-30T09:25:30.444318+0000","last_clean":"2023-09-30T09:25:30.444318+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:25:30.444318+0000","last_undegraded":"2023-09-30T09:25:30.444318+0000","last_fullsized":"2023-09-30T09:25:30.444318+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":130,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451227+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:25:30.451227+0000","last_peered":"2023-09-30T09:25:30.451227+0000","last_clean":"2023-09-30T09:25:30.451227+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:25:30.451227+0000","last_undegraded":"2023-09-30T09:25:30.451227+0000","last_fullsized":"2023-09-30T09:25:30.451227+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":134,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451272+0000","last_change":"2023-09-30T09:23:03.986755+0000","last_active":"2023-09-30T09:25:30.451272+0000","last_peered":"2023-09-30T09:25:30.451272+0000","last_clean":"2023-09-30T09:25:30.451272+0000","last_became_active":"2023-09-30T09:23:03.986411+0000","last_became_peered":"2023-09-30T09:23:03.986411+0000","last_unstale":"2023-09-30T09:25:30.451272+0000","last_undegraded":"2023-09-30T09:25:30.451272+0000","last_fullsized":"2023-09-30T09:25:30.451272+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:53:23.627823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":128,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.476607+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:25:30.476607+0000","last_peered":"2023-09-30T09:25:30.476607+0000","last_clean":"2023-09-30T09:25:30.476607+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:25:30.476607+0000","last_undegraded":"2023-09-30T09:25:30.476607+0000","last_fullsized":"2023-09-30T09:25:30.476607+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":143,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.437807+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:25:30.437807+0000","last_peered":"2023-09-30T09:25:30.437807+0000","last_clean":"2023-09-30T09:25:30.437807+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:25:30.437807+0000","last_undegraded":"2023-09-30T09:25:30.437807+0000","last_fullsized":"2023-09-30T09:25:30.437807+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":57,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:34.042306+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:25:34.042306+0000","last_peered":"2023-09-30T09:25:34.042306+0000","last_clean":"2023-09-30T09:25:34.042306+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:25:34.042306+0000","last_undegraded":"2023-09-30T09:25:34.042306+0000","last_fullsized":"2023-09-30T09:25:34.042306+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451595+0000","last_change":"2023-09-30T09:25:14.358999+0000","last_active":"2023-09-30T09:25:30.451595+0000","last_peered":"2023-09-30T09:25:30.451595+0000","last_clean":"2023-09-30T09:25:30.451595+0000","last_became_active":"2023-09-30T09:25:14.358650+0000","last_became_peered":"2023-09-30T09:25:14.358650+0000","last_unstale":"2023-09-30T09:25:30.451595+0000","last_undegraded":"2023-09-30T09:25:30.451595+0000","last_fullsized":"2023-09-30T09:25:30.451595+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:57:41.373342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.478083+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:25:30.478083+0000","last_peered":"2023-09-30T09:25:30.478083+0000","last_clean":"2023-09-30T09:25:30.478083+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:25:30.478083+0000","last_undegraded":"2023-09-30T09:25:30.478083+0000","last_fullsized":"2023-09-30T09:25:30.478083+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":57,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:34.048932+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:25:34.048932+0000","last_peered":"2023-09-30T09:25:34.048932+0000","last_clean":"2023-09-30T09:25:34.048932+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:25:34.048932+0000","last_undegraded":"2023-09-30T09:25:34.048932+0000","last_fullsized":"2023-09-30T09:25:34.048932+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":214,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451125+0000","last_change":"2023-09-30T09:25:14.389693+0000","last_active":"2023-09-30T09:25:30.451125+0000","last_peered":"2023-09-30T09:25:30.451125+0000","last_clean":"2023-09-30T09:25:30.451125+0000","last_became_active":"2023-09-30T09:25:14.365482+0000","last_became_peered":"2023-09-30T09:25:14.365482+0000","last_unstale":"2023-09-30T09:25:30.451125+0000","last_undegraded":"2023-09-30T09:25:30.451125+0000","last_fullsized":"2023-09-30T09:25:30.451125+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:09:23.017579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":143,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438364+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:25:30.438364+0000","last_peered":"2023-09-30T09:25:30.438364+0000","last_clean":"2023-09-30T09:25:30.438364+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:25:30.438364+0000","last_undegraded":"2023-09-30T09:25:30.438364+0000","last_fullsized":"2023-09-30T09:25:30.438364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"166'6358","reported_seq":6419,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:43.779610+0000","last_change":"2023-09-30T09:22:48.114186+0000","last_active":"2023-09-30T09:25:43.779610+0000","last_peered":"2023-09-30T09:25:43.779610+0000","last_clean":"2023-09-30T09:25:43.779610+0000","last_became_active":"2023-09-30T09:22:48.113857+0000","last_became_peered":"2023-09-30T09:22:48.113857+0000","last_unstale":"2023-09-30T09:25:43.779610+0000","last_undegraded":"2023-09-30T09:25:43.779610+0000","last_fullsized":"2023-09-30T09:25:43.779610+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6358,"log_dups_size":0,"ondisk_log_size":6358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:18.973595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259129344,"num_objects":3954,"num_object_clones":0,"num_object_copies":11862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6358,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":155,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.476831+0000","last_change":"2023-09-30T09:23:08.038825+0000","last_active":"2023-09-30T09:25:30.476831+0000","last_peered":"2023-09-30T09:25:30.476831+0000","last_clean":"2023-09-30T09:25:30.476831+0000","last_became_active":"2023-09-30T09:23:08.029150+0000","last_became_peered":"2023-09-30T09:23:08.029150+0000","last_unstale":"2023-09-30T09:25:30.476831+0000","last_undegraded":"2023-09-30T09:25:30.476831+0000","last_fullsized":"2023-09-30T09:25:30.476831+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:14.557254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.450724+0000","last_change":"2023-09-30T09:25:14.359647+0000","last_active":"2023-09-30T09:25:30.450724+0000","last_peered":"2023-09-30T09:25:30.450724+0000","last_clean":"2023-09-30T09:25:30.450724+0000","last_became_active":"2023-09-30T09:25:14.359350+0000","last_became_peered":"2023-09-30T09:25:14.359350+0000","last_unstale":"2023-09-30T09:25:30.450724+0000","last_undegraded":"2023-09-30T09:25:30.450724+0000","last_fullsized":"2023-09-30T09:25:30.450724+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:09.489014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"166'6480","reported_seq":9639,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:44.553647+0000","last_change":"2023-09-30T09:24:25.563637+0000","last_active":"2023-09-30T09:25:44.553647+0000","last_peered":"2023-09-30T09:25:44.553647+0000","last_clean":"2023-09-30T09:25:44.553647+0000","last_became_active":"2023-09-30T09:24:25.554649+0000","last_became_peered":"2023-09-30T09:24:25.554649+0000","last_unstale":"2023-09-30T09:25:44.553647+0000","last_undegraded":"2023-09-30T09:25:44.553647+0000","last_fullsized":"2023-09-30T09:25:44.553647+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6480,"log_dups_size":0,"ondisk_log_size":6480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:09:56.974492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262537216,"num_objects":4006,"num_object_clones":0,"num_object_copies":12018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6480,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1538,"num_bytes_recovered":100794368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":156,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.476803+0000","last_change":"2023-09-30T09:25:04.304869+0000","last_active":"2023-09-30T09:25:30.476803+0000","last_peered":"2023-09-30T09:25:30.476803+0000","last_clean":"2023-09-30T09:25:30.476803+0000","last_became_active":"2023-09-30T09:25:04.285159+0000","last_became_peered":"2023-09-30T09:25:04.285159+0000","last_unstale":"2023-09-30T09:25:30.476803+0000","last_undegraded":"2023-09-30T09:25:30.476803+0000","last_fullsized":"2023-09-30T09:25:30.476803+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:03:13.971240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":141,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451810+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:25:30.451810+0000","last_peered":"2023-09-30T09:25:30.451810+0000","last_clean":"2023-09-30T09:25:30.451810+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:25:30.451810+0000","last_undegraded":"2023-09-30T09:25:30.451810+0000","last_fullsized":"2023-09-30T09:25:30.451810+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"166'6385","reported_seq":8906,"reported_epoch":166,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:25:44.762054+0000","last_change":"2023-09-30T09:25:14.561014+0000","last_active":"2023-09-30T09:25:44.762054+0000","last_peered":"2023-09-30T09:25:44.762054+0000","last_clean":"2023-09-30T09:23:07.000618+0000","last_became_active":"2023-09-30T09:25:14.392643+0000","last_became_peered":"2023-09-30T09:25:14.392643+0000","last_unstale":"2023-09-30T09:25:44.762054+0000","last_undegraded":"2023-09-30T09:25:44.762054+0000","last_fullsized":"2023-09-30T09:25:14.356532+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6385,"log_dups_size":0,"ondisk_log_size":6385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:05:55.317362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258801664,"num_objects":3949,"num_object_clones":0,"num_object_copies":11847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":435,"num_objects_unfound":0,"num_objects_dirty":3949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6385,"num_write_kb":330688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1219,"num_bytes_recovered":79822848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,2","objects":3514},{"shards":"0,2,7","objects":152},{"shards":"2,5,7","objects":283}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":138,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.477260+0000","last_change":"2023-09-30T09:25:14.385912+0000","last_active":"2023-09-30T09:25:30.477260+0000","last_peered":"2023-09-30T09:25:30.477260+0000","last_clean":"2023-09-30T09:25:30.477260+0000","last_became_active":"2023-09-30T09:25:14.359789+0000","last_became_peered":"2023-09-30T09:25:14.359789+0000","last_unstale":"2023-09-30T09:25:30.477260+0000","last_undegraded":"2023-09-30T09:25:30.477260+0000","last_fullsized":"2023-09-30T09:25:30.477260+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:54.213336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":135,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.452156+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:25:30.452156+0000","last_peered":"2023-09-30T09:25:30.452156+0000","last_clean":"2023-09-30T09:25:30.452156+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:25:30.452156+0000","last_undegraded":"2023-09-30T09:25:30.452156+0000","last_fullsized":"2023-09-30T09:25:30.452156+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"166'6291","reported_seq":7234,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:44.482362+0000","last_change":"2023-09-30T09:25:16.371393+0000","last_active":"2023-09-30T09:25:44.482362+0000","last_peered":"2023-09-30T09:25:44.482362+0000","last_clean":"2023-09-30T09:25:44.482362+0000","last_became_active":"2023-09-30T09:25:16.364820+0000","last_became_peered":"2023-09-30T09:25:16.364820+0000","last_unstale":"2023-09-30T09:25:44.482362+0000","last_undegraded":"2023-09-30T09:25:44.482362+0000","last_fullsized":"2023-09-30T09:25:44.482362+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6291,"log_dups_size":0,"ondisk_log_size":6291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:19:41.535739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254738432,"num_objects":3887,"num_object_clones":0,"num_object_copies":11661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6291,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":429,"num_bytes_recovered":9240576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":125,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451310+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:25:30.451310+0000","last_peered":"2023-09-30T09:25:30.451310+0000","last_clean":"2023-09-30T09:25:30.451310+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:25:30.451310+0000","last_undegraded":"2023-09-30T09:25:30.451310+0000","last_fullsized":"2023-09-30T09:25:30.451310+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":54,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.434877+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:25:30.434877+0000","last_peered":"2023-09-30T09:25:30.434877+0000","last_clean":"2023-09-30T09:25:30.434877+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:25:30.434877+0000","last_undegraded":"2023-09-30T09:25:30.434877+0000","last_fullsized":"2023-09-30T09:25:30.434877+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"166'6503","reported_seq":7753,"reported_epoch":166,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:25:44.766814+0000","last_change":"2023-09-30T09:25:14.421586+0000","last_active":"2023-09-30T09:25:44.766814+0000","last_peered":"2023-09-30T09:25:44.766814+0000","last_clean":"2023-09-30T09:24:01.611293+0000","last_became_active":"2023-09-30T09:25:14.386329+0000","last_became_peered":"2023-09-30T09:25:14.386329+0000","last_unstale":"2023-09-30T09:25:44.766814+0000","last_undegraded":"2023-09-30T09:25:14.364354+0000","last_fullsized":"2023-09-30T09:25:14.355372+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6503,"log_dups_size":0,"ondisk_log_size":6503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:11:34.305355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260702232,"num_objects":3979,"num_object_clones":0,"num_object_copies":11937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3979,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6503,"num_write_kb":335361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":586,"num_bytes_recovered":38404096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":127},{"shards":"2,7","objects":3852}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":151,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.437535+0000","last_change":"2023-09-30T09:23:08.029556+0000","last_active":"2023-09-30T09:25:30.437535+0000","last_peered":"2023-09-30T09:25:30.437535+0000","last_clean":"2023-09-30T09:25:30.437535+0000","last_became_active":"2023-09-30T09:23:08.014483+0000","last_became_peered":"2023-09-30T09:23:08.014483+0000","last_unstale":"2023-09-30T09:25:30.437535+0000","last_undegraded":"2023-09-30T09:25:30.437535+0000","last_fullsized":"2023-09-30T09:25:30.437535+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:07:01.455742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":124,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451086+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:25:30.451086+0000","last_peered":"2023-09-30T09:25:30.451086+0000","last_clean":"2023-09-30T09:25:30.451086+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:25:30.451086+0000","last_undegraded":"2023-09-30T09:25:30.451086+0000","last_fullsized":"2023-09-30T09:25:30.451086+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"166'6256","reported_seq":9247,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:46.646786+0000","last_change":"2023-09-30T09:24:20.547592+0000","last_active":"2023-09-30T09:25:46.646786+0000","last_peered":"2023-09-30T09:25:46.646786+0000","last_clean":"2023-09-30T09:25:46.646786+0000","last_became_active":"2023-09-30T09:24:20.539203+0000","last_became_peered":"2023-09-30T09:24:20.539203+0000","last_unstale":"2023-09-30T09:25:46.646786+0000","last_undegraded":"2023-09-30T09:25:46.646786+0000","last_fullsized":"2023-09-30T09:25:46.646786+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6256,"log_dups_size":0,"ondisk_log_size":6256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:33.219264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251265024,"num_objects":3834,"num_object_clones":0,"num_object_copies":11502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6256,"num_write_kb":322880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1455,"num_bytes_recovered":95354880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.452239+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:25:30.452239+0000","last_peered":"2023-09-30T09:25:30.452239+0000","last_clean":"2023-09-30T09:25:30.452239+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:25:30.452239+0000","last_undegraded":"2023-09-30T09:25:30.452239+0000","last_fullsized":"2023-09-30T09:25:30.452239+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":20,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451591+0000","last_change":"2023-09-30T09:25:04.290019+0000","last_active":"2023-09-30T09:25:30.451591+0000","last_peered":"2023-09-30T09:25:30.451591+0000","last_clean":"2023-09-30T09:25:30.451591+0000","last_became_active":"2023-09-30T09:25:04.289710+0000","last_became_peered":"2023-09-30T09:25:04.289710+0000","last_unstale":"2023-09-30T09:25:30.451591+0000","last_undegraded":"2023-09-30T09:25:30.451591+0000","last_fullsized":"2023-09-30T09:25:30.451591+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:22:01.874727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"166'6423","reported_seq":6484,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:43.912230+0000","last_change":"2023-09-30T09:22:48.115158+0000","last_active":"2023-09-30T09:25:43.912230+0000","last_peered":"2023-09-30T09:25:43.912230+0000","last_clean":"2023-09-30T09:25:43.912230+0000","last_became_active":"2023-09-30T09:22:48.114453+0000","last_became_peered":"2023-09-30T09:22:48.114453+0000","last_unstale":"2023-09-30T09:25:43.912230+0000","last_undegraded":"2023-09-30T09:25:43.912230+0000","last_fullsized":"2023-09-30T09:25:43.912230+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6423,"log_dups_size":0,"ondisk_log_size":6423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:31:35.934276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266665984,"num_objects":4069,"num_object_clones":0,"num_object_copies":12207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6423,"num_write_kb":335744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":125,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451666+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:25:30.451666+0000","last_peered":"2023-09-30T09:25:30.451666+0000","last_clean":"2023-09-30T09:25:30.451666+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:25:30.451666+0000","last_undegraded":"2023-09-30T09:25:30.451666+0000","last_fullsized":"2023-09-30T09:25:30.451666+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":124,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451269+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:25:30.451269+0000","last_peered":"2023-09-30T09:25:30.451269+0000","last_clean":"2023-09-30T09:25:30.451269+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:25:30.451269+0000","last_undegraded":"2023-09-30T09:25:30.451269+0000","last_fullsized":"2023-09-30T09:25:30.451269+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"166'6289","reported_seq":9330,"reported_epoch":166,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:25:44.049834+0000","last_change":"2023-09-30T09:25:14.555041+0000","last_active":"2023-09-30T09:25:44.049834+0000","last_peered":"2023-09-30T09:25:44.049834+0000","last_clean":"2023-09-30T09:24:01.532724+0000","last_became_active":"2023-09-30T09:25:04.306365+0000","last_became_peered":"2023-09-30T09:25:04.306365+0000","last_unstale":"2023-09-30T09:25:44.049834+0000","last_undegraded":"2023-09-30T09:25:14.554738+0000","last_fullsized":"2023-09-30T09:25:04.272594+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6289,"log_dups_size":0,"ondisk_log_size":6289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:11.652193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262602752,"num_objects":4007,"num_object_clones":0,"num_object_copies":12021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6289,"num_write_kb":329472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1471,"num_bytes_recovered":96403456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,2,5","objects":145},{"shards":"0,5","objects":3862}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":130,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438754+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:25:30.438754+0000","last_peered":"2023-09-30T09:25:30.438754+0000","last_clean":"2023-09-30T09:25:30.438754+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:25:30.438754+0000","last_undegraded":"2023-09-30T09:25:30.438754+0000","last_fullsized":"2023-09-30T09:25:30.438754+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":104,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.444638+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:25:30.444638+0000","last_peered":"2023-09-30T09:25:30.444638+0000","last_clean":"2023-09-30T09:25:30.444638+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:25:30.444638+0000","last_undegraded":"2023-09-30T09:25:30.444638+0000","last_fullsized":"2023-09-30T09:25:30.444638+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":508,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.444040+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:25:30.444040+0000","last_peered":"2023-09-30T09:25:30.444040+0000","last_clean":"2023-09-30T09:25:30.444040+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:25:30.444040+0000","last_undegraded":"2023-09-30T09:25:30.444040+0000","last_fullsized":"2023-09-30T09:25:30.444040+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":142,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451703+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:25:30.451703+0000","last_peered":"2023-09-30T09:25:30.451703+0000","last_clean":"2023-09-30T09:25:30.451703+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:25:30.451703+0000","last_undegraded":"2023-09-30T09:25:30.451703+0000","last_fullsized":"2023-09-30T09:25:30.451703+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.444366+0000","last_change":"2023-09-30T09:25:14.364474+0000","last_active":"2023-09-30T09:25:30.444366+0000","last_peered":"2023-09-30T09:25:30.444366+0000","last_clean":"2023-09-30T09:25:30.444366+0000","last_became_active":"2023-09-30T09:25:14.364128+0000","last_became_peered":"2023-09-30T09:25:14.364128+0000","last_unstale":"2023-09-30T09:25:30.444366+0000","last_undegraded":"2023-09-30T09:25:30.444366+0000","last_fullsized":"2023-09-30T09:25:30.444366+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:41:17.197492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"166'6345","reported_seq":8766,"reported_epoch":166,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:25:44.051349+0000","last_change":"2023-09-30T09:25:17.149514+0000","last_active":"2023-09-30T09:25:44.051349+0000","last_peered":"2023-09-30T09:25:44.051349+0000","last_clean":"2023-09-30T09:23:06.986974+0000","last_became_active":"2023-09-30T09:25:16.112832+0000","last_became_peered":"2023-09-30T09:25:16.112832+0000","last_unstale":"2023-09-30T09:25:44.051349+0000","last_undegraded":"2023-09-30T09:25:16.092762+0000","last_fullsized":"2023-09-30T09:25:15.359533+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6345,"log_dups_size":0,"ondisk_log_size":6345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:19.810956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261947392,"num_objects":3997,"num_object_clones":0,"num_object_copies":11991,"num_objects_missing_on_primary":317,"num_objects_missing":317,"num_objects_degraded":806,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6345,"num_write_kb":330944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1164,"num_bytes_recovered":76218368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,5],"avail_no_missing":["6"],"object_location_counts":[{"shards":"0,3,5,6","objects":2},{"shards":"0,3,6","objects":130},{"shards":"0,5","objects":3548},{"shards":"3,5,6","objects":316},{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"166'6377","reported_seq":13418,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:43.994394+0000","last_change":"2023-09-30T09:25:17.387236+0000","last_active":"2023-09-30T09:25:43.994394+0000","last_peered":"2023-09-30T09:25:43.994394+0000","last_clean":"2023-09-30T09:25:43.994394+0000","last_became_active":"2023-09-30T09:25:17.376539+0000","last_became_peered":"2023-09-30T09:25:17.376539+0000","last_unstale":"2023-09-30T09:25:43.994394+0000","last_undegraded":"2023-09-30T09:25:43.994394+0000","last_fullsized":"2023-09-30T09:25:43.994394+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6377,"log_dups_size":0,"ondisk_log_size":6377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:15:31.918233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260636672,"num_objects":3977,"num_object_clones":0,"num_object_copies":11931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6377,"num_write_kb":331328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3466,"num_bytes_recovered":207814656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":312,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:31.646671+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:25:31.646671+0000","last_peered":"2023-09-30T09:25:31.646671+0000","last_clean":"2023-09-30T09:25:31.646671+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:25:31.646671+0000","last_undegraded":"2023-09-30T09:25:31.646671+0000","last_fullsized":"2023-09-30T09:25:31.646671+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":145,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451359+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:25:30.451359+0000","last_peered":"2023-09-30T09:25:30.451359+0000","last_clean":"2023-09-30T09:25:30.451359+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:25:30.451359+0000","last_undegraded":"2023-09-30T09:25:30.451359+0000","last_fullsized":"2023-09-30T09:25:30.451359+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":126,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.440384+0000","last_change":"2023-09-30T09:23:23.238851+0000","last_active":"2023-09-30T09:25:30.440384+0000","last_peered":"2023-09-30T09:25:30.440384+0000","last_clean":"2023-09-30T09:25:30.440384+0000","last_became_active":"2023-09-30T09:23:23.238582+0000","last_became_peered":"2023-09-30T09:23:23.238582+0000","last_unstale":"2023-09-30T09:25:30.440384+0000","last_undegraded":"2023-09-30T09:25:30.440384+0000","last_fullsized":"2023-09-30T09:25:30.440384+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:52.808966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"166'6238","reported_seq":6303,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:44.760726+0000","last_change":"2023-09-30T09:22:48.108513+0000","last_active":"2023-09-30T09:25:44.760726+0000","last_peered":"2023-09-30T09:25:44.760726+0000","last_clean":"2023-09-30T09:25:44.760726+0000","last_became_active":"2023-09-30T09:22:48.108231+0000","last_became_peered":"2023-09-30T09:22:48.108231+0000","last_unstale":"2023-09-30T09:25:44.760726+0000","last_undegraded":"2023-09-30T09:25:44.760726+0000","last_fullsized":"2023-09-30T09:25:44.760726+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6238,"log_dups_size":0,"ondisk_log_size":6238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:09.723671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257294336,"num_objects":3926,"num_object_clones":0,"num_object_copies":11778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6238,"num_write_kb":325248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":543,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.439704+0000","last_change":"2023-09-30T09:23:11.172147+0000","last_active":"2023-09-30T09:25:30.439704+0000","last_peered":"2023-09-30T09:25:30.439704+0000","last_clean":"2023-09-30T09:25:30.439704+0000","last_became_active":"2023-09-30T09:23:11.171540+0000","last_became_peered":"2023-09-30T09:23:11.171540+0000","last_unstale":"2023-09-30T09:25:30.439704+0000","last_undegraded":"2023-09-30T09:25:30.439704+0000","last_fullsized":"2023-09-30T09:25:30.439704+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:24.074307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":103,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.444104+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:25:30.444104+0000","last_peered":"2023-09-30T09:25:30.444104+0000","last_clean":"2023-09-30T09:25:30.444104+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:25:30.444104+0000","last_undegraded":"2023-09-30T09:25:30.444104+0000","last_fullsized":"2023-09-30T09:25:30.444104+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.444595+0000","last_change":"2023-09-30T09:23:08.027904+0000","last_active":"2023-09-30T09:25:30.444595+0000","last_peered":"2023-09-30T09:25:30.444595+0000","last_clean":"2023-09-30T09:25:30.444595+0000","last_became_active":"2023-09-30T09:23:08.027557+0000","last_became_peered":"2023-09-30T09:23:08.027557+0000","last_unstale":"2023-09-30T09:25:30.444595+0000","last_undegraded":"2023-09-30T09:25:30.444595+0000","last_fullsized":"2023-09-30T09:25:30.444595+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:19:26.977141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":573,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451533+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:25:30.451533+0000","last_peered":"2023-09-30T09:25:30.451533+0000","last_clean":"2023-09-30T09:25:30.451533+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:25:30.451533+0000","last_undegraded":"2023-09-30T09:25:30.451533+0000","last_fullsized":"2023-09-30T09:25:30.451533+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":559,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.476239+0000","last_change":"2023-09-30T09:23:27.263806+0000","last_active":"2023-09-30T09:25:30.476239+0000","last_peered":"2023-09-30T09:25:30.476239+0000","last_clean":"2023-09-30T09:25:30.476239+0000","last_became_active":"2023-09-30T09:23:27.263242+0000","last_became_peered":"2023-09-30T09:23:27.263242+0000","last_unstale":"2023-09-30T09:25:30.476239+0000","last_undegraded":"2023-09-30T09:25:30.476239+0000","last_fullsized":"2023-09-30T09:25:30.476239+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:38:00.839832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":80,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:37.133121+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:25:37.133121+0000","last_peered":"2023-09-30T09:25:37.133121+0000","last_clean":"2023-09-30T09:25:37.133121+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:25:37.133121+0000","last_undegraded":"2023-09-30T09:25:37.133121+0000","last_fullsized":"2023-09-30T09:25:37.133121+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":108,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451833+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:25:30.451833+0000","last_peered":"2023-09-30T09:25:30.451833+0000","last_clean":"2023-09-30T09:25:30.451833+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:25:30.451833+0000","last_undegraded":"2023-09-30T09:25:30.451833+0000","last_fullsized":"2023-09-30T09:25:30.451833+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"166'6336","reported_seq":6397,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:43.788313+0000","last_change":"2023-09-30T09:22:48.113213+0000","last_active":"2023-09-30T09:25:43.788313+0000","last_peered":"2023-09-30T09:25:43.788313+0000","last_clean":"2023-09-30T09:25:43.788313+0000","last_became_active":"2023-09-30T09:22:48.112904+0000","last_became_peered":"2023-09-30T09:22:48.112904+0000","last_unstale":"2023-09-30T09:25:43.788313+0000","last_undegraded":"2023-09-30T09:25:43.788313+0000","last_fullsized":"2023-09-30T09:25:43.788313+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6336,"log_dups_size":0,"ondisk_log_size":6336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:37:37.122003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258605056,"num_objects":3946,"num_object_clones":0,"num_object_copies":11838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6336,"num_write_kb":329024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"166'6283","reported_seq":17018,"reported_epoch":166,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:25:44.044026+0000","last_change":"2023-09-30T09:25:15.494362+0000","last_active":"2023-09-30T09:25:44.044026+0000","last_peered":"2023-09-30T09:25:44.044026+0000","last_clean":"2023-09-30T09:25:13.127337+0000","last_became_active":"2023-09-30T09:25:15.484895+0000","last_became_peered":"2023-09-30T09:25:15.484895+0000","last_unstale":"2023-09-30T09:25:44.044026+0000","last_undegraded":"2023-09-30T09:25:44.044026+0000","last_fullsized":"2023-09-30T09:25:44.044026+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:16.127121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259325952,"num_objects":3957,"num_object_clones":0,"num_object_copies":11871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3957,"num_objects_unfound":0,"num_objects_dirty":3957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6283,"num_write_kb":327680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5313,"num_bytes_recovered":348192768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3957}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":726,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.476264+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:25:30.476264+0000","last_peered":"2023-09-30T09:25:30.476264+0000","last_clean":"2023-09-30T09:25:30.476264+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:25:30.476264+0000","last_undegraded":"2023-09-30T09:25:30.476264+0000","last_fullsized":"2023-09-30T09:25:30.476264+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.434333+0000","last_change":"2023-09-30T09:25:04.283467+0000","last_active":"2023-09-30T09:25:30.434333+0000","last_peered":"2023-09-30T09:25:30.434333+0000","last_clean":"2023-09-30T09:25:30.434333+0000","last_became_active":"2023-09-30T09:25:04.283162+0000","last_became_peered":"2023-09-30T09:25:04.283162+0000","last_unstale":"2023-09-30T09:25:30.434333+0000","last_undegraded":"2023-09-30T09:25:30.434333+0000","last_fullsized":"2023-09-30T09:25:30.434333+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:12:35.333268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":57,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.434143+0000","last_change":"2023-09-30T09:23:08.024242+0000","last_active":"2023-09-30T09:25:30.434143+0000","last_peered":"2023-09-30T09:25:30.434143+0000","last_clean":"2023-09-30T09:25:30.434143+0000","last_became_active":"2023-09-30T09:23:08.023891+0000","last_became_peered":"2023-09-30T09:23:08.023891+0000","last_unstale":"2023-09-30T09:25:30.434143+0000","last_undegraded":"2023-09-30T09:25:30.434143+0000","last_fullsized":"2023-09-30T09:25:30.434143+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:19:36.687182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":806,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.477139+0000","last_change":"2023-09-30T09:23:11.125442+0000","last_active":"2023-09-30T09:25:30.477139+0000","last_peered":"2023-09-30T09:25:30.477139+0000","last_clean":"2023-09-30T09:25:30.477139+0000","last_became_active":"2023-09-30T09:23:11.124962+0000","last_became_peered":"2023-09-30T09:23:11.124962+0000","last_unstale":"2023-09-30T09:25:30.477139+0000","last_undegraded":"2023-09-30T09:25:30.477139+0000","last_fullsized":"2023-09-30T09:25:30.477139+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:53.386815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":119,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.443960+0000","last_change":"2023-09-30T09:25:14.359586+0000","last_active":"2023-09-30T09:25:30.443960+0000","last_peered":"2023-09-30T09:25:30.443960+0000","last_clean":"2023-09-30T09:25:30.443960+0000","last_became_active":"2023-09-30T09:25:14.359198+0000","last_became_peered":"2023-09-30T09:25:14.359198+0000","last_unstale":"2023-09-30T09:25:30.443960+0000","last_undegraded":"2023-09-30T09:25:30.443960+0000","last_fullsized":"2023-09-30T09:25:30.443960+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:48.533039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":53,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438474+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:25:30.438474+0000","last_peered":"2023-09-30T09:25:30.438474+0000","last_clean":"2023-09-30T09:25:30.438474+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:25:30.438474+0000","last_undegraded":"2023-09-30T09:25:30.438474+0000","last_fullsized":"2023-09-30T09:25:30.438474+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"166'6546","reported_seq":9735,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:43.784204+0000","last_change":"2023-09-30T09:24:12.672495+0000","last_active":"2023-09-30T09:25:43.784204+0000","last_peered":"2023-09-30T09:25:43.784204+0000","last_clean":"2023-09-30T09:25:43.784204+0000","last_became_active":"2023-09-30T09:24:12.668774+0000","last_became_peered":"2023-09-30T09:24:12.668774+0000","last_unstale":"2023-09-30T09:25:43.784204+0000","last_undegraded":"2023-09-30T09:25:43.784204+0000","last_fullsized":"2023-09-30T09:25:43.784204+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6546,"log_dups_size":0,"ondisk_log_size":6546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:01:27.375538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271450112,"num_objects":4142,"num_object_clones":0,"num_object_copies":12426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6546,"num_write_kb":342016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1553,"num_bytes_recovered":101777408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":151,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438477+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:25:30.438477+0000","last_peered":"2023-09-30T09:25:30.438477+0000","last_clean":"2023-09-30T09:25:30.438477+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:25:30.438477+0000","last_undegraded":"2023-09-30T09:25:30.438477+0000","last_fullsized":"2023-09-30T09:25:30.438477+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1113,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.444226+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:25:30.444226+0000","last_peered":"2023-09-30T09:25:30.444226+0000","last_clean":"2023-09-30T09:25:30.444226+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:25:30.444226+0000","last_undegraded":"2023-09-30T09:25:30.444226+0000","last_fullsized":"2023-09-30T09:25:30.444226+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":141,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438836+0000","last_change":"2023-09-30T09:23:08.028741+0000","last_active":"2023-09-30T09:25:30.438836+0000","last_peered":"2023-09-30T09:25:30.438836+0000","last_clean":"2023-09-30T09:25:30.438836+0000","last_became_active":"2023-09-30T09:23:08.028167+0000","last_became_peered":"2023-09-30T09:23:08.028167+0000","last_unstale":"2023-09-30T09:25:30.438836+0000","last_undegraded":"2023-09-30T09:25:30.438836+0000","last_fullsized":"2023-09-30T09:25:30.438836+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:47:39.241913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"166'6463","reported_seq":6524,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:43.759211+0000","last_change":"2023-09-30T09:22:48.113865+0000","last_active":"2023-09-30T09:25:43.759211+0000","last_peered":"2023-09-30T09:25:43.759211+0000","last_clean":"2023-09-30T09:25:43.759211+0000","last_became_active":"2023-09-30T09:22:48.113578+0000","last_became_peered":"2023-09-30T09:22:48.113578+0000","last_unstale":"2023-09-30T09:25:43.759211+0000","last_undegraded":"2023-09-30T09:25:43.759211+0000","last_fullsized":"2023-09-30T09:25:43.759211+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6463,"log_dups_size":0,"ondisk_log_size":6463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:17.848647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264962048,"num_objects":4043,"num_object_clones":0,"num_object_copies":12129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6463,"num_write_kb":336192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":419,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.450912+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:25:30.450912+0000","last_peered":"2023-09-30T09:25:30.450912+0000","last_clean":"2023-09-30T09:25:30.450912+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:25:30.450912+0000","last_undegraded":"2023-09-30T09:25:30.450912+0000","last_fullsized":"2023-09-30T09:25:30.450912+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":61,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.434695+0000","last_change":"2023-09-30T09:23:08.031587+0000","last_active":"2023-09-30T09:25:30.434695+0000","last_peered":"2023-09-30T09:25:30.434695+0000","last_clean":"2023-09-30T09:25:30.434695+0000","last_became_active":"2023-09-30T09:23:08.016566+0000","last_became_peered":"2023-09-30T09:23:08.016566+0000","last_unstale":"2023-09-30T09:25:30.434695+0000","last_undegraded":"2023-09-30T09:25:30.434695+0000","last_fullsized":"2023-09-30T09:25:30.434695+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:44:10.092575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":107,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451740+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:25:30.451740+0000","last_peered":"2023-09-30T09:25:30.451740+0000","last_clean":"2023-09-30T09:25:30.451740+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:25:30.451740+0000","last_undegraded":"2023-09-30T09:25:30.451740+0000","last_fullsized":"2023-09-30T09:25:30.451740+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"166'6234","reported_seq":6299,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:42.023846+0000","last_change":"2023-09-30T09:22:48.113933+0000","last_active":"2023-09-30T09:25:42.023846+0000","last_peered":"2023-09-30T09:25:42.023846+0000","last_clean":"2023-09-30T09:25:42.023846+0000","last_became_active":"2023-09-30T09:22:48.113640+0000","last_became_peered":"2023-09-30T09:22:48.113640+0000","last_unstale":"2023-09-30T09:25:42.023846+0000","last_undegraded":"2023-09-30T09:25:42.023846+0000","last_fullsized":"2023-09-30T09:25:42.023846+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6234,"log_dups_size":0,"ondisk_log_size":6234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:16.995917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266862592,"num_objects":4072,"num_object_clones":0,"num_object_copies":12216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6234,"num_write_kb":329792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":53,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.050283+0000","last_change":"2023-09-30T09:23:08.024830+0000","last_active":"2023-09-30T09:25:39.050283+0000","last_peered":"2023-09-30T09:25:39.050283+0000","last_clean":"2023-09-30T09:25:39.050283+0000","last_became_active":"2023-09-30T09:23:08.024489+0000","last_became_peered":"2023-09-30T09:23:08.024489+0000","last_unstale":"2023-09-30T09:25:39.050283+0000","last_undegraded":"2023-09-30T09:25:39.050283+0000","last_fullsized":"2023-09-30T09:25:39.050283+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:41.769725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.440246+0000","last_change":"2023-09-30T09:21:29.096256+0000","last_active":"2023-09-30T09:25:30.440246+0000","last_peered":"2023-09-30T09:25:30.440246+0000","last_clean":"2023-09-30T09:25:30.440246+0000","last_became_active":"2023-09-30T09:21:29.095930+0000","last_became_peered":"2023-09-30T09:21:29.095930+0000","last_unstale":"2023-09-30T09:25:30.440246+0000","last_undegraded":"2023-09-30T09:25:30.440246+0000","last_fullsized":"2023-09-30T09:25:30.440246+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:25.270756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":127,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451402+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:25:30.451402+0000","last_peered":"2023-09-30T09:25:30.451402+0000","last_clean":"2023-09-30T09:25:30.451402+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:25:30.451402+0000","last_undegraded":"2023-09-30T09:25:30.451402+0000","last_fullsized":"2023-09-30T09:25:30.451402+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":20,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451065+0000","last_change":"2023-09-30T09:25:04.284772+0000","last_active":"2023-09-30T09:25:30.451065+0000","last_peered":"2023-09-30T09:25:30.451065+0000","last_clean":"2023-09-30T09:25:30.451065+0000","last_became_active":"2023-09-30T09:25:04.284405+0000","last_became_peered":"2023-09-30T09:25:04.284405+0000","last_unstale":"2023-09-30T09:25:30.451065+0000","last_undegraded":"2023-09-30T09:25:30.451065+0000","last_fullsized":"2023-09-30T09:25:30.451065+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:44.112365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":110,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.451783+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:25:30.451783+0000","last_peered":"2023-09-30T09:25:30.451783+0000","last_clean":"2023-09-30T09:25:30.451783+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:25:30.451783+0000","last_undegraded":"2023-09-30T09:25:30.451783+0000","last_fullsized":"2023-09-30T09:25:30.451783+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.452080+0000","last_change":"2023-09-30T09:25:04.306048+0000","last_active":"2023-09-30T09:25:30.452080+0000","last_peered":"2023-09-30T09:25:30.452080+0000","last_clean":"2023-09-30T09:25:30.452080+0000","last_became_active":"2023-09-30T09:25:04.305804+0000","last_became_peered":"2023-09-30T09:25:04.305804+0000","last_unstale":"2023-09-30T09:25:30.452080+0000","last_undegraded":"2023-09-30T09:25:30.452080+0000","last_fullsized":"2023-09-30T09:25:30.452080+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:32.116748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":136,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438136+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:25:30.438136+0000","last_peered":"2023-09-30T09:25:30.438136+0000","last_clean":"2023-09-30T09:25:30.438136+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:25:30.438136+0000","last_undegraded":"2023-09-30T09:25:30.438136+0000","last_fullsized":"2023-09-30T09:25:30.438136+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":166,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.438838+0000","last_change":"2023-09-30T09:23:08.021721+0000","last_active":"2023-09-30T09:25:30.438838+0000","last_peered":"2023-09-30T09:25:30.438838+0000","last_clean":"2023-09-30T09:25:30.438838+0000","last_became_active":"2023-09-30T09:23:08.021401+0000","last_became_peered":"2023-09-30T09:23:08.021401+0000","last_unstale":"2023-09-30T09:25:30.438838+0000","last_undegraded":"2023-09-30T09:25:30.438838+0000","last_fullsized":"2023-09-30T09:25:30.438838+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:33:33.611604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":63,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.435484+0000","last_change":"2023-09-30T09:23:08.039181+0000","last_active":"2023-09-30T09:25:30.435484+0000","last_peered":"2023-09-30T09:25:30.435484+0000","last_clean":"2023-09-30T09:25:30.435484+0000","last_became_active":"2023-09-30T09:23:08.025651+0000","last_became_peered":"2023-09-30T09:23:08.025651+0000","last_unstale":"2023-09-30T09:25:30.435484+0000","last_undegraded":"2023-09-30T09:25:30.435484+0000","last_fullsized":"2023-09-30T09:25:30.435484+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:27.721207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":124,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-30T09:25:30.450900+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:25:30.450900+0000","last_peered":"2023-09-30T09:25:30.450900+0000","last_clean":"2023-09-30T09:25:30.450900+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:25:30.450900+0000","last_undegraded":"2023-09-30T09:25:30.450900+0000","last_fullsized":"2023-09-30T09:25:30.450900+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":4177526808,"num_objects":63745,"num_object_clones":0,"num_object_copies":191235,"num_objects_missing_on_primary":317,"num_objects_missing":317,"num_objects_degraded":1761,"num_objects_misplaced":4392,"num_objects_unfound":0,"num_objects_dirty":63745,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101807,"num_write_kb":5297601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18194,"num_bytes_recovered":1154023424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13358411776,"data_stored":13358399560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101807,"ondisk_log_size":101807,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":738,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234169,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1421164,"kb_used_data":1370980,"kb_used_omap":0,"kb_used_meta":50176,"kb_avail":92323988,"statfs":{"total":95995035648,"available":94539763712,"internally_reserved":0,"allocated":1403883520,"data_stored":1403164258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":51380224},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:17 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.78700000000000003,"15min":0.68100000000000005},"min":{"1min":0.30099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":1.444,"5min":7.3559999999999999,"15min":7.3559999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.441,"5min":0.629,"15min":0.65000000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.59499999999999997,"5min":4.8289999999999997,"15min":4.8289999999999997},"last":0.433}]},{"osd":1,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":2.4289999999999998,"15min":1.175},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.212,"5min":82.025999999999996,"15min":82.025999999999996},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":2.3980000000000001,"15min":1.1799999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":82.012,"15min":82.012},"last":0.55500000000000005}]},{"osd":2,"last update":"Sat Sep 30 09:25:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":0.92900000000000005,"15min":0.71299999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":5.9800000000000004,"5min":11.955,"15min":11.955},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.054,"15min":0.75700000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.875,"5min":15.443,"15min":15.443},"last":1.5580000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:25:40 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.91000000000000003,"15min":0.71699999999999997},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0090000000000003,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.753,"15min":0.67500000000000004},"min":{"1min":0.40200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.3540000000000001,"5min":4.8150000000000004,"15min":4.8150000000000004},"last":0.59099999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:24:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.54300000000000004,"15min":0.48699999999999999},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.4790000000000001,"5min":2.3599999999999999,"15min":2.3599999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.61799999999999999,"15min":0.53400000000000003},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.093,"5min":3,"15min":3},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:24:56 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.65200000000000002,"15min":0.55500000000000005},"min":{"1min":0.33700000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.1619999999999999,"5min":5.968,"15min":5.968},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.66200000000000003,"15min":0.58799999999999997},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.5640000000000001,"5min":5.452,"15min":5.452},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:24:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.68200000000000005,"15min":0.59899999999999998},"min":{"1min":0.41999999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.8999999999999999,"5min":2.8809999999999998,"15min":2.8809999999999998},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.67500000000000004,"15min":0.59199999999999997},"min":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":1.1200000000000001,"5min":2.919,"15min":2.919},"last":0.46800000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495806,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1234200,"kb_used_data":1177680,"kb_used_omap":0,"kb_used_meta":56512,"kb_avail":92510952,"statfs":{"total":95995035648,"available":94731214848,"internally_reserved":0,"allocated":1205944320,"data_stored":1205244799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":57868288},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.83999999999999997,"15min":0.70299999999999996},"min":{"1min":0.23599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.4249999999999998,"5min":6.649,"15min":6.649},"last":0.253},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.84499999999999997,"15min":0.71899999999999997},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.327,"5min":8.327,"15min":8.327},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.83399999999999996,"15min":0.69999999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.2970000000000002,"5min":6.5090000000000003,"15min":6.5090000000000003},"last":0.313},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.81299999999999994,"15min":0.69399999999999995},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2810000000000001,"5min":4.944,"15min":4.944},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:25:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":0.95199999999999996,"15min":0.69799999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.4730000000000008,"5min":8.4730000000000008,"15min":8.4730000000000008},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.95999999999999996,"15min":0.72399999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.9399999999999999,"5min":8.6159999999999997,"15min":8.6159999999999997},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.878,"15min":0.68999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.387,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.0309999999999999,"15min":0.73599999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7759999999999998,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.82899999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":1.2210000000000001,"15min":0.73499999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":0.59299999999999997,"5min":73.656000000000006,"15min":73.656000000000006},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":1.2829999999999999,"15min":0.77000000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58199999999999996,"5min":73.882999999999996,"15min":73.882999999999996},"last":0.46600000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.64100000000000001,"15min":0.54200000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":0.48999999999999999,"5min":4.2430000000000003,"15min":4.2430000000000003},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.71399999999999997,"15min":0.60899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28000000000000003},"max":{"1min":0.60299999999999998,"5min":6.5999999999999996,"15min":6.5999999999999996},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:24:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.67600000000000005,"15min":0.60999999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.26500000000000001},"max":{"1min":0.98899999999999999,"5min":3.7959999999999998,"15min":3.7959999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.91500000000000004,"15min":0.70999999999999996},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.31900000000000001},"max":{"1min":8.2690000000000001,"5min":10.339,"15min":10.339},"last":0.439}]}]},{"osd":1,"up_from":99,"seq":425201762371,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1318668,"kb_used_data":1255108,"kb_used_omap":5,"kb_used_meta":63546,"kb_avail":92426484,"statfs":{"total":95995035648,"available":94644719616,"internally_reserved":0,"allocated":1285230592,"data_stored":1284311898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":65071286},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":4.2450000000000001,"5min":2.0859999999999999,"15min":0.94199999999999995},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":62.935000000000002,"5min":72.007000000000005,"15min":72.007000000000005},"last":0.20100000000000001},{"interface":"front","average":{"1min":3.907,"5min":2.1779999999999999,"15min":0.997},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":63.088000000000001,"5min":72.322999999999993,"15min":72.322999999999993},"last":0.224}]},{"osd":2,"last update":"Sat Sep 30 09:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.626,"15min":0.49099999999999999},"min":{"1min":0.254,"5min":0.249,"15min":0.249},"max":{"1min":2.9980000000000002,"5min":7.9790000000000001,"15min":7.9790000000000001},"last":95.245000000000005},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.73199999999999998,"15min":0.50900000000000001},"min":{"1min":0.27700000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.988,"5min":11.099,"15min":11.099},"last":95.019000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:25:36 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.74199999999999999,"15min":0.52100000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.845000000000001,"5min":11.845000000000001,"15min":11.845000000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.72099999999999997,"15min":0.502},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.6499999999999999,"5min":11.06,"15min":11.06},"last":0.27500000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.80500000000000005,"15min":0.69799999999999995},"min":{"1min":0.317,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.569,"5min":5.6340000000000003,"15min":5.6340000000000003},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.81299999999999994,"15min":0.72499999999999998},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.9470000000000001,"5min":4.3869999999999996,"15min":4.3869999999999996},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.73499999999999999,"15min":0.65900000000000003},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.619,"5min":5.0140000000000002,"15min":5.0140000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.83399999999999996,"15min":0.70999999999999996},"min":{"1min":0.439,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.9849999999999999,"5min":2.9929999999999999,"15min":2.9929999999999999},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.81399999999999995,"15min":0.68600000000000005},"min":{"1min":0.44900000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":2.851,"5min":3.0150000000000001,"15min":3.0150000000000001},"last":0.378},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.91100000000000003,"15min":0.76000000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.8140000000000001,"5min":7.8140000000000001,"15min":7.8140000000000001},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.96499999999999997,"15min":0.77500000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.9079999999999999,"5min":7.4820000000000002,"15min":7.4820000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.018,"15min":0.80200000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":7.7729999999999997,"5min":7.7729999999999997,"15min":7.7729999999999997},"last":0.437}]}]},{"osd":0,"up_from":94,"seq":403726925903,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2256376,"kb_used_data":2076144,"kb_used_omap":5,"kb_used_meta":180154,"kb_avail":91488776,"statfs":{"total":95995035648,"available":93684506624,"internally_reserved":0,"allocated":2125971456,"data_stored":2125014567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":184477949},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.81599999999999995,"15min":0.51200000000000001},"min":{"1min":0.22,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.5859999999999999,"5min":15.9,"15min":15.9},"last":1.204},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.92500000000000004,"15min":0.56399999999999995},"min":{"1min":0.27800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.8849999999999998,"5min":13.824999999999999,"15min":13.824999999999999},"last":0.19800000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:25:01 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":0.98999999999999999,"15min":0.58199999999999996},"min":{"1min":0.28100000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":18.096,"5min":18.096,"15min":18.096},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.5669999999999999,"5min":0.99099999999999999,"15min":0.621},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":18.149000000000001,"5min":18.149000000000001,"15min":18.149000000000001},"last":0.34399999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":0.98199999999999998,"15min":0.65100000000000002},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.9630000000000001,"5min":13.02,"15min":13.02},"last":1.752},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.96799999999999997,"15min":0.60999999999999999},"min":{"1min":0.29999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.847,"5min":10.128,"15min":10.128},"last":1.232}]},{"osd":4,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.86099999999999999,"15min":0.78500000000000003},"min":{"1min":0.38700000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.369,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.96999999999999997,"15min":0.84699999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":3.169,"5min":12.99,"15min":12.99},"last":1.786}]},{"osd":5,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.1719999999999999,"15min":0.94399999999999995},"min":{"1min":0.38400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.266,"5min":12.949999999999999,"15min":12.949999999999999},"last":2.2639999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.151,"15min":0.94199999999999995},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.105,"5min":12.823,"15min":12.823},"last":2.71}]},{"osd":6,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.91000000000000003,"15min":0.84599999999999997},"min":{"1min":0.432,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.038,"5min":6.9660000000000002,"15min":6.9660000000000002},"last":1.3049999999999999},{"interface":"front","average":{"1min":0.745,"5min":0.85999999999999999,"15min":0.871},"min":{"1min":0.48799999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.5430000000000001,"5min":7.556,"15min":7.556},"last":1.1659999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.96099999999999997,"15min":0.83999999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.1699999999999999,"5min":7.0339999999999998,"15min":7.0339999999999998},"last":2.8100000000000001},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.85799999999999998,"15min":0.86599999999999999},"min":{"1min":0.47399999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.504,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.40400000000000003}]}]},{"osd":2,"up_from":104,"seq":446676598844,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1797568,"kb_used_data":1711800,"kb_used_omap":5,"kb_used_meta":85754,"kb_avail":91947584,"statfs":{"total":95995035648,"available":94154326016,"internally_reserved":0,"allocated":1752883200,"data_stored":1751974179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":87812749},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.88600000000000001,"15min":0.53900000000000003},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.2009999999999996,"5min":15.843999999999999,"15min":15.843999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.86099999999999999,"15min":0.58899999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.0259999999999998,"5min":10.065,"15min":10.065},"last":0.73199999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.66400000000000003,"15min":0.47799999999999998},"min":{"1min":0.23300000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":1.454,"5min":5.4640000000000004,"15min":5.4640000000000004},"last":1.018},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.73999999999999999,"15min":0.52800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.1059999999999999,"5min":6.5369999999999999,"15min":6.5369999999999999},"last":0.42199999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0009999999999999,"15min":0.63100000000000001},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.4009999999999998,"5min":18.635999999999999,"15min":18.635999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.173,"15min":0.72599999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.996,"5min":18.026,"15min":18.026},"last":0.57499999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.81299999999999994,"15min":0.67500000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.1719999999999997,"5min":4.1719999999999997,"15min":4.1719999999999997},"last":0.377},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.0700000000000001,"15min":0.78700000000000003},"min":{"1min":0.39500000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.3440000000000003,"5min":9.9550000000000001,"15min":9.9550000000000001},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.81499999999999995,"15min":0.70499999999999996},"min":{"1min":0.41099999999999998,"5min":0.371,"15min":0.371},"max":{"1min":1.484,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.0049999999999999,"15min":0.77300000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":4.1429999999999998,"5min":10.359,"15min":10.359},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.238,"15min":0.871},"min":{"1min":0.42699999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":4.4560000000000004,"5min":10.037000000000001,"15min":10.037000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.90400000000000003,"15min":0.753},"min":{"1min":0.442,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.5189999999999999,"5min":5.4939999999999998,"15min":5.4939999999999998},"last":0.95099999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":1.1599999999999999,"15min":0.83999999999999997},"min":{"1min":0.44900000000000001,"5min":0.44,"15min":0.44},"max":{"1min":4.5010000000000003,"5min":10.003,"15min":10.003},"last":2.5649999999999999},{"interface":"front","average":{"1min":0.746,"5min":0.90300000000000002,"15min":0.77800000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.609,"5min":5.3979999999999997,"15min":5.3979999999999997},"last":0.98699999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435317,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920580,"kb_used_data":862648,"kb_used_omap":2,"kb_used_meta":57917,"kb_avail":92824572,"statfs":{"total":95995035648,"available":95052361728,"internally_reserved":0,"allocated":883351552,"data_stored":882440545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":59307367},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":2.5920000000000001,"15min":1.121},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.5619999999999998,"5min":130.946,"15min":130.946},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":2.5169999999999999,"15min":1.125},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.7470000000000001,"5min":130.96899999999999,"15min":130.96899999999999},"last":0.96099999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.70799999999999996,"15min":0.52000000000000002},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":6.0099999999999998,"5min":6.0099999999999998,"15min":6.0099999999999998},"last":0.377},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.64100000000000001,"15min":0.48699999999999999},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.159,"5min":3.9340000000000002,"15min":3.9340000000000002},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.831,"15min":1.2470000000000001},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1779999999999999,"5min":145.75899999999999,"15min":145.75899999999999},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.47199999999999998,"5min":2.7909999999999999,"15min":1.2170000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.1120000000000001,"5min":150.358,"15min":150.358},"last":0.73699999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.71299999999999997,"15min":0.64800000000000002},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.312},"max":{"1min":1.042,"5min":3.9119999999999999,"15min":3.9119999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.74099999999999999,"15min":0.65100000000000002},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.27,"5min":4.0309999999999997,"15min":4.0309999999999997},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.85799999999999998,"15min":0.70699999999999996},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.2390000000000001,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":0.5},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.86899999999999999,"15min":0.75800000000000001},"min":{"1min":0.375,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9100000000000001,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.71299999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.75900000000000001,"15min":0.70299999999999996},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.147,"5min":3.9590000000000001,"15min":3.9590000000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.86499999999999999,"15min":0.746},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":3.0230000000000001,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.81100000000000005,"15min":0.72199999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.948,"5min":4.1639999999999997,"15min":4.1639999999999997},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.74199999999999999,"15min":0.70299999999999996},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.372},"max":{"1min":1.1479999999999999,"5min":3.9910000000000001,"15min":3.9910000000000001},"last":0.67900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019079,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2402456,"kb_used_data":2327952,"kb_used_omap":0,"kb_used_meta":74496,"kb_avail":91342696,"statfs":{"total":95995035648,"available":93534920704,"internally_reserved":0,"allocated":2383822848,"data_stored":2383122898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":76283904},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":2.0950000000000002,"15min":1.1279999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.69799999999999995,"5min":128.79300000000001,"15min":128.79300000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.40799999999999997,"5min":1.883,"15min":1.0589999999999999},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":0.53400000000000003,"5min":128.38800000000001,"15min":128.38800000000001},"last":0.55200000000000005}]},{"osd":1,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":1.0569999999999999,"15min":0.73399999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.71899999999999997,"5min":31.469999999999999,"15min":31.469999999999999},"last":0.438},{"interface":"front","average":{"1min":0.748,"5min":1.149,"15min":0.75700000000000001},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.2729999999999997,"5min":31.962,"15min":31.962},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.234,"15min":0.81899999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.333,"5min":29.457000000000001,"15min":29.457000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.256,"15min":0.81599999999999995},"min":{"1min":0.35199999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.2370000000000001,"5min":29.504999999999999,"15min":29.504999999999999},"last":0.44800000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.607,"15min":0.93300000000000005},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.0019999999999998,"5min":86.620000000000005,"15min":86.620000000000005},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.747,"5min":1.5940000000000001,"15min":0.94399999999999995},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9929999999999999,"5min":86.786000000000001,"15min":86.786000000000001},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:25:14 2023","interfaces":[{"interface":"back","average":{"1min":4.4969999999999999,"5min":1.3160000000000001,"15min":0.75700000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":86.691000000000003,"5min":86.691000000000003,"15min":86.691000000000003},"last":0.28699999999999998},{"interface":"front","average":{"1min":4.5309999999999997,"5min":1.3879999999999999,"15min":0.80300000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":86.822000000000003,"5min":86.822000000000003,"15min":86.822000000000003},"last":0.31900000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:25:39 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.55900000000000005,"15min":0.52600000000000002},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71299999999999997,"5min":3.218,"15min":3.218},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.58599999999999997,"15min":0.56799999999999995},"min":{"1min":0.33900000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74199999999999999,"5min":2.0209999999999999,"15min":2.0209999999999999},"last":0.33900000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:25:01 2023","interfaces":[{"interface":"back","average":{"1min":4.7469999999999999,"5min":1.4970000000000001,"15min":0.876},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":86.728999999999999,"5min":86.728999999999999,"15min":86.728999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.69399999999999995,"15min":0.61599999999999999},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":3.1800000000000002,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":0.48399999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757443,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2452560,"kb_used_data":2373576,"kb_used_omap":0,"kb_used_meta":78976,"kb_avail":91292592,"statfs":{"total":95995035648,"available":93483614208,"internally_reserved":0,"allocated":2430541824,"data_stored":2429846048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":80871424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.048,"15min":0.73599999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":27.573,"5min":27.573,"15min":27.573},"last":2.2879999999999998},{"interface":"front","average":{"1min":2.2309999999999999,"5min":1.1830000000000001,"15min":0.80200000000000005},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":27.588999999999999,"5min":27.588999999999999,"15min":27.588999999999999},"last":0.72099999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":1.9359999999999999,"15min":1.03},"min":{"1min":0.33900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.24,"5min":100.593,"15min":100.593},"last":0.75},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.875,"15min":1.056},"min":{"1min":0.35099999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.5369999999999999,"5min":100.575,"15min":100.575},"last":0.27400000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:25:00 2023","interfaces":[{"interface":"back","average":{"1min":5.6529999999999996,"5min":2.2759999999999998,"15min":1.157},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":59.758000000000003,"5min":59.758000000000003,"15min":59.758000000000003},"last":2.0099999999999998},{"interface":"front","average":{"1min":5.8490000000000002,"5min":2.4689999999999999,"15min":1.2529999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":56.100000000000001,"5min":56.100000000000001,"15min":56.100000000000001},"last":2.1619999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:25:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.446,"15min":0.91000000000000003},"min":{"1min":0.39700000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":28.015999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":2.133},{"interface":"front","average":{"1min":2.024,"5min":1.29,"15min":0.85999999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":28.039999999999999,"5min":28.039999999999999,"15min":28.039999999999999},"last":2.1429999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:25:13 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.56000000000000005,"15min":0.49199999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.72399999999999998,"5min":4.2199999999999998,"15min":4.2199999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.68100000000000005,"15min":0.56799999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25900000000000001},"max":{"1min":0.68300000000000005,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":0.38500000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.59099999999999997,"15min":0.54200000000000004},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.745,"5min":2.125,"15min":2.125},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.79100000000000004,"15min":0.66300000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":0.66600000000000004,"5min":4.46,"15min":4.46},"last":0.64900000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.76600000000000001,"15min":0.64100000000000001},"min":{"1min":0.32800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.194,"5min":4.194,"15min":4.194},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.72299999999999998,"15min":0.621},"min":{"1min":0.40999999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":2.1070000000000002,"5min":6.2300000000000004,"15min":6.2300000000000004},"last":0.48399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2105610240,"data_stored":2105606168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1271332864,"data_stored":1271332864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1739460608,"data_stored":1739456536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":880148480,"data_stored":880148480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2361720832,"data_stored":2361720832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2416508928,"data_stored":2416508928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1195311104,"data_stored":1195311104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1388318720,"data_stored":1388314648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:25:50.224 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:25:50.250 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:50.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:50.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:50.361 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:50.442 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:50.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:50 smithi019 ceph-mon[216815]: pgmap v372: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1761/187551 objects degraded (0.939%); 4392/187551 objects misplaced (2.342%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:50.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:50 smithi019 ceph-mon[224043]: pgmap v372: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1761/187551 objects degraded (0.939%); 4392/187551 objects misplaced (2.342%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:50.855 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:50.901 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:50.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:50.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:51.008 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:25:51.009 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:25:51.009 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:25:51.009 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:25:51.009 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:25:51.009 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:51.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:50 smithi167 ceph-mon[192880]: pgmap v372: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1761/187551 objects degraded (0.939%); 4392/187551 objects misplaced (2.342%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:25:51.041 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:51.119 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:51.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:51.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:51.228 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:51.270 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:51.339 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:51.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:51.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:51.447 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:51.521 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:51.589 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:25:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2184712388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:51 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2184712388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:51.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:51 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2184712388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:51.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:51 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:25:52.713 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:52.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:52.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:52.819 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:52.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1565/181857 objects degraded (0.861%); 3987/181857 objects misplaced (2.192%); 5.2 MiB/s, 83 objects/s recovering 2023-09-30T09:25:52.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:52.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:52.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: osdmap e167: 8 total, 8 up, 8 in 2023-09-30T09:25:52.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[216815]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1565/181857 objects degraded (0.861%); 3987/181857 objects misplaced (2.192%); 5.2 MiB/s, 83 objects/s recovering 2023-09-30T09:25:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: osdmap e167: 8 total, 8 up, 8 in 2023-09-30T09:25:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:52 smithi019 ceph-mon[224043]: failed to encode map e167 with expected crc 2023-09-30T09:25:52.900 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:52.901 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:52.936 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:25:53.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1565/181857 objects degraded (0.861%); 3987/181857 objects misplaced (2.192%); 5.2 MiB/s, 83 objects/s recovering 2023-09-30T09:25:53.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:25:53.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:25:53.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: osdmap e167: 8 total, 8 up, 8 in 2023-09-30T09:25:53.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: failed to encode map e167 with expected crc 2023-09-30T09:25:53.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: failed to encode map e167 with expected crc 2023-09-30T09:25:53.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: failed to encode map e167 with expected crc 2023-09-30T09:25:53.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: failed to encode map e167 with expected crc 2023-09-30T09:25:53.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:52 smithi167 ceph-mon[192880]: failed to encode map e167 with expected crc 2023-09-30T09:25:53.097 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:53.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:53.222 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:25:53.306 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:53.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:25:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:25:53] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.43.0" 2023-09-30T09:25:53.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1565/181857 objects degraded (0.861%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:53.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1565/181857 objects degraded (0.861%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:53.963 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:25:53.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:53.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:25:54.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1565/181857 objects degraded (0.861%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:54.068 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:54.138 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:54.251 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:54.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:54.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:54.358 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:54.435 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:54.605 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:54.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:54.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:54.615 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:54 smithi019 ceph-mon[224043]: pgmap v375: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 1565/180498 objects degraded (0.867%); 3987/180498 objects misplaced (2.209%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:25:54.615 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:54 smithi019 ceph-mon[216815]: pgmap v375: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 1565/180498 objects degraded (0.867%); 3987/180498 objects misplaced (2.209%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:25:54.717 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:25:54.783 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:54.847 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:25:54.856 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:25:54.858 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:54 smithi167 ceph-mon[192880]: pgmap v375: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 1565/180498 objects degraded (0.867%); 3987/180498 objects misplaced (2.209%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:25:54.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:54.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:25:54.960 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:54.990 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:55.102 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:55.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:55.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:55.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:55.269 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:55.456 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:55.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:55.511 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:55.561 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:55.592 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:55.708 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:55.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:55.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:55.830 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:55.861 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:55.916 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:25:55.957 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:25:55.960 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:55.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:56.004 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:25:56.037 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:25:56.066 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:25:56.073 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:25:56.114 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:25:56.152 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:56.162 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:25:56.208 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:25:56.210 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:25:56.210 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:25:56.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:25:56.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:25:56.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:25:56.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:25:56.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:25:56.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:25:56.266 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:25:56.895 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:25:56.903 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:25:56.932 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:56 smithi019 ceph-mon[216815]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 1444/170655 objects degraded (0.846%); 3523/170655 objects misplaced (2.064%); 4.7 MiB/s, 79 objects/s recovering 2023-09-30T09:25:56.933 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1396554377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:56.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:56 smithi019 ceph-mon[224043]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 1444/170655 objects degraded (0.846%); 3523/170655 objects misplaced (2.064%); 4.7 MiB/s, 79 objects/s recovering 2023-09-30T09:25:56.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1396554377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:56 smithi167 ceph-mon[192880]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 1444/170655 objects degraded (0.846%); 3523/170655 objects misplaced (2.064%); 4.7 MiB/s, 79 objects/s recovering 2023-09-30T09:25:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1396554377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:25:57.065 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:25:57.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:57.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:25:57.172 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:25:57.199 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:57.243 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:25:57.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:57.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:25:57.350 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:57.376 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:57.417 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:57.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:57.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:25:57.522 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:57.554 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:58.622 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:58.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:58.729 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:58.774 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:58 smithi019 ceph-mon[224043]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:25:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:58 smithi019 ceph-mon[224043]: pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 1444/170010 objects degraded (0.849%); 3523/170010 objects misplaced (2.072%); 4.7 MiB/s, 79 objects/s recovering 2023-09-30T09:25:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:25:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1444/170010 objects degraded (0.849%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:58.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:58 smithi019 ceph-mon[216815]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:25:58.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:58 smithi019 ceph-mon[216815]: pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 1444/170010 objects degraded (0.849%); 3523/170010 objects misplaced (2.072%); 4.7 MiB/s, 79 objects/s recovering 2023-09-30T09:25:58.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:25:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1444/170010 objects degraded (0.849%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:58 smithi167 ceph-mon[192880]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:25:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:58 smithi167 ceph-mon[192880]: pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 1444/170010 objects degraded (0.849%); 3523/170010 objects misplaced (2.072%); 4.7 MiB/s, 79 objects/s recovering 2023-09-30T09:25:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:25:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1444/170010 objects degraded (0.849%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:25:59.522 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:59.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:59.562 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":376,"stamp":"2023-09-30T09:25:55.052429+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3730905023,"num_objects":56885,"num_object_clones":0,"num_object_copies":170655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1444,"num_objects_misplaced":3523,"num_objects_unfound":0,"num_objects_dirty":56885,"num_whiteouts":0,"num_read":6443,"num_read_kb":13071,"num_write":109841,"num_write_kb":5312742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19107,"num_bytes_recovered":1214778754,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109887,"ondisk_log_size":109887,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":281,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12592084,"kb_used_data":11920528,"kb_used_omap":19,"kb_used_meta":671212,"kb_avail":737369132,"statfs":{"total":767960285184,"available":755065991168,"internally_reserved":0,"allocated":12206620672,"data_stored":12200071578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":687321511},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,5,6,1,0,3,7],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-462356480,"num_objects":-7055,"num_object_clones":0,"num_object_copies":-21165,"num_objects_missing_on_primary":-317,"num_objects_missing":-317,"num_objects_degraded":-317,"num_objects_misplaced":-869,"num_objects_unfound":0,"num_objects_dirty":-7055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7055,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":791,"num_bytes_recovered":49414144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.012470"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":144,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514450+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:25:39.514450+0000","last_peered":"2023-09-30T09:25:39.514450+0000","last_clean":"2023-09-30T09:25:39.514450+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:25:39.514450+0000","last_undegraded":"2023-09-30T09:25:39.514450+0000","last_fullsized":"2023-09-30T09:25:39.514450+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":130,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.503622+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:25:39.503622+0000","last_peered":"2023-09-30T09:25:39.503622+0000","last_clean":"2023-09-30T09:25:39.503622+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:25:39.503622+0000","last_undegraded":"2023-09-30T09:25:39.503622+0000","last_fullsized":"2023-09-30T09:25:39.503622+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":120,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.506275+0000","last_change":"2023-09-30T09:25:04.281698+0000","last_active":"2023-09-30T09:25:39.506275+0000","last_peered":"2023-09-30T09:25:39.506275+0000","last_clean":"2023-09-30T09:25:39.506275+0000","last_became_active":"2023-09-30T09:25:04.281351+0000","last_became_peered":"2023-09-30T09:25:04.281351+0000","last_unstale":"2023-09-30T09:25:39.506275+0000","last_undegraded":"2023-09-30T09:25:39.506275+0000","last_fullsized":"2023-09-30T09:25:39.506275+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:17:40.382369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":58,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.508016+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:25:39.508016+0000","last_peered":"2023-09-30T09:25:39.508016+0000","last_clean":"2023-09-30T09:25:39.508016+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:25:39.508016+0000","last_undegraded":"2023-09-30T09:25:39.508016+0000","last_fullsized":"2023-09-30T09:25:39.508016+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":164,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.501558+0000","last_change":"2023-09-30T09:23:08.014781+0000","last_active":"2023-09-30T09:25:39.501558+0000","last_peered":"2023-09-30T09:25:39.501558+0000","last_clean":"2023-09-30T09:25:39.501558+0000","last_became_active":"2023-09-30T09:23:08.014387+0000","last_became_peered":"2023-09-30T09:23:08.014387+0000","last_unstale":"2023-09-30T09:25:39.501558+0000","last_undegraded":"2023-09-30T09:25:39.501558+0000","last_fullsized":"2023-09-30T09:25:39.501558+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:52:13.669946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":126,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.521838+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:25:39.521838+0000","last_peered":"2023-09-30T09:25:39.521838+0000","last_clean":"2023-09-30T09:25:39.521838+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:25:39.521838+0000","last_undegraded":"2023-09-30T09:25:39.521838+0000","last_fullsized":"2023-09-30T09:25:39.521838+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":58,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.508107+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:25:39.508107+0000","last_peered":"2023-09-30T09:25:39.508107+0000","last_clean":"2023-09-30T09:25:39.508107+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:25:39.508107+0000","last_undegraded":"2023-09-30T09:25:39.508107+0000","last_fullsized":"2023-09-30T09:25:39.508107+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":130,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.501633+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:25:39.501633+0000","last_peered":"2023-09-30T09:25:39.501633+0000","last_clean":"2023-09-30T09:25:39.501633+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:25:39.501633+0000","last_undegraded":"2023-09-30T09:25:39.501633+0000","last_fullsized":"2023-09-30T09:25:39.501633+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":18,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.519425+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:25:39.519425+0000","last_peered":"2023-09-30T09:25:39.519425+0000","last_clean":"2023-09-30T09:25:39.519425+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:25:39.519425+0000","last_undegraded":"2023-09-30T09:25:39.519425+0000","last_fullsized":"2023-09-30T09:25:39.519425+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":81,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.502639+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:25:39.502639+0000","last_peered":"2023-09-30T09:25:39.502639+0000","last_clean":"2023-09-30T09:25:39.502639+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:25:39.502639+0000","last_undegraded":"2023-09-30T09:25:39.502639+0000","last_fullsized":"2023-09-30T09:25:39.502639+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":130,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.510948+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:25:39.510948+0000","last_peered":"2023-09-30T09:25:39.510948+0000","last_clean":"2023-09-30T09:25:39.510948+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:25:39.510948+0000","last_undegraded":"2023-09-30T09:25:39.510948+0000","last_fullsized":"2023-09-30T09:25:39.510948+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.506287+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:25:39.506287+0000","last_peered":"2023-09-30T09:25:39.506287+0000","last_clean":"2023-09-30T09:25:39.506287+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:25:39.506287+0000","last_undegraded":"2023-09-30T09:25:39.506287+0000","last_fullsized":"2023-09-30T09:25:39.506287+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":131,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.515482+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:25:39.515482+0000","last_peered":"2023-09-30T09:25:39.515482+0000","last_clean":"2023-09-30T09:25:39.515482+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:25:39.515482+0000","last_undegraded":"2023-09-30T09:25:39.515482+0000","last_fullsized":"2023-09-30T09:25:39.515482+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":135,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514671+0000","last_change":"2023-09-30T09:23:03.986755+0000","last_active":"2023-09-30T09:25:39.514671+0000","last_peered":"2023-09-30T09:25:39.514671+0000","last_clean":"2023-09-30T09:25:39.514671+0000","last_became_active":"2023-09-30T09:23:03.986411+0000","last_became_peered":"2023-09-30T09:23:03.986411+0000","last_unstale":"2023-09-30T09:25:39.514671+0000","last_undegraded":"2023-09-30T09:25:39.514671+0000","last_fullsized":"2023-09-30T09:25:39.514671+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:53:23.627823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":129,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.510476+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:25:39.510476+0000","last_peered":"2023-09-30T09:25:39.510476+0000","last_clean":"2023-09-30T09:25:39.510476+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:25:39.510476+0000","last_undegraded":"2023-09-30T09:25:39.510476+0000","last_fullsized":"2023-09-30T09:25:39.510476+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":144,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.502281+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:25:39.502281+0000","last_peered":"2023-09-30T09:25:39.502281+0000","last_clean":"2023-09-30T09:25:39.502281+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:25:39.502281+0000","last_undegraded":"2023-09-30T09:25:39.502281+0000","last_fullsized":"2023-09-30T09:25:39.502281+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.507621+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:25:39.507621+0000","last_peered":"2023-09-30T09:25:39.507621+0000","last_clean":"2023-09-30T09:25:39.507621+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:25:39.507621+0000","last_undegraded":"2023-09-30T09:25:39.507621+0000","last_fullsized":"2023-09-30T09:25:39.507621+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":18,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.519389+0000","last_change":"2023-09-30T09:25:14.358999+0000","last_active":"2023-09-30T09:25:39.519389+0000","last_peered":"2023-09-30T09:25:39.519389+0000","last_clean":"2023-09-30T09:25:39.519389+0000","last_became_active":"2023-09-30T09:25:14.358650+0000","last_became_peered":"2023-09-30T09:25:14.358650+0000","last_unstale":"2023-09-30T09:25:39.519389+0000","last_undegraded":"2023-09-30T09:25:39.519389+0000","last_fullsized":"2023-09-30T09:25:39.519389+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:57:41.373342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":110,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.511523+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:25:39.511523+0000","last_peered":"2023-09-30T09:25:39.511523+0000","last_clean":"2023-09-30T09:25:39.511523+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:25:39.511523+0000","last_undegraded":"2023-09-30T09:25:39.511523+0000","last_fullsized":"2023-09-30T09:25:39.511523+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":58,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.507842+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:25:39.507842+0000","last_peered":"2023-09-30T09:25:39.507842+0000","last_clean":"2023-09-30T09:25:39.507842+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:25:39.507842+0000","last_undegraded":"2023-09-30T09:25:39.507842+0000","last_fullsized":"2023-09-30T09:25:39.507842+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":215,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.520051+0000","last_change":"2023-09-30T09:25:14.389693+0000","last_active":"2023-09-30T09:25:39.520051+0000","last_peered":"2023-09-30T09:25:39.520051+0000","last_clean":"2023-09-30T09:25:39.520051+0000","last_became_active":"2023-09-30T09:25:14.365482+0000","last_became_peered":"2023-09-30T09:25:14.365482+0000","last_unstale":"2023-09-30T09:25:39.520051+0000","last_undegraded":"2023-09-30T09:25:39.520051+0000","last_fullsized":"2023-09-30T09:25:39.520051+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:09:23.017579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":144,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.501916+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:25:39.501916+0000","last_peered":"2023-09-30T09:25:39.501916+0000","last_clean":"2023-09-30T09:25:39.501916+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:25:39.501916+0000","last_undegraded":"2023-09-30T09:25:39.501916+0000","last_fullsized":"2023-09-30T09:25:39.501916+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"167'6814","reported_seq":6876,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:53.922860+0000","last_change":"2023-09-30T09:22:48.114186+0000","last_active":"2023-09-30T09:25:53.922860+0000","last_peered":"2023-09-30T09:25:53.922860+0000","last_clean":"2023-09-30T09:25:53.922860+0000","last_became_active":"2023-09-30T09:22:48.113857+0000","last_became_peered":"2023-09-30T09:22:48.113857+0000","last_unstale":"2023-09-30T09:25:53.922860+0000","last_undegraded":"2023-09-30T09:25:53.922860+0000","last_fullsized":"2023-09-30T09:25:53.922860+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6814,"log_dups_size":0,"ondisk_log_size":6814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:18.973595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229244928,"num_objects":3498,"num_object_clones":0,"num_object_copies":10494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6814,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":156,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.510711+0000","last_change":"2023-09-30T09:23:08.038825+0000","last_active":"2023-09-30T09:25:39.510711+0000","last_peered":"2023-09-30T09:25:39.510711+0000","last_clean":"2023-09-30T09:25:39.510711+0000","last_became_active":"2023-09-30T09:23:08.029150+0000","last_became_peered":"2023-09-30T09:23:08.029150+0000","last_unstale":"2023-09-30T09:25:39.510711+0000","last_undegraded":"2023-09-30T09:25:39.510711+0000","last_fullsized":"2023-09-30T09:25:39.510711+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:14.557254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.517715+0000","last_change":"2023-09-30T09:25:14.359647+0000","last_active":"2023-09-30T09:25:39.517715+0000","last_peered":"2023-09-30T09:25:39.517715+0000","last_clean":"2023-09-30T09:25:39.517715+0000","last_became_active":"2023-09-30T09:25:14.359350+0000","last_became_peered":"2023-09-30T09:25:14.359350+0000","last_unstale":"2023-09-30T09:25:39.517715+0000","last_undegraded":"2023-09-30T09:25:39.517715+0000","last_fullsized":"2023-09-30T09:25:39.517715+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:09.489014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"167'6924","reported_seq":10084,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:54.682812+0000","last_change":"2023-09-30T09:24:25.563637+0000","last_active":"2023-09-30T09:25:54.682812+0000","last_peered":"2023-09-30T09:25:54.682812+0000","last_clean":"2023-09-30T09:25:54.682812+0000","last_became_active":"2023-09-30T09:24:25.554649+0000","last_became_peered":"2023-09-30T09:24:25.554649+0000","last_unstale":"2023-09-30T09:25:54.682812+0000","last_undegraded":"2023-09-30T09:25:54.682812+0000","last_fullsized":"2023-09-30T09:25:54.682812+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6924,"log_dups_size":0,"ondisk_log_size":6924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:09:56.974492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233439232,"num_objects":3562,"num_object_clones":0,"num_object_copies":10686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6924,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1538,"num_bytes_recovered":100794368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":157,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.511295+0000","last_change":"2023-09-30T09:25:04.304869+0000","last_active":"2023-09-30T09:25:39.511295+0000","last_peered":"2023-09-30T09:25:39.511295+0000","last_clean":"2023-09-30T09:25:39.511295+0000","last_became_active":"2023-09-30T09:25:04.285159+0000","last_became_peered":"2023-09-30T09:25:04.285159+0000","last_unstale":"2023-09-30T09:25:39.511295+0000","last_undegraded":"2023-09-30T09:25:39.511295+0000","last_fullsized":"2023-09-30T09:25:39.511295+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:03:13.971240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":142,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514588+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:25:39.514588+0000","last_peered":"2023-09-30T09:25:39.514588+0000","last_clean":"2023-09-30T09:25:39.514588+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:25:39.514588+0000","last_undegraded":"2023-09-30T09:25:39.514588+0000","last_fullsized":"2023-09-30T09:25:39.514588+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"167'6823","reported_seq":10290,"reported_epoch":167,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:25:54.760448+0000","last_change":"2023-09-30T09:25:14.561014+0000","last_active":"2023-09-30T09:25:54.760448+0000","last_peered":"2023-09-30T09:25:54.760448+0000","last_clean":"2023-09-30T09:23:07.000618+0000","last_became_active":"2023-09-30T09:25:14.392643+0000","last_became_peered":"2023-09-30T09:25:14.392643+0000","last_unstale":"2023-09-30T09:25:54.760448+0000","last_undegraded":"2023-09-30T09:25:54.760448+0000","last_fullsized":"2023-09-30T09:25:14.356532+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6823,"log_dups_size":0,"ondisk_log_size":6823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:05:55.317362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230096896,"num_objects":3511,"num_object_clones":0,"num_object_copies":10533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6823,"num_write_kb":330688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1691,"num_bytes_recovered":108331008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":3511}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":139,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.511105+0000","last_change":"2023-09-30T09:25:14.385912+0000","last_active":"2023-09-30T09:25:39.511105+0000","last_peered":"2023-09-30T09:25:39.511105+0000","last_clean":"2023-09-30T09:25:39.511105+0000","last_became_active":"2023-09-30T09:25:14.359789+0000","last_became_peered":"2023-09-30T09:25:14.359789+0000","last_unstale":"2023-09-30T09:25:39.511105+0000","last_undegraded":"2023-09-30T09:25:39.511105+0000","last_fullsized":"2023-09-30T09:25:39.511105+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:54.213336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":136,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.520081+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:25:39.520081+0000","last_peered":"2023-09-30T09:25:39.520081+0000","last_clean":"2023-09-30T09:25:39.520081+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:25:39.520081+0000","last_undegraded":"2023-09-30T09:25:39.520081+0000","last_fullsized":"2023-09-30T09:25:39.520081+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"167'6746","reported_seq":7690,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:54.749696+0000","last_change":"2023-09-30T09:25:16.371393+0000","last_active":"2023-09-30T09:25:54.749696+0000","last_peered":"2023-09-30T09:25:54.749696+0000","last_clean":"2023-09-30T09:25:54.749696+0000","last_became_active":"2023-09-30T09:25:16.364820+0000","last_became_peered":"2023-09-30T09:25:16.364820+0000","last_unstale":"2023-09-30T09:25:54.749696+0000","last_undegraded":"2023-09-30T09:25:54.749696+0000","last_fullsized":"2023-09-30T09:25:54.749696+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6746,"log_dups_size":0,"ondisk_log_size":6746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:19:41.535739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224919552,"num_objects":3432,"num_object_clones":0,"num_object_copies":10296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6746,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":429,"num_bytes_recovered":9240576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":126,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.519316+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:25:39.519316+0000","last_peered":"2023-09-30T09:25:39.519316+0000","last_clean":"2023-09-30T09:25:39.519316+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:25:39.519316+0000","last_undegraded":"2023-09-30T09:25:39.519316+0000","last_fullsized":"2023-09-30T09:25:39.519316+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":55,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.508663+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:25:39.508663+0000","last_peered":"2023-09-30T09:25:39.508663+0000","last_clean":"2023-09-30T09:25:39.508663+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:25:39.508663+0000","last_undegraded":"2023-09-30T09:25:39.508663+0000","last_fullsized":"2023-09-30T09:25:39.508663+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"167'6972","reported_seq":8224,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:25:54.755549+0000","last_change":"2023-09-30T09:25:14.421586+0000","last_active":"2023-09-30T09:25:54.755549+0000","last_peered":"2023-09-30T09:25:54.755549+0000","last_clean":"2023-09-30T09:24:01.611293+0000","last_became_active":"2023-09-30T09:25:14.386329+0000","last_became_peered":"2023-09-30T09:25:14.386329+0000","last_unstale":"2023-09-30T09:25:54.755549+0000","last_undegraded":"2023-09-30T09:25:14.364354+0000","last_fullsized":"2023-09-30T09:25:14.355372+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6972,"log_dups_size":0,"ondisk_log_size":6972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:11:34.305355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229965848,"num_objects":3510,"num_object_clones":0,"num_object_copies":10530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3510,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6972,"num_write_kb":335361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":586,"num_bytes_recovered":38404096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":127},{"shards":"2,7","objects":3383}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":152,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.502259+0000","last_change":"2023-09-30T09:23:08.029556+0000","last_active":"2023-09-30T09:25:39.502259+0000","last_peered":"2023-09-30T09:25:39.502259+0000","last_clean":"2023-09-30T09:25:39.502259+0000","last_became_active":"2023-09-30T09:23:08.014483+0000","last_became_peered":"2023-09-30T09:23:08.014483+0000","last_unstale":"2023-09-30T09:25:39.502259+0000","last_undegraded":"2023-09-30T09:25:39.502259+0000","last_fullsized":"2023-09-30T09:25:39.502259+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:07:01.455742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":125,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.515353+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:25:39.515353+0000","last_peered":"2023-09-30T09:25:39.515353+0000","last_clean":"2023-09-30T09:25:39.515353+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:25:39.515353+0000","last_undegraded":"2023-09-30T09:25:39.515353+0000","last_fullsized":"2023-09-30T09:25:39.515353+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"167'6502","reported_seq":9494,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:51.639926+0000","last_change":"2023-09-30T09:24:20.547592+0000","last_active":"2023-09-30T09:25:51.639926+0000","last_peered":"2023-09-30T09:25:51.639926+0000","last_clean":"2023-09-30T09:25:51.639926+0000","last_became_active":"2023-09-30T09:24:20.539203+0000","last_became_peered":"2023-09-30T09:24:20.539203+0000","last_unstale":"2023-09-30T09:25:51.639926+0000","last_undegraded":"2023-09-30T09:25:51.639926+0000","last_fullsized":"2023-09-30T09:25:51.639926+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6502,"log_dups_size":0,"ondisk_log_size":6502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:33.219264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235143168,"num_objects":3588,"num_object_clones":0,"num_object_copies":10764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":322880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1455,"num_bytes_recovered":95354880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.520175+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:25:39.520175+0000","last_peered":"2023-09-30T09:25:39.520175+0000","last_clean":"2023-09-30T09:25:39.520175+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:25:39.520175+0000","last_undegraded":"2023-09-30T09:25:39.520175+0000","last_fullsized":"2023-09-30T09:25:39.520175+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":21,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514929+0000","last_change":"2023-09-30T09:25:04.290019+0000","last_active":"2023-09-30T09:25:39.514929+0000","last_peered":"2023-09-30T09:25:39.514929+0000","last_clean":"2023-09-30T09:25:39.514929+0000","last_became_active":"2023-09-30T09:25:04.289710+0000","last_became_peered":"2023-09-30T09:25:04.289710+0000","last_unstale":"2023-09-30T09:25:39.514929+0000","last_undegraded":"2023-09-30T09:25:39.514929+0000","last_fullsized":"2023-09-30T09:25:39.514929+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:22:01.874727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"167'6886","reported_seq":6948,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:53.870594+0000","last_change":"2023-09-30T09:22:48.115158+0000","last_active":"2023-09-30T09:25:53.870594+0000","last_peered":"2023-09-30T09:25:53.870594+0000","last_clean":"2023-09-30T09:25:53.870594+0000","last_became_active":"2023-09-30T09:22:48.114453+0000","last_became_peered":"2023-09-30T09:22:48.114453+0000","last_unstale":"2023-09-30T09:25:53.870594+0000","last_undegraded":"2023-09-30T09:25:53.870594+0000","last_fullsized":"2023-09-30T09:25:53.870594+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6886,"log_dups_size":0,"ondisk_log_size":6886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:31:35.934276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236322816,"num_objects":3606,"num_object_clones":0,"num_object_copies":10818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6886,"num_write_kb":335744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":126,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514447+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:25:39.514447+0000","last_peered":"2023-09-30T09:25:39.514447+0000","last_clean":"2023-09-30T09:25:39.514447+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:25:39.514447+0000","last_undegraded":"2023-09-30T09:25:39.514447+0000","last_fullsized":"2023-09-30T09:25:39.514447+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":125,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.515662+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:25:39.515662+0000","last_peered":"2023-09-30T09:25:39.515662+0000","last_clean":"2023-09-30T09:25:39.515662+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:25:39.515662+0000","last_undegraded":"2023-09-30T09:25:39.515662+0000","last_fullsized":"2023-09-30T09:25:39.515662+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"167'6729","reported_seq":9771,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:25:54.050691+0000","last_change":"2023-09-30T09:25:14.555041+0000","last_active":"2023-09-30T09:25:54.050691+0000","last_peered":"2023-09-30T09:25:54.050691+0000","last_clean":"2023-09-30T09:24:01.532724+0000","last_became_active":"2023-09-30T09:25:04.306365+0000","last_became_peered":"2023-09-30T09:25:04.306365+0000","last_unstale":"2023-09-30T09:25:54.050691+0000","last_undegraded":"2023-09-30T09:25:14.554738+0000","last_fullsized":"2023-09-30T09:25:04.272594+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:11.652193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233766912,"num_objects":3567,"num_object_clones":0,"num_object_copies":10701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6729,"num_write_kb":329472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1471,"num_bytes_recovered":96403456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,2,5","objects":145},{"shards":"0,5","objects":3422}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":132,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:47.137012+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:25:47.137012+0000","last_peered":"2023-09-30T09:25:47.137012+0000","last_clean":"2023-09-30T09:25:47.137012+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:25:47.137012+0000","last_undegraded":"2023-09-30T09:25:47.137012+0000","last_fullsized":"2023-09-30T09:25:47.137012+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":105,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.506950+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:25:39.506950+0000","last_peered":"2023-09-30T09:25:39.506950+0000","last_clean":"2023-09-30T09:25:39.506950+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:25:39.506950+0000","last_undegraded":"2023-09-30T09:25:39.506950+0000","last_fullsized":"2023-09-30T09:25:39.506950+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":509,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.507531+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:25:39.507531+0000","last_peered":"2023-09-30T09:25:39.507531+0000","last_clean":"2023-09-30T09:25:39.507531+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:25:39.507531+0000","last_undegraded":"2023-09-30T09:25:39.507531+0000","last_fullsized":"2023-09-30T09:25:39.507531+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.519522+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:25:39.519522+0000","last_peered":"2023-09-30T09:25:39.519522+0000","last_clean":"2023-09-30T09:25:39.519522+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:25:39.519522+0000","last_undegraded":"2023-09-30T09:25:39.519522+0000","last_fullsized":"2023-09-30T09:25:39.519522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":116,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.506451+0000","last_change":"2023-09-30T09:25:14.364474+0000","last_active":"2023-09-30T09:25:39.506451+0000","last_peered":"2023-09-30T09:25:39.506451+0000","last_clean":"2023-09-30T09:25:39.506451+0000","last_became_active":"2023-09-30T09:25:14.364128+0000","last_became_peered":"2023-09-30T09:25:14.364128+0000","last_unstale":"2023-09-30T09:25:39.506451+0000","last_undegraded":"2023-09-30T09:25:39.506451+0000","last_fullsized":"2023-09-30T09:25:39.506451+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:41:17.197492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"167'6776","reported_seq":9834,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:25:54.060558+0000","last_change":"2023-09-30T09:25:17.149514+0000","last_active":"2023-09-30T09:25:54.060558+0000","last_peered":"2023-09-30T09:25:54.060558+0000","last_clean":"2023-09-30T09:23:06.986974+0000","last_became_active":"2023-09-30T09:25:16.112832+0000","last_became_peered":"2023-09-30T09:25:16.112832+0000","last_unstale":"2023-09-30T09:25:54.060558+0000","last_undegraded":"2023-09-30T09:25:16.092762+0000","last_fullsized":"2023-09-30T09:25:15.359533+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6776,"log_dups_size":0,"ondisk_log_size":6776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:19.810956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233701376,"num_objects":3566,"num_object_clones":0,"num_object_copies":10698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":489,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6776,"num_write_kb":330944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97124352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,5],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,3,6","objects":130},{"shards":"0,5","objects":3436}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"167'6794","reported_seq":13836,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:54.027663+0000","last_change":"2023-09-30T09:25:17.387236+0000","last_active":"2023-09-30T09:25:54.027663+0000","last_peered":"2023-09-30T09:25:54.027663+0000","last_clean":"2023-09-30T09:25:54.027663+0000","last_became_active":"2023-09-30T09:25:17.376539+0000","last_became_peered":"2023-09-30T09:25:17.376539+0000","last_unstale":"2023-09-30T09:25:54.027663+0000","last_undegraded":"2023-09-30T09:25:54.027663+0000","last_fullsized":"2023-09-30T09:25:54.027663+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6794,"log_dups_size":0,"ondisk_log_size":6794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:15:31.918233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233308160,"num_objects":3560,"num_object_clones":0,"num_object_copies":10680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6794,"num_write_kb":331328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3466,"num_bytes_recovered":207814656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":313,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.506537+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:25:39.506537+0000","last_peered":"2023-09-30T09:25:39.506537+0000","last_clean":"2023-09-30T09:25:39.506537+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:25:39.506537+0000","last_undegraded":"2023-09-30T09:25:39.506537+0000","last_fullsized":"2023-09-30T09:25:39.506537+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":146,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514738+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:25:39.514738+0000","last_peered":"2023-09-30T09:25:39.514738+0000","last_clean":"2023-09-30T09:25:39.514738+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:25:39.514738+0000","last_undegraded":"2023-09-30T09:25:39.514738+0000","last_fullsized":"2023-09-30T09:25:39.514738+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":127,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.517216+0000","last_change":"2023-09-30T09:23:23.238851+0000","last_active":"2023-09-30T09:25:39.517216+0000","last_peered":"2023-09-30T09:25:39.517216+0000","last_clean":"2023-09-30T09:25:39.517216+0000","last_became_active":"2023-09-30T09:23:23.238582+0000","last_became_peered":"2023-09-30T09:23:23.238582+0000","last_unstale":"2023-09-30T09:25:39.517216+0000","last_undegraded":"2023-09-30T09:25:39.517216+0000","last_fullsized":"2023-09-30T09:25:39.517216+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:52.808966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"167'6701","reported_seq":6767,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:54.755293+0000","last_change":"2023-09-30T09:22:48.108513+0000","last_active":"2023-09-30T09:25:54.755293+0000","last_peered":"2023-09-30T09:25:54.755293+0000","last_clean":"2023-09-30T09:25:54.755293+0000","last_became_active":"2023-09-30T09:22:48.108231+0000","last_became_peered":"2023-09-30T09:22:48.108231+0000","last_unstale":"2023-09-30T09:25:54.755293+0000","last_undegraded":"2023-09-30T09:25:54.755293+0000","last_fullsized":"2023-09-30T09:25:54.755293+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6701,"log_dups_size":0,"ondisk_log_size":6701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:09.723671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226951168,"num_objects":3463,"num_object_clones":0,"num_object_copies":10389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6701,"num_write_kb":325248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":544,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.515281+0000","last_change":"2023-09-30T09:23:11.172147+0000","last_active":"2023-09-30T09:25:39.515281+0000","last_peered":"2023-09-30T09:25:39.515281+0000","last_clean":"2023-09-30T09:25:39.515281+0000","last_became_active":"2023-09-30T09:23:11.171540+0000","last_became_peered":"2023-09-30T09:23:11.171540+0000","last_unstale":"2023-09-30T09:25:39.515281+0000","last_undegraded":"2023-09-30T09:25:39.515281+0000","last_fullsized":"2023-09-30T09:25:39.515281+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:24.074307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":104,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.506374+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:25:39.506374+0000","last_peered":"2023-09-30T09:25:39.506374+0000","last_clean":"2023-09-30T09:25:39.506374+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:25:39.506374+0000","last_undegraded":"2023-09-30T09:25:39.506374+0000","last_fullsized":"2023-09-30T09:25:39.506374+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":152,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.506878+0000","last_change":"2023-09-30T09:23:08.027904+0000","last_active":"2023-09-30T09:25:39.506878+0000","last_peered":"2023-09-30T09:25:39.506878+0000","last_clean":"2023-09-30T09:25:39.506878+0000","last_became_active":"2023-09-30T09:23:08.027557+0000","last_became_peered":"2023-09-30T09:23:08.027557+0000","last_unstale":"2023-09-30T09:25:39.506878+0000","last_undegraded":"2023-09-30T09:25:39.506878+0000","last_fullsized":"2023-09-30T09:25:39.506878+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:19:26.977141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":574,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.518185+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:25:39.518185+0000","last_peered":"2023-09-30T09:25:39.518185+0000","last_clean":"2023-09-30T09:25:39.518185+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:25:39.518185+0000","last_undegraded":"2023-09-30T09:25:39.518185+0000","last_fullsized":"2023-09-30T09:25:39.518185+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":560,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.510106+0000","last_change":"2023-09-30T09:23:27.263806+0000","last_active":"2023-09-30T09:25:39.510106+0000","last_peered":"2023-09-30T09:25:39.510106+0000","last_clean":"2023-09-30T09:25:39.510106+0000","last_became_active":"2023-09-30T09:23:27.263242+0000","last_became_peered":"2023-09-30T09:23:27.263242+0000","last_unstale":"2023-09-30T09:25:39.510106+0000","last_undegraded":"2023-09-30T09:25:39.510106+0000","last_fullsized":"2023-09-30T09:25:39.510106+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:38:00.839832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.502441+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:25:39.502441+0000","last_peered":"2023-09-30T09:25:39.502441+0000","last_clean":"2023-09-30T09:25:39.502441+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:25:39.502441+0000","last_undegraded":"2023-09-30T09:25:39.502441+0000","last_fullsized":"2023-09-30T09:25:39.502441+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":109,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.521825+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:25:39.521825+0000","last_peered":"2023-09-30T09:25:39.521825+0000","last_clean":"2023-09-30T09:25:39.521825+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:25:39.521825+0000","last_undegraded":"2023-09-30T09:25:39.521825+0000","last_fullsized":"2023-09-30T09:25:39.521825+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"167'6765","reported_seq":6827,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:53.918163+0000","last_change":"2023-09-30T09:22:48.113213+0000","last_active":"2023-09-30T09:25:53.918163+0000","last_peered":"2023-09-30T09:25:53.918163+0000","last_clean":"2023-09-30T09:25:53.918163+0000","last_became_active":"2023-09-30T09:22:48.112904+0000","last_became_peered":"2023-09-30T09:22:48.112904+0000","last_unstale":"2023-09-30T09:25:53.918163+0000","last_undegraded":"2023-09-30T09:25:53.918163+0000","last_fullsized":"2023-09-30T09:25:53.918163+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:37:37.122003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230490112,"num_objects":3517,"num_object_clones":0,"num_object_copies":10551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6765,"num_write_kb":329024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"167'6717","reported_seq":17453,"reported_epoch":167,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:25:54.052146+0000","last_change":"2023-09-30T09:25:15.494362+0000","last_active":"2023-09-30T09:25:54.052146+0000","last_peered":"2023-09-30T09:25:54.052146+0000","last_clean":"2023-09-30T09:25:13.127337+0000","last_became_active":"2023-09-30T09:25:15.484895+0000","last_became_peered":"2023-09-30T09:25:15.484895+0000","last_unstale":"2023-09-30T09:25:54.052146+0000","last_undegraded":"2023-09-30T09:25:54.052146+0000","last_fullsized":"2023-09-30T09:25:54.052146+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6717,"log_dups_size":0,"ondisk_log_size":6717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:16.127121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230883328,"num_objects":3523,"num_object_clones":0,"num_object_copies":10569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3523,"num_objects_unfound":0,"num_objects_dirty":3523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6717,"num_write_kb":327680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5313,"num_bytes_recovered":348192768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3523}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":727,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.510185+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:25:39.510185+0000","last_peered":"2023-09-30T09:25:39.510185+0000","last_clean":"2023-09-30T09:25:39.510185+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:25:39.510185+0000","last_undegraded":"2023-09-30T09:25:39.510185+0000","last_fullsized":"2023-09-30T09:25:39.510185+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":74,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.504297+0000","last_change":"2023-09-30T09:25:04.283467+0000","last_active":"2023-09-30T09:25:39.504297+0000","last_peered":"2023-09-30T09:25:39.504297+0000","last_clean":"2023-09-30T09:25:39.504297+0000","last_became_active":"2023-09-30T09:25:04.283162+0000","last_became_peered":"2023-09-30T09:25:04.283162+0000","last_unstale":"2023-09-30T09:25:39.504297+0000","last_undegraded":"2023-09-30T09:25:39.504297+0000","last_fullsized":"2023-09-30T09:25:39.504297+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:12:35.333268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":58,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.507557+0000","last_change":"2023-09-30T09:23:08.024242+0000","last_active":"2023-09-30T09:25:39.507557+0000","last_peered":"2023-09-30T09:25:39.507557+0000","last_clean":"2023-09-30T09:25:39.507557+0000","last_became_active":"2023-09-30T09:23:08.023891+0000","last_became_peered":"2023-09-30T09:23:08.023891+0000","last_unstale":"2023-09-30T09:25:39.507557+0000","last_undegraded":"2023-09-30T09:25:39.507557+0000","last_fullsized":"2023-09-30T09:25:39.507557+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:19:36.687182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":807,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.511021+0000","last_change":"2023-09-30T09:23:11.125442+0000","last_active":"2023-09-30T09:25:39.511021+0000","last_peered":"2023-09-30T09:25:39.511021+0000","last_clean":"2023-09-30T09:25:39.511021+0000","last_became_active":"2023-09-30T09:23:11.124962+0000","last_became_peered":"2023-09-30T09:23:11.124962+0000","last_unstale":"2023-09-30T09:25:39.511021+0000","last_undegraded":"2023-09-30T09:25:39.511021+0000","last_fullsized":"2023-09-30T09:25:39.511021+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:53.386815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":120,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.507445+0000","last_change":"2023-09-30T09:25:14.359586+0000","last_active":"2023-09-30T09:25:39.507445+0000","last_peered":"2023-09-30T09:25:39.507445+0000","last_clean":"2023-09-30T09:25:39.507445+0000","last_became_active":"2023-09-30T09:25:14.359198+0000","last_became_peered":"2023-09-30T09:25:14.359198+0000","last_unstale":"2023-09-30T09:25:39.507445+0000","last_undegraded":"2023-09-30T09:25:39.507445+0000","last_fullsized":"2023-09-30T09:25:39.507445+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:48.533039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.509176+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:25:39.509176+0000","last_peered":"2023-09-30T09:25:39.509176+0000","last_clean":"2023-09-30T09:25:39.509176+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:25:39.509176+0000","last_undegraded":"2023-09-30T09:25:39.509176+0000","last_fullsized":"2023-09-30T09:25:39.509176+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"167'7000","reported_seq":10190,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:53.859773+0000","last_change":"2023-09-30T09:24:12.672495+0000","last_active":"2023-09-30T09:25:53.859773+0000","last_peered":"2023-09-30T09:25:53.859773+0000","last_clean":"2023-09-30T09:25:53.859773+0000","last_became_active":"2023-09-30T09:24:12.668774+0000","last_became_peered":"2023-09-30T09:24:12.668774+0000","last_unstale":"2023-09-30T09:25:53.859773+0000","last_undegraded":"2023-09-30T09:25:53.859773+0000","last_fullsized":"2023-09-30T09:25:53.859773+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:01:27.375538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241696768,"num_objects":3688,"num_object_clones":0,"num_object_copies":11064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7000,"num_write_kb":342016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1553,"num_bytes_recovered":101777408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":153,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:47.137137+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:25:47.137137+0000","last_peered":"2023-09-30T09:25:47.137137+0000","last_clean":"2023-09-30T09:25:47.137137+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:25:47.137137+0000","last_undegraded":"2023-09-30T09:25:47.137137+0000","last_fullsized":"2023-09-30T09:25:47.137137+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1114,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.506812+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:25:39.506812+0000","last_peered":"2023-09-30T09:25:39.506812+0000","last_clean":"2023-09-30T09:25:39.506812+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:25:39.506812+0000","last_undegraded":"2023-09-30T09:25:39.506812+0000","last_fullsized":"2023-09-30T09:25:39.506812+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":142,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.501467+0000","last_change":"2023-09-30T09:23:08.028741+0000","last_active":"2023-09-30T09:25:39.501467+0000","last_peered":"2023-09-30T09:25:39.501467+0000","last_clean":"2023-09-30T09:25:39.501467+0000","last_became_active":"2023-09-30T09:23:08.028167+0000","last_became_peered":"2023-09-30T09:23:08.028167+0000","last_unstale":"2023-09-30T09:25:39.501467+0000","last_undegraded":"2023-09-30T09:25:39.501467+0000","last_fullsized":"2023-09-30T09:25:39.501467+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:47:39.241913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"167'6915","reported_seq":6977,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:53.910359+0000","last_change":"2023-09-30T09:22:48.113865+0000","last_active":"2023-09-30T09:25:53.910359+0000","last_peered":"2023-09-30T09:25:53.910359+0000","last_clean":"2023-09-30T09:25:53.910359+0000","last_became_active":"2023-09-30T09:22:48.113578+0000","last_became_peered":"2023-09-30T09:22:48.113578+0000","last_unstale":"2023-09-30T09:25:53.910359+0000","last_undegraded":"2023-09-30T09:25:53.910359+0000","last_fullsized":"2023-09-30T09:25:53.910359+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6915,"log_dups_size":0,"ondisk_log_size":6915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:17.848647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235339776,"num_objects":3591,"num_object_clones":0,"num_object_copies":10773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6915,"num_write_kb":336192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":420,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.521765+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:25:39.521765+0000","last_peered":"2023-09-30T09:25:39.521765+0000","last_clean":"2023-09-30T09:25:39.521765+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:25:39.521765+0000","last_undegraded":"2023-09-30T09:25:39.521765+0000","last_fullsized":"2023-09-30T09:25:39.521765+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":62,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.507933+0000","last_change":"2023-09-30T09:23:08.031587+0000","last_active":"2023-09-30T09:25:39.507933+0000","last_peered":"2023-09-30T09:25:39.507933+0000","last_clean":"2023-09-30T09:25:39.507933+0000","last_became_active":"2023-09-30T09:23:08.016566+0000","last_became_peered":"2023-09-30T09:23:08.016566+0000","last_unstale":"2023-09-30T09:25:39.507933+0000","last_undegraded":"2023-09-30T09:25:39.507933+0000","last_fullsized":"2023-09-30T09:25:39.507933+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:44:10.092575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":108,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514517+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:25:39.514517+0000","last_peered":"2023-09-30T09:25:39.514517+0000","last_clean":"2023-09-30T09:25:39.514517+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:25:39.514517+0000","last_undegraded":"2023-09-30T09:25:39.514517+0000","last_fullsized":"2023-09-30T09:25:39.514517+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"167'6631","reported_seq":6694,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:52.082508+0000","last_change":"2023-09-30T09:22:48.113933+0000","last_active":"2023-09-30T09:25:52.082508+0000","last_peered":"2023-09-30T09:25:52.082508+0000","last_clean":"2023-09-30T09:25:52.082508+0000","last_became_active":"2023-09-30T09:22:48.113640+0000","last_became_peered":"2023-09-30T09:22:48.113640+0000","last_unstale":"2023-09-30T09:25:52.082508+0000","last_undegraded":"2023-09-30T09:25:52.082508+0000","last_fullsized":"2023-09-30T09:25:52.082508+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":6631,"log_dups_size":0,"ondisk_log_size":6631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:16.995917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240844800,"num_objects":3675,"num_object_clones":0,"num_object_copies":11025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6631,"num_write_kb":329792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":54,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.512715+0000","last_change":"2023-09-30T09:23:08.024830+0000","last_active":"2023-09-30T09:25:39.512715+0000","last_peered":"2023-09-30T09:25:39.512715+0000","last_clean":"2023-09-30T09:25:39.512715+0000","last_became_active":"2023-09-30T09:23:08.024489+0000","last_became_peered":"2023-09-30T09:23:08.024489+0000","last_unstale":"2023-09-30T09:25:39.512715+0000","last_undegraded":"2023-09-30T09:25:39.512715+0000","last_fullsized":"2023-09-30T09:25:39.512715+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:41.769725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":149,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-30T09:25:51.579483+0000","last_change":"2023-09-30T09:21:29.096256+0000","last_active":"2023-09-30T09:25:51.579483+0000","last_peered":"2023-09-30T09:25:51.579483+0000","last_clean":"2023-09-30T09:25:51.579483+0000","last_became_active":"2023-09-30T09:21:29.095930+0000","last_became_peered":"2023-09-30T09:21:29.095930+0000","last_unstale":"2023-09-30T09:25:51.579483+0000","last_undegraded":"2023-09-30T09:25:51.579483+0000","last_fullsized":"2023-09-30T09:25:51.579483+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:25.270756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":128,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.519398+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:25:39.519398+0000","last_peered":"2023-09-30T09:25:39.519398+0000","last_clean":"2023-09-30T09:25:39.519398+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:25:39.519398+0000","last_undegraded":"2023-09-30T09:25:39.519398+0000","last_fullsized":"2023-09-30T09:25:39.519398+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514494+0000","last_change":"2023-09-30T09:25:04.284772+0000","last_active":"2023-09-30T09:25:39.514494+0000","last_peered":"2023-09-30T09:25:39.514494+0000","last_clean":"2023-09-30T09:25:39.514494+0000","last_became_active":"2023-09-30T09:25:04.284405+0000","last_became_peered":"2023-09-30T09:25:04.284405+0000","last_unstale":"2023-09-30T09:25:39.514494+0000","last_undegraded":"2023-09-30T09:25:39.514494+0000","last_fullsized":"2023-09-30T09:25:39.514494+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:44.112365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":111,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.519544+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:25:39.519544+0000","last_peered":"2023-09-30T09:25:39.519544+0000","last_clean":"2023-09-30T09:25:39.519544+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:25:39.519544+0000","last_undegraded":"2023-09-30T09:25:39.519544+0000","last_fullsized":"2023-09-30T09:25:39.519544+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514522+0000","last_change":"2023-09-30T09:25:04.306048+0000","last_active":"2023-09-30T09:25:39.514522+0000","last_peered":"2023-09-30T09:25:39.514522+0000","last_clean":"2023-09-30T09:25:39.514522+0000","last_became_active":"2023-09-30T09:25:04.305804+0000","last_became_peered":"2023-09-30T09:25:04.305804+0000","last_unstale":"2023-09-30T09:25:39.514522+0000","last_undegraded":"2023-09-30T09:25:39.514522+0000","last_fullsized":"2023-09-30T09:25:39.514522+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:32.116748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":137,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.501696+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:25:39.501696+0000","last_peered":"2023-09-30T09:25:39.501696+0000","last_clean":"2023-09-30T09:25:39.501696+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:25:39.501696+0000","last_undegraded":"2023-09-30T09:25:39.501696+0000","last_fullsized":"2023-09-30T09:25:39.501696+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":167,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.502492+0000","last_change":"2023-09-30T09:23:08.021721+0000","last_active":"2023-09-30T09:25:39.502492+0000","last_peered":"2023-09-30T09:25:39.502492+0000","last_clean":"2023-09-30T09:25:39.502492+0000","last_became_active":"2023-09-30T09:23:08.021401+0000","last_became_peered":"2023-09-30T09:23:08.021401+0000","last_unstale":"2023-09-30T09:25:39.502492+0000","last_undegraded":"2023-09-30T09:25:39.502492+0000","last_fullsized":"2023-09-30T09:25:39.502492+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:33:33.611604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":64,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.508072+0000","last_change":"2023-09-30T09:23:08.039181+0000","last_active":"2023-09-30T09:25:39.508072+0000","last_peered":"2023-09-30T09:25:39.508072+0000","last_clean":"2023-09-30T09:25:39.508072+0000","last_became_active":"2023-09-30T09:23:08.025651+0000","last_became_peered":"2023-09-30T09:23:08.025651+0000","last_unstale":"2023-09-30T09:25:39.508072+0000","last_undegraded":"2023-09-30T09:25:39.508072+0000","last_fullsized":"2023-09-30T09:25:39.508072+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:27.721207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":125,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-30T09:25:39.514165+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:25:39.514165+0000","last_peered":"2023-09-30T09:25:39.514165+0000","last_clean":"2023-09-30T09:25:39.514165+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:25:39.514165+0000","last_undegraded":"2023-09-30T09:25:39.514165+0000","last_fullsized":"2023-09-30T09:25:39.514165+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":3726114840,"num_objects":56857,"num_object_clones":0,"num_object_copies":170571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1444,"num_objects_misplaced":3523,"num_objects_unfound":0,"num_objects_dirty":56857,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108695,"num_write_kb":5297601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18985,"num_bytes_recovered":1203437568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12069449728,"data_stored":12069437512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108695,"ondisk_log_size":108695,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":738,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234171,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1277176,"kb_used_data":1233648,"kb_used_omap":0,"kb_used_meta":43520,"kb_avail":92467976,"statfs":{"total":95995035648,"available":94687207424,"internally_reserved":0,"allocated":1263255552,"data_stored":1262532311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44564480},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:17 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.78700000000000003,"15min":0.68100000000000005},"min":{"1min":0.30099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":1.444,"5min":7.3559999999999999,"15min":7.3559999999999999},"last":4.093},{"interface":"front","average":{"1min":0.441,"5min":0.629,"15min":0.65000000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.59499999999999997,"5min":4.8289999999999997,"15min":4.8289999999999997},"last":3.698}]},{"osd":1,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":2.4289999999999998,"15min":1.175},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.212,"5min":82.025999999999996,"15min":82.025999999999996},"last":5.9480000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":2.3980000000000001,"15min":1.1799999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":82.012,"15min":82.012},"last":3.1949999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:25:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":0.92900000000000005,"15min":0.71299999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":5.9800000000000004,"5min":11.955,"15min":11.955},"last":3.762},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.054,"15min":0.75700000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.875,"5min":15.443,"15min":15.443},"last":3.972}]},{"osd":3,"last update":"Sat Sep 30 09:25:40 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.91000000000000003,"15min":0.71699999999999997},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0090000000000003,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":5.2709999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.753,"15min":0.67500000000000004},"min":{"1min":0.40200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.3540000000000001,"5min":4.8150000000000004,"15min":4.8150000000000004},"last":7.1479999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:24:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.54300000000000004,"15min":0.48699999999999999},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.4790000000000001,"5min":2.3599999999999999,"15min":2.3599999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.61799999999999999,"15min":0.53400000000000003},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.093,"5min":3,"15min":3},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:24:56 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.65200000000000002,"15min":0.55500000000000005},"min":{"1min":0.33700000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.1619999999999999,"5min":5.968,"15min":5.968},"last":1.1499999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.66200000000000003,"15min":0.58799999999999997},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.5640000000000001,"5min":5.452,"15min":5.452},"last":0.497}]},{"osd":6,"last update":"Sat Sep 30 09:24:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.68200000000000005,"15min":0.59899999999999998},"min":{"1min":0.41999999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.8999999999999999,"5min":2.8809999999999998,"15min":2.8809999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.67500000000000004,"15min":0.59199999999999997},"min":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":1.1200000000000001,"5min":2.919,"15min":2.919},"last":0.48199999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495808,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1093732,"kb_used_data":1043804,"kb_used_omap":0,"kb_used_meta":49792,"kb_avail":92651420,"statfs":{"total":95995035648,"available":94875054080,"internally_reserved":0,"allocated":1068855296,"data_stored":1068151796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":50987008},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.83999999999999997,"15min":0.70299999999999996},"min":{"1min":0.23599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.4249999999999998,"5min":6.649,"15min":6.649},"last":2.5299999999999998},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.84499999999999997,"15min":0.71899999999999997},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.327,"5min":8.327,"15min":8.327},"last":1.145}]},{"osd":1,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.83399999999999996,"15min":0.69999999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.2970000000000002,"5min":6.5090000000000003,"15min":6.5090000000000003},"last":2.3889999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.81299999999999994,"15min":0.69399999999999995},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2810000000000001,"5min":4.944,"15min":4.944},"last":2.403}]},{"osd":2,"last update":"Sat Sep 30 09:25:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":0.95199999999999996,"15min":0.69799999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.4730000000000008,"5min":8.4730000000000008,"15min":8.4730000000000008},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.95999999999999996,"15min":0.72399999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.9399999999999999,"5min":8.6159999999999997,"15min":8.6159999999999997},"last":2.3010000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.878,"15min":0.68999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.387,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":4.0970000000000004},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.0309999999999999,"15min":0.73599999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7759999999999998,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":1.196}]},{"osd":4,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":1.2210000000000001,"15min":0.73499999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":0.59299999999999997,"5min":73.656000000000006,"15min":73.656000000000006},"last":1.226},{"interface":"front","average":{"1min":0.40799999999999997,"5min":1.2829999999999999,"15min":0.77000000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58199999999999996,"5min":73.882999999999996,"15min":73.882999999999996},"last":0.32000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.64100000000000001,"15min":0.54200000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":0.48999999999999999,"5min":4.2430000000000003,"15min":4.2430000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.71399999999999997,"15min":0.60899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28000000000000003},"max":{"1min":0.60299999999999998,"5min":6.5999999999999996,"15min":6.5999999999999996},"last":0.377}]},{"osd":7,"last update":"Sat Sep 30 09:24:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.67600000000000005,"15min":0.60999999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.26500000000000001},"max":{"1min":0.98899999999999999,"5min":3.7959999999999998,"15min":3.7959999999999998},"last":1.1719999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.91500000000000004,"15min":0.70999999999999996},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.31900000000000001},"max":{"1min":8.2690000000000001,"5min":10.339,"15min":10.339},"last":2.1389999999999998}]}]},{"osd":1,"up_from":99,"seq":425201762372,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251980,"kb_used_data":1184132,"kb_used_omap":5,"kb_used_meta":67706,"kb_avail":92493172,"statfs":{"total":95995035648,"available":94713008128,"internally_reserved":0,"allocated":1212551168,"data_stored":1211632474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":69331126},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":4.2450000000000001,"5min":2.0859999999999999,"15min":0.94199999999999995},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":62.935000000000002,"5min":72.007000000000005,"15min":72.007000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":3.907,"5min":2.1779999999999999,"15min":0.997},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":63.088000000000001,"5min":72.322999999999993,"15min":72.322999999999993},"last":0.63}]},{"osd":2,"last update":"Sat Sep 30 09:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.626,"15min":0.49099999999999999},"min":{"1min":0.254,"5min":0.249,"15min":0.249},"max":{"1min":2.9980000000000002,"5min":7.9790000000000001,"15min":7.9790000000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.73199999999999998,"15min":0.50900000000000001},"min":{"1min":0.27700000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.988,"5min":11.099,"15min":11.099},"last":1.3009999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:25:36 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.74199999999999999,"15min":0.52100000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.845000000000001,"5min":11.845000000000001,"15min":11.845000000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.72099999999999997,"15min":0.502},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.6499999999999999,"5min":11.06,"15min":11.06},"last":0.64700000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.80500000000000005,"15min":0.69799999999999995},"min":{"1min":0.317,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.569,"5min":5.6340000000000003,"15min":5.6340000000000003},"last":1.732},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.81299999999999994,"15min":0.72499999999999998},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.9470000000000001,"5min":4.3869999999999996,"15min":4.3869999999999996},"last":4.7670000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.73499999999999999,"15min":0.65900000000000003},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.619,"5min":5.0140000000000002,"15min":5.0140000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.83399999999999996,"15min":0.70999999999999996},"min":{"1min":0.439,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.9849999999999999,"5min":2.9929999999999999,"15min":2.9929999999999999},"last":4.8129999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.81399999999999995,"15min":0.68600000000000005},"min":{"1min":0.44900000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":2.851,"5min":3.0150000000000001,"15min":3.0150000000000001},"last":4.843},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.91100000000000003,"15min":0.76000000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.8140000000000001,"5min":7.8140000000000001,"15min":7.8140000000000001},"last":0.70799999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.96499999999999997,"15min":0.77500000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.9079999999999999,"5min":7.4820000000000002,"15min":7.4820000000000002},"last":4.8680000000000003},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.018,"15min":0.80200000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":7.7729999999999997,"5min":7.7729999999999997,"15min":7.7729999999999997},"last":0.88400000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925905,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2075144,"kb_used_data":1894976,"kb_used_omap":5,"kb_used_meta":180154,"kb_avail":91670008,"statfs":{"total":95995035648,"available":93870088192,"internally_reserved":0,"allocated":1940455424,"data_stored":1939490849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":184477949},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,3,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.81599999999999995,"15min":0.51200000000000001},"min":{"1min":0.22,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.5859999999999999,"5min":15.9,"15min":15.9},"last":5.8739999999999997},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.92500000000000004,"15min":0.56399999999999995},"min":{"1min":0.27800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.8849999999999998,"5min":13.824999999999999,"15min":13.824999999999999},"last":0.72599999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:25:01 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":0.98999999999999999,"15min":0.58199999999999996},"min":{"1min":0.28100000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":18.096,"5min":18.096,"15min":18.096},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.5669999999999999,"5min":0.99099999999999999,"15min":0.621},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":18.149000000000001,"5min":18.149000000000001,"15min":18.149000000000001},"last":1.8500000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":0.98199999999999998,"15min":0.65100000000000002},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.9630000000000001,"5min":13.02,"15min":13.02},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.96799999999999997,"15min":0.60999999999999999},"min":{"1min":0.29999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.847,"5min":10.128,"15min":10.128},"last":4.835}]},{"osd":4,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.86099999999999999,"15min":0.78500000000000003},"min":{"1min":0.38700000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.369,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.96999999999999997,"15min":0.84699999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":3.169,"5min":12.99,"15min":12.99},"last":0.56200000000000006}]},{"osd":5,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.1719999999999999,"15min":0.94399999999999995},"min":{"1min":0.38400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.266,"5min":12.949999999999999,"15min":12.949999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.151,"15min":0.94199999999999995},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.105,"5min":12.823,"15min":12.823},"last":5.8490000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.91000000000000003,"15min":0.84599999999999997},"min":{"1min":0.432,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.038,"5min":6.9660000000000002,"15min":6.9660000000000002},"last":5.8220000000000001},{"interface":"front","average":{"1min":0.745,"5min":0.85999999999999999,"15min":0.871},"min":{"1min":0.48799999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.5430000000000001,"5min":7.556,"15min":7.556},"last":0.439}]},{"osd":7,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.96099999999999997,"15min":0.83999999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.1699999999999999,"5min":7.0339999999999998,"15min":7.0339999999999998},"last":5.7850000000000001},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.85799999999999998,"15min":0.86599999999999999},"min":{"1min":0.47399999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.504,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.46400000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598846,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1660560,"kb_used_data":1544264,"kb_used_omap":5,"kb_used_meta":116282,"kb_avail":92084592,"statfs":{"total":95995035648,"available":94294622208,"internally_reserved":0,"allocated":1581326336,"data_stored":1580409629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":119073421},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2,7],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.88600000000000001,"15min":0.53900000000000003},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.2009999999999996,"5min":15.843999999999999,"15min":15.843999999999999},"last":3.0859999999999999},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.86099999999999999,"15min":0.58899999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.0259999999999998,"5min":10.065,"15min":10.065},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.66400000000000003,"15min":0.47799999999999998},"min":{"1min":0.23300000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":1.454,"5min":5.4640000000000004,"15min":5.4640000000000004},"last":1.748},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.73999999999999999,"15min":0.52800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.1059999999999999,"5min":6.5369999999999999,"15min":6.5369999999999999},"last":2.6880000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0009999999999999,"15min":0.63100000000000001},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.4009999999999998,"5min":18.635999999999999,"15min":18.635999999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.173,"15min":0.72599999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.996,"5min":18.026,"15min":18.026},"last":3.052}]},{"osd":4,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.81299999999999994,"15min":0.67500000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.1719999999999997,"5min":4.1719999999999997,"15min":4.1719999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.0700000000000001,"15min":0.78700000000000003},"min":{"1min":0.39500000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.3440000000000003,"5min":9.9550000000000001,"15min":9.9550000000000001},"last":3.1139999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.81499999999999995,"15min":0.70499999999999996},"min":{"1min":0.41099999999999998,"5min":0.371,"15min":0.371},"max":{"1min":1.484,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":1.7769999999999999},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.0049999999999999,"15min":0.77300000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":4.1429999999999998,"5min":10.359,"15min":10.359},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.238,"15min":0.871},"min":{"1min":0.42699999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":4.4560000000000004,"5min":10.037000000000001,"15min":10.037000000000001},"last":3.1699999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.90400000000000003,"15min":0.753},"min":{"1min":0.442,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.5189999999999999,"5min":5.4939999999999998,"15min":5.4939999999999998},"last":1.8109999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":1.1599999999999999,"15min":0.83999999999999997},"min":{"1min":0.44900000000000001,"5min":0.44,"15min":0.44},"max":{"1min":4.5010000000000003,"5min":10.003,"15min":10.003},"last":3.1419999999999999},{"interface":"front","average":{"1min":0.746,"5min":0.90300000000000002,"15min":0.77800000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.609,"5min":5.3979999999999997,"15min":5.3979999999999997},"last":1.8360000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435319,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":828628,"kb_used_data":783176,"kb_used_omap":2,"kb_used_meta":45437,"kb_avail":92916524,"statfs":{"total":95995035648,"available":95146520576,"internally_reserved":0,"allocated":801972224,"data_stored":801053531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":46527847},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":2.5920000000000001,"15min":1.121},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.5619999999999998,"5min":130.946,"15min":130.946},"last":1.262},{"interface":"front","average":{"1min":0.61499999999999999,"5min":2.5169999999999999,"15min":1.125},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.7470000000000001,"5min":130.96899999999999,"15min":130.96899999999999},"last":0.93700000000000006}]},{"osd":1,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.70799999999999996,"15min":0.52000000000000002},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":6.0099999999999998,"5min":6.0099999999999998,"15min":6.0099999999999998},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.64100000000000001,"15min":0.48699999999999999},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.159,"5min":3.9340000000000002,"15min":3.9340000000000002},"last":1.075}]},{"osd":2,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.831,"15min":1.2470000000000001},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1779999999999999,"5min":145.75899999999999,"15min":145.75899999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":2.7909999999999999,"15min":1.2170000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.1120000000000001,"5min":150.358,"15min":150.358},"last":0.314}]},{"osd":4,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.71299999999999997,"15min":0.64800000000000002},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.312},"max":{"1min":1.042,"5min":3.9119999999999999,"15min":3.9119999999999999},"last":0.374},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.74099999999999999,"15min":0.65100000000000002},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.27,"5min":4.0309999999999997,"15min":4.0309999999999997},"last":0.96499999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.85799999999999998,"15min":0.70699999999999996},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.2390000000000001,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":1.002},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.86899999999999999,"15min":0.75800000000000001},"min":{"1min":0.375,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9100000000000001,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.38500000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.75900000000000001,"15min":0.70299999999999996},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.147,"5min":3.9590000000000001,"15min":3.9590000000000001},"last":1.095},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.86499999999999999,"15min":0.746},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":3.0230000000000001,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":1.0389999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.81100000000000005,"15min":0.72199999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.948,"5min":4.1639999999999997,"15min":4.1639999999999997},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.74199999999999999,"15min":0.70299999999999996},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.372},"max":{"1min":1.1479999999999999,"5min":3.9910000000000001,"15min":3.9910000000000001},"last":1.0589999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019081,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2167332,"kb_used_data":2077724,"kb_used_omap":0,"kb_used_meta":89600,"kb_avail":91577820,"statfs":{"total":95995035648,"available":93775687680,"internally_reserved":0,"allocated":2127589376,"data_stored":2126885447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":91750400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":2.0950000000000002,"15min":1.1279999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.69799999999999995,"5min":128.79300000000001,"15min":128.79300000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":1.883,"15min":1.0589999999999999},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":0.53400000000000003,"5min":128.38800000000001,"15min":128.38800000000001},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":1.0569999999999999,"15min":0.73399999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.71899999999999997,"5min":31.469999999999999,"15min":31.469999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.748,"5min":1.149,"15min":0.75700000000000001},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.2729999999999997,"5min":31.962,"15min":31.962},"last":0.86499999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.234,"15min":0.81899999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.333,"5min":29.457000000000001,"15min":29.457000000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.256,"15min":0.81599999999999995},"min":{"1min":0.35199999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.2370000000000001,"5min":29.504999999999999,"15min":29.504999999999999},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.607,"15min":0.93300000000000005},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.0019999999999998,"5min":86.620000000000005,"15min":86.620000000000005},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.747,"5min":1.5940000000000001,"15min":0.94399999999999995},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9929999999999999,"5min":86.786000000000001,"15min":86.786000000000001},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:25:14 2023","interfaces":[{"interface":"back","average":{"1min":4.4969999999999999,"5min":1.3160000000000001,"15min":0.75700000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":86.691000000000003,"5min":86.691000000000003,"15min":86.691000000000003},"last":0.438},{"interface":"front","average":{"1min":4.5309999999999997,"5min":1.3879999999999999,"15min":0.80300000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":86.822000000000003,"5min":86.822000000000003,"15min":86.822000000000003},"last":0.77000000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:25:39 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.55900000000000005,"15min":0.52600000000000002},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71299999999999997,"5min":3.218,"15min":3.218},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.58599999999999997,"15min":0.56799999999999995},"min":{"1min":0.33900000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74199999999999999,"5min":2.0209999999999999,"15min":2.0209999999999999},"last":1.538}]},{"osd":7,"last update":"Sat Sep 30 09:25:01 2023","interfaces":[{"interface":"back","average":{"1min":4.7469999999999999,"5min":1.4970000000000001,"15min":0.876},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":86.728999999999999,"5min":86.728999999999999,"15min":86.728999999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.69399999999999995,"15min":0.61599999999999999},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":3.1800000000000002,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":0.55900000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757445,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2237532,"kb_used_data":2158804,"kb_used_omap":0,"kb_used_meta":78720,"kb_avail":91507620,"statfs":{"total":95995035648,"available":93703802880,"internally_reserved":0,"allocated":2210615296,"data_stored":2209915541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":80609280},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.048,"15min":0.73599999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":27.573,"5min":27.573,"15min":27.573},"last":2.0699999999999998},{"interface":"front","average":{"1min":2.2309999999999999,"5min":1.1830000000000001,"15min":0.80200000000000005},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":27.588999999999999,"5min":27.588999999999999,"15min":27.588999999999999},"last":0.95699999999999996}]},{"osd":1,"last update":"Sat Sep 30 09:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":1.9359999999999999,"15min":1.03},"min":{"1min":0.33900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.24,"5min":100.593,"15min":100.593},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.875,"15min":1.056},"min":{"1min":0.35099999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.5369999999999999,"5min":100.575,"15min":100.575},"last":1.2}]},{"osd":2,"last update":"Sat Sep 30 09:25:00 2023","interfaces":[{"interface":"back","average":{"1min":5.6529999999999996,"5min":2.2759999999999998,"15min":1.157},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":59.758000000000003,"5min":59.758000000000003,"15min":59.758000000000003},"last":7.0810000000000004},{"interface":"front","average":{"1min":5.8490000000000002,"5min":2.4689999999999999,"15min":1.2529999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":56.100000000000001,"5min":56.100000000000001,"15min":56.100000000000001},"last":1.913}]},{"osd":3,"last update":"Sat Sep 30 09:25:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.446,"15min":0.91000000000000003},"min":{"1min":0.39700000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":28.015999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":1.2649999999999999},{"interface":"front","average":{"1min":2.024,"5min":1.29,"15min":0.85999999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":28.039999999999999,"5min":28.039999999999999,"15min":28.039999999999999},"last":1.1699999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:25:13 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.56000000000000005,"15min":0.49199999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.72399999999999998,"5min":4.2199999999999998,"15min":4.2199999999999998},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.68100000000000005,"15min":0.56799999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25900000000000001},"max":{"1min":0.68300000000000005,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":1.8759999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.59099999999999997,"15min":0.54200000000000004},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.745,"5min":2.125,"15min":2.125},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.499,"5min":0.79100000000000004,"15min":0.66300000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":0.66600000000000004,"5min":4.46,"15min":4.46},"last":0.92000000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:25:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.75800000000000001,"15min":0.64200000000000002},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32500000000000001},"max":{"1min":1.1419999999999999,"5min":4.194,"15min":4.194},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.72199999999999998,"15min":0.623},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.605,"5min":6.2300000000000004,"15min":6.2300000000000004},"last":2.605}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1918701568,"data_stored":1918697496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1198063616,"data_stored":1198063616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1560547328,"data_stored":1560543256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":797966336,"data_stored":797966336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2101149696,"data_stored":2101149696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2193162240,"data_stored":2193162240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1057488896,"data_stored":1057488896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1242370048,"data_stored":1242365976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:25:59.570 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:25:59.628 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:25:59.673 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:59.774 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:25:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:59.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:25:59.880 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:25:59.909 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:25:59.952 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:25:59.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:25:59.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:00.059 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:00.084 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:00.128 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:00.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:00.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:00.232 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:00.292 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:00.419 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:00.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:00.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:00.436 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:00 smithi167 ceph-mon[192880]: pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 1444/165582 objects degraded (0.872%); 3523/165582 objects misplaced (2.128%); 4.7 MiB/s, 78 objects/s recovering 2023-09-30T09:26:00.436 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:00 smithi167 ceph-mon[192880]: osdmap e168: 8 total, 8 up, 8 in 2023-09-30T09:26:00.436 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:00 smithi167 ceph-mon[192880]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.437 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:00 smithi167 ceph-mon[192880]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.437 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:00 smithi167 ceph-mon[192880]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.437 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:00 smithi167 ceph-mon[192880]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.437 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:00 smithi167 ceph-mon[192880]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.437 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:00 smithi167 ceph-mon[192880]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[216815]: pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 1444/165582 objects degraded (0.872%); 3523/165582 objects misplaced (2.128%); 4.7 MiB/s, 78 objects/s recovering 2023-09-30T09:26:00.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[216815]: osdmap e168: 8 total, 8 up, 8 in 2023-09-30T09:26:00.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[216815]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[216815]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[216815]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[216815]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[216815]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[216815]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.508 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[224043]: pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 1444/165582 objects degraded (0.872%); 3523/165582 objects misplaced (2.128%); 4.7 MiB/s, 78 objects/s recovering 2023-09-30T09:26:00.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[224043]: osdmap e168: 8 total, 8 up, 8 in 2023-09-30T09:26:00.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[224043]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[224043]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[224043]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[224043]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[224043]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:00 smithi019 ceph-mon[224043]: failed to encode map e168 with expected crc 2023-09-30T09:26:00.533 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:00.566 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:00.617 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:26:00.652 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:00.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:00.763 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:00.842 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:26:00.843 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:26:00.886 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:00.914 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:01 smithi167 ceph-mon[192880]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:01 smithi167 ceph-mon[192880]: osdmap e169: 8 total, 8 up, 8 in 2023-09-30T09:26:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:01 smithi167 ceph-mon[192880]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:01 smithi167 ceph-mon[192880]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:01 smithi167 ceph-mon[192880]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[224043]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[224043]: osdmap e169: 8 total, 8 up, 8 in 2023-09-30T09:26:01.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[224043]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[224043]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[224043]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[216815]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[216815]: osdmap e169: 8 total, 8 up, 8 in 2023-09-30T09:26:01.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[216815]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[216815]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:01 smithi019 ceph-mon[216815]: failed to encode map e169 with expected crc 2023-09-30T09:26:01.825 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:01.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:01.932 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:01.969 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:26:02.013 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:02.341 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:02.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:02.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:02.465 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:02 smithi019 ceph-mon[224043]: pgmap v381: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 955/159255 objects degraded (0.600%); 3361/159255 objects misplaced (2.110%); 3.2 MiB/s, 97 objects/s recovering 2023-09-30T09:26:02.465 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:02 smithi019 ceph-mon[216815]: pgmap v381: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 955/159255 objects degraded (0.600%); 3361/159255 objects misplaced (2.110%); 3.2 MiB/s, 97 objects/s recovering 2023-09-30T09:26:02.465 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:02.515 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:02 smithi167 ceph-mon[192880]: pgmap v381: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 955/159255 objects degraded (0.600%); 3361/159255 objects misplaced (2.110%); 3.2 MiB/s, 97 objects/s recovering 2023-09-30T09:26:02.521 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:02.603 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:02.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:02.722 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:26:02.788 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:03.213 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:26:03.371 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:26:03.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:03.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:26:03.478 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:03.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2745981912' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:03.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:03.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:03.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 955/159255 objects degraded (0.600%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:26:03.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:03.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:26:03.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[224043]: osdmap e170: 8 total, 8 up, 8 in 2023-09-30T09:26:03.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2745981912' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:03.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:03.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:03.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 955/159255 objects degraded (0.600%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:26:03.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:03.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:26:03.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:03 smithi019 ceph-mon[216815]: osdmap e170: 8 total, 8 up, 8 in 2023-09-30T09:26:03.514 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:03.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2745981912' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 955/159255 objects degraded (0.600%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:26:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:03 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:26:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:03 smithi167 ceph-mon[192880]: osdmap e170: 8 total, 8 up, 8 in 2023-09-30T09:26:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:26:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:26:03] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.43.0" 2023-09-30T09:26:04.146 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:04.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:04.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:04.224 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:04.249 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:04.294 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:04.414 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:04.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:04.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:04.436 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: pgmap v382: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 955/158112 objects degraded (0.604%); 3361/158112 objects misplaced (2.126%); 480 KiB/s, 49 objects/s recovering 2023-09-30T09:26:04.437 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.437 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.437 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.437 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.437 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.437 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[224043]: osdmap e171: 8 total, 8 up, 8 in 2023-09-30T09:26:04.448 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: pgmap v382: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 955/158112 objects degraded (0.604%); 3361/158112 objects misplaced (2.126%); 480 KiB/s, 49 objects/s recovering 2023-09-30T09:26:04.448 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.448 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.450 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:04 smithi019 ceph-mon[216815]: osdmap e171: 8 total, 8 up, 8 in 2023-09-30T09:26:04.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: pgmap v382: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 955/158112 objects degraded (0.604%); 3361/158112 objects misplaced (2.126%); 480 KiB/s, 49 objects/s recovering 2023-09-30T09:26:04.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: failed to encode map e170 with expected crc 2023-09-30T09:26:04.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:04 smithi167 ceph-mon[192880]: osdmap e171: 8 total, 8 up, 8 in 2023-09-30T09:26:04.528 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:04.639 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:04.731 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:04.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:04.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:04.741 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:04.839 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:04.927 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:05.194 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:05.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:05.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:05.288 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:26:05.288 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 518/150756 objects degraded (0.344%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":503,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":171,"num_osds":8,"num_up_osds":8,"osd_up_since":1696065688,"num_in_osds":8,"osd_in_since":1696065786,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"peering","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":50252,"data_bytes":3296204735,"bytes_used":11472478208,"bytes_avail":756487806976,"bytes_total":767960285184,"inactive_pgs_ratio":0.01123595517128706,"degraded_objects":518,"degraded_total":150756,"degraded_ratio":0.0034360158136326254,"misplaced_objects":3127,"misplaced_total":150756,"misplaced_ratio":0.020742126349863357,"recovering_objects_per_sec":176,"recovering_bytes_per_sec":2393237,"recovering_keys_per_sec":0,"num_objects_recovered":1042,"num_bytes_recovered":14090240,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":668},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":20,"modified":"2023-09-30T09:22:28.956591+0000","services":{}},"progress_events":{"1fc8495a-b88c-4d2b-8c9c-8116c16ace1c":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 9s)","progress":0.94382023811340332,"add_to_ceph_s":true},"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-30T09:26:05.304 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:05.360 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:05.402 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:05.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:05.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:05.508 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:05.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:05 smithi167 ceph-mon[192880]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:05 smithi167 ceph-mon[192880]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:05 smithi167 ceph-mon[192880]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:05 smithi167 ceph-mon[192880]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:05 smithi019 ceph-mon[216815]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:05 smithi019 ceph-mon[216815]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:05 smithi019 ceph-mon[216815]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:05 smithi019 ceph-mon[216815]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:05 smithi019 ceph-mon[224043]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:05 smithi019 ceph-mon[224043]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:05 smithi019 ceph-mon[224043]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:05 smithi019 ceph-mon[224043]: failed to encode map e171 with expected crc 2023-09-30T09:26:05.561 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:05.669 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:05.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:05.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:05.778 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:05.850 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:06.004 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:06.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:06.111 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:06.163 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:06.209 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:06.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:06.231 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:06 smithi019 ceph-mon[216815]: pgmap v385: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 518/150756 objects degraded (0.344%); 3127/150756 objects misplaced (2.074%); 2.3 MiB/s, 176 objects/s recovering 2023-09-30T09:26:06.231 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1403434965' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:26:06.270 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:26:06.314 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:06.384 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:06 smithi167 ceph-mon[192880]: pgmap v385: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 518/150756 objects degraded (0.344%); 3127/150756 objects misplaced (2.074%); 2.3 MiB/s, 176 objects/s recovering 2023-09-30T09:26:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1403434965' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:26:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:06 smithi019 ceph-mon[224043]: pgmap v385: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 518/150756 objects degraded (0.344%); 3127/150756 objects misplaced (2.074%); 2.3 MiB/s, 176 objects/s recovering 2023-09-30T09:26:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1403434965' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:26:06.751 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:06.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:06.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:06.820 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:26:06.820 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:26:06.855 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:06.897 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:07.472 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:07 smithi019 ceph-mon[216815]: osdmap e172: 8 total, 8 up, 8 in 2023-09-30T09:26:07.472 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:07 smithi019 ceph-mon[216815]: failed to encode map e172 with expected crc 2023-09-30T09:26:07.472 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:07 smithi019 ceph-mon[216815]: failed to encode map e172 with expected crc 2023-09-30T09:26:07.472 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:07 smithi019 ceph-mon[224043]: osdmap e172: 8 total, 8 up, 8 in 2023-09-30T09:26:07.473 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:07 smithi019 ceph-mon[224043]: failed to encode map e172 with expected crc 2023-09-30T09:26:07.473 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:07 smithi019 ceph-mon[224043]: failed to encode map e172 with expected crc 2023-09-30T09:26:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:07 smithi167 ceph-mon[192880]: osdmap e172: 8 total, 8 up, 8 in 2023-09-30T09:26:07.599 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:07 smithi167 ceph-mon[192880]: failed to encode map e172 with expected crc 2023-09-30T09:26:07.600 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:07 smithi167 ceph-mon[192880]: failed to encode map e172 with expected crc 2023-09-30T09:26:07.738 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:26:07.738 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:26:07.738 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:26:07.738 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:26:07.768 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:26:07.809 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:26:07.862 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:26:07.895 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:26:08.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:26:08.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:26:08.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:26:08.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:26:08.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:26:08.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:26:08.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:26:08.001 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:26:08.237 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:08.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:08.348 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:08.428 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:08.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:08 smithi167 ceph-mon[192880]: pgmap v387: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 518/150153 objects degraded (0.345%); 3127/150153 objects misplaced (2.083%); 2.2 MiB/s, 117 objects/s recovering 2023-09-30T09:26:08.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:08 smithi167 ceph-mon[192880]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:08 smithi167 ceph-mon[192880]: osdmap e173: 8 total, 8 up, 8 in 2023-09-30T09:26:08.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:08 smithi167 ceph-mon[192880]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:08 smithi167 ceph-mon[192880]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:08 smithi167 ceph-mon[192880]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3925806394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:08.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 518/150153 objects degraded (0.345%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:26:08.543 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:08.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[224043]: pgmap v387: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 518/150153 objects degraded (0.345%); 3127/150153 objects misplaced (2.083%); 2.2 MiB/s, 117 objects/s recovering 2023-09-30T09:26:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[224043]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[224043]: osdmap e173: 8 total, 8 up, 8 in 2023-09-30T09:26:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[224043]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[224043]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[224043]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3925806394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 518/150153 objects degraded (0.345%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:26:08.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[216815]: pgmap v387: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 518/150153 objects degraded (0.345%); 3127/150153 objects misplaced (2.083%); 2.2 MiB/s, 117 objects/s recovering 2023-09-30T09:26:08.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[216815]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[216815]: osdmap e173: 8 total, 8 up, 8 in 2023-09-30T09:26:08.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[216815]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[216815]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[216815]: failed to encode map e173 with expected crc 2023-09-30T09:26:08.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3925806394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:08.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 518/150153 objects degraded (0.345%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:26:08.658 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:08.717 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:08.871 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:08.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:08.982 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:09.031 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:09.040 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:09.100 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:09.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:09.211 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:09.283 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:09.333 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:09.398 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:26:09.398 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:26:09.398 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.398 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.44600.0:118866 5.a e173/160 5:58745ec7:::benchmark_data_smithi019_287194_object36088:head v 173'7342, mlcod=173'7340)", 2023-09-30T09:26:09.398 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:09.384209+0000", 2023-09-30T09:26:09.398 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0024280069999999998, 2023-09-30T09:26:09.398 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024713309999999998, 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384209+0000", 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384209+0000", 2023-09-30T09:26:09.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384210+0000", 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.322e-06 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384220+0000", 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6579999999999997e-06 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384244+0000", 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3485999999999998e-05 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384254+0000", 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0553999999999999e-05 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384333+0000", 2023-09-30T09:26:09.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8614000000000003e-05 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384357+0000", 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4088999999999999e-05 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.44600.0:118874 5.a e173/160)", 2023-09-30T09:26:09.402 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:09.384285+0000", 2023-09-30T09:26:09.403 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0023519299999999999, 2023-09-30T09:26:09.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002459536, 2023-09-30T09:26:09.403 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:26:09.403 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "queued for pg", 2023-09-30T09:26:09.403 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:26:09.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384285+0000", 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384285+0000", 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384286+0000", 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7500000000000001e-07 2023-09-30T09:26:09.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384292+0000", 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.959e-06 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384300+0000", 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.123e-06 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:26:09.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:09.384305+0000", 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7520000000000002e-06 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 2 2023-09-30T09:26:09.406 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:26:09.446 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:09.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:09.559 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:09.614 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:09.725 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:09.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:09.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:09.839 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:26:09.899 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:10 smithi019 ceph-mon[224043]: pgmap v389: 89 pgs: 1 peering, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 518/146100 objects degraded (0.355%); 3127/146100 objects misplaced (2.140%); 2.3 MiB/s, 146 objects/s recovering 2023-09-30T09:26:10.499 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:10 smithi019 ceph-mon[216815]: pgmap v389: 89 pgs: 1 peering, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 518/146100 objects degraded (0.355%); 3127/146100 objects misplaced (2.140%); 2.3 MiB/s, 146 objects/s recovering 2023-09-30T09:26:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:10 smithi167 ceph-mon[192880]: pgmap v389: 89 pgs: 1 peering, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 518/146100 objects degraded (0.355%); 3127/146100 objects misplaced (2.140%); 2.3 MiB/s, 146 objects/s recovering 2023-09-30T09:26:10.705 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:26:10.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:10.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:26:10.809 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:10.835 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:10.877 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:26:10.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:10.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:10.954 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:26:10.983 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:11.031 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:11.395 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:11.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:11.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:11.501 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:11 smithi167 ceph-mon[192880]: osdmap e174: 8 total, 8 up, 8 in 2023-09-30T09:26:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:11 smithi167 ceph-mon[192880]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:11 smithi167 ceph-mon[192880]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:11 smithi167 ceph-mon[192880]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:11 smithi019 ceph-mon[216815]: osdmap e174: 8 total, 8 up, 8 in 2023-09-30T09:26:11.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:11 smithi019 ceph-mon[216815]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:11 smithi019 ceph-mon[216815]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:11 smithi019 ceph-mon[216815]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:11 smithi019 ceph-mon[224043]: osdmap e174: 8 total, 8 up, 8 in 2023-09-30T09:26:11.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:11 smithi019 ceph-mon[224043]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:11 smithi019 ceph-mon[224043]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:11 smithi019 ceph-mon[224043]: failed to encode map e174 with expected crc 2023-09-30T09:26:11.571 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:12.262 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[216815]: pgmap v391: 89 pgs: 1 active+recovering+undersized+remapped, 87 active+clean, 1 active+remapped+backfill_wait; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 2906/140946 objects misplaced (2.062%); 1.3 MiB/s, 106 objects/s recovering 2023-09-30T09:26:12.262 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 518/146100 objects degraded (0.355%), 1 pg degraded) 2023-09-30T09:26:12.262 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[216815]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.263 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[216815]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.263 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[216815]: osdmap e175: 8 total, 8 up, 8 in 2023-09-30T09:26:12.263 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[216815]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.263 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[216815]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[216815]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.520 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:12.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:12 smithi167 ceph-mon[192880]: pgmap v391: 89 pgs: 1 active+recovering+undersized+remapped, 87 active+clean, 1 active+remapped+backfill_wait; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 2906/140946 objects misplaced (2.062%); 1.3 MiB/s, 106 objects/s recovering 2023-09-30T09:26:12.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:12 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 518/146100 objects degraded (0.355%), 1 pg degraded) 2023-09-30T09:26:12.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:12 smithi167 ceph-mon[192880]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:12 smithi167 ceph-mon[192880]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:12 smithi167 ceph-mon[192880]: osdmap e175: 8 total, 8 up, 8 in 2023-09-30T09:26:12.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:12 smithi167 ceph-mon[192880]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:12 smithi167 ceph-mon[192880]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:12 smithi167 ceph-mon[192880]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:12.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:12.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[224043]: pgmap v391: 89 pgs: 1 active+recovering+undersized+remapped, 87 active+clean, 1 active+remapped+backfill_wait; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 2906/140946 objects misplaced (2.062%); 1.3 MiB/s, 106 objects/s recovering 2023-09-30T09:26:12.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 518/146100 objects degraded (0.355%), 1 pg degraded) 2023-09-30T09:26:12.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[224043]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[224043]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[224043]: osdmap e175: 8 total, 8 up, 8 in 2023-09-30T09:26:12.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[224043]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[224043]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:12 smithi019 ceph-mon[224043]: failed to encode map e175 with expected crc 2023-09-30T09:26:12.629 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:12.721 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:12.848 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:12.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:12.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:12.932 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:12.958 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:13.022 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:13.161 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:13.557 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:13.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:13.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:13.665 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:13.718 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:13.809 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:13.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:13.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:13.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:26:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:26:13] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.43.0" 2023-09-30T09:26:13.919 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:13.953 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:14.020 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:14.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:14.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:14.047 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:26:14.059 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:26:14.125 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:14.194 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:14.283 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:14.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:14 smithi019 ceph-mon[224043]: pgmap v393: 89 pgs: 1 active+recovering+undersized+remapped, 87 active+clean, 1 active+remapped+backfill_wait; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 2906/139833 objects misplaced (2.078%); 1.3 MiB/s, 106 objects/s recovering 2023-09-30T09:26:14.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1423653151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:14.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:14 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:14.284 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:14 smithi167 ceph-mon[192880]: pgmap v393: 89 pgs: 1 active+recovering+undersized+remapped, 87 active+clean, 1 active+remapped+backfill_wait; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 2906/139833 objects misplaced (2.078%); 1.3 MiB/s, 106 objects/s recovering 2023-09-30T09:26:14.285 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1423653151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:14.285 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:14 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:14.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:14 smithi019 ceph-mon[216815]: pgmap v393: 89 pgs: 1 active+recovering+undersized+remapped, 87 active+clean, 1 active+remapped+backfill_wait; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 2906/139833 objects misplaced (2.078%); 1.3 MiB/s, 106 objects/s recovering 2023-09-30T09:26:14.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1423653151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:14.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:14 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:14.296 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:26:14.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:14.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:14.399 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:14.462 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:14.510 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:14.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:14.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:14.616 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:14.706 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:14.793 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:14.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:14.897 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:14.956 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:15.062 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:15.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:15.170 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:15.230 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:15.308 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:15.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:15.417 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:15.481 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:26:15.481 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:26:15.481 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:26:15.481 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:15.481 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: osdmap e176: 8 total, 8 up, 8 in 2023-09-30T09:26:15.482 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.482 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.482 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.482 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.482 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:15 smithi167 ceph-mon[192880]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.485 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:15.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:26:15.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:26:15.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:26:15.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:15.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: osdmap e176: 8 total, 8 up, 8 in 2023-09-30T09:26:15.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[224043]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:26:15.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:26:15.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:26:15.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:15.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: osdmap e176: 8 total, 8 up, 8 in 2023-09-30T09:26:15.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:15 smithi019 ceph-mon[216815]: failed to encode map e176 with expected crc 2023-09-30T09:26:15.600 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:15.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:15.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:15.712 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:15.777 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:16.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:16 smithi019 ceph-mon[216815]: pgmap v395: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 2275/131163 objects misplaced (1.734%); 5.8 MiB/s, 158 objects/s recovering 2023-09-30T09:26:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:16 smithi019 ceph-mon[224043]: pgmap v395: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 2275/131163 objects misplaced (1.734%); 5.8 MiB/s, 158 objects/s recovering 2023-09-30T09:26:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:16 smithi167 ceph-mon[192880]: pgmap v395: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 2275/131163 objects misplaced (1.734%); 5.8 MiB/s, 158 objects/s recovering 2023-09-30T09:26:17.002 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:17.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:17.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:17.108 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:17.138 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:17.688 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:17.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:17.791 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:17.819 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:17.863 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:17.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:17.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:17.968 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:18.048 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:18.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:18 smithi019 ceph-mon[216815]: pgmap v396: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 2275/130545 objects misplaced (1.743%); 4.0 MiB/s, 69 objects/s recovering 2023-09-30T09:26:18.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:18.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:18.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:18 smithi019 ceph-mon[224043]: pgmap v396: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 2275/130545 objects misplaced (1.743%); 4.0 MiB/s, 69 objects/s recovering 2023-09-30T09:26:18.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:18.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:18 smithi167 ceph-mon[192880]: pgmap v396: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 2275/130545 objects misplaced (1.743%); 4.0 MiB/s, 69 objects/s recovering 2023-09-30T09:26:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:19.496 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":393,"stamp":"2023-09-30T09:26:13.059015+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3057588159,"num_objects":46611,"num_object_clones":0,"num_object_copies":139833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2906,"num_objects_unfound":0,"num_objects_dirty":46611,"num_whiteouts":0,"num_read":6443,"num_read_kb":13071,"num_write":120115,"num_write_kb":5312742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20847,"num_bytes_recovered":1241058714,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119645,"ondisk_log_size":119645,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":277,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10505192,"kb_used_data":9770908,"kb_used_omap":19,"kb_used_meta":733996,"kb_avail":739456024,"statfs":{"total":767960285184,"available":757202968576,"internally_reserved":0,"allocated":10005409792,"data_stored":9998558492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":751612327},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-225443840,"num_objects":-3440,"num_object_clones":0,"num_object_copies":-10320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-518,"num_objects_misplaced":-221,"num_objects_unfound":0,"num_objects_dirty":-3440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3440,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":638,"num_bytes_recovered":8257560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001679"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":150,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.252335+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:26:06.252335+0000","last_peered":"2023-09-30T09:26:06.252335+0000","last_clean":"2023-09-30T09:26:06.252335+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:26:06.252335+0000","last_undegraded":"2023-09-30T09:26:06.252335+0000","last_fullsized":"2023-09-30T09:26:06.252335+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":138,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254571+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:26:10.254571+0000","last_peered":"2023-09-30T09:26:10.254571+0000","last_clean":"2023-09-30T09:26:10.254571+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:26:10.254571+0000","last_undegraded":"2023-09-30T09:26:10.254571+0000","last_fullsized":"2023-09-30T09:26:10.254571+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":128,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.378448+0000","last_change":"2023-09-30T09:25:04.281698+0000","last_active":"2023-09-30T09:26:10.378448+0000","last_peered":"2023-09-30T09:26:10.378448+0000","last_clean":"2023-09-30T09:26:10.378448+0000","last_became_active":"2023-09-30T09:25:04.281351+0000","last_became_peered":"2023-09-30T09:25:04.281351+0000","last_unstale":"2023-09-30T09:26:10.378448+0000","last_undegraded":"2023-09-30T09:26:10.378448+0000","last_fullsized":"2023-09-30T09:26:10.378448+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:17:40.382369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":64,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.256751+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:26:06.256751+0000","last_peered":"2023-09-30T09:26:06.256751+0000","last_clean":"2023-09-30T09:26:06.256751+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:26:06.256751+0000","last_undegraded":"2023-09-30T09:26:06.256751+0000","last_fullsized":"2023-09-30T09:26:06.256751+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":172,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254307+0000","last_change":"2023-09-30T09:23:08.014781+0000","last_active":"2023-09-30T09:26:10.254307+0000","last_peered":"2023-09-30T09:26:10.254307+0000","last_clean":"2023-09-30T09:26:10.254307+0000","last_became_active":"2023-09-30T09:23:08.014387+0000","last_became_peered":"2023-09-30T09:23:08.014387+0000","last_unstale":"2023-09-30T09:26:10.254307+0000","last_undegraded":"2023-09-30T09:26:10.254307+0000","last_fullsized":"2023-09-30T09:26:10.254307+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:52:13.669946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":132,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.272187+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:26:06.272187+0000","last_peered":"2023-09-30T09:26:06.272187+0000","last_clean":"2023-09-30T09:26:06.272187+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:26:06.272187+0000","last_undegraded":"2023-09-30T09:26:06.272187+0000","last_fullsized":"2023-09-30T09:26:06.272187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253904+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:26:06.253904+0000","last_peered":"2023-09-30T09:26:06.253904+0000","last_clean":"2023-09-30T09:26:06.253904+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:26:06.253904+0000","last_undegraded":"2023-09-30T09:26:06.253904+0000","last_fullsized":"2023-09-30T09:26:06.253904+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":139,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254307+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:26:10.254307+0000","last_peered":"2023-09-30T09:26:10.254307+0000","last_clean":"2023-09-30T09:26:10.254307+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:26:10.254307+0000","last_undegraded":"2023-09-30T09:26:10.254307+0000","last_fullsized":"2023-09-30T09:26:10.254307+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":24,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.260355+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:26:06.260355+0000","last_peered":"2023-09-30T09:26:06.260355+0000","last_clean":"2023-09-30T09:26:06.260355+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:26:06.260355+0000","last_undegraded":"2023-09-30T09:26:06.260355+0000","last_fullsized":"2023-09-30T09:26:06.260355+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":90,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254356+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:26:10.254356+0000","last_peered":"2023-09-30T09:26:10.254356+0000","last_clean":"2023-09-30T09:26:10.254356+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:26:10.254356+0000","last_undegraded":"2023-09-30T09:26:10.254356+0000","last_fullsized":"2023-09-30T09:26:10.254356+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":136,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.263251+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:26:06.263251+0000","last_peered":"2023-09-30T09:26:06.263251+0000","last_clean":"2023-09-30T09:26:06.263251+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:26:06.263251+0000","last_undegraded":"2023-09-30T09:26:06.263251+0000","last_fullsized":"2023-09-30T09:26:06.263251+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":113,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.324967+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:26:10.324967+0000","last_peered":"2023-09-30T09:26:10.324967+0000","last_clean":"2023-09-30T09:26:10.324967+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:26:10.324967+0000","last_undegraded":"2023-09-30T09:26:10.324967+0000","last_fullsized":"2023-09-30T09:26:10.324967+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":137,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253981+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:26:06.253981+0000","last_peered":"2023-09-30T09:26:06.253981+0000","last_clean":"2023-09-30T09:26:06.253981+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:26:06.253981+0000","last_undegraded":"2023-09-30T09:26:06.253981+0000","last_fullsized":"2023-09-30T09:26:06.253981+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":141,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.252725+0000","last_change":"2023-09-30T09:23:03.986755+0000","last_active":"2023-09-30T09:26:06.252725+0000","last_peered":"2023-09-30T09:26:06.252725+0000","last_clean":"2023-09-30T09:26:06.252725+0000","last_became_active":"2023-09-30T09:23:03.986411+0000","last_became_peered":"2023-09-30T09:23:03.986411+0000","last_unstale":"2023-09-30T09:26:06.252725+0000","last_undegraded":"2023-09-30T09:26:06.252725+0000","last_fullsized":"2023-09-30T09:26:06.252725+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:53:23.627823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":135,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.262687+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:26:06.262687+0000","last_peered":"2023-09-30T09:26:06.262687+0000","last_clean":"2023-09-30T09:26:06.262687+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:26:06.262687+0000","last_undegraded":"2023-09-30T09:26:06.262687+0000","last_fullsized":"2023-09-30T09:26:06.262687+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":153,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254991+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:26:10.254991+0000","last_peered":"2023-09-30T09:26:10.254991+0000","last_clean":"2023-09-30T09:26:10.254991+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:26:10.254991+0000","last_undegraded":"2023-09-30T09:26:10.254991+0000","last_fullsized":"2023-09-30T09:26:10.254991+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":64,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.254979+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:26:06.254979+0000","last_peered":"2023-09-30T09:26:06.254979+0000","last_clean":"2023-09-30T09:26:06.254979+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:26:06.254979+0000","last_undegraded":"2023-09-30T09:26:06.254979+0000","last_fullsized":"2023-09-30T09:26:06.254979+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":24,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.260260+0000","last_change":"2023-09-30T09:25:14.358999+0000","last_active":"2023-09-30T09:26:06.260260+0000","last_peered":"2023-09-30T09:26:06.260260+0000","last_clean":"2023-09-30T09:26:06.260260+0000","last_became_active":"2023-09-30T09:25:14.358650+0000","last_became_peered":"2023-09-30T09:25:14.358650+0000","last_unstale":"2023-09-30T09:26:06.260260+0000","last_undegraded":"2023-09-30T09:26:06.260260+0000","last_fullsized":"2023-09-30T09:26:06.260260+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:57:41.373342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":116,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.262449+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:26:06.262449+0000","last_peered":"2023-09-30T09:26:06.262449+0000","last_clean":"2023-09-30T09:26:06.262449+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:26:06.262449+0000","last_undegraded":"2023-09-30T09:26:06.262449+0000","last_fullsized":"2023-09-30T09:26:06.262449+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":64,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.256596+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:26:06.256596+0000","last_peered":"2023-09-30T09:26:06.256596+0000","last_clean":"2023-09-30T09:26:06.256596+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:26:06.256596+0000","last_undegraded":"2023-09-30T09:26:06.256596+0000","last_fullsized":"2023-09-30T09:26:06.256596+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":221,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.271713+0000","last_change":"2023-09-30T09:25:14.389693+0000","last_active":"2023-09-30T09:26:06.271713+0000","last_peered":"2023-09-30T09:26:06.271713+0000","last_clean":"2023-09-30T09:26:06.271713+0000","last_became_active":"2023-09-30T09:25:14.365482+0000","last_became_peered":"2023-09-30T09:25:14.365482+0000","last_unstale":"2023-09-30T09:26:06.271713+0000","last_undegraded":"2023-09-30T09:26:06.271713+0000","last_fullsized":"2023-09-30T09:26:06.271713+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:09:23.017579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":152,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254760+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:26:10.254760+0000","last_peered":"2023-09-30T09:26:10.254760+0000","last_clean":"2023-09-30T09:26:10.254760+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:26:10.254760+0000","last_undegraded":"2023-09-30T09:26:10.254760+0000","last_fullsized":"2023-09-30T09:26:10.254760+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"173'7422","reported_seq":7490,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:08.899245+0000","last_change":"2023-09-30T09:22:48.114186+0000","last_active":"2023-09-30T09:26:08.899245+0000","last_peered":"2023-09-30T09:26:08.899245+0000","last_clean":"2023-09-30T09:26:08.899245+0000","last_became_active":"2023-09-30T09:22:48.113857+0000","last_became_peered":"2023-09-30T09:22:48.113857+0000","last_unstale":"2023-09-30T09:26:08.899245+0000","last_undegraded":"2023-09-30T09:26:08.899245+0000","last_fullsized":"2023-09-30T09:26:08.899245+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7422,"log_dups_size":0,"ondisk_log_size":7422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:18.973595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189399040,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7422,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":162,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.262960+0000","last_change":"2023-09-30T09:23:08.038825+0000","last_active":"2023-09-30T09:26:06.262960+0000","last_peered":"2023-09-30T09:26:06.262960+0000","last_clean":"2023-09-30T09:26:06.262960+0000","last_became_active":"2023-09-30T09:23:08.029150+0000","last_became_peered":"2023-09-30T09:23:08.029150+0000","last_unstale":"2023-09-30T09:26:06.262960+0000","last_undegraded":"2023-09-30T09:26:06.262960+0000","last_fullsized":"2023-09-30T09:26:06.262960+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:14.557254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":24,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.259935+0000","last_change":"2023-09-30T09:25:14.359647+0000","last_active":"2023-09-30T09:26:06.259935+0000","last_peered":"2023-09-30T09:26:06.259935+0000","last_clean":"2023-09-30T09:26:06.259935+0000","last_became_active":"2023-09-30T09:25:14.359350+0000","last_became_peered":"2023-09-30T09:25:14.359350+0000","last_unstale":"2023-09-30T09:26:06.259935+0000","last_undegraded":"2023-09-30T09:26:06.259935+0000","last_fullsized":"2023-09-30T09:26:06.259935+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:09.489014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"173'7568","reported_seq":10734,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:09.748384+0000","last_change":"2023-09-30T09:24:25.563637+0000","last_active":"2023-09-30T09:26:09.748384+0000","last_peered":"2023-09-30T09:26:09.748384+0000","last_clean":"2023-09-30T09:26:09.748384+0000","last_became_active":"2023-09-30T09:24:25.554649+0000","last_became_peered":"2023-09-30T09:24:25.554649+0000","last_unstale":"2023-09-30T09:26:09.748384+0000","last_undegraded":"2023-09-30T09:26:09.748384+0000","last_fullsized":"2023-09-30T09:26:09.748384+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:09:56.974492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191234048,"num_objects":2918,"num_object_clones":0,"num_object_copies":8754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7568,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1538,"num_bytes_recovered":100794368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":163,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.267610+0000","last_change":"2023-09-30T09:25:04.304869+0000","last_active":"2023-09-30T09:26:06.267610+0000","last_peered":"2023-09-30T09:26:06.267610+0000","last_clean":"2023-09-30T09:26:06.267610+0000","last_became_active":"2023-09-30T09:25:04.285159+0000","last_became_peered":"2023-09-30T09:25:04.285159+0000","last_unstale":"2023-09-30T09:26:06.267610+0000","last_undegraded":"2023-09-30T09:26:06.267610+0000","last_fullsized":"2023-09-30T09:26:06.267610+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:03:13.971240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":148,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.255368+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:26:06.255368+0000","last_peered":"2023-09-30T09:26:06.255368+0000","last_clean":"2023-09-30T09:26:06.255368+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:26:06.255368+0000","last_undegraded":"2023-09-30T09:26:06.255368+0000","last_fullsized":"2023-09-30T09:26:06.255368+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"173'7422","reported_seq":11576,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:09.780311+0000","last_change":"2023-09-30T09:26:00.198334+0000","last_active":"2023-09-30T09:26:09.780311+0000","last_peered":"2023-09-30T09:26:09.780311+0000","last_clean":"2023-09-30T09:26:09.780311+0000","last_became_active":"2023-09-30T09:26:00.182970+0000","last_became_peered":"2023-09-30T09:26:00.182970+0000","last_unstale":"2023-09-30T09:26:09.780311+0000","last_undegraded":"2023-09-30T09:26:09.780311+0000","last_fullsized":"2023-09-30T09:26:09.780311+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7422,"log_dups_size":0,"ondisk_log_size":7422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:34.041047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190840832,"num_objects":2912,"num_object_clones":0,"num_object_copies":8736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7422,"num_write_kb":330688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2027,"num_bytes_recovered":108331008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":145,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.263430+0000","last_change":"2023-09-30T09:25:14.385912+0000","last_active":"2023-09-30T09:26:06.263430+0000","last_peered":"2023-09-30T09:26:06.263430+0000","last_clean":"2023-09-30T09:26:06.263430+0000","last_became_active":"2023-09-30T09:25:14.359789+0000","last_became_peered":"2023-09-30T09:25:14.359789+0000","last_unstale":"2023-09-30T09:26:06.263430+0000","last_undegraded":"2023-09-30T09:26:06.263430+0000","last_fullsized":"2023-09-30T09:26:06.263430+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:54.213336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":142,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.272435+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:26:06.272435+0000","last_peered":"2023-09-30T09:26:06.272435+0000","last_clean":"2023-09-30T09:26:06.272435+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:26:06.272435+0000","last_undegraded":"2023-09-30T09:26:06.272435+0000","last_fullsized":"2023-09-30T09:26:06.272435+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"173'7360","reported_seq":8310,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:09.736222+0000","last_change":"2023-09-30T09:25:16.371393+0000","last_active":"2023-09-30T09:26:09.736222+0000","last_peered":"2023-09-30T09:26:09.736222+0000","last_clean":"2023-09-30T09:26:09.736222+0000","last_became_active":"2023-09-30T09:25:16.364820+0000","last_became_peered":"2023-09-30T09:25:16.364820+0000","last_unstale":"2023-09-30T09:26:09.736222+0000","last_undegraded":"2023-09-30T09:26:09.736222+0000","last_fullsized":"2023-09-30T09:26:09.736222+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7360,"log_dups_size":0,"ondisk_log_size":7360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:19:41.535739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184680448,"num_objects":2818,"num_object_clones":0,"num_object_copies":8454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7360,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":429,"num_bytes_recovered":9240576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":132,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.260172+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:26:06.260172+0000","last_peered":"2023-09-30T09:26:06.260172+0000","last_clean":"2023-09-30T09:26:06.260172+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:26:06.260172+0000","last_undegraded":"2023-09-30T09:26:06.260172+0000","last_fullsized":"2023-09-30T09:26:06.260172+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":61,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253245+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:26:06.253245+0000","last_peered":"2023-09-30T09:26:06.253245+0000","last_clean":"2023-09-30T09:26:06.253245+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:26:06.253245+0000","last_undegraded":"2023-09-30T09:26:06.253245+0000","last_fullsized":"2023-09-30T09:26:06.253245+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"173'7615","reported_seq":9830,"reported_epoch":173,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:26:09.777773+0000","last_change":"2023-09-30T09:26:05.580520+0000","last_active":"2023-09-30T09:26:09.777773+0000","last_peered":"2023-09-30T09:26:09.777773+0000","last_clean":"2023-09-30T09:24:01.611293+0000","last_became_active":"2023-09-30T09:25:14.386329+0000","last_became_peered":"2023-09-30T09:25:14.386329+0000","last_unstale":"2023-09-30T09:26:09.777773+0000","last_undegraded":"2023-09-30T09:26:09.777773+0000","last_fullsized":"2023-09-30T09:25:14.355372+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7615,"log_dups_size":0,"ondisk_log_size":7615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:11:34.305355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187826200,"num_objects":2867,"num_object_clones":0,"num_object_copies":8601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2867,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7615,"num_write_kb":335361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1064,"num_bytes_recovered":46661656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2867}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":161,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254662+0000","last_change":"2023-09-30T09:23:08.029556+0000","last_active":"2023-09-30T09:26:10.254662+0000","last_peered":"2023-09-30T09:26:10.254662+0000","last_clean":"2023-09-30T09:26:10.254662+0000","last_became_active":"2023-09-30T09:23:08.014483+0000","last_became_peered":"2023-09-30T09:23:08.014483+0000","last_unstale":"2023-09-30T09:26:10.254662+0000","last_undegraded":"2023-09-30T09:26:10.254662+0000","last_fullsized":"2023-09-30T09:26:10.254662+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:07:01.455742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":131,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253851+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:26:06.253851+0000","last_peered":"2023-09-30T09:26:06.253851+0000","last_clean":"2023-09-30T09:26:06.253851+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:26:06.253851+0000","last_undegraded":"2023-09-30T09:26:06.253851+0000","last_fullsized":"2023-09-30T09:26:06.253851+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"175'7269","reported_seq":10269,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-30T09:26:11.551374+0000","last_change":"2023-09-30T09:24:20.547592+0000","last_active":"2023-09-30T09:26:11.551374+0000","last_peered":"2023-09-30T09:26:11.551374+0000","last_clean":"2023-09-30T09:26:11.551374+0000","last_became_active":"2023-09-30T09:24:20.539203+0000","last_became_peered":"2023-09-30T09:24:20.539203+0000","last_unstale":"2023-09-30T09:26:11.551374+0000","last_undegraded":"2023-09-30T09:26:11.551374+0000","last_fullsized":"2023-09-30T09:26:11.551374+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7269,"log_dups_size":0,"ondisk_log_size":7269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:33.219264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184877056,"num_objects":2821,"num_object_clones":0,"num_object_copies":8463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7269,"num_write_kb":322880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1455,"num_bytes_recovered":95354880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":133,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.272551+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:26:06.272551+0000","last_peered":"2023-09-30T09:26:06.272551+0000","last_clean":"2023-09-30T09:26:06.272551+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:26:06.272551+0000","last_undegraded":"2023-09-30T09:26:06.272551+0000","last_fullsized":"2023-09-30T09:26:06.272551+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":27,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.252965+0000","last_change":"2023-09-30T09:25:04.290019+0000","last_active":"2023-09-30T09:26:06.252965+0000","last_peered":"2023-09-30T09:26:06.252965+0000","last_clean":"2023-09-30T09:26:06.252965+0000","last_became_active":"2023-09-30T09:25:04.289710+0000","last_became_peered":"2023-09-30T09:25:04.289710+0000","last_unstale":"2023-09-30T09:26:06.252965+0000","last_undegraded":"2023-09-30T09:26:06.252965+0000","last_fullsized":"2023-09-30T09:26:06.252965+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:22:01.874727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"173'7508","reported_seq":7576,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:08.911666+0000","last_change":"2023-09-30T09:22:48.115158+0000","last_active":"2023-09-30T09:26:08.911666+0000","last_peered":"2023-09-30T09:26:08.911666+0000","last_clean":"2023-09-30T09:26:08.911666+0000","last_became_active":"2023-09-30T09:22:48.114453+0000","last_became_peered":"2023-09-30T09:22:48.114453+0000","last_unstale":"2023-09-30T09:26:08.911666+0000","last_undegraded":"2023-09-30T09:26:08.911666+0000","last_fullsized":"2023-09-30T09:26:08.911666+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7508,"log_dups_size":0,"ondisk_log_size":7508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:31:35.934276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195559424,"num_objects":2984,"num_object_clones":0,"num_object_copies":8952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7508,"num_write_kb":335744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":132,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.255075+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:26:06.255075+0000","last_peered":"2023-09-30T09:26:06.255075+0000","last_clean":"2023-09-30T09:26:06.255075+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:26:06.255075+0000","last_undegraded":"2023-09-30T09:26:06.255075+0000","last_fullsized":"2023-09-30T09:26:06.255075+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":131,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.254424+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:26:06.254424+0000","last_peered":"2023-09-30T09:26:06.254424+0000","last_clean":"2023-09-30T09:26:06.254424+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:26:06.254424+0000","last_undegraded":"2023-09-30T09:26:06.254424+0000","last_fullsized":"2023-09-30T09:26:06.254424+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"173'7362","reported_seq":11297,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:08.913511+0000","last_change":"2023-09-30T09:26:07.270277+0000","last_active":"2023-09-30T09:26:08.913511+0000","last_peered":"2023-09-30T09:26:08.913511+0000","last_clean":"2023-09-30T09:26:08.913511+0000","last_became_active":"2023-09-30T09:26:07.253252+0000","last_became_peered":"2023-09-30T09:26:07.253252+0000","last_unstale":"2023-09-30T09:26:08.913511+0000","last_undegraded":"2023-09-30T09:26:08.913511+0000","last_fullsized":"2023-09-30T09:26:08.913511+0000","mapping_epoch":172,"log_start":"117'216","ondisk_log_start":"117'216","created":113,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7146,"log_dups_size":0,"ondisk_log_size":7146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:38:20.303668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192282624,"num_objects":2934,"num_object_clones":0,"num_object_copies":8802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7362,"num_write_kb":329472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1908,"num_bytes_recovered":105906176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":140,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254167+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:26:10.254167+0000","last_peered":"2023-09-30T09:26:10.254167+0000","last_clean":"2023-09-30T09:26:10.254167+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:26:10.254167+0000","last_undegraded":"2023-09-30T09:26:10.254167+0000","last_fullsized":"2023-09-30T09:26:10.254167+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":113,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.395603+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:26:10.395603+0000","last_peered":"2023-09-30T09:26:10.395603+0000","last_clean":"2023-09-30T09:26:10.395603+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:26:10.395603+0000","last_undegraded":"2023-09-30T09:26:10.395603+0000","last_fullsized":"2023-09-30T09:26:10.395603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":517,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.325558+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:26:10.325558+0000","last_peered":"2023-09-30T09:26:10.325558+0000","last_clean":"2023-09-30T09:26:10.325558+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:26:10.325558+0000","last_undegraded":"2023-09-30T09:26:10.325558+0000","last_fullsized":"2023-09-30T09:26:10.325558+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":149,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.262198+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:26:06.262198+0000","last_peered":"2023-09-30T09:26:06.262198+0000","last_clean":"2023-09-30T09:26:06.262198+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:26:06.262198+0000","last_undegraded":"2023-09-30T09:26:06.262198+0000","last_fullsized":"2023-09-30T09:26:06.262198+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":124,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.325049+0000","last_change":"2023-09-30T09:25:14.364474+0000","last_active":"2023-09-30T09:26:10.325049+0000","last_peered":"2023-09-30T09:26:10.325049+0000","last_clean":"2023-09-30T09:26:10.325049+0000","last_became_active":"2023-09-30T09:25:14.364128+0000","last_became_peered":"2023-09-30T09:25:14.364128+0000","last_unstale":"2023-09-30T09:26:10.325049+0000","last_undegraded":"2023-09-30T09:26:10.325049+0000","last_fullsized":"2023-09-30T09:26:10.325049+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:41:17.197492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"173'7403","reported_seq":11453,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:09.055455+0000","last_change":"2023-09-30T09:26:04.207784+0000","last_active":"2023-09-30T09:26:09.055455+0000","last_peered":"2023-09-30T09:26:09.055455+0000","last_clean":"2023-09-30T09:26:09.055455+0000","last_became_active":"2023-09-30T09:26:04.186571+0000","last_became_peered":"2023-09-30T09:26:04.186571+0000","last_unstale":"2023-09-30T09:26:09.055455+0000","last_undegraded":"2023-09-30T09:26:09.055455+0000","last_fullsized":"2023-09-30T09:26:09.055455+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7403,"log_dups_size":0,"ondisk_log_size":7403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:24.650075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192610304,"num_objects":2939,"num_object_clones":0,"num_object_copies":8817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7403,"num_write_kb":330944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1972,"num_bytes_recovered":105644032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"173'7447","reported_seq":14495,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:09.057501+0000","last_change":"2023-09-30T09:25:17.387236+0000","last_active":"2023-09-30T09:26:09.057501+0000","last_peered":"2023-09-30T09:26:09.057501+0000","last_clean":"2023-09-30T09:26:09.057501+0000","last_became_active":"2023-09-30T09:25:17.376539+0000","last_became_peered":"2023-09-30T09:25:17.376539+0000","last_unstale":"2023-09-30T09:26:09.057501+0000","last_undegraded":"2023-09-30T09:26:09.057501+0000","last_fullsized":"2023-09-30T09:26:09.057501+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7447,"log_dups_size":0,"ondisk_log_size":7447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:15:31.918233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190513152,"num_objects":2907,"num_object_clones":0,"num_object_copies":8721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7447,"num_write_kb":331328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3466,"num_bytes_recovered":207814656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":321,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.325192+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:26:10.325192+0000","last_peered":"2023-09-30T09:26:10.325192+0000","last_clean":"2023-09-30T09:26:10.325192+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:26:10.325192+0000","last_undegraded":"2023-09-30T09:26:10.325192+0000","last_fullsized":"2023-09-30T09:26:10.325192+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":152,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.252790+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:26:06.252790+0000","last_peered":"2023-09-30T09:26:06.252790+0000","last_clean":"2023-09-30T09:26:06.252790+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:26:06.252790+0000","last_undegraded":"2023-09-30T09:26:06.252790+0000","last_fullsized":"2023-09-30T09:26:06.252790+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":133,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253560+0000","last_change":"2023-09-30T09:23:23.238851+0000","last_active":"2023-09-30T09:26:06.253560+0000","last_peered":"2023-09-30T09:26:06.253560+0000","last_clean":"2023-09-30T09:26:06.253560+0000","last_became_active":"2023-09-30T09:23:23.238582+0000","last_became_peered":"2023-09-30T09:23:23.238582+0000","last_unstale":"2023-09-30T09:26:06.253560+0000","last_undegraded":"2023-09-30T09:26:06.253560+0000","last_fullsized":"2023-09-30T09:26:06.253560+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:52.808966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"173'7292","reported_seq":7364,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:09.761506+0000","last_change":"2023-09-30T09:22:48.108513+0000","last_active":"2023-09-30T09:26:09.761506+0000","last_peered":"2023-09-30T09:26:09.761506+0000","last_clean":"2023-09-30T09:26:09.761506+0000","last_became_active":"2023-09-30T09:22:48.108231+0000","last_became_peered":"2023-09-30T09:22:48.108231+0000","last_unstale":"2023-09-30T09:26:09.761506+0000","last_undegraded":"2023-09-30T09:26:09.761506+0000","last_fullsized":"2023-09-30T09:26:09.761506+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7292,"log_dups_size":0,"ondisk_log_size":7292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:09.723671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188219392,"num_objects":2872,"num_object_clones":0,"num_object_copies":8616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7292,"num_write_kb":325248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":550,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253183+0000","last_change":"2023-09-30T09:23:11.172147+0000","last_active":"2023-09-30T09:26:06.253183+0000","last_peered":"2023-09-30T09:26:06.253183+0000","last_clean":"2023-09-30T09:26:06.253183+0000","last_became_active":"2023-09-30T09:23:11.171540+0000","last_became_peered":"2023-09-30T09:23:11.171540+0000","last_unstale":"2023-09-30T09:26:06.253183+0000","last_undegraded":"2023-09-30T09:26:06.253183+0000","last_fullsized":"2023-09-30T09:26:06.253183+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:24.074307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":112,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.324764+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:26:10.324764+0000","last_peered":"2023-09-30T09:26:10.324764+0000","last_clean":"2023-09-30T09:26:10.324764+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:26:10.324764+0000","last_undegraded":"2023-09-30T09:26:10.324764+0000","last_fullsized":"2023-09-30T09:26:10.324764+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.378123+0000","last_change":"2023-09-30T09:23:08.027904+0000","last_active":"2023-09-30T09:26:10.378123+0000","last_peered":"2023-09-30T09:26:10.378123+0000","last_clean":"2023-09-30T09:26:10.378123+0000","last_became_active":"2023-09-30T09:23:08.027557+0000","last_became_peered":"2023-09-30T09:23:08.027557+0000","last_unstale":"2023-09-30T09:26:10.378123+0000","last_undegraded":"2023-09-30T09:26:10.378123+0000","last_fullsized":"2023-09-30T09:26:10.378123+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:19:26.977141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":580,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.260591+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:26:06.260591+0000","last_peered":"2023-09-30T09:26:06.260591+0000","last_clean":"2023-09-30T09:26:06.260591+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:26:06.260591+0000","last_undegraded":"2023-09-30T09:26:06.260591+0000","last_fullsized":"2023-09-30T09:26:06.260591+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":566,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.262270+0000","last_change":"2023-09-30T09:23:27.263806+0000","last_active":"2023-09-30T09:26:06.262270+0000","last_peered":"2023-09-30T09:26:06.262270+0000","last_clean":"2023-09-30T09:26:06.262270+0000","last_became_active":"2023-09-30T09:23:27.263242+0000","last_became_peered":"2023-09-30T09:23:27.263242+0000","last_unstale":"2023-09-30T09:26:06.262270+0000","last_undegraded":"2023-09-30T09:26:06.262270+0000","last_fullsized":"2023-09-30T09:26:06.262270+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:38:00.839832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":89,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254364+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:26:10.254364+0000","last_peered":"2023-09-30T09:26:10.254364+0000","last_clean":"2023-09-30T09:26:10.254364+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:26:10.254364+0000","last_undegraded":"2023-09-30T09:26:10.254364+0000","last_fullsized":"2023-09-30T09:26:10.254364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":115,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.271889+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:26:06.271889+0000","last_peered":"2023-09-30T09:26:06.271889+0000","last_clean":"2023-09-30T09:26:06.271889+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:26:06.271889+0000","last_undegraded":"2023-09-30T09:26:06.271889+0000","last_fullsized":"2023-09-30T09:26:06.271889+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"173'7392","reported_seq":7460,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:08.912040+0000","last_change":"2023-09-30T09:22:48.113213+0000","last_active":"2023-09-30T09:26:08.912040+0000","last_peered":"2023-09-30T09:26:08.912040+0000","last_clean":"2023-09-30T09:26:08.912040+0000","last_became_active":"2023-09-30T09:22:48.112904+0000","last_became_peered":"2023-09-30T09:22:48.112904+0000","last_unstale":"2023-09-30T09:26:08.912040+0000","last_undegraded":"2023-09-30T09:26:08.912040+0000","last_fullsized":"2023-09-30T09:26:08.912040+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7392,"log_dups_size":0,"ondisk_log_size":7392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:37:37.122003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189399040,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7392,"num_write_kb":329024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"173'7334","reported_seq":18077,"reported_epoch":173,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:26:09.083983+0000","last_change":"2023-09-30T09:25:15.494362+0000","last_active":"2023-09-30T09:26:09.083983+0000","last_peered":"2023-09-30T09:26:09.083983+0000","last_clean":"2023-09-30T09:25:13.127337+0000","last_became_active":"2023-09-30T09:25:15.484895+0000","last_became_peered":"2023-09-30T09:25:15.484895+0000","last_unstale":"2023-09-30T09:26:09.083983+0000","last_undegraded":"2023-09-30T09:26:09.083983+0000","last_fullsized":"2023-09-30T09:26:09.083983+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7334,"log_dups_size":0,"ondisk_log_size":7334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:16.127121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190447616,"num_objects":2906,"num_object_clones":0,"num_object_copies":8718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2906,"num_objects_unfound":0,"num_objects_dirty":2906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7334,"num_write_kb":327680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5313,"num_bytes_recovered":348192768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2906}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":733,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.262358+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:26:06.262358+0000","last_peered":"2023-09-30T09:26:06.262358+0000","last_clean":"2023-09-30T09:26:06.262358+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:26:06.262358+0000","last_undegraded":"2023-09-30T09:26:06.262358+0000","last_fullsized":"2023-09-30T09:26:06.262358+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":80,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.254363+0000","last_change":"2023-09-30T09:25:04.283467+0000","last_active":"2023-09-30T09:26:06.254363+0000","last_peered":"2023-09-30T09:26:06.254363+0000","last_clean":"2023-09-30T09:26:06.254363+0000","last_became_active":"2023-09-30T09:25:04.283162+0000","last_became_peered":"2023-09-30T09:25:04.283162+0000","last_unstale":"2023-09-30T09:26:06.254363+0000","last_undegraded":"2023-09-30T09:26:06.254363+0000","last_fullsized":"2023-09-30T09:26:06.254363+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:12:35.333268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":64,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.254301+0000","last_change":"2023-09-30T09:23:08.024242+0000","last_active":"2023-09-30T09:26:06.254301+0000","last_peered":"2023-09-30T09:26:06.254301+0000","last_clean":"2023-09-30T09:26:06.254301+0000","last_became_active":"2023-09-30T09:23:08.023891+0000","last_became_peered":"2023-09-30T09:23:08.023891+0000","last_unstale":"2023-09-30T09:26:06.254301+0000","last_undegraded":"2023-09-30T09:26:06.254301+0000","last_fullsized":"2023-09-30T09:26:06.254301+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:19:36.687182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":813,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.263347+0000","last_change":"2023-09-30T09:23:11.125442+0000","last_active":"2023-09-30T09:26:06.263347+0000","last_peered":"2023-09-30T09:26:06.263347+0000","last_clean":"2023-09-30T09:26:06.263347+0000","last_became_active":"2023-09-30T09:23:11.124962+0000","last_became_peered":"2023-09-30T09:23:11.124962+0000","last_unstale":"2023-09-30T09:26:06.263347+0000","last_undegraded":"2023-09-30T09:26:06.263347+0000","last_fullsized":"2023-09-30T09:26:06.263347+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:53.386815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":128,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.325471+0000","last_change":"2023-09-30T09:25:14.359586+0000","last_active":"2023-09-30T09:26:10.325471+0000","last_peered":"2023-09-30T09:26:10.325471+0000","last_clean":"2023-09-30T09:26:10.325471+0000","last_became_active":"2023-09-30T09:25:14.359198+0000","last_became_peered":"2023-09-30T09:25:14.359198+0000","last_unstale":"2023-09-30T09:26:10.325471+0000","last_undegraded":"2023-09-30T09:26:10.325471+0000","last_fullsized":"2023-09-30T09:26:10.325471+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:48.533039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":60,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.255641+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:26:06.255641+0000","last_peered":"2023-09-30T09:26:06.255641+0000","last_clean":"2023-09-30T09:26:06.255641+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:26:06.255641+0000","last_undegraded":"2023-09-30T09:26:06.255641+0000","last_fullsized":"2023-09-30T09:26:06.255641+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"173'7611","reported_seq":10807,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:08.891927+0000","last_change":"2023-09-30T09:24:12.672495+0000","last_active":"2023-09-30T09:26:08.891927+0000","last_peered":"2023-09-30T09:26:08.891927+0000","last_clean":"2023-09-30T09:26:08.891927+0000","last_became_active":"2023-09-30T09:24:12.668774+0000","last_became_peered":"2023-09-30T09:24:12.668774+0000","last_unstale":"2023-09-30T09:26:08.891927+0000","last_undegraded":"2023-09-30T09:26:08.891927+0000","last_fullsized":"2023-09-30T09:26:08.891927+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7611,"log_dups_size":0,"ondisk_log_size":7611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:01:27.375538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201654272,"num_objects":3077,"num_object_clones":0,"num_object_copies":9231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7611,"num_write_kb":342016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1553,"num_bytes_recovered":101777408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":161,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254845+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:26:10.254845+0000","last_peered":"2023-09-30T09:26:10.254845+0000","last_clean":"2023-09-30T09:26:10.254845+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:26:10.254845+0000","last_undegraded":"2023-09-30T09:26:10.254845+0000","last_fullsized":"2023-09-30T09:26:10.254845+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1122,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.325486+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:26:10.325486+0000","last_peered":"2023-09-30T09:26:10.325486+0000","last_clean":"2023-09-30T09:26:10.325486+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:26:10.325486+0000","last_undegraded":"2023-09-30T09:26:10.325486+0000","last_fullsized":"2023-09-30T09:26:10.325486+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":150,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254167+0000","last_change":"2023-09-30T09:23:08.028741+0000","last_active":"2023-09-30T09:26:10.254167+0000","last_peered":"2023-09-30T09:26:10.254167+0000","last_clean":"2023-09-30T09:26:10.254167+0000","last_became_active":"2023-09-30T09:23:08.028167+0000","last_became_peered":"2023-09-30T09:23:08.028167+0000","last_unstale":"2023-09-30T09:26:10.254167+0000","last_undegraded":"2023-09-30T09:26:10.254167+0000","last_fullsized":"2023-09-30T09:26:10.254167+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:47:39.241913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"173'7533","reported_seq":7601,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-30T09:26:08.898992+0000","last_change":"2023-09-30T09:22:48.113865+0000","last_active":"2023-09-30T09:26:08.898992+0000","last_peered":"2023-09-30T09:26:08.898992+0000","last_clean":"2023-09-30T09:26:08.898992+0000","last_became_active":"2023-09-30T09:22:48.113578+0000","last_became_peered":"2023-09-30T09:22:48.113578+0000","last_unstale":"2023-09-30T09:26:08.898992+0000","last_undegraded":"2023-09-30T09:26:08.898992+0000","last_fullsized":"2023-09-30T09:26:08.898992+0000","mapping_epoch":113,"log_start":"117'300","ondisk_log_start":"117'300","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7233,"log_dups_size":0,"ondisk_log_size":7233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:17.848647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194838528,"num_objects":2973,"num_object_clones":0,"num_object_copies":8919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7533,"num_write_kb":336192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":426,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.272118+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:26:06.272118+0000","last_peered":"2023-09-30T09:26:06.272118+0000","last_clean":"2023-09-30T09:26:06.272118+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:26:06.272118+0000","last_undegraded":"2023-09-30T09:26:06.272118+0000","last_fullsized":"2023-09-30T09:26:06.272118+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":68,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.256680+0000","last_change":"2023-09-30T09:23:08.031587+0000","last_active":"2023-09-30T09:26:06.256680+0000","last_peered":"2023-09-30T09:26:06.256680+0000","last_clean":"2023-09-30T09:26:06.256680+0000","last_became_active":"2023-09-30T09:23:08.016566+0000","last_became_peered":"2023-09-30T09:23:08.016566+0000","last_unstale":"2023-09-30T09:26:06.256680+0000","last_undegraded":"2023-09-30T09:26:06.256680+0000","last_fullsized":"2023-09-30T09:26:06.256680+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:44:10.092575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":114,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.255148+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:26:06.255148+0000","last_peered":"2023-09-30T09:26:06.255148+0000","last_clean":"2023-09-30T09:26:06.255148+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:26:06.255148+0000","last_undegraded":"2023-09-30T09:26:06.255148+0000","last_fullsized":"2023-09-30T09:26:06.255148+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"175'7431","reported_seq":7505,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-30T09:26:12.091057+0000","last_change":"2023-09-30T09:22:48.113933+0000","last_active":"2023-09-30T09:26:12.091057+0000","last_peered":"2023-09-30T09:26:12.091057+0000","last_clean":"2023-09-30T09:26:12.091057+0000","last_became_active":"2023-09-30T09:22:48.113640+0000","last_became_peered":"2023-09-30T09:22:48.113640+0000","last_unstale":"2023-09-30T09:26:12.091057+0000","last_undegraded":"2023-09-30T09:26:12.091057+0000","last_fullsized":"2023-09-30T09:26:12.091057+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7431,"log_dups_size":0,"ondisk_log_size":7431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:16.995917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188416000,"num_objects":2875,"num_object_clones":0,"num_object_copies":8625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7431,"num_write_kb":329792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":60,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.255793+0000","last_change":"2023-09-30T09:23:08.024830+0000","last_active":"2023-09-30T09:26:06.255793+0000","last_peered":"2023-09-30T09:26:06.255793+0000","last_clean":"2023-09-30T09:26:06.255793+0000","last_became_active":"2023-09-30T09:23:08.024489+0000","last_became_peered":"2023-09-30T09:23:08.024489+0000","last_unstale":"2023-09-30T09:26:06.255793+0000","last_undegraded":"2023-09-30T09:26:06.255793+0000","last_fullsized":"2023-09-30T09:26:06.255793+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:41.769725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":155,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253421+0000","last_change":"2023-09-30T09:21:29.096256+0000","last_active":"2023-09-30T09:26:06.253421+0000","last_peered":"2023-09-30T09:26:06.253421+0000","last_clean":"2023-09-30T09:26:06.253421+0000","last_became_active":"2023-09-30T09:21:29.095930+0000","last_became_peered":"2023-09-30T09:21:29.095930+0000","last_unstale":"2023-09-30T09:26:06.253421+0000","last_undegraded":"2023-09-30T09:26:06.253421+0000","last_fullsized":"2023-09-30T09:26:06.253421+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:25.270756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":134,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.260271+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:26:06.260271+0000","last_peered":"2023-09-30T09:26:06.260271+0000","last_clean":"2023-09-30T09:26:06.260271+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:26:06.260271+0000","last_undegraded":"2023-09-30T09:26:06.260271+0000","last_fullsized":"2023-09-30T09:26:06.260271+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":27,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.252551+0000","last_change":"2023-09-30T09:25:04.284772+0000","last_active":"2023-09-30T09:26:06.252551+0000","last_peered":"2023-09-30T09:26:06.252551+0000","last_clean":"2023-09-30T09:26:06.252551+0000","last_became_active":"2023-09-30T09:25:04.284405+0000","last_became_peered":"2023-09-30T09:25:04.284405+0000","last_unstale":"2023-09-30T09:26:06.252551+0000","last_undegraded":"2023-09-30T09:26:06.252551+0000","last_fullsized":"2023-09-30T09:26:06.252551+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:44.112365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":117,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.262317+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:26:06.262317+0000","last_peered":"2023-09-30T09:26:06.262317+0000","last_clean":"2023-09-30T09:26:06.262317+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:26:06.262317+0000","last_undegraded":"2023-09-30T09:26:06.262317+0000","last_fullsized":"2023-09-30T09:26:06.262317+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.252414+0000","last_change":"2023-09-30T09:25:04.306048+0000","last_active":"2023-09-30T09:26:06.252414+0000","last_peered":"2023-09-30T09:26:06.252414+0000","last_clean":"2023-09-30T09:26:06.252414+0000","last_became_active":"2023-09-30T09:25:04.305804+0000","last_became_peered":"2023-09-30T09:25:04.305804+0000","last_unstale":"2023-09-30T09:26:06.252414+0000","last_undegraded":"2023-09-30T09:26:06.252414+0000","last_fullsized":"2023-09-30T09:26:06.252414+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:32.116748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":145,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254582+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:26:10.254582+0000","last_peered":"2023-09-30T09:26:10.254582+0000","last_clean":"2023-09-30T09:26:10.254582+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:26:10.254582+0000","last_undegraded":"2023-09-30T09:26:10.254582+0000","last_fullsized":"2023-09-30T09:26:10.254582+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":175,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-30T09:26:10.254251+0000","last_change":"2023-09-30T09:23:08.021721+0000","last_active":"2023-09-30T09:26:10.254251+0000","last_peered":"2023-09-30T09:26:10.254251+0000","last_clean":"2023-09-30T09:26:10.254251+0000","last_became_active":"2023-09-30T09:23:08.021401+0000","last_became_peered":"2023-09-30T09:23:08.021401+0000","last_unstale":"2023-09-30T09:26:10.254251+0000","last_undegraded":"2023-09-30T09:26:10.254251+0000","last_fullsized":"2023-09-30T09:26:10.254251+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:33:33.611604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":70,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253830+0000","last_change":"2023-09-30T09:23:08.039181+0000","last_active":"2023-09-30T09:26:06.253830+0000","last_peered":"2023-09-30T09:26:06.253830+0000","last_clean":"2023-09-30T09:26:06.253830+0000","last_became_active":"2023-09-30T09:23:08.025651+0000","last_became_peered":"2023-09-30T09:23:08.025651+0000","last_unstale":"2023-09-30T09:26:06.253830+0000","last_undegraded":"2023-09-30T09:26:06.253830+0000","last_fullsized":"2023-09-30T09:26:06.253830+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:27.721207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":131,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-30T09:26:06.253325+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:26:06.253325+0000","last_peered":"2023-09-30T09:26:06.253325+0000","last_clean":"2023-09-30T09:26:06.253325+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:26:06.253325+0000","last_undegraded":"2023-09-30T09:26:06.253325+0000","last_fullsized":"2023-09-30T09:26:06.253325+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":3052797976,"num_objects":46583,"num_object_clones":0,"num_object_copies":139749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2906,"num_objects_unfound":0,"num_objects_dirty":46583,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":118969,"num_write_kb":5297601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20725,"num_bytes_recovered":1229717528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9888022528,"data_stored":9888006240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118453,"ondisk_log_size":118453,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":738,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234174,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1063104,"kb_used_data":1004472,"kb_used_omap":0,"kb_used_meta":58560,"kb_avail":92682048,"statfs":{"total":95995035648,"available":94906417152,"internally_reserved":0,"allocated":1028579328,"data_stored":1027831865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":59965440},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:17 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.78700000000000003,"15min":0.68100000000000005},"min":{"1min":0.30099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":1.444,"5min":7.3559999999999999,"15min":7.3559999999999999},"last":2.5329999999999999},{"interface":"front","average":{"1min":0.441,"5min":0.629,"15min":0.65000000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.59499999999999997,"5min":4.8289999999999997,"15min":4.8289999999999997},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":2.4289999999999998,"15min":1.175},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.212,"5min":82.025999999999996,"15min":82.025999999999996},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":2.3980000000000001,"15min":1.1799999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":82.012,"15min":82.012},"last":0.308}]},{"osd":2,"last update":"Sat Sep 30 09:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4009999999999998,"5min":1.288,"15min":0.83299999999999996},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":17.838000000000001,"5min":17.838000000000001,"15min":17.838000000000001},"last":0.495},{"interface":"front","average":{"1min":2.6469999999999998,"5min":1.462,"15min":0.89300000000000002},"min":{"1min":0.34799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":17.805,"5min":17.805,"15min":17.805},"last":0.34799999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:25:40 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.91000000000000003,"15min":0.71699999999999997},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0090000000000003,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.753,"15min":0.67500000000000004},"min":{"1min":0.40200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.3540000000000001,"5min":4.8150000000000004,"15min":4.8150000000000004},"last":2.6360000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:25:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.56399999999999995,"15min":0.497},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.3580000000000001,"5min":3.3580000000000001,"15min":3.3580000000000001},"last":0.252},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.69599999999999995,"15min":0.56399999999999995},"min":{"1min":0.28999999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.282,"5min":6.282,"15min":6.282},"last":0.32200000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:25:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.67400000000000004,"15min":0.56599999999999995},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.504,"5min":5.968,"15min":5.968},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.72299999999999998,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.4790000000000001,"5min":5.452,"15min":5.452},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:25:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.70699999999999996,"15min":0.61199999999999999},"min":{"1min":0.39100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.4580000000000002,"5min":3.4580000000000002,"15min":3.4580000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.016,"5min":0.751,"15min":0.626},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.3529999999999998,"5min":4.3529999999999998,"15min":4.3529999999999998},"last":0.375}]}]},{"osd":6,"up_from":46,"seq":197568495811,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920748,"kb_used_data":860964,"kb_used_omap":0,"kb_used_meta":59776,"kb_avail":92824404,"statfs":{"total":95995035648,"available":95052189696,"internally_reserved":0,"allocated":881627136,"data_stored":880899414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":61210624},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.83999999999999997,"15min":0.70299999999999996},"min":{"1min":0.23599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.4249999999999998,"5min":6.649,"15min":6.649},"last":2.8069999999999999},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.84499999999999997,"15min":0.71899999999999997},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.327,"5min":8.327,"15min":8.327},"last":2.907}]},{"osd":1,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.83399999999999996,"15min":0.69999999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.2970000000000002,"5min":6.5090000000000003,"15min":6.5090000000000003},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.81299999999999994,"15min":0.69399999999999995},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2810000000000001,"5min":4.944,"15min":4.944},"last":3.0510000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:26:04 2023","interfaces":[{"interface":"back","average":{"1min":4.5720000000000001,"5min":1.752,"15min":0.96499999999999997},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":37.978000000000002,"5min":37.978000000000002,"15min":37.978000000000002},"last":7.5979999999999999},{"interface":"front","average":{"1min":3.7549999999999999,"5min":1.589,"15min":0.93400000000000005},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":37.759999999999998,"5min":37.759999999999998,"15min":37.759999999999998},"last":0.379}]},{"osd":3,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.878,"15min":0.68999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.387,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.437},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.0309999999999999,"15min":0.73599999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7759999999999998,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":6.7999999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":1.2210000000000001,"15min":0.73499999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":0.59299999999999997,"5min":73.656000000000006,"15min":73.656000000000006},"last":6.8399999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":1.2829999999999999,"15min":0.77000000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58199999999999996,"5min":73.882999999999996,"15min":73.882999999999996},"last":3.4129999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.64100000000000001,"15min":0.54200000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":0.48999999999999999,"5min":4.2430000000000003,"15min":4.2430000000000003},"last":1.3779999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.71399999999999997,"15min":0.60899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28000000000000003},"max":{"1min":0.60299999999999998,"5min":6.5999999999999996,"15min":6.5999999999999996},"last":11.725}]},{"osd":7,"last update":"Sat Sep 30 09:25:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.79500000000000004,"15min":0.65100000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.26500000000000001},"max":{"1min":12.709,"5min":12.709,"15min":12.709},"last":3.339},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.077,"15min":0.76600000000000001},"min":{"1min":0.40899999999999997,"5min":0.35599999999999998,"15min":0.31900000000000001},"max":{"1min":9.5640000000000001,"5min":10.339,"15min":10.339},"last":3.355}]}]},{"osd":1,"up_from":99,"seq":425201762376,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":989132,"kb_used_data":921412,"kb_used_omap":5,"kb_used_meta":67706,"kb_avail":92756020,"statfs":{"total":95995035648,"available":94982164480,"internally_reserved":0,"allocated":943525888,"data_stored":942545382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":69331126},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":4.2450000000000001,"5min":2.0859999999999999,"15min":0.94199999999999995},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":62.935000000000002,"5min":72.007000000000005,"15min":72.007000000000005},"last":0.33400000000000002},{"interface":"front","average":{"1min":3.907,"5min":2.1779999999999999,"15min":0.997},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":63.088000000000001,"5min":72.322999999999993,"15min":72.322999999999993},"last":0.24299999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:26:01 2023","interfaces":[{"interface":"back","average":{"1min":11.202999999999999,"5min":2.7810000000000001,"15min":1.2090000000000001},"min":{"1min":0.25600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":95.245000000000005,"5min":95.245000000000005,"15min":95.245000000000005},"last":0.372},{"interface":"front","average":{"1min":9.6969999999999992,"5min":2.5910000000000002,"15min":1.129},"min":{"1min":0.29099999999999998,"5min":0.254,"15min":0.254},"max":{"1min":95.019000000000005,"5min":95.019000000000005,"15min":95.019000000000005},"last":0.309}]},{"osd":3,"last update":"Sat Sep 30 09:25:36 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.74199999999999999,"15min":0.52100000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.845000000000001,"5min":11.845000000000001,"15min":11.845000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.72099999999999997,"15min":0.502},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.6499999999999999,"5min":11.06,"15min":11.06},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.80500000000000005,"15min":0.69799999999999995},"min":{"1min":0.317,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.569,"5min":5.6340000000000003,"15min":5.6340000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.81299999999999994,"15min":0.72499999999999998},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.9470000000000001,"5min":4.3869999999999996,"15min":4.3869999999999996},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.73499999999999999,"15min":0.65900000000000003},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.619,"5min":5.0140000000000002,"15min":5.0140000000000002},"last":0.432},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.83399999999999996,"15min":0.70999999999999996},"min":{"1min":0.439,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.9849999999999999,"5min":2.9929999999999999,"15min":2.9929999999999999},"last":0.47799999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.81399999999999995,"15min":0.68600000000000005},"min":{"1min":0.44900000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":2.851,"5min":3.0150000000000001,"15min":3.0150000000000001},"last":0.501},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.91100000000000003,"15min":0.76000000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.8140000000000001,"5min":7.8140000000000001,"15min":7.8140000000000001},"last":0.58599999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.96499999999999997,"15min":0.77500000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.9079999999999999,"5min":7.4820000000000002,"15min":7.4820000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.018,"15min":0.80200000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":7.7729999999999997,"5min":7.7729999999999997,"15min":7.7729999999999997},"last":0.56399999999999995}]}]},{"osd":0,"up_from":94,"seq":403726925908,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1747888,"kb_used_data":1567712,"kb_used_omap":5,"kb_used_meta":180154,"kb_avail":91997264,"statfs":{"total":95995035648,"available":94205198336,"internally_reserved":0,"allocated":1605337088,"data_stored":1604326160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":184477949},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.81599999999999995,"15min":0.51200000000000001},"min":{"1min":0.22,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.5859999999999999,"5min":15.9,"15min":15.9},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.92500000000000004,"15min":0.56399999999999995},"min":{"1min":0.27800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.8849999999999998,"5min":13.824999999999999,"15min":13.824999999999999},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:26:02 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.1359999999999999,"15min":0.63100000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":3.7669999999999999,"5min":18.096,"15min":18.096},"last":30.359999999999999},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.167,"15min":0.67900000000000005},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.258,"5min":18.149000000000001,"15min":18.149000000000001},"last":31.661000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":0.98199999999999998,"15min":0.65100000000000002},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.9630000000000001,"5min":13.02,"15min":13.02},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.96799999999999997,"15min":0.60999999999999999},"min":{"1min":0.29999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.847,"5min":10.128,"15min":10.128},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.86099999999999999,"15min":0.78500000000000003},"min":{"1min":0.38700000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.369,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":1.4159999999999999},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.96999999999999997,"15min":0.84699999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":3.169,"5min":12.99,"15min":12.99},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.1719999999999999,"15min":0.94399999999999995},"min":{"1min":0.38400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.266,"5min":12.949999999999999,"15min":12.949999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.151,"15min":0.94199999999999995},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.105,"5min":12.823,"15min":12.823},"last":0.79500000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.91000000000000003,"15min":0.84599999999999997},"min":{"1min":0.432,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.038,"5min":6.9660000000000002,"15min":6.9660000000000002},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.745,"5min":0.85999999999999999,"15min":0.871},"min":{"1min":0.48799999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.5430000000000001,"5min":7.556,"15min":7.556},"last":1.4490000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.96099999999999997,"15min":0.83999999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.1699999999999999,"5min":7.0339999999999998,"15min":7.0339999999999998},"last":0.747},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.85799999999999998,"15min":0.86599999999999999},"min":{"1min":0.47399999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.504,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.60899999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598849,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1415464,"kb_used_data":1303328,"kb_used_omap":5,"kb_used_meta":112122,"kb_avail":92329688,"statfs":{"total":95995035648,"available":94545600512,"internally_reserved":0,"allocated":1334607872,"data_stored":1333652210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":114813581},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.268,"15min":0.66600000000000004},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.172000000000001,"5min":15.843999999999999,"15min":15.843999999999999},"last":4.2149999999999999},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.03,"15min":0.64500000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.3620000000000001,"5min":10.065,"15min":10.065},"last":1.8899999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0960000000000001,"5min":1.006,"15min":0.59199999999999997},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.9120000000000008,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":8.9120000000000008},{"interface":"front","average":{"1min":1.5029999999999999,"5min":0.95599999999999996,"15min":0.59999999999999998},"min":{"1min":0.28999999999999998,"5min":0.249,"15min":0.249},"max":{"1min":5.016,"5min":6.5369999999999999,"15min":6.5369999999999999},"last":1.917}]},{"osd":3,"last update":"Sat Sep 30 09:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0009999999999999,"15min":0.63100000000000001},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.4009999999999998,"5min":18.635999999999999,"15min":18.635999999999999},"last":1.9950000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.173,"15min":0.72599999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.996,"5min":18.026,"15min":18.026},"last":1.712}]},{"osd":4,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.86899999999999999,"15min":0.69399999999999995},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.3069999999999999,"5min":4.1719999999999997,"15min":4.1719999999999997},"last":1.849},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.2669999999999999,"15min":0.85299999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.125999999999999,"5min":11.125999999999999,"15min":11.125999999999999},"last":1.367}]},{"osd":5,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.0840000000000001,"15min":0.79500000000000004},"min":{"1min":0.38300000000000001,"5min":0.371,"15min":0.371},"max":{"1min":8.8640000000000008,"5min":8.8640000000000008,"15min":8.8640000000000008},"last":8.8640000000000008},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.1990000000000001,"15min":0.83699999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":11.866,"5min":11.866,"15min":11.866},"last":11.866}]},{"osd":6,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.504,"15min":0.95999999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":11.247999999999999,"5min":11.247999999999999,"15min":11.247999999999999},"last":1.3360000000000001},{"interface":"front","average":{"1min":2.2850000000000001,"5min":1.226,"15min":0.85999999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":8.9619999999999997,"5min":8.9619999999999997,"15min":8.9619999999999997},"last":8.9619999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.409,"15min":0.92300000000000004},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":1.391},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.163,"15min":0.86499999999999999},"min":{"1min":0.45100000000000001,"5min":0.439,"15min":0.439},"max":{"1min":4.923,"5min":5.3979999999999997,"15min":5.3979999999999997},"last":2.3279999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435323,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":692868,"kb_used_data":621048,"kb_used_omap":2,"kb_used_meta":71805,"kb_avail":93052284,"statfs":{"total":95995035648,"available":95285538816,"internally_reserved":0,"allocated":635953152,"data_stored":634980333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":73528679},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":2.5920000000000001,"15min":1.121},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.5619999999999998,"5min":130.946,"15min":130.946},"last":3.5910000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":2.5169999999999999,"15min":1.125},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.7470000000000001,"5min":130.96899999999999,"15min":130.96899999999999},"last":3.6970000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.70799999999999996,"15min":0.52000000000000002},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":6.0099999999999998,"5min":6.0099999999999998,"15min":6.0099999999999998},"last":4.1420000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.64100000000000001,"15min":0.48699999999999999},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.159,"5min":3.9340000000000002,"15min":3.9340000000000002},"last":4.3179999999999996}]},{"osd":2,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.831,"15min":1.2470000000000001},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1779999999999999,"5min":145.75899999999999,"15min":145.75899999999999},"last":1.593},{"interface":"front","average":{"1min":0.47199999999999998,"5min":2.7909999999999999,"15min":1.2170000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.1120000000000001,"5min":150.358,"15min":150.358},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.71299999999999997,"15min":0.64800000000000002},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.312},"max":{"1min":1.042,"5min":3.9119999999999999,"15min":3.9119999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.74099999999999999,"15min":0.65100000000000002},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.27,"5min":4.0309999999999997,"15min":4.0309999999999997},"last":0.61099999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.85799999999999998,"15min":0.70699999999999996},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.2390000000000001,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.86899999999999999,"15min":0.75800000000000001},"min":{"1min":0.375,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9100000000000001,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":1.6379999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.75900000000000001,"15min":0.70299999999999996},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.147,"5min":3.9590000000000001,"15min":3.9590000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.86499999999999999,"15min":0.746},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":3.0230000000000001,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.81100000000000005,"15min":0.72199999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.948,"5min":4.1639999999999997,"15min":4.1639999999999997},"last":1.6739999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.74199999999999999,"15min":0.70299999999999996},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.372},"max":{"1min":1.1479999999999999,"5min":3.9910000000000001,"15min":3.9910000000000001},"last":0.42599999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019084,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1778796,"kb_used_data":1692388,"kb_used_omap":0,"kb_used_meta":86272,"kb_avail":91966356,"statfs":{"total":95995035648,"available":94173548544,"internally_reserved":0,"allocated":1733005312,"data_stored":1732277161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":88342528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":2.0950000000000002,"15min":1.1279999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.69799999999999995,"5min":128.79300000000001,"15min":128.79300000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":1.883,"15min":1.0589999999999999},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":0.53400000000000003,"5min":128.38800000000001,"15min":128.38800000000001},"last":0.42199999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":1.0569999999999999,"15min":0.73399999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.71899999999999997,"5min":31.469999999999999,"15min":31.469999999999999},"last":0.375},{"interface":"front","average":{"1min":0.748,"5min":1.149,"15min":0.75700000000000001},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.2729999999999997,"5min":31.962,"15min":31.962},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:26:03 2023","interfaces":[{"interface":"back","average":{"1min":12.256,"5min":3.5630000000000002,"15min":1.595},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":140.601,"5min":140.601,"15min":140.601},"last":0.44500000000000001},{"interface":"front","average":{"1min":12.117000000000001,"5min":3.5600000000000001,"15min":1.5840000000000001},"min":{"1min":0.40500000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":140.54900000000001,"5min":140.54900000000001,"15min":140.54900000000001},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.607,"15min":0.93300000000000005},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.0019999999999998,"5min":86.620000000000005,"15min":86.620000000000005},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.747,"5min":1.5940000000000001,"15min":0.94399999999999995},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9929999999999999,"5min":86.786000000000001,"15min":86.786000000000001},"last":0.45200000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:25:14 2023","interfaces":[{"interface":"back","average":{"1min":4.4969999999999999,"5min":1.3160000000000001,"15min":0.75700000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":86.691000000000003,"5min":86.691000000000003,"15min":86.691000000000003},"last":0.318},{"interface":"front","average":{"1min":4.5309999999999997,"5min":1.3879999999999999,"15min":0.80300000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":86.822000000000003,"5min":86.822000000000003,"15min":86.822000000000003},"last":0.40200000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:25:39 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.55900000000000005,"15min":0.52600000000000002},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71299999999999997,"5min":3.218,"15min":3.218},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.58599999999999997,"15min":0.56799999999999995},"min":{"1min":0.33900000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74199999999999999,"5min":2.0209999999999999,"15min":2.0209999999999999},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:26:02 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.514,"15min":0.88600000000000001},"min":{"1min":0.373,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.5419999999999998,"5min":86.728999999999999,"15min":86.728999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.69899999999999995,"15min":0.623},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":3.0710000000000002,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":0.435}]}]},{"osd":5,"up_from":38,"seq":163208757448,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1897192,"kb_used_data":1799584,"kb_used_omap":0,"kb_used_meta":97600,"kb_avail":91847960,"statfs":{"total":95995035648,"available":94052311040,"internally_reserved":0,"allocated":1842774016,"data_stored":1842045967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":99942400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.048,"15min":0.73599999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":27.573,"5min":27.573,"15min":27.573},"last":0.23899999999999999},{"interface":"front","average":{"1min":2.2309999999999999,"5min":1.1830000000000001,"15min":0.80200000000000005},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":27.588999999999999,"5min":27.588999999999999,"15min":27.588999999999999},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":1.9359999999999999,"15min":1.03},"min":{"1min":0.33900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.24,"5min":100.593,"15min":100.593},"last":0.251},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.875,"15min":1.056},"min":{"1min":0.35099999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.5369999999999999,"5min":100.575,"15min":100.575},"last":0.377}]},{"osd":2,"last update":"Sat Sep 30 09:26:01 2023","interfaces":[{"interface":"back","average":{"1min":10.221,"5min":4.2000000000000002,"15min":1.798},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":165.768,"5min":165.768,"15min":165.768},"last":0.314},{"interface":"front","average":{"1min":18.170000000000002,"5min":5.9740000000000002,"15min":2.4209999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":166.24000000000001,"5min":166.24000000000001,"15min":166.24000000000001},"last":0.28499999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:25:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.446,"15min":0.91000000000000003},"min":{"1min":0.39700000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":28.015999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":2.8490000000000002},{"interface":"front","average":{"1min":2.024,"5min":1.29,"15min":0.85999999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":28.039999999999999,"5min":28.039999999999999,"15min":28.039999999999999},"last":2.8929999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:25:13 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.56000000000000005,"15min":0.49199999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.72399999999999998,"5min":4.2199999999999998,"15min":4.2199999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.68100000000000005,"15min":0.56799999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25900000000000001},"max":{"1min":0.68300000000000005,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.59099999999999997,"15min":0.54200000000000004},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.745,"5min":2.125,"15min":2.125},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.79100000000000004,"15min":0.66300000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":0.66600000000000004,"5min":4.46,"15min":4.46},"last":0.44}]},{"osd":7,"last update":"Sat Sep 30 09:25:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.75800000000000001,"15min":0.64200000000000002},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32500000000000001},"max":{"1min":1.1419999999999999,"5min":4.194,"15min":4.194},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.72199999999999998,"15min":0.623},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.605,"5min":6.2300000000000004,"15min":6.2300000000000004},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1583288320,"data_stored":1583284248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":930152448,"data_stored":930152448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1317933056,"data_stored":1317928984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":622592000,"data_stored":622592000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1728053248,"data_stored":1728053248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1818365952,"data_stored":1818361880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":866058240,"data_stored":866058240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1021579264,"data_stored":1021575192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:26:19.505 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:19.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:19.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:19.619 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:19.680 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:19.889 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:19.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:20.001 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:20.026 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:26:20.026 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:26:20.026 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:26:20.026 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:26:20.026 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:26:20.076 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:20.319 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:20 smithi019 ceph-mon[224043]: pgmap v397: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 2275/126792 objects misplaced (1.794%); 3.4 MiB/s, 59 objects/s recovering 2023-09-30T09:26:20.320 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:20 smithi019 ceph-mon[216815]: pgmap v397: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 2275/126792 objects misplaced (1.794%); 3.4 MiB/s, 59 objects/s recovering 2023-09-30T09:26:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:20 smithi167 ceph-mon[192880]: pgmap v397: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 2275/126792 objects misplaced (1.794%); 3.4 MiB/s, 59 objects/s recovering 2023-09-30T09:26:20.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:26:20.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:26:20.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:26:20.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:26:20.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:26:20.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:26:20.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:26:20.833 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:26:20.834 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:26:20.834 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:26:20.834 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:26:20.835 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:26:20.835 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:26:20.835 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:26:20.835 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:26:20.835 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:26:20.865 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:20.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:20.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:20.970 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:20.997 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:21.037 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:21.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:21.143 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:21.178 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:21.535 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:26:21.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[216815]: osdmap e177: 8 total, 8 up, 8 in 2023-09-30T09:26:21.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[216815]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[216815]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[216815]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[216815]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[216815]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[216815]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4275260766' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[224043]: osdmap e177: 8 total, 8 up, 8 in 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[224043]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[224043]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.592 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[224043]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.593 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[224043]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.593 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[224043]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.593 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[224043]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.593 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4275260766' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:21 smithi167 ceph-mon[192880]: osdmap e177: 8 total, 8 up, 8 in 2023-09-30T09:26:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:21 smithi167 ceph-mon[192880]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:21 smithi167 ceph-mon[192880]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:21 smithi167 ceph-mon[192880]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:21 smithi167 ceph-mon[192880]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:21 smithi167 ceph-mon[192880]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:21 smithi167 ceph-mon[192880]: failed to encode map e177 with expected crc 2023-09-30T09:26:21.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4275260766' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:22.474 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[216815]: pgmap v399: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 471/123120 objects misplaced (0.383%); 4.2 MiB/s, 72 objects/s recovering 2023-09-30T09:26:22.474 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[216815]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.474 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[216815]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.474 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[216815]: osdmap e178: 8 total, 8 up, 8 in 2023-09-30T09:26:22.475 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[216815]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.475 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[216815]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.475 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[216815]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.475 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[224043]: pgmap v399: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 471/123120 objects misplaced (0.383%); 4.2 MiB/s, 72 objects/s recovering 2023-09-30T09:26:22.475 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[224043]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.475 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[224043]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.475 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[224043]: osdmap e178: 8 total, 8 up, 8 in 2023-09-30T09:26:22.475 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[224043]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.476 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[224043]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.476 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:22 smithi019 ceph-mon[224043]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.506 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:26:22.514 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:26:22.661 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:22.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:22.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:22.768 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:22.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:22 smithi167 ceph-mon[192880]: pgmap v399: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 471/123120 objects misplaced (0.383%); 4.2 MiB/s, 72 objects/s recovering 2023-09-30T09:26:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:22 smithi167 ceph-mon[192880]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:22 smithi167 ceph-mon[192880]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:22 smithi167 ceph-mon[192880]: osdmap e178: 8 total, 8 up, 8 in 2023-09-30T09:26:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:22 smithi167 ceph-mon[192880]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:22 smithi167 ceph-mon[192880]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:22 smithi167 ceph-mon[192880]: failed to encode map e178 with expected crc 2023-09-30T09:26:22.870 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:22.976 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:22.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:23.092 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:23.096 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:23.137 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:23.222 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:23.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:23.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:23.339 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:23.387 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:23.479 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:23.501 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:23.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:23.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:23.538 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:23.614 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:23.681 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:23.748 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:23.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:23.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:23.853 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:23.932 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:24.058 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:26:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:26:23] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.43.0" 2023-09-30T09:26:24.227 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:24.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:24.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:24.340 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:24.395 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:24.506 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:24.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:24.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:24.616 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:24.679 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:24 smithi019 ceph-mon[224043]: pgmap v401: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 471/122253 objects misplaced (0.385%); 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:24.679 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:24 smithi019 ceph-mon[216815]: pgmap v401: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 471/122253 objects misplaced (0.385%); 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:24.682 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:24.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:24 smithi167 ceph-mon[192880]: pgmap v401: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 471/122253 objects misplaced (0.385%); 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:24.803 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:26:25.571 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:25.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:25.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:25.683 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:25.745 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:25.854 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:25.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:25.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:25.975 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:26.046 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:26.128 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:26.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:26.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:26.244 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:26:26.319 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:26.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:26 smithi019 ceph-mon[216815]: pgmap v402: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:26.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:26 smithi019 ceph-mon[224043]: pgmap v402: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:26.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:26 smithi167 ceph-mon[192880]: pgmap v402: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:27.050 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:26:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:27.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:26:27.163 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:27.249 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:27.698 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:27.713 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:27.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:27.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:27.817 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:27.854 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:27.925 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:27.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:27.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:27.988 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:26:28.014 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:26:28.055 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:28.188 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:28.253 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:26:28.254 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:26:28.620 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:28 smithi019 ceph-mon[216815]: pgmap v403: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:28.917 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1053285332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:26:28.917 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:28 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:26:28.918 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:28 smithi167 ceph-mon[192880]: pgmap v403: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:28.930 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1053285332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:26:28.930 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:28 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:26:28.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:28 smithi019 ceph-mon[224043]: pgmap v403: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 848 KiB/s, 13 objects/s recovering 2023-09-30T09:26:28.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1053285332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:26:28.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:28 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:26:29.209 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:26:29.506 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:29 smithi019 ceph-mon[216815]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:26:29.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:29 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:29.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:29 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:26:29.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:29 smithi019 ceph-mon[216815]: osdmap e179: 8 total, 8 up, 8 in 2023-09-30T09:26:29.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:29 smithi019 ceph-mon[224043]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:26:29.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:29 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:29.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:29 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:26:29.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:29 smithi019 ceph-mon[224043]: osdmap e179: 8 total, 8 up, 8 in 2023-09-30T09:26:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:29 smithi167 ceph-mon[192880]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:26:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:29 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:29 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:26:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:29 smithi167 ceph-mon[192880]: osdmap e179: 8 total, 8 up, 8 in 2023-09-30T09:26:29.919 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":403,"stamp":"2023-09-30T09:26:27.065038+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2478249919,"num_objects":37771,"num_object_clones":0,"num_object_copies":113313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37771,"num_whiteouts":0,"num_read":6443,"num_read_kb":13071,"num_write":128955,"num_write_kb":5312742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21426,"num_bytes_recovered":1276382618,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127385,"ondisk_log_size":127385,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8549472,"kb_used_data":7824280,"kb_used_omap":19,"kb_used_meta":724972,"kb_avail":741411744,"statfs":{"total":767960285184,"available":759205625856,"internally_reserved":0,"allocated":8012062720,"data_stored":8005065832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":742371751},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,3,3,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-294453248,"num_objects":-4493,"num_object_clones":0,"num_object_copies":-13479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2275,"num_objects_unfound":0,"num_objects_dirty":-4493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4493,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":106,"num_bytes_recovered":6946816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004511"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":155,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.352310+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:26:20.352310+0000","last_peered":"2023-09-30T09:26:20.352310+0000","last_clean":"2023-09-30T09:26:20.352310+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:26:20.352310+0000","last_undegraded":"2023-09-30T09:26:20.352310+0000","last_fullsized":"2023-09-30T09:26:20.352310+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":141,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.330846+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:26:20.330846+0000","last_peered":"2023-09-30T09:26:20.330846+0000","last_clean":"2023-09-30T09:26:20.330846+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:26:20.330846+0000","last_undegraded":"2023-09-30T09:26:20.330846+0000","last_fullsized":"2023-09-30T09:26:20.330846+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336420+0000","last_change":"2023-09-30T09:25:04.281698+0000","last_active":"2023-09-30T09:26:20.336420+0000","last_peered":"2023-09-30T09:26:20.336420+0000","last_clean":"2023-09-30T09:26:20.336420+0000","last_became_active":"2023-09-30T09:25:04.281351+0000","last_became_peered":"2023-09-30T09:25:04.281351+0000","last_unstale":"2023-09-30T09:26:20.336420+0000","last_undegraded":"2023-09-30T09:26:20.336420+0000","last_fullsized":"2023-09-30T09:26:20.336420+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:17:40.382369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":69,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.335060+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:26:20.335060+0000","last_peered":"2023-09-30T09:26:20.335060+0000","last_clean":"2023-09-30T09:26:20.335060+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:26:20.335060+0000","last_undegraded":"2023-09-30T09:26:20.335060+0000","last_fullsized":"2023-09-30T09:26:20.335060+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":176,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.330770+0000","last_change":"2023-09-30T09:23:08.014781+0000","last_active":"2023-09-30T09:26:20.330770+0000","last_peered":"2023-09-30T09:26:20.330770+0000","last_clean":"2023-09-30T09:26:20.330770+0000","last_became_active":"2023-09-30T09:23:08.014387+0000","last_became_peered":"2023-09-30T09:23:08.014387+0000","last_unstale":"2023-09-30T09:26:20.330770+0000","last_undegraded":"2023-09-30T09:26:20.330770+0000","last_fullsized":"2023-09-30T09:26:20.330770+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:52:13.669946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.346620+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:26:20.346620+0000","last_peered":"2023-09-30T09:26:20.346620+0000","last_clean":"2023-09-30T09:26:20.346620+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:26:20.346620+0000","last_undegraded":"2023-09-30T09:26:20.346620+0000","last_fullsized":"2023-09-30T09:26:20.346620+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":69,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.335326+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:26:20.335326+0000","last_peered":"2023-09-30T09:26:20.335326+0000","last_clean":"2023-09-30T09:26:20.335326+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:26:20.335326+0000","last_undegraded":"2023-09-30T09:26:20.335326+0000","last_fullsized":"2023-09-30T09:26:20.335326+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.330708+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:26:20.330708+0000","last_peered":"2023-09-30T09:26:20.330708+0000","last_clean":"2023-09-30T09:26:20.330708+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:26:20.330708+0000","last_undegraded":"2023-09-30T09:26:20.330708+0000","last_fullsized":"2023-09-30T09:26:20.330708+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.353742+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:26:20.353742+0000","last_peered":"2023-09-30T09:26:20.353742+0000","last_clean":"2023-09-30T09:26:20.353742+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:26:20.353742+0000","last_undegraded":"2023-09-30T09:26:20.353742+0000","last_fullsized":"2023-09-30T09:26:20.353742+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":93,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.331784+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:26:20.331784+0000","last_peered":"2023-09-30T09:26:20.331784+0000","last_clean":"2023-09-30T09:26:20.331784+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:26:20.331784+0000","last_undegraded":"2023-09-30T09:26:20.331784+0000","last_fullsized":"2023-09-30T09:26:20.331784+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":141,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.345176+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:26:20.345176+0000","last_peered":"2023-09-30T09:26:20.345176+0000","last_clean":"2023-09-30T09:26:20.345176+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:26:20.345176+0000","last_undegraded":"2023-09-30T09:26:20.345176+0000","last_fullsized":"2023-09-30T09:26:20.345176+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":116,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336537+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:26:20.336537+0000","last_peered":"2023-09-30T09:26:20.336537+0000","last_clean":"2023-09-30T09:26:20.336537+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:26:20.336537+0000","last_undegraded":"2023-09-30T09:26:20.336537+0000","last_fullsized":"2023-09-30T09:26:20.336537+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.350462+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:26:20.350462+0000","last_peered":"2023-09-30T09:26:20.350462+0000","last_clean":"2023-09-30T09:26:20.350462+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:26:20.350462+0000","last_undegraded":"2023-09-30T09:26:20.350462+0000","last_fullsized":"2023-09-30T09:26:20.350462+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.349564+0000","last_change":"2023-09-30T09:23:03.986755+0000","last_active":"2023-09-30T09:26:20.349564+0000","last_peered":"2023-09-30T09:26:20.349564+0000","last_clean":"2023-09-30T09:26:20.349564+0000","last_became_active":"2023-09-30T09:23:03.986411+0000","last_became_peered":"2023-09-30T09:23:03.986411+0000","last_unstale":"2023-09-30T09:26:20.349564+0000","last_undegraded":"2023-09-30T09:26:20.349564+0000","last_fullsized":"2023-09-30T09:26:20.349564+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:53:23.627823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":140,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.344395+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:26:20.344395+0000","last_peered":"2023-09-30T09:26:20.344395+0000","last_clean":"2023-09-30T09:26:20.344395+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:26:20.344395+0000","last_undegraded":"2023-09-30T09:26:20.344395+0000","last_fullsized":"2023-09-30T09:26:20.344395+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":156,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.330984+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:26:20.330984+0000","last_peered":"2023-09-30T09:26:20.330984+0000","last_clean":"2023-09-30T09:26:20.330984+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:26:20.330984+0000","last_undegraded":"2023-09-30T09:26:20.330984+0000","last_fullsized":"2023-09-30T09:26:20.330984+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.334710+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:26:20.334710+0000","last_peered":"2023-09-30T09:26:20.334710+0000","last_clean":"2023-09-30T09:26:20.334710+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:26:20.334710+0000","last_undegraded":"2023-09-30T09:26:20.334710+0000","last_fullsized":"2023-09-30T09:26:20.334710+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.353658+0000","last_change":"2023-09-30T09:25:14.358999+0000","last_active":"2023-09-30T09:26:20.353658+0000","last_peered":"2023-09-30T09:26:20.353658+0000","last_clean":"2023-09-30T09:26:20.353658+0000","last_became_active":"2023-09-30T09:25:14.358650+0000","last_became_peered":"2023-09-30T09:25:14.358650+0000","last_unstale":"2023-09-30T09:26:20.353658+0000","last_undegraded":"2023-09-30T09:26:20.353658+0000","last_fullsized":"2023-09-30T09:26:20.353658+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:57:41.373342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.344236+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:26:20.344236+0000","last_peered":"2023-09-30T09:26:20.344236+0000","last_clean":"2023-09-30T09:26:20.344236+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:26:20.344236+0000","last_undegraded":"2023-09-30T09:26:20.344236+0000","last_fullsized":"2023-09-30T09:26:20.344236+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":69,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.334910+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:26:20.334910+0000","last_peered":"2023-09-30T09:26:20.334910+0000","last_clean":"2023-09-30T09:26:20.334910+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:26:20.334910+0000","last_undegraded":"2023-09-30T09:26:20.334910+0000","last_fullsized":"2023-09-30T09:26:20.334910+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":226,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.343270+0000","last_change":"2023-09-30T09:25:14.389693+0000","last_active":"2023-09-30T09:26:20.343270+0000","last_peered":"2023-09-30T09:26:20.343270+0000","last_clean":"2023-09-30T09:26:20.343270+0000","last_became_active":"2023-09-30T09:25:14.365482+0000","last_became_peered":"2023-09-30T09:25:14.365482+0000","last_unstale":"2023-09-30T09:26:20.343270+0000","last_undegraded":"2023-09-30T09:26:20.343270+0000","last_fullsized":"2023-09-30T09:26:20.343270+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:09:23.017579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":156,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.331005+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:26:20.331005+0000","last_peered":"2023-09-30T09:26:20.331005+0000","last_clean":"2023-09-30T09:26:20.331005+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:26:20.331005+0000","last_undegraded":"2023-09-30T09:26:20.331005+0000","last_fullsized":"2023-09-30T09:26:20.331005+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"178'7959","reported_seq":8032,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:23.910503+0000","last_change":"2023-09-30T09:22:48.114186+0000","last_active":"2023-09-30T09:26:23.910503+0000","last_peered":"2023-09-30T09:26:23.910503+0000","last_clean":"2023-09-30T09:26:23.910503+0000","last_became_active":"2023-09-30T09:22:48.113857+0000","last_became_peered":"2023-09-30T09:22:48.113857+0000","last_unstale":"2023-09-30T09:26:23.910503+0000","last_undegraded":"2023-09-30T09:26:23.910503+0000","last_fullsized":"2023-09-30T09:26:23.910503+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7959,"log_dups_size":0,"ondisk_log_size":7959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:18.973595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154206208,"num_objects":2353,"num_object_clones":0,"num_object_copies":7059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7959,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":167,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.345364+0000","last_change":"2023-09-30T09:23:08.038825+0000","last_active":"2023-09-30T09:26:20.345364+0000","last_peered":"2023-09-30T09:26:20.345364+0000","last_clean":"2023-09-30T09:26:20.345364+0000","last_became_active":"2023-09-30T09:23:08.029150+0000","last_became_peered":"2023-09-30T09:23:08.029150+0000","last_unstale":"2023-09-30T09:26:20.345364+0000","last_undegraded":"2023-09-30T09:26:20.345364+0000","last_fullsized":"2023-09-30T09:26:20.345364+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:14.557254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.353383+0000","last_change":"2023-09-30T09:25:14.359647+0000","last_active":"2023-09-30T09:26:20.353383+0000","last_peered":"2023-09-30T09:26:20.353383+0000","last_clean":"2023-09-30T09:26:20.353383+0000","last_became_active":"2023-09-30T09:25:14.359350+0000","last_became_peered":"2023-09-30T09:25:14.359350+0000","last_unstale":"2023-09-30T09:26:20.353383+0000","last_undegraded":"2023-09-30T09:26:20.353383+0000","last_fullsized":"2023-09-30T09:26:20.353383+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:09.489014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"178'8130","reported_seq":11301,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:24.783227+0000","last_change":"2023-09-30T09:24:25.563637+0000","last_active":"2023-09-30T09:26:24.783227+0000","last_peered":"2023-09-30T09:26:24.783227+0000","last_clean":"2023-09-30T09:26:24.783227+0000","last_became_active":"2023-09-30T09:24:25.554649+0000","last_became_peered":"2023-09-30T09:24:25.554649+0000","last_unstale":"2023-09-30T09:26:24.783227+0000","last_undegraded":"2023-09-30T09:26:24.783227+0000","last_fullsized":"2023-09-30T09:26:24.783227+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:09:56.974492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154402816,"num_objects":2356,"num_object_clones":0,"num_object_copies":7068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8130,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1538,"num_bytes_recovered":100794368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":168,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.346213+0000","last_change":"2023-09-30T09:25:04.304869+0000","last_active":"2023-09-30T09:26:20.346213+0000","last_peered":"2023-09-30T09:26:20.346213+0000","last_clean":"2023-09-30T09:26:20.346213+0000","last_became_active":"2023-09-30T09:25:04.285159+0000","last_became_peered":"2023-09-30T09:25:04.285159+0000","last_unstale":"2023-09-30T09:26:20.346213+0000","last_undegraded":"2023-09-30T09:26:20.346213+0000","last_fullsized":"2023-09-30T09:26:20.346213+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:03:13.971240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.349115+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:26:20.349115+0000","last_peered":"2023-09-30T09:26:20.349115+0000","last_clean":"2023-09-30T09:26:20.349115+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:26:20.349115+0000","last_undegraded":"2023-09-30T09:26:20.349115+0000","last_fullsized":"2023-09-30T09:26:20.349115+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"178'7960","reported_seq":12118,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:24.671391+0000","last_change":"2023-09-30T09:26:00.198334+0000","last_active":"2023-09-30T09:26:24.671391+0000","last_peered":"2023-09-30T09:26:24.671391+0000","last_clean":"2023-09-30T09:26:24.671391+0000","last_became_active":"2023-09-30T09:26:00.182970+0000","last_became_peered":"2023-09-30T09:26:00.182970+0000","last_unstale":"2023-09-30T09:26:24.671391+0000","last_undegraded":"2023-09-30T09:26:24.671391+0000","last_fullsized":"2023-09-30T09:26:24.671391+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7960,"log_dups_size":0,"ondisk_log_size":7960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:34.041047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155582464,"num_objects":2374,"num_object_clones":0,"num_object_copies":7122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7960,"num_write_kb":330688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2027,"num_bytes_recovered":108331008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":150,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.345358+0000","last_change":"2023-09-30T09:25:14.385912+0000","last_active":"2023-09-30T09:26:20.345358+0000","last_peered":"2023-09-30T09:26:20.345358+0000","last_clean":"2023-09-30T09:26:20.345358+0000","last_became_active":"2023-09-30T09:25:14.359789+0000","last_became_peered":"2023-09-30T09:25:14.359789+0000","last_unstale":"2023-09-30T09:26:20.345358+0000","last_undegraded":"2023-09-30T09:26:20.345358+0000","last_fullsized":"2023-09-30T09:26:20.345358+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:54.213336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.343542+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:26:20.343542+0000","last_peered":"2023-09-30T09:26:20.343542+0000","last_clean":"2023-09-30T09:26:20.343542+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:26:20.343542+0000","last_undegraded":"2023-09-30T09:26:20.343542+0000","last_fullsized":"2023-09-30T09:26:20.343542+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"178'7887","reported_seq":8842,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:24.764119+0000","last_change":"2023-09-30T09:25:16.371393+0000","last_active":"2023-09-30T09:26:24.764119+0000","last_peered":"2023-09-30T09:26:24.764119+0000","last_clean":"2023-09-30T09:26:24.764119+0000","last_became_active":"2023-09-30T09:25:16.364820+0000","last_became_peered":"2023-09-30T09:25:16.364820+0000","last_unstale":"2023-09-30T09:26:24.764119+0000","last_undegraded":"2023-09-30T09:26:24.764119+0000","last_fullsized":"2023-09-30T09:26:24.764119+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7887,"log_dups_size":0,"ondisk_log_size":7887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:19:41.535739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150142976,"num_objects":2291,"num_object_clones":0,"num_object_copies":6873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7887,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":429,"num_bytes_recovered":9240576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.356447+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:26:20.356447+0000","last_peered":"2023-09-30T09:26:20.356447+0000","last_clean":"2023-09-30T09:26:20.356447+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:26:20.356447+0000","last_undegraded":"2023-09-30T09:26:20.356447+0000","last_fullsized":"2023-09-30T09:26:20.356447+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336696+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:26:20.336696+0000","last_peered":"2023-09-30T09:26:20.336696+0000","last_clean":"2023-09-30T09:26:20.336696+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:26:20.336696+0000","last_undegraded":"2023-09-30T09:26:20.336696+0000","last_fullsized":"2023-09-30T09:26:20.336696+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"178'8181","reported_seq":10488,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:24.751885+0000","last_change":"2023-09-30T09:26:11.289797+0000","last_active":"2023-09-30T09:26:24.751885+0000","last_peered":"2023-09-30T09:26:24.751885+0000","last_clean":"2023-09-30T09:26:24.751885+0000","last_became_active":"2023-09-30T09:26:11.276208+0000","last_became_peered":"2023-09-30T09:26:11.276208+0000","last_unstale":"2023-09-30T09:26:24.751885+0000","last_undegraded":"2023-09-30T09:26:24.751885+0000","last_fullsized":"2023-09-30T09:26:24.751885+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":8181,"log_dups_size":0,"ondisk_log_size":8181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:35:30.421912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150732824,"num_objects":2301,"num_object_clones":0,"num_object_copies":6903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2301,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8181,"num_write_kb":335361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1104,"num_bytes_recovered":46661656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":164,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.330764+0000","last_change":"2023-09-30T09:23:08.029556+0000","last_active":"2023-09-30T09:26:20.330764+0000","last_peered":"2023-09-30T09:26:20.330764+0000","last_clean":"2023-09-30T09:26:20.330764+0000","last_became_active":"2023-09-30T09:23:08.014483+0000","last_became_peered":"2023-09-30T09:23:08.014483+0000","last_unstale":"2023-09-30T09:26:20.330764+0000","last_undegraded":"2023-09-30T09:26:20.330764+0000","last_fullsized":"2023-09-30T09:26:20.330764+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:07:01.455742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":136,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.350322+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:26:20.350322+0000","last_peered":"2023-09-30T09:26:20.350322+0000","last_clean":"2023-09-30T09:26:20.350322+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:26:20.350322+0000","last_undegraded":"2023-09-30T09:26:20.350322+0000","last_fullsized":"2023-09-30T09:26:20.350322+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"178'7825","reported_seq":10828,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:26.626490+0000","last_change":"2023-09-30T09:24:20.547592+0000","last_active":"2023-09-30T09:26:26.626490+0000","last_peered":"2023-09-30T09:26:26.626490+0000","last_clean":"2023-09-30T09:26:26.626490+0000","last_became_active":"2023-09-30T09:24:20.539203+0000","last_became_peered":"2023-09-30T09:24:20.539203+0000","last_unstale":"2023-09-30T09:26:26.626490+0000","last_undegraded":"2023-09-30T09:26:26.626490+0000","last_fullsized":"2023-09-30T09:26:26.626490+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7825,"log_dups_size":0,"ondisk_log_size":7825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:33.219264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148439040,"num_objects":2265,"num_object_clones":0,"num_object_copies":6795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7825,"num_write_kb":322880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1455,"num_bytes_recovered":95354880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":138,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.343669+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:26:20.343669+0000","last_peered":"2023-09-30T09:26:20.343669+0000","last_clean":"2023-09-30T09:26:20.343669+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:26:20.343669+0000","last_undegraded":"2023-09-30T09:26:20.343669+0000","last_fullsized":"2023-09-30T09:26:20.343669+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":32,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.349827+0000","last_change":"2023-09-30T09:25:04.290019+0000","last_active":"2023-09-30T09:26:20.349827+0000","last_peered":"2023-09-30T09:26:20.349827+0000","last_clean":"2023-09-30T09:26:20.349827+0000","last_became_active":"2023-09-30T09:25:04.289710+0000","last_became_peered":"2023-09-30T09:25:04.289710+0000","last_unstale":"2023-09-30T09:26:20.349827+0000","last_undegraded":"2023-09-30T09:26:20.349827+0000","last_fullsized":"2023-09-30T09:26:20.349827+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:22:01.874727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"178'8065","reported_seq":8138,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:23.920529+0000","last_change":"2023-09-30T09:22:48.115158+0000","last_active":"2023-09-30T09:26:23.920529+0000","last_peered":"2023-09-30T09:26:23.920529+0000","last_clean":"2023-09-30T09:26:23.920529+0000","last_became_active":"2023-09-30T09:22:48.114453+0000","last_became_peered":"2023-09-30T09:22:48.114453+0000","last_unstale":"2023-09-30T09:26:23.920529+0000","last_undegraded":"2023-09-30T09:26:23.920529+0000","last_fullsized":"2023-09-30T09:26:23.920529+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":8065,"log_dups_size":0,"ondisk_log_size":8065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:31:35.934276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159055872,"num_objects":2427,"num_object_clones":0,"num_object_copies":7281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8065,"num_write_kb":335744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.348963+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:26:20.348963+0000","last_peered":"2023-09-30T09:26:20.348963+0000","last_clean":"2023-09-30T09:26:20.348963+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:26:20.348963+0000","last_undegraded":"2023-09-30T09:26:20.348963+0000","last_fullsized":"2023-09-30T09:26:20.348963+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":136,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.349665+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:26:20.349665+0000","last_peered":"2023-09-30T09:26:20.349665+0000","last_clean":"2023-09-30T09:26:20.349665+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:26:20.349665+0000","last_undegraded":"2023-09-30T09:26:20.349665+0000","last_fullsized":"2023-09-30T09:26:20.349665+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"178'7952","reported_seq":11892,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:23.904641+0000","last_change":"2023-09-30T09:26:07.270277+0000","last_active":"2023-09-30T09:26:23.904641+0000","last_peered":"2023-09-30T09:26:23.904641+0000","last_clean":"2023-09-30T09:26:23.904641+0000","last_became_active":"2023-09-30T09:26:07.253252+0000","last_became_peered":"2023-09-30T09:26:07.253252+0000","last_unstale":"2023-09-30T09:26:23.904641+0000","last_undegraded":"2023-09-30T09:26:23.904641+0000","last_fullsized":"2023-09-30T09:26:23.904641+0000","mapping_epoch":172,"log_start":"117'716","ondisk_log_start":"117'716","created":113,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7236,"log_dups_size":0,"ondisk_log_size":7236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:38:20.303668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153616384,"num_objects":2344,"num_object_clones":0,"num_object_copies":7032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7952,"num_write_kb":329472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1908,"num_bytes_recovered":105906176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.330515+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:26:20.330515+0000","last_peered":"2023-09-30T09:26:20.330515+0000","last_clean":"2023-09-30T09:26:20.330515+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:26:20.330515+0000","last_undegraded":"2023-09-30T09:26:20.330515+0000","last_fullsized":"2023-09-30T09:26:20.330515+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":116,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.337000+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:26:20.337000+0000","last_peered":"2023-09-30T09:26:20.337000+0000","last_clean":"2023-09-30T09:26:20.337000+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:26:20.337000+0000","last_undegraded":"2023-09-30T09:26:20.337000+0000","last_fullsized":"2023-09-30T09:26:20.337000+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":520,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.337146+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:26:20.337146+0000","last_peered":"2023-09-30T09:26:20.337146+0000","last_clean":"2023-09-30T09:26:20.337146+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:26:20.337146+0000","last_undegraded":"2023-09-30T09:26:20.337146+0000","last_fullsized":"2023-09-30T09:26:20.337146+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":154,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.353638+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:26:20.353638+0000","last_peered":"2023-09-30T09:26:20.353638+0000","last_clean":"2023-09-30T09:26:20.353638+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:26:20.353638+0000","last_undegraded":"2023-09-30T09:26:20.353638+0000","last_fullsized":"2023-09-30T09:26:20.353638+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336638+0000","last_change":"2023-09-30T09:25:14.364474+0000","last_active":"2023-09-30T09:26:20.336638+0000","last_peered":"2023-09-30T09:26:20.336638+0000","last_clean":"2023-09-30T09:26:20.336638+0000","last_became_active":"2023-09-30T09:25:14.364128+0000","last_became_peered":"2023-09-30T09:25:14.364128+0000","last_unstale":"2023-09-30T09:26:20.336638+0000","last_undegraded":"2023-09-30T09:26:20.336638+0000","last_fullsized":"2023-09-30T09:26:20.336638+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:41:17.197492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"178'7998","reported_seq":12054,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:24.347881+0000","last_change":"2023-09-30T09:26:04.207784+0000","last_active":"2023-09-30T09:26:24.347881+0000","last_peered":"2023-09-30T09:26:24.347881+0000","last_clean":"2023-09-30T09:26:24.347881+0000","last_became_active":"2023-09-30T09:26:04.186571+0000","last_became_peered":"2023-09-30T09:26:04.186571+0000","last_unstale":"2023-09-30T09:26:24.347881+0000","last_undegraded":"2023-09-30T09:26:24.347881+0000","last_fullsized":"2023-09-30T09:26:24.347881+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7998,"log_dups_size":0,"ondisk_log_size":7998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:24.650075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153616384,"num_objects":2344,"num_object_clones":0,"num_object_copies":7032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7998,"num_write_kb":330944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1972,"num_bytes_recovered":105644032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"178'8014","reported_seq":15062,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:24.334843+0000","last_change":"2023-09-30T09:25:17.387236+0000","last_active":"2023-09-30T09:26:24.334843+0000","last_peered":"2023-09-30T09:26:24.334843+0000","last_clean":"2023-09-30T09:26:24.334843+0000","last_became_active":"2023-09-30T09:25:17.376539+0000","last_became_peered":"2023-09-30T09:25:17.376539+0000","last_unstale":"2023-09-30T09:26:24.334843+0000","last_undegraded":"2023-09-30T09:26:24.334843+0000","last_fullsized":"2023-09-30T09:26:24.334843+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":8014,"log_dups_size":0,"ondisk_log_size":8014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:15:31.918233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153354240,"num_objects":2340,"num_object_clones":0,"num_object_copies":7020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8014,"num_write_kb":331328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3466,"num_bytes_recovered":207814656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":324,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336799+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:26:20.336799+0000","last_peered":"2023-09-30T09:26:20.336799+0000","last_clean":"2023-09-30T09:26:20.336799+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:26:20.336799+0000","last_undegraded":"2023-09-30T09:26:20.336799+0000","last_fullsized":"2023-09-30T09:26:20.336799+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.349634+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:26:20.349634+0000","last_peered":"2023-09-30T09:26:20.349634+0000","last_clean":"2023-09-30T09:26:20.349634+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:26:20.349634+0000","last_undegraded":"2023-09-30T09:26:20.349634+0000","last_fullsized":"2023-09-30T09:26:20.349634+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":138,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.341700+0000","last_change":"2023-09-30T09:23:23.238851+0000","last_active":"2023-09-30T09:26:20.341700+0000","last_peered":"2023-09-30T09:26:20.341700+0000","last_clean":"2023-09-30T09:26:20.341700+0000","last_became_active":"2023-09-30T09:23:23.238582+0000","last_became_peered":"2023-09-30T09:23:23.238582+0000","last_unstale":"2023-09-30T09:26:20.341700+0000","last_undegraded":"2023-09-30T09:26:20.341700+0000","last_fullsized":"2023-09-30T09:26:20.341700+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:52.808966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"178'7823","reported_seq":7900,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:24.796343+0000","last_change":"2023-09-30T09:22:48.108513+0000","last_active":"2023-09-30T09:26:24.796343+0000","last_peered":"2023-09-30T09:26:24.796343+0000","last_clean":"2023-09-30T09:26:24.796343+0000","last_became_active":"2023-09-30T09:22:48.108231+0000","last_became_peered":"2023-09-30T09:22:48.108231+0000","last_unstale":"2023-09-30T09:26:24.796343+0000","last_undegraded":"2023-09-30T09:26:24.796343+0000","last_fullsized":"2023-09-30T09:26:24.796343+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7823,"log_dups_size":0,"ondisk_log_size":7823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:09.723671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419776,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7823,"num_write_kb":325248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":555,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.341438+0000","last_change":"2023-09-30T09:23:11.172147+0000","last_active":"2023-09-30T09:26:20.341438+0000","last_peered":"2023-09-30T09:26:20.341438+0000","last_clean":"2023-09-30T09:26:20.341438+0000","last_became_active":"2023-09-30T09:23:11.171540+0000","last_became_peered":"2023-09-30T09:23:11.171540+0000","last_unstale":"2023-09-30T09:26:20.341438+0000","last_undegraded":"2023-09-30T09:26:20.341438+0000","last_fullsized":"2023-09-30T09:26:20.341438+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:24.074307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":116,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:21.344440+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:26:21.344440+0000","last_peered":"2023-09-30T09:26:21.344440+0000","last_clean":"2023-09-30T09:26:21.344440+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:26:21.344440+0000","last_undegraded":"2023-09-30T09:26:21.344440+0000","last_fullsized":"2023-09-30T09:26:21.344440+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":163,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336906+0000","last_change":"2023-09-30T09:23:08.027904+0000","last_active":"2023-09-30T09:26:20.336906+0000","last_peered":"2023-09-30T09:26:20.336906+0000","last_clean":"2023-09-30T09:26:20.336906+0000","last_became_active":"2023-09-30T09:23:08.027557+0000","last_became_peered":"2023-09-30T09:23:08.027557+0000","last_unstale":"2023-09-30T09:26:20.336906+0000","last_undegraded":"2023-09-30T09:26:20.336906+0000","last_fullsized":"2023-09-30T09:26:20.336906+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:19:26.977141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":585,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.353531+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:26:20.353531+0000","last_peered":"2023-09-30T09:26:20.353531+0000","last_clean":"2023-09-30T09:26:20.353531+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:26:20.353531+0000","last_undegraded":"2023-09-30T09:26:20.353531+0000","last_fullsized":"2023-09-30T09:26:20.353531+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":571,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.343965+0000","last_change":"2023-09-30T09:23:27.263806+0000","last_active":"2023-09-30T09:26:20.343965+0000","last_peered":"2023-09-30T09:26:20.343965+0000","last_clean":"2023-09-30T09:26:20.343965+0000","last_became_active":"2023-09-30T09:23:27.263242+0000","last_became_peered":"2023-09-30T09:23:27.263242+0000","last_unstale":"2023-09-30T09:26:20.343965+0000","last_undegraded":"2023-09-30T09:26:20.343965+0000","last_fullsized":"2023-09-30T09:26:20.343965+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:38:00.839832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":92,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.331409+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:26:20.331409+0000","last_peered":"2023-09-30T09:26:20.331409+0000","last_clean":"2023-09-30T09:26:20.331409+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:26:20.331409+0000","last_undegraded":"2023-09-30T09:26:20.331409+0000","last_fullsized":"2023-09-30T09:26:20.331409+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":120,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.344339+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:26:20.344339+0000","last_peered":"2023-09-30T09:26:20.344339+0000","last_clean":"2023-09-30T09:26:20.344339+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:26:20.344339+0000","last_undegraded":"2023-09-30T09:26:20.344339+0000","last_fullsized":"2023-09-30T09:26:20.344339+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"178'7977","reported_seq":8050,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:23.872537+0000","last_change":"2023-09-30T09:22:48.113213+0000","last_active":"2023-09-30T09:26:23.872537+0000","last_peered":"2023-09-30T09:26:23.872537+0000","last_clean":"2023-09-30T09:26:23.872537+0000","last_became_active":"2023-09-30T09:22:48.112904+0000","last_became_peered":"2023-09-30T09:22:48.112904+0000","last_unstale":"2023-09-30T09:26:23.872537+0000","last_undegraded":"2023-09-30T09:26:23.872537+0000","last_fullsized":"2023-09-30T09:26:23.872537+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:37:37.122003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151060480,"num_objects":2305,"num_object_clones":0,"num_object_copies":6915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7977,"num_write_kb":329024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"178'7912","reported_seq":19755,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:23.891141+0000","last_change":"2023-09-30T09:26:21.389918+0000","last_active":"2023-09-30T09:26:23.891141+0000","last_peered":"2023-09-30T09:26:23.891141+0000","last_clean":"2023-09-30T09:26:23.891141+0000","last_became_active":"2023-09-30T09:26:21.345095+0000","last_became_peered":"2023-09-30T09:26:21.345095+0000","last_unstale":"2023-09-30T09:26:23.891141+0000","last_undegraded":"2023-09-30T09:26:23.891141+0000","last_fullsized":"2023-09-30T09:26:23.891141+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7912,"log_dups_size":0,"ondisk_log_size":7912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:27.510109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152567808,"num_objects":2328,"num_object_clones":0,"num_object_copies":6984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7912,"num_write_kb":327680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5852,"num_bytes_recovered":383516672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":738,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.344011+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:26:20.344011+0000","last_peered":"2023-09-30T09:26:20.344011+0000","last_clean":"2023-09-30T09:26:20.344011+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:26:20.344011+0000","last_undegraded":"2023-09-30T09:26:20.344011+0000","last_fullsized":"2023-09-30T09:26:20.344011+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":85,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.334312+0000","last_change":"2023-09-30T09:25:04.283467+0000","last_active":"2023-09-30T09:26:20.334312+0000","last_peered":"2023-09-30T09:26:20.334312+0000","last_clean":"2023-09-30T09:26:20.334312+0000","last_became_active":"2023-09-30T09:25:04.283162+0000","last_became_peered":"2023-09-30T09:25:04.283162+0000","last_unstale":"2023-09-30T09:26:20.334312+0000","last_undegraded":"2023-09-30T09:26:20.334312+0000","last_fullsized":"2023-09-30T09:26:20.334312+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:12:35.333268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":69,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.334789+0000","last_change":"2023-09-30T09:23:08.024242+0000","last_active":"2023-09-30T09:26:20.334789+0000","last_peered":"2023-09-30T09:26:20.334789+0000","last_clean":"2023-09-30T09:26:20.334789+0000","last_became_active":"2023-09-30T09:23:08.023891+0000","last_became_peered":"2023-09-30T09:23:08.023891+0000","last_unstale":"2023-09-30T09:26:20.334789+0000","last_undegraded":"2023-09-30T09:26:20.334789+0000","last_fullsized":"2023-09-30T09:26:20.334789+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:19:36.687182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":818,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.345265+0000","last_change":"2023-09-30T09:23:11.125442+0000","last_active":"2023-09-30T09:26:20.345265+0000","last_peered":"2023-09-30T09:26:20.345265+0000","last_clean":"2023-09-30T09:26:20.345265+0000","last_became_active":"2023-09-30T09:23:11.124962+0000","last_became_peered":"2023-09-30T09:23:11.124962+0000","last_unstale":"2023-09-30T09:26:20.345265+0000","last_undegraded":"2023-09-30T09:26:20.345265+0000","last_fullsized":"2023-09-30T09:26:20.345265+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:53.386815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.337127+0000","last_change":"2023-09-30T09:25:14.359586+0000","last_active":"2023-09-30T09:26:20.337127+0000","last_peered":"2023-09-30T09:26:20.337127+0000","last_clean":"2023-09-30T09:26:20.337127+0000","last_became_active":"2023-09-30T09:25:14.359198+0000","last_became_peered":"2023-09-30T09:25:14.359198+0000","last_unstale":"2023-09-30T09:26:20.337127+0000","last_undegraded":"2023-09-30T09:26:20.337127+0000","last_fullsized":"2023-09-30T09:26:20.337127+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:48.533039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336867+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:26:20.336867+0000","last_peered":"2023-09-30T09:26:20.336867+0000","last_clean":"2023-09-30T09:26:20.336867+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:26:20.336867+0000","last_undegraded":"2023-09-30T09:26:20.336867+0000","last_fullsized":"2023-09-30T09:26:20.336867+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"178'8208","reported_seq":11407,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:23.911993+0000","last_change":"2023-09-30T09:24:12.672495+0000","last_active":"2023-09-30T09:26:23.911993+0000","last_peered":"2023-09-30T09:26:23.911993+0000","last_clean":"2023-09-30T09:26:23.911993+0000","last_became_active":"2023-09-30T09:24:12.668774+0000","last_became_peered":"2023-09-30T09:24:12.668774+0000","last_unstale":"2023-09-30T09:26:23.911993+0000","last_undegraded":"2023-09-30T09:26:23.911993+0000","last_fullsized":"2023-09-30T09:26:23.911993+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":8208,"log_dups_size":0,"ondisk_log_size":8208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:01:27.375538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162529280,"num_objects":2480,"num_object_clones":0,"num_object_copies":7440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8208,"num_write_kb":342016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1553,"num_bytes_recovered":101777408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":164,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.331113+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:26:20.331113+0000","last_peered":"2023-09-30T09:26:20.331113+0000","last_clean":"2023-09-30T09:26:20.331113+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:26:20.331113+0000","last_undegraded":"2023-09-30T09:26:20.331113+0000","last_fullsized":"2023-09-30T09:26:20.331113+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1125,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336602+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:26:20.336602+0000","last_peered":"2023-09-30T09:26:20.336602+0000","last_clean":"2023-09-30T09:26:20.336602+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:26:20.336602+0000","last_undegraded":"2023-09-30T09:26:20.336602+0000","last_fullsized":"2023-09-30T09:26:20.336602+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":153,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.330605+0000","last_change":"2023-09-30T09:23:08.028741+0000","last_active":"2023-09-30T09:26:20.330605+0000","last_peered":"2023-09-30T09:26:20.330605+0000","last_clean":"2023-09-30T09:26:20.330605+0000","last_became_active":"2023-09-30T09:23:08.028167+0000","last_became_peered":"2023-09-30T09:23:08.028167+0000","last_unstale":"2023-09-30T09:26:20.330605+0000","last_undegraded":"2023-09-30T09:26:20.330605+0000","last_fullsized":"2023-09-30T09:26:20.330605+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:47:39.241913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"178'8106","reported_seq":8179,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:23.914390+0000","last_change":"2023-09-30T09:22:48.113865+0000","last_active":"2023-09-30T09:26:23.914390+0000","last_peered":"2023-09-30T09:26:23.914390+0000","last_clean":"2023-09-30T09:26:23.914390+0000","last_became_active":"2023-09-30T09:22:48.113578+0000","last_became_peered":"2023-09-30T09:22:48.113578+0000","last_unstale":"2023-09-30T09:26:23.914390+0000","last_undegraded":"2023-09-30T09:26:23.914390+0000","last_fullsized":"2023-09-30T09:26:23.914390+0000","mapping_epoch":113,"log_start":"117'900","ondisk_log_start":"117'900","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7206,"log_dups_size":0,"ondisk_log_size":7206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:17.848647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157286400,"num_objects":2400,"num_object_clones":0,"num_object_copies":7200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8106,"num_write_kb":336192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":431,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.346539+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:26:20.346539+0000","last_peered":"2023-09-30T09:26:20.346539+0000","last_clean":"2023-09-30T09:26:20.346539+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:26:20.346539+0000","last_undegraded":"2023-09-30T09:26:20.346539+0000","last_fullsized":"2023-09-30T09:26:20.346539+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":73,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.334990+0000","last_change":"2023-09-30T09:23:08.031587+0000","last_active":"2023-09-30T09:26:20.334990+0000","last_peered":"2023-09-30T09:26:20.334990+0000","last_clean":"2023-09-30T09:26:20.334990+0000","last_became_active":"2023-09-30T09:23:08.016566+0000","last_became_peered":"2023-09-30T09:23:08.016566+0000","last_unstale":"2023-09-30T09:26:20.334990+0000","last_undegraded":"2023-09-30T09:26:20.334990+0000","last_fullsized":"2023-09-30T09:26:20.334990+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:44:10.092575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":119,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.349041+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:26:20.349041+0000","last_peered":"2023-09-30T09:26:20.349041+0000","last_clean":"2023-09-30T09:26:20.349041+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:26:20.349041+0000","last_undegraded":"2023-09-30T09:26:20.349041+0000","last_fullsized":"2023-09-30T09:26:20.349041+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"178'7812","reported_seq":7889,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-30T09:26:22.117628+0000","last_change":"2023-09-30T09:22:48.113933+0000","last_active":"2023-09-30T09:26:22.117628+0000","last_peered":"2023-09-30T09:26:22.117628+0000","last_clean":"2023-09-30T09:26:22.117628+0000","last_became_active":"2023-09-30T09:22:48.113640+0000","last_became_peered":"2023-09-30T09:22:48.113640+0000","last_unstale":"2023-09-30T09:26:22.117628+0000","last_undegraded":"2023-09-30T09:26:22.117628+0000","last_fullsized":"2023-09-30T09:26:22.117628+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7812,"log_dups_size":0,"ondisk_log_size":7812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:16.995917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163446784,"num_objects":2494,"num_object_clones":0,"num_object_copies":7482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7812,"num_write_kb":329792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":65,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.336958+0000","last_change":"2023-09-30T09:23:08.024830+0000","last_active":"2023-09-30T09:26:20.336958+0000","last_peered":"2023-09-30T09:26:20.336958+0000","last_clean":"2023-09-30T09:26:20.336958+0000","last_became_active":"2023-09-30T09:23:08.024489+0000","last_became_peered":"2023-09-30T09:23:08.024489+0000","last_unstale":"2023-09-30T09:26:20.336958+0000","last_undegraded":"2023-09-30T09:26:20.336958+0000","last_fullsized":"2023-09-30T09:26:20.336958+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:41.769725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":160,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.341608+0000","last_change":"2023-09-30T09:21:29.096256+0000","last_active":"2023-09-30T09:26:20.341608+0000","last_peered":"2023-09-30T09:26:20.341608+0000","last_clean":"2023-09-30T09:26:20.341608+0000","last_became_active":"2023-09-30T09:21:29.095930+0000","last_became_peered":"2023-09-30T09:21:29.095930+0000","last_unstale":"2023-09-30T09:26:20.341608+0000","last_undegraded":"2023-09-30T09:26:20.341608+0000","last_fullsized":"2023-09-30T09:26:20.341608+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:25.270756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":139,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.356535+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:26:20.356535+0000","last_peered":"2023-09-30T09:26:20.356535+0000","last_clean":"2023-09-30T09:26:20.356535+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:26:20.356535+0000","last_undegraded":"2023-09-30T09:26:20.356535+0000","last_fullsized":"2023-09-30T09:26:20.356535+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":32,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.349392+0000","last_change":"2023-09-30T09:25:04.284772+0000","last_active":"2023-09-30T09:26:20.349392+0000","last_peered":"2023-09-30T09:26:20.349392+0000","last_clean":"2023-09-30T09:26:20.349392+0000","last_became_active":"2023-09-30T09:25:04.284405+0000","last_became_peered":"2023-09-30T09:25:04.284405+0000","last_unstale":"2023-09-30T09:26:20.349392+0000","last_undegraded":"2023-09-30T09:26:20.349392+0000","last_fullsized":"2023-09-30T09:26:20.349392+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:44.112365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":122,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.353729+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:26:20.353729+0000","last_peered":"2023-09-30T09:26:20.353729+0000","last_clean":"2023-09-30T09:26:20.353729+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:26:20.353729+0000","last_undegraded":"2023-09-30T09:26:20.353729+0000","last_fullsized":"2023-09-30T09:26:20.353729+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":32,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.352359+0000","last_change":"2023-09-30T09:25:04.306048+0000","last_active":"2023-09-30T09:26:20.352359+0000","last_peered":"2023-09-30T09:26:20.352359+0000","last_clean":"2023-09-30T09:26:20.352359+0000","last_became_active":"2023-09-30T09:25:04.305804+0000","last_became_peered":"2023-09-30T09:25:04.305804+0000","last_unstale":"2023-09-30T09:26:20.352359+0000","last_undegraded":"2023-09-30T09:26:20.352359+0000","last_fullsized":"2023-09-30T09:26:20.352359+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:32.116748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":148,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.331041+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:26:20.331041+0000","last_peered":"2023-09-30T09:26:20.331041+0000","last_clean":"2023-09-30T09:26:20.331041+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:26:20.331041+0000","last_undegraded":"2023-09-30T09:26:20.331041+0000","last_fullsized":"2023-09-30T09:26:20.331041+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":179,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.331639+0000","last_change":"2023-09-30T09:23:08.021721+0000","last_active":"2023-09-30T09:26:20.331639+0000","last_peered":"2023-09-30T09:26:20.331639+0000","last_clean":"2023-09-30T09:26:20.331639+0000","last_became_active":"2023-09-30T09:23:08.021401+0000","last_became_peered":"2023-09-30T09:23:08.021401+0000","last_unstale":"2023-09-30T09:26:20.331639+0000","last_undegraded":"2023-09-30T09:26:20.331639+0000","last_fullsized":"2023-09-30T09:26:20.331639+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:33:33.611604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":75,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.335254+0000","last_change":"2023-09-30T09:23:08.039181+0000","last_active":"2023-09-30T09:26:20.335254+0000","last_peered":"2023-09-30T09:26:20.335254+0000","last_clean":"2023-09-30T09:26:20.335254+0000","last_became_active":"2023-09-30T09:23:08.025651+0000","last_became_peered":"2023-09-30T09:23:08.025651+0000","last_unstale":"2023-09-30T09:26:20.335254+0000","last_undegraded":"2023-09-30T09:26:20.335254+0000","last_fullsized":"2023-09-30T09:26:20.335254+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:27.721207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":136,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-30T09:26:20.347092+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:26:20.347092+0000","last_peered":"2023-09-30T09:26:20.347092+0000","last_clean":"2023-09-30T09:26:20.347092+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:26:20.347092+0000","last_undegraded":"2023-09-30T09:26:20.347092+0000","last_fullsized":"2023-09-30T09:26:20.347092+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2473459736,"num_objects":37743,"num_object_clones":0,"num_object_copies":113229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37743,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127809,"num_write_kb":5297601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21304,"num_bytes_recovered":1265041432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7864397824,"data_stored":7864385608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126193,"ondisk_log_size":126193,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":738,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234177,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":851060,"kb_used_data":798892,"kb_used_omap":0,"kb_used_meta":52160,"kb_avail":92894092,"statfs":{"total":95995035648,"available":95123550208,"internally_reserved":0,"allocated":818065408,"data_stored":817305596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":53411840},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8159999999999998,"5min":1.244,"15min":0.82699999999999996},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":17.914000000000001,"5min":17.914000000000001,"15min":17.914000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.5910000000000002,"5min":1.046,"15min":0.77900000000000003},"min":{"1min":0.246,"5min":0.245,"15min":0.245},"max":{"1min":17.858000000000001,"5min":17.858000000000001,"15min":17.858000000000001},"last":4.2210000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":2.4289999999999998,"15min":1.175},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.212,"5min":82.025999999999996,"15min":82.025999999999996},"last":2.5659999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":2.3980000000000001,"15min":1.1799999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":82.012,"15min":82.012},"last":2.335}]},{"osd":2,"last update":"Sat Sep 30 09:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4009999999999998,"5min":1.288,"15min":0.83299999999999996},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":17.838000000000001,"5min":17.838000000000001,"15min":17.838000000000001},"last":4.3410000000000002},{"interface":"front","average":{"1min":2.6469999999999998,"5min":1.462,"15min":0.89300000000000002},"min":{"1min":0.34799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":17.805,"5min":17.805,"15min":17.805},"last":3.9769999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:25:40 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.91000000000000003,"15min":0.71699999999999997},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0090000000000003,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":2.2120000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.753,"15min":0.67500000000000004},"min":{"1min":0.40200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.3540000000000001,"5min":4.8150000000000004,"15min":4.8150000000000004},"last":1.484}]},{"osd":4,"last update":"Sat Sep 30 09:25:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.56399999999999995,"15min":0.497},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.3580000000000001,"5min":3.3580000000000001,"15min":3.3580000000000001},"last":2.5129999999999999},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.69599999999999995,"15min":0.56399999999999995},"min":{"1min":0.28999999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.282,"5min":6.282,"15min":6.282},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:25:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.67400000000000004,"15min":0.56599999999999995},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.504,"5min":5.968,"15min":5.968},"last":2.5310000000000001},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.72299999999999998,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.4790000000000001,"5min":5.452,"15min":5.452},"last":1.915}]},{"osd":6,"last update":"Sat Sep 30 09:25:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.70699999999999996,"15min":0.61199999999999999},"min":{"1min":0.39100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.4580000000000002,"5min":3.4580000000000002,"15min":3.4580000000000002},"last":1.881},{"interface":"front","average":{"1min":1.016,"5min":0.751,"15min":0.626},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.3529999999999998,"5min":4.3529999999999998,"15min":4.3529999999999998},"last":0.46800000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495814,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":738656,"kb_used_data":685656,"kb_used_omap":0,"kb_used_meta":52992,"kb_avail":93006496,"statfs":{"total":95995035648,"available":95238651904,"internally_reserved":0,"allocated":702111744,"data_stored":701371673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":54263808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.83999999999999997,"15min":0.70299999999999996},"min":{"1min":0.23599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.4249999999999998,"5min":6.649,"15min":6.649},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.84499999999999997,"15min":0.71899999999999997},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.327,"5min":8.327,"15min":8.327},"last":1.841}]},{"osd":1,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.83399999999999996,"15min":0.69999999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.2970000000000002,"5min":6.5090000000000003,"15min":6.5090000000000003},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.81299999999999994,"15min":0.69399999999999995},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2810000000000001,"5min":4.944,"15min":4.944},"last":1.8089999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:26:04 2023","interfaces":[{"interface":"back","average":{"1min":4.5720000000000001,"5min":1.752,"15min":0.96499999999999997},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":37.978000000000002,"5min":37.978000000000002,"15min":37.978000000000002},"last":1.5900000000000001},{"interface":"front","average":{"1min":3.7549999999999999,"5min":1.589,"15min":0.93400000000000005},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":37.759999999999998,"5min":37.759999999999998,"15min":37.759999999999998},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.878,"15min":0.68999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.387,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.0309999999999999,"15min":0.73599999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7759999999999998,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":1.899}]},{"osd":4,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":1.2210000000000001,"15min":0.73499999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":0.59299999999999997,"5min":73.656000000000006,"15min":73.656000000000006},"last":1.7649999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":1.2829999999999999,"15min":0.77000000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58199999999999996,"5min":73.882999999999996,"15min":73.882999999999996},"last":1.698}]},{"osd":5,"last update":"Sat Sep 30 09:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.64100000000000001,"15min":0.54200000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":0.48999999999999999,"5min":4.2430000000000003,"15min":4.2430000000000003},"last":1.5529999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.71399999999999997,"15min":0.60899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28000000000000003},"max":{"1min":0.60299999999999998,"5min":6.5999999999999996,"15min":6.5999999999999996},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:25:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.79500000000000004,"15min":0.65100000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.26500000000000001},"max":{"1min":12.709,"5min":12.709,"15min":12.709},"last":1.8720000000000001},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.077,"15min":0.76600000000000001},"min":{"1min":0.40899999999999997,"5min":0.35599999999999998,"15min":0.31900000000000001},"max":{"1min":9.5640000000000001,"5min":10.339,"15min":10.339},"last":0.35499999999999998}]}]},{"osd":1,"up_from":99,"seq":425201762379,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807808,"kb_used_data":731704,"kb_used_omap":5,"kb_used_meta":76026,"kb_avail":92937344,"statfs":{"total":95995035648,"available":95167840256,"internally_reserved":0,"allocated":749264896,"data_stored":748265052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":77850806},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":4.2450000000000001,"5min":2.0859999999999999,"15min":0.94199999999999995},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":62.935000000000002,"5min":72.007000000000005,"15min":72.007000000000005},"last":3.8069999999999999},{"interface":"front","average":{"1min":3.907,"5min":2.1779999999999999,"15min":0.997},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":63.088000000000001,"5min":72.322999999999993,"15min":72.322999999999993},"last":4.2270000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:26:01 2023","interfaces":[{"interface":"back","average":{"1min":11.202999999999999,"5min":2.7810000000000001,"15min":1.2090000000000001},"min":{"1min":0.25600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":95.245000000000005,"5min":95.245000000000005,"15min":95.245000000000005},"last":0.76500000000000001},{"interface":"front","average":{"1min":9.6969999999999992,"5min":2.5910000000000002,"15min":1.129},"min":{"1min":0.29099999999999998,"5min":0.254,"15min":0.254},"max":{"1min":95.019000000000005,"5min":95.019000000000005,"15min":95.019000000000005},"last":3.7309999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:25:36 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.74199999999999999,"15min":0.52100000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.845000000000001,"5min":11.845000000000001,"15min":11.845000000000001},"last":0.371},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.72099999999999997,"15min":0.502},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.6499999999999999,"5min":11.06,"15min":11.06},"last":0.26500000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.80500000000000005,"15min":0.69799999999999995},"min":{"1min":0.317,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.569,"5min":5.6340000000000003,"15min":5.6340000000000003},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.81299999999999994,"15min":0.72499999999999998},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.9470000000000001,"5min":4.3869999999999996,"15min":4.3869999999999996},"last":0.65600000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.73499999999999999,"15min":0.65900000000000003},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.619,"5min":5.0140000000000002,"15min":5.0140000000000002},"last":2.1190000000000002},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.83399999999999996,"15min":0.70999999999999996},"min":{"1min":0.439,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.9849999999999999,"5min":2.9929999999999999,"15min":2.9929999999999999},"last":0.72699999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.81399999999999995,"15min":0.68600000000000005},"min":{"1min":0.44900000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":2.851,"5min":3.0150000000000001,"15min":3.0150000000000001},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.91100000000000003,"15min":0.76000000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.8140000000000001,"5min":7.8140000000000001,"15min":7.8140000000000001},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.96499999999999997,"15min":0.77500000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.9079999999999999,"5min":7.4820000000000002,"15min":7.4820000000000002},"last":3.198},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.018,"15min":0.80200000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":7.7729999999999997,"5min":7.7729999999999997,"15min":7.7729999999999997},"last":3.1829999999999998}]}]},{"osd":0,"up_from":94,"seq":403726925911,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1431728,"kb_used_data":1251492,"kb_used_omap":5,"kb_used_meta":180154,"kb_avail":92313424,"statfs":{"total":95995035648,"available":94528946176,"internally_reserved":0,"allocated":1281527808,"data_stored":1280490142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":184477949},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.81599999999999995,"15min":0.51200000000000001},"min":{"1min":0.22,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.5859999999999999,"5min":15.9,"15min":15.9},"last":4.2240000000000002},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.92500000000000004,"15min":0.56399999999999995},"min":{"1min":0.27800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.8849999999999998,"5min":13.824999999999999,"15min":13.824999999999999},"last":3.2130000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:26:02 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.1359999999999999,"15min":0.63100000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":3.7669999999999999,"5min":18.096,"15min":18.096},"last":4.0810000000000004},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.167,"15min":0.67900000000000005},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.258,"5min":18.149000000000001,"15min":18.149000000000001},"last":3.2370000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":0.98199999999999998,"15min":0.65100000000000002},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.9630000000000001,"5min":13.02,"15min":13.02},"last":16.670000000000002},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.96799999999999997,"15min":0.60999999999999999},"min":{"1min":0.29999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.847,"5min":10.128,"15min":10.128},"last":11.802}]},{"osd":4,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.86099999999999999,"15min":0.78500000000000003},"min":{"1min":0.38700000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.369,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":3.1219999999999999},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.96999999999999997,"15min":0.84699999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":3.169,"5min":12.99,"15min":12.99},"last":3.8849999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.1719999999999999,"15min":0.94399999999999995},"min":{"1min":0.38400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.266,"5min":12.949999999999999,"15min":12.949999999999999},"last":3.8450000000000002},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.151,"15min":0.94199999999999995},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.105,"5min":12.823,"15min":12.823},"last":4.1749999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.91000000000000003,"15min":0.84599999999999997},"min":{"1min":0.432,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.038,"5min":6.9660000000000002,"15min":6.9660000000000002},"last":4.1449999999999996},{"interface":"front","average":{"1min":0.745,"5min":0.85999999999999999,"15min":0.871},"min":{"1min":0.48799999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.5430000000000001,"5min":7.556,"15min":7.556},"last":3.1579999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.96099999999999997,"15min":0.83999999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.1699999999999999,"5min":7.0339999999999998,"15min":7.0339999999999998},"last":4.1989999999999998},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.85799999999999998,"15min":0.86599999999999999},"min":{"1min":0.47399999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.504,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":3.1850000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598852,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1145008,"kb_used_data":1037032,"kb_used_omap":5,"kb_used_meta":107962,"kb_avail":92600144,"statfs":{"total":95995035648,"available":94822547456,"internally_reserved":0,"allocated":1061920768,"data_stored":1060934296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":110553741},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.268,"15min":0.66600000000000004},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.172000000000001,"5min":15.843999999999999,"15min":15.843999999999999},"last":2.4769999999999999},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.03,"15min":0.64500000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.3620000000000001,"5min":10.065,"15min":10.065},"last":7.2389999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0960000000000001,"5min":1.006,"15min":0.59199999999999997},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.9120000000000008,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":3.5960000000000001},{"interface":"front","average":{"1min":1.5029999999999999,"5min":0.95599999999999996,"15min":0.59999999999999998},"min":{"1min":0.28999999999999998,"5min":0.249,"15min":0.249},"max":{"1min":5.016,"5min":6.5369999999999999,"15min":6.5369999999999999},"last":8.3810000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0009999999999999,"15min":0.63100000000000001},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.4009999999999998,"5min":18.635999999999999,"15min":18.635999999999999},"last":3.9929999999999999},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.173,"15min":0.72599999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.996,"5min":18.026,"15min":18.026},"last":7.0359999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.86899999999999999,"15min":0.69399999999999995},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.3069999999999999,"5min":4.1719999999999997,"15min":4.1719999999999997},"last":7.282},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.2669999999999999,"15min":0.85299999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.125999999999999,"5min":11.125999999999999,"15min":11.125999999999999},"last":6.9870000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.0840000000000001,"15min":0.79500000000000004},"min":{"1min":0.38300000000000001,"5min":0.371,"15min":0.371},"max":{"1min":8.8640000000000008,"5min":8.8640000000000008,"15min":8.8640000000000008},"last":3.6379999999999999},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.1990000000000001,"15min":0.83699999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":11.866,"5min":11.866,"15min":11.866},"last":7.3159999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.504,"15min":0.95999999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":11.247999999999999,"5min":11.247999999999999,"15min":11.247999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.2850000000000001,"5min":1.226,"15min":0.85999999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":8.9619999999999997,"5min":8.9619999999999997,"15min":8.9619999999999997},"last":4.0540000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:26:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.409,"15min":0.92300000000000004},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":2.117},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.163,"15min":0.86499999999999999},"min":{"1min":0.45100000000000001,"5min":0.439,"15min":0.439},"max":{"1min":4.923,"5min":5.3979999999999997,"15min":5.3979999999999997},"last":7.0979999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435325,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":606320,"kb_used_data":530340,"kb_used_omap":2,"kb_used_meta":75965,"kb_avail":93138832,"statfs":{"total":95995035648,"available":95374163968,"internally_reserved":0,"allocated":543068160,"data_stored":542076035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":77788519},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":2.5920000000000001,"15min":1.121},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.5619999999999998,"5min":130.946,"15min":130.946},"last":0.251},{"interface":"front","average":{"1min":0.61499999999999999,"5min":2.5169999999999999,"15min":1.125},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.7470000000000001,"5min":130.96899999999999,"15min":130.96899999999999},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.70799999999999996,"15min":0.52000000000000002},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":6.0099999999999998,"5min":6.0099999999999998,"15min":6.0099999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.64100000000000001,"15min":0.48699999999999999},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.159,"5min":3.9340000000000002,"15min":3.9340000000000002},"last":0.36499999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.831,"15min":1.2470000000000001},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1779999999999999,"5min":145.75899999999999,"15min":145.75899999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":2.7909999999999999,"15min":1.2170000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.1120000000000001,"5min":150.358,"15min":150.358},"last":0.439}]},{"osd":4,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.71299999999999997,"15min":0.64800000000000002},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.312},"max":{"1min":1.042,"5min":3.9119999999999999,"15min":3.9119999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.74099999999999999,"15min":0.65100000000000002},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.27,"5min":4.0309999999999997,"15min":4.0309999999999997},"last":0.505}]},{"osd":5,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.85799999999999998,"15min":0.70699999999999996},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.2390000000000001,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.86899999999999999,"15min":0.75800000000000001},"min":{"1min":0.375,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9100000000000001,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.75900000000000001,"15min":0.70299999999999996},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.147,"5min":3.9590000000000001,"15min":3.9590000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.86499999999999999,"15min":0.746},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":3.0230000000000001,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.81100000000000005,"15min":0.72199999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.948,"5min":4.1639999999999997,"15min":4.1639999999999997},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.74199999999999999,"15min":0.70299999999999996},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.372},"max":{"1min":1.1479999999999999,"5min":3.9910000000000001,"15min":3.9910000000000001},"last":0.54400000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019087,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487776,"kb_used_data":1398488,"kb_used_omap":0,"kb_used_meta":89280,"kb_avail":92257376,"statfs":{"total":95995035648,"available":94471553024,"internally_reserved":0,"allocated":1432051712,"data_stored":1431311212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":91422720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:26:17 2023","interfaces":[{"interface":"back","average":{"1min":12.593,"5min":4.5010000000000003,"15min":1.925},"min":{"1min":0.22800000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":119.986,"5min":128.79300000000001,"15min":128.79300000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":12.821999999999999,"5min":4.335,"15min":1.871},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.248},"max":{"1min":119.863,"5min":128.38800000000001,"15min":128.38800000000001},"last":0.443}]},{"osd":1,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":1.0569999999999999,"15min":0.73399999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.71899999999999997,"5min":31.469999999999999,"15min":31.469999999999999},"last":2.0419999999999998},{"interface":"front","average":{"1min":0.748,"5min":1.149,"15min":0.75700000000000001},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.2729999999999997,"5min":31.962,"15min":31.962},"last":0.65700000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:26:03 2023","interfaces":[{"interface":"back","average":{"1min":12.256,"5min":3.5630000000000002,"15min":1.595},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":140.601,"5min":140.601,"15min":140.601},"last":2.3170000000000002},{"interface":"front","average":{"1min":12.117000000000001,"5min":3.5600000000000001,"15min":1.5840000000000001},"min":{"1min":0.40500000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":140.54900000000001,"5min":140.54900000000001,"15min":140.54900000000001},"last":2.2810000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:25:34 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.607,"15min":0.93300000000000005},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.0019999999999998,"5min":86.620000000000005,"15min":86.620000000000005},"last":3.8460000000000001},{"interface":"front","average":{"1min":0.747,"5min":1.5940000000000001,"15min":0.94399999999999995},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9929999999999999,"5min":86.786000000000001,"15min":86.786000000000001},"last":3.835}]},{"osd":5,"last update":"Sat Sep 30 09:26:17 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":1.387,"15min":0.78800000000000003},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":4.6680000000000001,"5min":86.691000000000003,"15min":86.691000000000003},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.999,"5min":1.4770000000000001,"15min":0.84199999999999997},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.6680000000000001,"5min":86.822000000000003,"15min":86.822000000000003},"last":0.57599999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:25:39 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.55900000000000005,"15min":0.52600000000000002},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71299999999999997,"5min":3.218,"15min":3.218},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.58599999999999997,"15min":0.56799999999999995},"min":{"1min":0.33900000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74199999999999999,"5min":2.0209999999999999,"15min":2.0209999999999999},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:26:02 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.514,"15min":0.88600000000000001},"min":{"1min":0.373,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.5419999999999998,"5min":86.728999999999999,"15min":86.728999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.69899999999999995,"15min":0.623},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":3.0710000000000002,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":0.59599999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757451,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1481116,"kb_used_data":1390676,"kb_used_omap":0,"kb_used_meta":90432,"kb_avail":92264036,"statfs":{"total":95995035648,"available":94478372864,"internally_reserved":0,"allocated":1424052224,"data_stored":1423311826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":92602368},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,3,3,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:26:20 2023","interfaces":[{"interface":"back","average":{"1min":15.002000000000001,"5min":3.9420000000000002,"15min":1.6970000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":148.79900000000001,"5min":148.79900000000001,"15min":148.79900000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":15.145,"5min":4.0970000000000004,"15min":1.7709999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":148.779,"5min":148.779,"15min":148.779},"last":0.39100000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":1.9359999999999999,"15min":1.03},"min":{"1min":0.33900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.24,"5min":100.593,"15min":100.593},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.875,"15min":1.056},"min":{"1min":0.35099999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.5369999999999999,"5min":100.575,"15min":100.575},"last":0.70399999999999996}]},{"osd":2,"last update":"Sat Sep 30 09:26:01 2023","interfaces":[{"interface":"back","average":{"1min":10.221,"5min":4.2000000000000002,"15min":1.798},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":165.768,"5min":165.768,"15min":165.768},"last":0.41399999999999998},{"interface":"front","average":{"1min":18.170000000000002,"5min":5.9740000000000002,"15min":2.4209999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":166.24000000000001,"5min":166.24000000000001,"15min":166.24000000000001},"last":0.376}]},{"osd":3,"last update":"Sat Sep 30 09:25:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.446,"15min":0.91000000000000003},"min":{"1min":0.39700000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":28.015999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":2.5699999999999998},{"interface":"front","average":{"1min":2.024,"5min":1.29,"15min":0.85999999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":28.039999999999999,"5min":28.039999999999999,"15min":28.039999999999999},"last":2.5859999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:26:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.65000000000000002,"15min":0.53100000000000003},"min":{"1min":0.23699999999999999,"5min":0.222,"15min":0.222},"max":{"1min":8.5359999999999996,"5min":8.5359999999999996,"15min":8.5359999999999996},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.73599999999999999,"15min":0.59499999999999997},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.6200000000000001,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":0.33600000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.59099999999999997,"15min":0.54200000000000004},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.745,"5min":2.125,"15min":2.125},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.79100000000000004,"15min":0.66300000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":0.66600000000000004,"5min":4.46,"15min":4.46},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:25:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.75800000000000001,"15min":0.64200000000000002},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32500000000000001},"max":{"1min":1.1419999999999999,"5min":4.194,"15min":4.194},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.72199999999999998,"15min":0.623},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.605,"5min":6.2300000000000004,"15min":6.2300000000000004},"last":0.35699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1254359040,"data_stored":1254359040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":734199808,"data_stored":734199808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1042157568,"data_stored":1042153496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":531824640,"data_stored":531824640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1403387904,"data_stored":1403387904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1401688064,"data_stored":1401683992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":687603712,"data_stored":687603712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":809177088,"data_stored":809173016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:26:29.927 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-30T09:26:29.927 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:26:29.927 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-30T09:26:29.928 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-30T09:26:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:30 smithi167 ceph-mon[192880]: pgmap v404: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-09-30T09:26:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:30 smithi167 ceph-mon[192880]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:30 smithi167 ceph-mon[192880]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:30 smithi167 ceph-mon[192880]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:30 smithi167 ceph-mon[192880]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:30 smithi167 ceph-mon[192880]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[216815]: pgmap v404: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-09-30T09:26:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[216815]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[216815]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[216815]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[216815]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[216815]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[224043]: pgmap v404: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-09-30T09:26:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[224043]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[224043]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[224043]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[224043]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:30 smithi019 ceph-mon[224043]: failed to encode map e179 with expected crc 2023-09-30T09:26:30.926 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:30.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:30.970 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:26:31.030 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:31.074 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:31.200 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:31.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:31.311 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:31.381 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:31.390 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:31.491 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:31.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:31.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:31.611 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:31.685 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:26:31.700 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:31.791 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:31.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:31.895 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:31.965 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:32.031 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:26:32.072 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:26:32.075 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:32.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:32.111 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:26:32.149 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:26:32.184 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:32.199 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:26:32.231 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:32.238 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:26:32.271 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:26:32.309 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:26:32.314 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:26:32.314 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:26:32.314 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:26:32.314 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:26:32.314 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:26:32.314 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:26:32.315 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:26:32.315 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:26:32.335 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:32.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:32.447 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:26:32.519 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:32 smithi167 ceph-mon[192880]: pgmap v406: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-30T09:26:32.519 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4088793745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:32.523 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:32.615 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:32 smithi019 ceph-mon[216815]: pgmap v406: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-30T09:26:32.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4088793745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:32.616 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:32 smithi019 ceph-mon[224043]: pgmap v406: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-30T09:26:32.616 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4088793745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:33.027 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:26:33.846 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:26:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:33.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:26:33.958 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:34.003 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:34.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:26:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:26:33] "GET /metrics HTTP/1.1" 200 36291 "" "Prometheus/2.43.0" 2023-09-30T09:26:34.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[216815]: pgmap v407: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[224043]: pgmap v407: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:34.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:26:34.227 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:34.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:33 smithi167 ceph-mon[192880]: pgmap v407: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-30T09:26:34.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:34.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:26:34.669 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:34.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:34.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:34.777 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:34.847 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:34.958 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:34.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:34.996 INFO:teuthology.orchestra.run.smithi019.stdout:moved 0 / 267 (0%) 2023-09-30T09:26:34.996 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:26:34.996 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 4.02919 -> 4.02919 (expected baseline 5.40399) 2023-09-30T09:26:34.996 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.2 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-09-30T09:26:34.996 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.4 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-30T09:26:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:26:34.997 INFO:teuthology.orchestra.run.smithi019.stdout:oload 1000 2023-09-30T09:26:34.997 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 1 2023-09-30T09:26:34.997 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:26:34.997 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0103 2023-09-30T09:26:34.997 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.1028 2023-09-30T09:26:35.010 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:26:35.068 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:35.137 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:35.434 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:35.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:35.539 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:35.590 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:26:35.590 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:26:35.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.590 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:136095 5.e 5:7a043c5f:::benchmark_data_smithi019_287194_object53317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:26:35.590 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:35.554200+0000", 2023-09-30T09:26:35.590 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.034322136000000003, 2023-09-30T09:26:35.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.034347992000000001, 2023-09-30T09:26:35.590 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 136095 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.554200+0000", 2023-09-30T09:26:35.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.554200+0000", 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.554200+0000", 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.554203+0000", 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7779999999999999e-06 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.554397+0000", 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019438999999999999 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:35.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.554405+0000", 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2130000000000008e-06 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555518+0000", 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001113113 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555620+0000", 2023-09-30T09:26:35.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000102179 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555724+0000", 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103983 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.563287+0000", 2023-09-30T09:26:35.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075624819999999997 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.571809+0000", 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0085225270000000002 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.596 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:136096 5.e 5:7dd8321a:::benchmark_data_smithi019_287194_object53318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:35.555608+0000", 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.032914447999999999, 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.033010126000000001, 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 136096 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.597 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555608+0000", 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555608+0000", 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555607+0000", 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555613+0000", 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6010000000000002e-06 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.599 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555623+0000", 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0336999999999999e-05 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.555629+0000", 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0820000000000004e-06 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:35.600 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.559231+0000", 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036020269999999998 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.559342+0000", 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011059499999999999 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.559459+0000", 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011780199999999999 2023-09-30T09:26:35.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.564311+0000", 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048512989999999999 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.571847+0000", 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075358600000000001 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.602 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.44600.0:136112 5.f e179/113 5:f5d2323f:::benchmark_data_smithi019_287194_object53334:head v 179'8609, mlcod=179'8607)", 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:35.582113+0000", 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0064093759999999996, 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0065549830000000003, 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.582113+0000", 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.582113+0000", 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:35.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.582112+0000", 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.582119+0000", 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1629999999999999e-06 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.582134+0000", 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4256999999999999e-05 2023-09-30T09:26:35.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.582140+0000", 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1580000000000002e-06 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.582181+0000", 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1662000000000002e-05 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.582209+0000", 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7528000000000001e-05 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:136091 5.e 5:7c0c16aa:::benchmark_data_smithi019_287194_object53313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:35.548144+0000", 2023-09-30T09:26:35.607 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.040378543000000003, 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.040560387000000003, 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 136091 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.608 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.548144+0000", 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.548144+0000", 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.548143+0000", 2023-09-30T09:26:35.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.548147+0000", 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0380000000000001e-06 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.548158+0000", 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0186999999999999e-05 2023-09-30T09:26:35.610 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.548163+0000", 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9960000000000001e-06 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.551477+0000", 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003314703 2023-09-30T09:26:35.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.551606+0000", 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012871599999999999 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.551684+0000", 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7910000000000005e-05 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.612 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.563145+0000", 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011460705999999999 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:35.570129+0000", 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0069839710000000003 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.613 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:35.614 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.614 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:35.614 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:26:35.614 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 4 2023-09-30T09:26:35.614 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:26:35.614 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:35.813 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:35.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:35.917 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:35.973 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:36.023 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:26:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:36 smithi019 ceph-mon[216815]: from='client.44903 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:36 smithi019 ceph-mon[216815]: pgmap v408: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2023-09-30T09:26:36.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:36 smithi019 ceph-mon[224043]: from='client.44903 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:36.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:36 smithi019 ceph-mon[224043]: pgmap v408: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2023-09-30T09:26:36.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:36 smithi167 ceph-mon[192880]: from='client.44903 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:36.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:36 smithi167 ceph-mon[192880]: pgmap v408: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2023-09-30T09:26:36.639 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:26:36.652 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:36.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:36.719 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-30T09:26:36.719 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-30T09:26:36.787 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:36.902 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:37.112 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:37.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:37.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:37.216 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:37.248 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:37.297 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:37.540 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:26:37.540 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:37.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:37.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:37.647 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:37.728 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:37.847 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:37.877 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:37.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:37.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:37.987 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:38.047 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:38.152 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:26:38.298 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:38.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:38.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:38 smithi019 ceph-mon[216815]: pgmap v409: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-30T09:26:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2895634310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:38 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:38 smithi019 ceph-mon[224043]: pgmap v409: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-30T09:26:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2895634310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:38 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:38.402 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:38.431 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:38 smithi167 ceph-mon[192880]: pgmap v409: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-30T09:26:38.431 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2895634310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:38.431 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:38 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:38.434 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:39.106 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:39.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:39.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:39.149 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:39.149 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:39.149 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[216815]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.149 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[216815]: osdmap e180: 8 total, 8 up, 8 in 2023-09-30T09:26:39.149 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[216815]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.149 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[216815]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.209 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:39.257 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:39.445 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:39.445 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:39.445 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[224043]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.445 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[224043]: osdmap e180: 8 total, 8 up, 8 in 2023-09-30T09:26:39.445 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[224043]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.445 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:39 smithi019 ceph-mon[224043]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:39 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:39 smithi167 ceph-mon[192880]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:39 smithi167 ceph-mon[192880]: osdmap e180: 8 total, 8 up, 8 in 2023-09-30T09:26:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:39 smithi167 ceph-mon[192880]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:39 smithi167 ceph-mon[192880]: failed to encode map e180 with expected crc 2023-09-30T09:26:39.631 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:39.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:39.741 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:39.793 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:40.088 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:26:40.088 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:26:40.088 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:26:40.088 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:26:40.088 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:26:40.186 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:40.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:40.291 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:40 smithi019 ceph-mon[216815]: pgmap v411: 89 pgs: 89 active+clean; 1.8 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-30T09:26:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:40 smithi019 ceph-mon[224043]: pgmap v411: 89 pgs: 89 active+clean; 1.8 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-30T09:26:40.357 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:40.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:40 smithi167 ceph-mon[192880]: pgmap v411: 89 pgs: 89 active+clean; 1.8 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-30T09:26:40.756 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:26:40.756 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:26:40.756 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:26:40.769 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:26:40.800 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:26:40.834 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:26:40.872 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:26:40.914 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:26:40.924 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:26:40.925 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:26:40.925 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:26:40.925 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:26:40.925 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:26:40.925 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:26:40.925 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:26:40.925 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:26:41.220 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/651065855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:41.276 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:26:41.300 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:41.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:41.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:41.412 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:41.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/651065855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:41.512 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:41.583 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:26:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/651065855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:41.638 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:41.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:41.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:41.744 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:41.811 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:41.923 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:41.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:41.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:42.040 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:42.095 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:42.164 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:26:42.219 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:42.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:42.345 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:42.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:42 smithi167 ceph-mon[192880]: pgmap v412: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-30T09:26:42.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:42.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:42.448 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:42.556 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:42.568 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:42.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:42 smithi019 ceph-mon[224043]: pgmap v412: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-30T09:26:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:42 smithi019 ceph-mon[216815]: pgmap v412: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-30T09:26:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:42.662 INFO:teuthology.orchestra.run.smithi019.stdout:moved 0 / 267 (0%) 2023-09-30T09:26:42.662 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:26:42.662 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 4.02919 -> 4.02919 (expected baseline 5.40399) 2023-09-30T09:26:42.662 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.2 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-09-30T09:26:42.663 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.4 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-30T09:26:42.663 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:26:42.663 INFO:teuthology.orchestra.run.smithi019.stdout:oload 1000 2023-09-30T09:26:42.663 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 1 2023-09-30T09:26:42.663 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:26:42.663 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0085 2023-09-30T09:26:42.663 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0853 2023-09-30T09:26:42.672 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:42.680 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:26:42.734 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:43.028 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:43.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:43.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:43.136 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:43.163 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:43.206 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:43.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:43.309 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:43.396 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:43.479 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:43.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:43.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:43.582 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:43.639 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:43.810 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:43.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:43.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:43.845 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-30T09:26:43.845 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-30T09:26:43.918 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:43.939 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:26:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:26:43] "GET /metrics HTTP/1.1" 200 36291 "" "Prometheus/2.43.0" 2023-09-30T09:26:43.947 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:26:44.013 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:44.094 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:44.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:44.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:44.198 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:44.247 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:44.336 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:44.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:44.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:44.441 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:44 smithi167 ceph-mon[192880]: from='client.44921 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:44 smithi167 ceph-mon[192880]: pgmap v413: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-30T09:26:44.538 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:44.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:44 smithi019 ceph-mon[216815]: from='client.44921 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:44.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:44 smithi019 ceph-mon[216815]: pgmap v413: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-30T09:26:44.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:44 smithi019 ceph-mon[224043]: from='client.44921 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:44.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:44 smithi019 ceph-mon[224043]: pgmap v413: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-30T09:26:44.641 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:26:44.642 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:26:44.662 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:44.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:44.767 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:44.822 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:44.929 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:44.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:44.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:45.034 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:45.059 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:45.101 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:45.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:45.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:45.204 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:45.237 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:45.430 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:45.447 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:45.453 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:45.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:45.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:45.557 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:26:45.571 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:26:45.686 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:46.080 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:26:46.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:46.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:26:46.154 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:26:46.203 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:46.258 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:46.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[216815]: pgmap v414: 89 pgs: 89 active+clean; 1.5 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-30T09:26:46.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:46.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:46.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:26:46.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:26:46.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[224043]: pgmap v414: 89 pgs: 89 active+clean; 1.5 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-30T09:26:46.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:46.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:46.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:26:46.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:26:46.489 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:46.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:46.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:46 smithi167 ceph-mon[192880]: pgmap v414: 89 pgs: 89 active+clean; 1.5 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-30T09:26:46.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:46.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:46.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:26:46.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:26:46.597 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:46.670 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:46.933 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:46.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:46.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:47.037 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:47.101 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:47.315 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:26:47.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:47.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1458724341' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:47.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:47.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1458724341' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:47.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:26:47.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1458724341' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:26:47.722 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:47.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:47.825 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:47.879 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:47.935 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:47.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:48.039 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:48.081 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:48.297 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: pgmap v415: 89 pgs: 89 active+clean; 1.5 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1458724341' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: osdmap e181: 8 total, 8 up, 8 in 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.298 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.299 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:48.322 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:26:48.322 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:26:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: pgmap v415: 89 pgs: 89 active+clean; 1.5 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-30T09:26:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1458724341' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:26:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: osdmap e181: 8 total, 8 up, 8 in 2023-09-30T09:26:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: pgmap v415: 89 pgs: 89 active+clean; 1.5 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1458724341' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: osdmap e181: 8 total, 8 up, 8 in 2023-09-30T09:26:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: failed to encode map e181 with expected crc 2023-09-30T09:26:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:26:48.952 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:48.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:48.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:49.061 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:49.119 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:49.250 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:26:49.389 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:49.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:49.496 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:50.072 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:26:50.075 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:26:50.075 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:26:50.075 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:26:50.075 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:26:50.075 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:26:50.075 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:26:50.075 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:26:50.076 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:26:50.076 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:26:50.076 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:26:50.076 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:26:50.076 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:26:50.076 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:26:50.076 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:26:50.076 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:26:50.077 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:26:50.169 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:50.308 INFO:teuthology.orchestra.run.smithi019.stdout:moved 32 / 267 (11.985%) 2023-09-30T09:26:50.308 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:26:50.308 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 4.02919 -> 6.4214 (expected baseline 5.40399) 2023-09-30T09:26:50.308 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.2 with 26 -> 28 pgs (0.779026 -> 0.838951 * mean) 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.4 with 41 -> 35 pgs (1.22846 -> 1.04869 * mean) 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:oload 110 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 3 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0070 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0077 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 weight 1.0000 -> 0.6628 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 weight 1.0000 -> 0.7535 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 weight 1.0000 -> 0.7561 2023-09-30T09:26:50.309 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 weight 1.0000 -> 0.8893 2023-09-30T09:26:50.317 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:50 smithi019 ceph-mon[216815]: pgmap v417: 89 pgs: 89 active+clean; 1.4 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-30T09:26:50.317 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2160862305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:50.317 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'43438','2':'58280','4':'49379','5':'49555'}"}]: dispatch 2023-09-30T09:26:50.326 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:50.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:50.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:50.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:50.467 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:50.512 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:50.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:50.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:50.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:50 smithi167 ceph-mon[192880]: pgmap v417: 89 pgs: 89 active+clean; 1.4 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-30T09:26:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2160862305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'43438','2':'58280','4':'49379','5':'49555'}"}]: dispatch 2023-09-30T09:26:50.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:50 smithi019 ceph-mon[224043]: pgmap v417: 89 pgs: 89 active+clean; 1.4 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-30T09:26:50.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2160862305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:26:50.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'43438','2':'58280','4':'49379','5':'49555'}"}]: dispatch 2023-09-30T09:26:50.616 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:50.644 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:50.688 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:50.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:50.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:50.793 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:50.822 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:50.843 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:26:50.864 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:50.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:50.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:50.967 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:50.994 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:51.240 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:51.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:51.345 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:51.373 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:51.514 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: from='client.54817 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:51.515 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'43438','2':'58280','4':'49379','5':'49555'}"}]': finished 2023-09-30T09:26:51.515 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: osdmap e182: 8 total, 8 up, 8 in 2023-09-30T09:26:51.515 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.515 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.515 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.515 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.516 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.516 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:51 smithi167 ceph-mon[192880]: osdmap e183: 8 total, 8 up, 8 in 2023-09-30T09:26:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: from='client.54817 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'43438','2':'58280','4':'49379','5':'49555'}"}]': finished 2023-09-30T09:26:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: osdmap e182: 8 total, 8 up, 8 in 2023-09-30T09:26:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[216815]: osdmap e183: 8 total, 8 up, 8 in 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: from='client.54817 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'43438','2':'58280','4':'49379','5':'49555'}"}]': finished 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: osdmap e182: 8 total, 8 up, 8 in 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: failed to encode map e182 with expected crc 2023-09-30T09:26:51.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:51 smithi019 ceph-mon[224043]: osdmap e183: 8 total, 8 up, 8 in 2023-09-30T09:26:51.639 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:51.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:51.656 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-30T09:26:51.657 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-30T09:26:51.718 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:26:51.718 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:26:51.718 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:26:51.718 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:26:51.718 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:26:51.720 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:51.743 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:51.774 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:51.920 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:51.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:52.024 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:52.051 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:52.092 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:52.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:52.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:52.195 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:52.222 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:52.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: pgmap v419: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[216815]: osdmap e184: 8 total, 8 up, 8 in 2023-09-30T09:26:52.489 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:52.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:52.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:52.515 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:26:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: pgmap v419: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-09-30T09:26:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:52 smithi019 ceph-mon[224043]: osdmap e184: 8 total, 8 up, 8 in 2023-09-30T09:26:52.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: pgmap v419: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-09-30T09:26:52.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: failed to encode map e183 with expected crc 2023-09-30T09:26:52.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:52 smithi167 ceph-mon[192880]: osdmap e184: 8 total, 8 up, 8 in 2023-09-30T09:26:52.596 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:52.630 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:52.681 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:52.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:52.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:52.787 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:26:52.832 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:52.940 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:26:52.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:52.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:26:53.053 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:26:53.053 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:53.109 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:53.230 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:53.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:53.293 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:53.296 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:26:53.339 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:53.380 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:53.492 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:53.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:53.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[216815]: 5.e starting backfill to osd.3 from (0'0,0'0] MAX to 181'9081 2023-09-30T09:26:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[216815]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[216815]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[216815]: 5.7 starting backfill to osd.4 from (0'0,176'7771] MIN to 181'9056 2023-09-30T09:26:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[216815]: 5.5 starting backfill to osd.6 from (0'0,154'5472] MIN to 181'9073 2023-09-30T09:26:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[216815]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[216815]: 5.5 starting backfill to osd.7 from (0'0,0'0] MAX to 181'9073 2023-09-30T09:26:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[216815]: osdmap e185: 8 total, 8 up, 8 in 2023-09-30T09:26:53.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[224043]: 5.e starting backfill to osd.3 from (0'0,0'0] MAX to 181'9081 2023-09-30T09:26:53.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[224043]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[224043]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[224043]: 5.7 starting backfill to osd.4 from (0'0,176'7771] MIN to 181'9056 2023-09-30T09:26:53.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[224043]: 5.5 starting backfill to osd.6 from (0'0,154'5472] MIN to 181'9073 2023-09-30T09:26:53.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[224043]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[224043]: 5.5 starting backfill to osd.7 from (0'0,0'0] MAX to 181'9073 2023-09-30T09:26:53.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:53 smithi019 ceph-mon[224043]: osdmap e185: 8 total, 8 up, 8 in 2023-09-30T09:26:53.604 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:26:53.729 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:53 smithi167 ceph-mon[192880]: 5.e starting backfill to osd.3 from (0'0,0'0] MAX to 181'9081 2023-09-30T09:26:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:53 smithi167 ceph-mon[192880]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:53 smithi167 ceph-mon[192880]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:53 smithi167 ceph-mon[192880]: 5.7 starting backfill to osd.4 from (0'0,176'7771] MIN to 181'9056 2023-09-30T09:26:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:53 smithi167 ceph-mon[192880]: 5.5 starting backfill to osd.6 from (0'0,154'5472] MIN to 181'9073 2023-09-30T09:26:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:53 smithi167 ceph-mon[192880]: failed to encode map e184 with expected crc 2023-09-30T09:26:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:53 smithi167 ceph-mon[192880]: 5.5 starting backfill to osd.7 from (0'0,0'0] MAX to 181'9073 2023-09-30T09:26:53.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:53 smithi167 ceph-mon[192880]: osdmap e185: 8 total, 8 up, 8 in 2023-09-30T09:26:53.991 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:26:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:26:53] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2023-09-30T09:26:54.059 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:26:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:54.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:26:54.165 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:54.192 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:54.273 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:54.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:54.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:54.379 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:54.419 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: pgmap v422: 89 pgs: 2 remapped+peering, 87 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 0 B/s, 0 objects/s recovering 2023-09-30T09:26:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:54 smithi167 ceph-mon[192880]: osdmap e186: 8 total, 8 up, 8 in 2023-09-30T09:26:54.595 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:54.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:54.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:54.638 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: pgmap v422: 89 pgs: 2 remapped+peering, 87 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 0 B/s, 0 objects/s recovering 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.639 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[224043]: osdmap e186: 8 total, 8 up, 8 in 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: pgmap v422: 89 pgs: 2 remapped+peering, 87 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 0 B/s, 0 objects/s recovering 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: failed to encode map e185 with expected crc 2023-09-30T09:26:54.640 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:54 smithi019 ceph-mon[216815]: osdmap e186: 8 total, 8 up, 8 in 2023-09-30T09:26:54.707 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:26:54.789 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:54.973 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:26:54.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:54.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:26:55.079 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:55.147 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:55.255 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:26:55.255 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:26:55.255 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:26:55.255 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:26:55.255 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.255 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61518 5.7 5:e18009d3:::benchmark_data_smithi019_287194_object61517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.345480+0000", 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.90504210500001, 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0257004590000001, 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61518 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.256 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345480+0000", 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345480+0000", 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.257 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345479+0000", 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345523+0000", 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4019e-05 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.258 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345541+0000", 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7693000000000002e-05 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345548+0000", 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5120000000000002e-06 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345603+0000", 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4543000000000003e-05 2023-09-30T09:26:55.259 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345722+0000", 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011874700000000001 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.345851+0000", 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012963800000000001 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.260 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.347660+0000", 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018084659999999999 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.348395+0000", 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00073505900000000004 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.261 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.353323+0000", 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049284050000000003 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.353336+0000", 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3338e-05 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371124+0000", 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.017787105 2023-09-30T09:26:55.262 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371181+0000", 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7352000000000002e-05 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61530 5.7 5:ea633b77:::benchmark_data_smithi019_287194_object61529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.512460+0000", 2023-09-30T09:26:55.263 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.73806248400001, 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.877191371, 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61530 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.264 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512460+0000", 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512460+0000", 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512459+0000", 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:26:55.265 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512489+0000", 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9590999999999998e-05 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512497+0000", 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4209999999999995e-06 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.266 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512502+0000", 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0540000000000002e-06 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512572+0000", 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9250000000000003e-05 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.512581+0000", 2023-09-30T09:26:55.267 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6949999999999995e-06 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371197+0000", 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.85861520499999999 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371276+0000", 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9523000000000004e-05 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.268 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.371425+0000", 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014907200000000001 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.377793+0000", 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0063680660000000004 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.269 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389534+0000", 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011741122 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389564+0000", 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9206000000000001e-05 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389615+0000", 2023-09-30T09:26:55.270 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1591999999999997e-05 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389651+0000", 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6310999999999998e-05 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61551 5.7 5:eed4ef95:::benchmark_data_smithi019_287194_object61550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.271 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.536037+0000", 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.714485846, 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.85396402599999999, 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61551 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.272 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536037+0000", 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536037+0000", 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.273 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536035+0000", 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536072+0000", 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6541999999999997e-05 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.274 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536088+0000", 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5914999999999999e-05 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536094+0000", 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3520000000000003e-06 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536160+0000", 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5940999999999998e-05 2023-09-30T09:26:55.275 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.536171+0000", 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1406999999999999e-05 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.372958+0000", 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83678686899999999 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.276 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.373042+0000", 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4040000000000002e-05 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.373136+0000", 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3954000000000002e-05 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.377800+0000", 2023-09-30T09:26:55.277 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004663365 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389684+0000", 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011883906 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389944+0000", 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00026025100000000003 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.278 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389975+0000", 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1619999999999999e-05 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.390001+0000", 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5270999999999999e-05 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.279 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61563 5.7 5:e210ad0e:::benchmark_data_smithi019_287194_object61562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.542990+0000", 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.707532071, 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.84709957000000002, 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61563 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.280 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.542990+0000", 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.542990+0000", 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.281 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.542989+0000", 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543032+0000", 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3235000000000002e-05 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543046+0000", 2023-09-30T09:26:55.282 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4044e-05 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543053+0000", 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5930000000000002e-06 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543438+0000", 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00038565099999999998 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.283 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.543448+0000", 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5570000000000001e-06 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.374254+0000", 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83080556800000005 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.374332+0000", 2023-09-30T09:26:55.284 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8102000000000005e-05 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.374433+0000", 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101196 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379043+0000", 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046102929999999997 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.285 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389723+0000", 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010679615999999999 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.390039+0000", 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031628500000000003 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.390068+0000", 2023-09-30T09:26:55.286 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9300000000000001e-05 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.390090+0000", 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1701000000000001e-05 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61568 5.7 5:e495c4d5:::benchmark_data_smithi019_287194_object61567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.287 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.545646+0000", 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.70487621800001, 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.85827185699999997, 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61568 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.288 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545646+0000", 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545646+0000", 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.289 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545646+0000", 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545675+0000", 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9413000000000001e-05 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545681+0000", 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0109999999999997e-06 2023-09-30T09:26:55.290 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545685+0000", 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6059999999999999e-06 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545750+0000", 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5037999999999996e-05 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.291 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.545759+0000", 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2690000000000005e-06 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.375557+0000", 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82979780400000003 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.292 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.375662+0000", 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105048 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.375765+0000", 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000102701 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379053+0000", 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032880639999999998 2023-09-30T09:26:55.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389742+0000", 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010688939 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403825+0000", 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014083469 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.294 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403880+0000", 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5074000000000002e-05 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403918+0000", 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7787999999999998e-05 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.295 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61624 5.7 5:ea2cafff:::benchmark_data_smithi019_287194_object61623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.592059+0000", 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.65846386300001, 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81197014700000003, 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61624 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.296 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592059+0000", 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592059+0000", 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592057+0000", 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592149+0000", 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2001000000000002e-05 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592164+0000", 2023-09-30T09:26:55.298 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4605e-05 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.592171+0000", 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9630000000000001e-06 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.595232+0000", 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030614380000000001 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.299 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.595243+0000", 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0117e-05 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.376729+0000", 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.78148660800000003 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.300 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.376805+0000", 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5838000000000001e-05 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.376892+0000", 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7280000000000005e-05 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.381389+0000", 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044968070000000002 2023-09-30T09:26:55.301 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389842+0000", 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0084531889999999998 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403951+0000", 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014109116 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.302 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.403994+0000", 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2341e-05 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404029+0000", 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5113000000000001e-05 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.303 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61627 5.7 5:e02a1d6c:::benchmark_data_smithi019_287194_object61626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.596894+0000", 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.65362817100001, 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.80724527599999996, 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61627 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.304 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596894+0000", 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596894+0000", 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.305 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596893+0000", 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596950+0000", 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6727000000000001e-05 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596961+0000", 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1207e-05 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.306 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596968+0000", 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2500000000000003e-06 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597025+0000", 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7632e-05 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597034+0000", 2023-09-30T09:26:55.307 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8570000000000001e-06 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.377818+0000", 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.78078413999999996 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.377890+0000", 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.169e-05 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.308 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.378021+0000", 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013104300000000001 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383651+0000", 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0056301830000000004 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389885+0000", 2023-09-30T09:26:55.309 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062335450000000001 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404061+0000", 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014175956 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404105+0000", 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4431000000000002e-05 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.310 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404140+0000", 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4637000000000001e-05 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61628 5.7 5:ee236a29:::benchmark_data_smithi019_287194_object61627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.596983+0000", 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.65353995699999, 2023-09-30T09:26:55.311 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.80726483400000004, 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61628 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.312 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596983+0000", 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596983+0000", 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.596982+0000", 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:26:55.313 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597013+0000", 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1186000000000003e-05 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597028+0000", 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5129e-05 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.314 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597034+0000", 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.891e-06 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597075+0000", 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0327000000000002e-05 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.597083+0000", 2023-09-30T09:26:55.315 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5290000000000008e-06 2023-09-30T09:26:55.316 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.317 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.317 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.317 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379093+0000", 2023-09-30T09:26:55.317 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.78200961700000005 2023-09-30T09:26:55.317 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.317 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.317 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379208+0000", 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011480300000000001 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.379300+0000", 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2001000000000002e-05 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383663+0000", 2023-09-30T09:26:55.318 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043638419999999997 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389914+0000", 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062502749999999996 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404173+0000", 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014258966 2023-09-30T09:26:55.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404213+0000", 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0880999999999997e-05 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404247+0000", 2023-09-30T09:26:55.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3939000000000002e-05 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61650 5.7 5:e92d62dc:::benchmark_data_smithi019_287194_object61649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.615308+0000", 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.63521421900001, 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.78907863300000003, 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.321 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61650 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615308+0000", 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615308+0000", 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615306+0000", 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615343+0000", 2023-09-30T09:26:55.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6816000000000001e-05 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615361+0000", 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8280000000000001e-05 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615368+0000", 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9739999999999996e-06 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615441+0000", 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3008000000000003e-05 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.615451+0000", 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6490000000000003e-06 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.380228+0000", 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.76477674600000001 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.380321+0000", 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2794000000000006e-05 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.380444+0000", 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000123688 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.385970+0000", 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055259710000000002 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.393531+0000", 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075608689999999996 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404295+0000", 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010763477 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404350+0000", 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5593000000000001e-05 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404387+0000", 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6807000000000002e-05 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.328 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61667 5.7 5:e26b6ee9:::benchmark_data_smithi019_287194_object61666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.630906+0000", 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.619616799, 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.773589215, 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.329 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61667 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630906+0000", 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630906+0000", 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630905+0000", 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630952+0000", 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7675e-05 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630965+0000", 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3206e-05 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.630971+0000", 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1000000000000003e-06 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.631033+0000", 2023-09-30T09:26:55.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2259000000000001e-05 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.631043+0000", 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6919999999999997e-06 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.381434+0000", 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.75039142599999997 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.381539+0000", 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010470499999999999 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.381630+0000", 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1679000000000001e-05 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.385976+0000", 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043454490000000004 2023-09-30T09:26:55.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.393576+0000", 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075999409999999998 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404433+0000", 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010857232 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404473+0000", 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9950000000000002e-05 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404495+0000", 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2050999999999999e-05 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.336 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61696 5.7 5:e6913336:::benchmark_data_smithi019_287194_object61695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.658118+0000", 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.592404778, 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.74646133999999997, 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61696 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658118+0000", 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658118+0000", 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658116+0000", 2023-09-30T09:26:55.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658170+0000", 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3786000000000003e-05 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658180+0000", 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0378e-05 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658193+0000", 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2658e-05 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658254+0000", 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1359999999999995e-05 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.658263+0000", 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8990000000000007e-06 2023-09-30T09:26:55.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.382547+0000", 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.724284072 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.382670+0000", 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012278699999999999 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.382762+0000", 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1383000000000002e-05 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389213+0000", 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064515149999999997 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.396897+0000", 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076843409999999999 2023-09-30T09:26:55.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404518+0000", 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007620679 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404548+0000", 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9479999999999999e-05 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.404579+0000", 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1501000000000002e-05 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61735 5.7 5:e5c7b539:::benchmark_data_smithi019_287194_object61734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.698538+0000", 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.551984551, 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.70766760500000003, 2023-09-30T09:26:55.344 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61735 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698538+0000", 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698538+0000", 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698537+0000", 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698572+0000", 2023-09-30T09:26:55.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4150000000000003e-05 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698580+0000", 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3729999999999993e-06 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698585+0000", 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.13e-06 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698681+0000", 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6032000000000005e-05 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.698691+0000", 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7850000000000003e-06 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383703+0000", 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.68501200699999998 2023-09-30T09:26:55.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383793+0000", 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0301000000000002e-05 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.383885+0000", 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2262000000000005e-05 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389222+0000", 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0053361379999999998 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.396940+0000", 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077180720000000003 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406079+0000", 2023-09-30T09:26:55.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091388609999999999 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406151+0000", 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2158999999999998e-05 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406206+0000", 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4895999999999998e-05 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.351 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61739 5.7 5:efc4b11e:::benchmark_data_smithi019_287194_object61738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.701702+0000", 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.54882044300001, 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.70468040600000004, 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.352 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61739 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701702+0000", 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701702+0000", 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701701+0000", 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701760+0000", 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8833999999999999e-05 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701772+0000", 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1620999999999999e-05 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701777+0000", 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2429999999999996e-06 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701822+0000", 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4644999999999997e-05 2023-09-30T09:26:55.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701829+0000", 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4200000000000001e-06 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.384884+0000", 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.68305470300000004 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.384993+0000", 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010960500000000001 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.385082+0000", 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8086999999999999e-05 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389325+0000", 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042431379999999996 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.402848+0000", 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013523373 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406276+0000", 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034283489999999998 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406345+0000", 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8824000000000004e-05 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.406382+0000", 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7313e-05 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.359 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61741 5.7 5:e7992001:::benchmark_data_smithi019_287194_object61740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.701920+0000", 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.54860302, 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.712059569, 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61741 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701920+0000", 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701920+0000", 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701919+0000", 2023-09-30T09:26:55.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701991+0000", 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2441999999999995e-05 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.701999+0000", 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7209999999999994e-06 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.702006+0000", 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7259999999999996e-06 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.702038+0000", 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2450999999999998e-05 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.702044+0000", 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3680000000000001e-06 2023-09-30T09:26:55.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.386014+0000", 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.68397022399999996 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.386094+0000", 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9943000000000004e-05 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.386196+0000", 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101778 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389331+0000", 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031357960000000002 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.402875+0000", 2023-09-30T09:26:55.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013543914000000001 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.413867+0000", 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010991618999999999 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.413936+0000", 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8697999999999999e-05 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.413979+0000", 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3498999999999998e-05 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61746 5.7 5:e394340c:::benchmark_data_smithi019_287194_object61745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.711658+0000", 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.538864238, 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.70245204800000005, 2023-09-30T09:26:55.367 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61746 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711658+0000", 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711658+0000", 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711657+0000", 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711735+0000", 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7892000000000006e-05 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711751+0000", 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6310000000000001e-05 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711758+0000", 2023-09-30T09:26:55.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6769999999999998e-06 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711843+0000", 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5019999999999996e-05 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.711856+0000", 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2869e-05 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.387094+0000", 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.67523834800000004 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.387201+0000", 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010642899999999999 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.387290+0000", 2023-09-30T09:26:55.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9501999999999998e-05 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389336+0000", 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002045561 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.402893+0000", 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01355757 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414022+0000", 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011128895999999999 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414063+0000", 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0883000000000001e-05 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414110+0000", 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7240999999999997e-05 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:61753 5.7 5:e75af397:::benchmark_data_smithi019_287194_object61752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:40.728877+0000", 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 194.52164597300001, 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.68544973399999998, 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.375 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 61753 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728877+0000", 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728877+0000", 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728875+0000", 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728933+0000", 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7878999999999999e-05 2023-09-30T09:26:55.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728949+0000", 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5931e-05 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.728956+0000", 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9099999999999999e-06 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.729011+0000", 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5245999999999999e-05 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:40.729021+0000", 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7010000000000008e-06 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.388165+0000", 2023-09-30T09:26:55.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.65914413800000005 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.388241+0000", 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6187999999999996e-05 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.388331+0000", 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0103000000000001e-05 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.389689+0000", 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013576490000000001 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.402907+0000", 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013218647 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414240+0000", 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011332493000000001 2023-09-30T09:26:55.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414285+0000", 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5207000000000003e-05 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:41.414326+0000", 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1149000000000002e-05 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.382 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:80441 5.3 5:c3f79ee6:::benchmark_data_smithi019_287194_object80440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:24:01.607700+0000", 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 173.642822524, 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0053129009999999, 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.383 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 80441 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607700+0000", 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607700+0000", 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607699+0000", 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607729+0000", 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0666000000000001e-05 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607743+0000", 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3380999999999999e-05 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607752+0000", 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0180000000000007e-06 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607817+0000", 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5216999999999996e-05 2023-09-30T09:26:55.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.607989+0000", 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000171857 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,6", 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608168+0000", 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017956800000000001 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.615044+0000", 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006875613 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.615057+0000", 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3441000000000001e-05 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.612993+0000", 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99793548200000004 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613013+0000", 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0143e-05 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:80443 5.3 5.414b81e3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:24:01.608633+0000", 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 173.641889905, 2023-09-30T09:26:55.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.004423759, 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 80443 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608633+0000", 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608633+0000", 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608624+0000", 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999914 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608673+0000", 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9038e-05 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608686+0000", 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2653000000000001e-05 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.608693+0000", 2023-09-30T09:26:55.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9299999999999997e-06 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.615099+0000", 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006406747 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.615109+0000", 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7010000000000008e-06 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613038+0000", 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99792904500000001 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613056+0000", 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8216999999999999e-05 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:80448 5.3 5.44d7ee03 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:24:01.627487+0000", 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 173.623035969, 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98560211399999997, 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 80448 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.395 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627487+0000", 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627487+0000", 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627485+0000", 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627550+0000", 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4855000000000002e-05 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627566+0000", 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6478999999999999e-05 2023-09-30T09:26:55.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627574+0000", 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7209999999999994e-06 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627639+0000", 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5351999999999994e-05 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:01.627650+0000", 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0311e-05 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613079+0000", 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98542918400000001 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:24:02.613089+0000", 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8390000000000002e-06 2023-09-30T09:26:55.399 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:135213 5.7 5:e1737c38:::benchmark_data_smithi019_287194_object52435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.071277+0000", 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 21.179245577, 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33402298499999999, 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:26:55.400 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 135213 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.071277+0000", 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:55.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.071277+0000", 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.071277+0000", 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.071279+0000", 2023-09-30T09:26:55.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8559999999999998e-06 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.071286+0000", 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2160000000000003e-06 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.071290+0000", 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9960000000000004e-06 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.075880+0000", 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045904650000000002 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.075993+0000", 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112694 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.076064+0000", 2023-09-30T09:26:55.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1338e-05 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.080654+0000", 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045898420000000002 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.092473+0000", 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011818644 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.405204+0000", 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.31273095099999998 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.405254+0000", 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0866999999999999e-05 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:26:55.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.405300+0000", 2023-09-30T09:26:55.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5546000000000001e-05 2023-09-30T09:26:55.407 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.407 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.407 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.407 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:55.407 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:55.407 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:26:55.408 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:26:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:55.408 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:55.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:55.509 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:55.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2323561916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:55.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:55 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:55.581 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:55.622 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:55.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:55.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:55.726 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:26:55.755 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.756 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2323561916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: failed to encode map e186 with expected crc 2023-09-30T09:26:55.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2323561916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:55.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:55 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:26:55.760 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:56.148 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:26:56.200 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:26:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:56.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:26:56.309 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:26:56.382 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:56.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: pgmap v425: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 remapped+peering, 82 active+clean; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 5103/57501 objects degraded (8.875%); 2056/57501 objects misplaced (3.576%); 5.7 MiB/s, 96 objects/s recovering 2023-09-30T09:26:56.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 5103/57501 objects degraded (8.875%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:26:56.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:26:56.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:26:56.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:56.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: osdmap e187: 8 total, 8 up, 8 in 2023-09-30T09:26:56.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.399 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:56 smithi167 ceph-mon[192880]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: pgmap v425: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 remapped+peering, 82 active+clean; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 5103/57501 objects degraded (8.875%); 2056/57501 objects misplaced (3.576%); 5.7 MiB/s, 96 objects/s recovering 2023-09-30T09:26:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 5103/57501 objects degraded (8.875%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:26:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:26:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:26:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: osdmap e187: 8 total, 8 up, 8 in 2023-09-30T09:26:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[216815]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: pgmap v425: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 remapped+peering, 82 active+clean; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 5103/57501 objects degraded (8.875%); 2056/57501 objects misplaced (3.576%); 5.7 MiB/s, 96 objects/s recovering 2023-09-30T09:26:56.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 5103/57501 objects degraded (8.875%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:26:56.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:26:56.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:26:56.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:26:56.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: osdmap e187: 8 total, 8 up, 8 in 2023-09-30T09:26:56.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:56 smithi019 ceph-mon[224043]: failed to encode map e187 with expected crc 2023-09-30T09:26:56.689 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:26:57.230 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:26:57.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:57.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:26:57.341 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:26:57.467 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:57.539 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:26:57.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:57.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:26:57.645 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:57.681 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:26:57.701 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:57.997 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:58.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:58.111 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:58.195 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:58.210 INFO:teuthology.orchestra.run.smithi019.stdout:moved 1 / 267 (0.374532%) 2023-09-30T09:26:58.210 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:26:58.210 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 6.4214 -> 6.18339 (expected baseline 5.40399) 2023-09-30T09:26:58.210 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.0 with 23 -> 23 pgs (0.689139 -> 0.689139 * mean) 2023-09-30T09:26:58.210 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-30T09:26:58.210 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:26:58.211 INFO:teuthology.orchestra.run.smithi019.stdout:oload 1000 2023-09-30T09:26:58.211 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 1 2023-09-30T09:26:58.211 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:26:58.211 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0063 2023-09-30T09:26:58.211 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0626 2023-09-30T09:26:58.211 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 weight 0.8893 -> 0.9001 2023-09-30T09:26:58.211 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:26:58.260 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:58.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:58.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:26:58 smithi019 conmon[202899]: 2023-09-30T09:26:58.198+0000 7f1d921ed700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-30T09:26:58.365 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:58.415 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:58 smithi167 ceph-mon[192880]: pgmap v427: 89 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 82 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 5103/56388 objects degraded (9.050%); 4931/56388 objects misplaced (8.745%); 11 MiB/s, 177 objects/s recovering 2023-09-30T09:26:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:58 smithi167 ceph-mon[192880]: osdmap e188: 8 total, 8 up, 8 in 2023-09-30T09:26:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:58 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:58.584 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:58.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:58.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:58.647 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:26:58.647 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:26:58.647 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.647 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.44600.0:148684 5.f e188/113 5:f2671a8b:::benchmark_data_smithi019_287194_object65906:head v 188'9404, mlcod=188'9402)", 2023-09-30T09:26:58.647 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:58.642730+0000", 2023-09-30T09:26:58.647 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.003950851, 2023-09-30T09:26:58.647 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0039725689999999996, 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:58.642730+0000", 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:58.642730+0000", 2023-09-30T09:26:58.648 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:58.642729+0000", 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:58.642736+0000", 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5470000000000002e-06 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.649 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:58.642751+0000", 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4884e-05 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:58.642758+0000", 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3889999999999999e-06 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:58.642830+0000", 2023-09-30T09:26:58.650 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2423999999999996e-05 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:58.642852+0000", 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1739999999999999e-05 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 1 2023-09-30T09:26:58.651 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:26:58.693 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:26:58.723 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:58 smithi019 ceph-mon[224043]: pgmap v427: 89 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 82 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 5103/56388 objects degraded (9.050%); 4931/56388 objects misplaced (8.745%); 11 MiB/s, 177 objects/s recovering 2023-09-30T09:26:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:58 smithi019 ceph-mon[224043]: osdmap e188: 8 total, 8 up, 8 in 2023-09-30T09:26:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:58 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:58 smithi019 ceph-mon[216815]: pgmap v427: 89 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 82 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 5103/56388 objects degraded (9.050%); 4931/56388 objects misplaced (8.745%); 11 MiB/s, 177 objects/s recovering 2023-09-30T09:26:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:58 smithi019 ceph-mon[216815]: osdmap e188: 8 total, 8 up, 8 in 2023-09-30T09:26:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:58 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:58.871 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:26:58.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:58.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:26:58.974 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:59.001 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:59.044 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:59.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:59.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:26:59.147 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:26:59.180 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:59.385 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:26:59.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: from='client.35022 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:59.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[216815]: osdmap e189: 8 total, 8 up, 8 in 2023-09-30T09:26:59.509 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:26:59.538 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:26:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:59.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:26:59.641 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:26:59.667 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:59.710 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: from='client.35022 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:26:59 smithi167 ceph-mon[192880]: osdmap e189: 8 total, 8 up, 8 in 2023-09-30T09:26:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:59.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:26:59.818 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:26:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: from='client.35022 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:26:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: failed to encode map e188 with expected crc 2023-09-30T09:26:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:26:59 smithi019 ceph-mon[224043]: osdmap e189: 8 total, 8 up, 8 in 2023-09-30T09:26:59.846 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:26:59.889 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:26:59.893 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:26:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:26:59.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:00.003 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:00.033 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:00.438 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:00 smithi167 ceph-mon[192880]: pgmap v429: 89 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 5103/53835 objects degraded (9.479%); 4931/53835 objects misplaced (9.159%); 8.8 MiB/s, 144 objects/s recovering 2023-09-30T09:27:00.438 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:00 smithi167 ceph-mon[192880]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:00 smithi167 ceph-mon[192880]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:00 smithi167 ceph-mon[192880]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:00 smithi167 ceph-mon[192880]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:00 smithi167 ceph-mon[192880]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:00 smithi167 ceph-mon[192880]: osdmap e190: 8 total, 8 up, 8 in 2023-09-30T09:27:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[224043]: pgmap v429: 89 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 5103/53835 objects degraded (9.479%); 4931/53835 objects misplaced (9.159%); 8.8 MiB/s, 144 objects/s recovering 2023-09-30T09:27:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[224043]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[224043]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[224043]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[224043]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[224043]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[224043]: osdmap e190: 8 total, 8 up, 8 in 2023-09-30T09:27:00.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[216815]: pgmap v429: 89 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 5103/53835 objects degraded (9.479%); 4931/53835 objects misplaced (9.159%); 8.8 MiB/s, 144 objects/s recovering 2023-09-30T09:27:00.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[216815]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[216815]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[216815]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[216815]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[216815]: failed to encode map e189 with expected crc 2023-09-30T09:27:00.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:00 smithi019 ceph-mon[216815]: osdmap e190: 8 total, 8 up, 8 in 2023-09-30T09:27:01.388 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:01.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:01.502 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:27:01.581 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:01.708 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:27:01.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:01.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:01.712 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:01 smithi167 ceph-mon[192880]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.712 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:01 smithi167 ceph-mon[192880]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.712 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:01 smithi167 ceph-mon[192880]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.813 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:01.942 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:27:01.943 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:27:01.943 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:27:01.943 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:27:01.943 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:27:01.944 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:01 smithi019 ceph-mon[224043]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.944 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:01 smithi019 ceph-mon[224043]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.944 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:01 smithi019 ceph-mon[224043]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.944 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:01 smithi019 ceph-mon[216815]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.944 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:01 smithi019 ceph-mon[216815]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.944 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:01 smithi019 ceph-mon[216815]: failed to encode map e190 with expected crc 2023-09-30T09:27:01.947 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-30T09:27:01.947 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-30T09:27:02.012 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:02.173 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:02.221 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:02.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:02.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:02.334 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:02.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:02 smithi019 ceph-mon[224043]: pgmap v432: 89 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 84 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 5122/51150 objects degraded (10.014%); 3803/51150 objects misplaced (7.435%); 17 MiB/s, 267 objects/s recovering 2023-09-30T09:27:02.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:02 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5122/51150 objects degraded (10.014%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:02.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:02 smithi019 ceph-mon[216815]: pgmap v432: 89 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 84 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 5122/51150 objects degraded (10.014%); 3803/51150 objects misplaced (7.435%); 17 MiB/s, 267 objects/s recovering 2023-09-30T09:27:02.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:02 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5122/51150 objects degraded (10.014%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:02.685 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:27:02.707 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:02.745 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:02 smithi167 ceph-mon[192880]: pgmap v432: 89 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 84 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 5122/51150 objects degraded (10.014%); 3803/51150 objects misplaced (7.435%); 17 MiB/s, 267 objects/s recovering 2023-09-30T09:27:02.746 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:02 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5122/51150 objects degraded (10.014%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:02.902 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:27:02.967 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:27:02.987 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:02.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:03.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:27:03.077 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:27:03.098 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:03.147 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:27:03.181 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:03.187 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:27:03.230 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:27:03.270 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:27:03.285 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:27:03.286 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:27:03.286 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:27:03.286 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:27:03.286 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:27:03.287 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:27:03.287 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:27:03.287 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:27:03.287 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:27:03.301 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:03.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:03.408 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:03.480 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:03.631 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:03.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:03.737 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:27:03.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3028786302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:03.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:03.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:03.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:03.804 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3028786302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:03.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:03.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:03.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3028786302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:03.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:03.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:03.806 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:27:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:27:03] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.43.0" 2023-09-30T09:27:03.844 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:04.052 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:27:04.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:04.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:27:04.158 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:04.187 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:04.455 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:04 smithi019 ceph-mon[224043]: pgmap v433: 89 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 5122/49695 objects degraded (10.307%); 2866/49695 objects misplaced (5.767%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:27:04.712 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:04 smithi167 ceph-mon[192880]: pgmap v433: 89 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 5122/49695 objects degraded (10.307%); 2866/49695 objects misplaced (5.767%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:27:04.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:04 smithi019 ceph-mon[216815]: pgmap v433: 89 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 5122/49695 objects degraded (10.307%); 2866/49695 objects misplaced (5.767%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:27:05.190 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:05.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:05.294 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:05.314 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:27:05.314 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:27:05.322 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:05.423 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:05.580 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:05.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:05.701 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:27:05.732 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:05.777 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:27:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:05.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:27:05.879 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:05.941 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:06.220 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:06.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:06.227 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:27:06.327 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:06.416 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:06.476 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:06 smithi167 ceph-mon[192880]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 911 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 5122/43596 objects degraded (11.749%); 2640/43596 objects misplaced (6.056%); 12 MiB/s, 190 objects/s recovering 2023-09-30T09:27:06.647 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:06.651 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:06 smithi019 ceph-mon[216815]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 911 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 5122/43596 objects degraded (11.749%); 2640/43596 objects misplaced (6.056%); 12 MiB/s, 190 objects/s recovering 2023-09-30T09:27:06.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:06.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:06.654 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:06 smithi019 ceph-mon[224043]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 911 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 5122/43596 objects degraded (11.749%); 2640/43596 objects misplaced (6.056%); 12 MiB/s, 190 objects/s recovering 2023-09-30T09:27:06.754 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:06.814 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:27:06.816 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:06.911 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:06.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:06.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:07.022 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:07.083 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:07.167 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:07.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:07.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:07.270 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:07.346 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:07.480 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:07 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/447056775' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:27:07.480 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:07 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:27:07.482 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:07.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:07.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:07.526 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:27:07.592 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:07.655 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:07.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:27:07.724 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:07.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:07.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:07.783 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:07 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/447056775' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:27:07.783 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:07 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:27:07.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:07 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/447056775' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:27:07.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:07 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:27:07.828 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:07.928 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:08.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:08 smithi167 ceph-mon[192880]: pgmap v435: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 876 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 5122/41931 objects degraded (12.215%); 2266/41931 objects misplaced (5.404%); 10 MiB/s, 164 objects/s recovering 2023-09-30T09:27:08.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5122/41931 objects degraded (12.215%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:08.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:08 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:08.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:08 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:27:08.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:08 smithi167 ceph-mon[192880]: osdmap e191: 8 total, 8 up, 8 in 2023-09-30T09:27:08.585 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:08 smithi167 ceph-mon[192880]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.586 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:08 smithi167 ceph-mon[192880]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.586 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:08 smithi167 ceph-mon[192880]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[216815]: pgmap v435: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 876 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 5122/41931 objects degraded (12.215%); 2266/41931 objects misplaced (5.404%); 10 MiB/s, 164 objects/s recovering 2023-09-30T09:27:08.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5122/41931 objects degraded (12.215%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[216815]: osdmap e191: 8 total, 8 up, 8 in 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[216815]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[216815]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[216815]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[224043]: pgmap v435: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 876 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 5122/41931 objects degraded (12.215%); 2266/41931 objects misplaced (5.404%); 10 MiB/s, 164 objects/s recovering 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5122/41931 objects degraded (12.215%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:08.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:08.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:27:08.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[224043]: osdmap e191: 8 total, 8 up, 8 in 2023-09-30T09:27:08.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[224043]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[224043]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:08 smithi019 ceph-mon[224043]: failed to encode map e191 with expected crc 2023-09-30T09:27:08.848 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:08.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:08.890 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:27:08.890 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:27:08.967 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:27:09.017 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:09.124 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:27:09.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:09.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:09.228 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:09.285 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:09.389 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:09.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:09.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:09.457 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:09.494 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:27:09.534 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:09.602 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:27:09.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:09.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:09.707 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:09.801 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:09.896 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:27:09.912 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:09.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:09.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:09.953 INFO:teuthology.orchestra.run.smithi019.stdout:moved 19 / 267 (7.11611%) 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:avg 33.375 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 6.4214 -> 10.3675 (expected baseline 5.40399) 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.0 with 23 -> 15 pgs (0.689139 -> 0.449438 * mean) 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.1 with 41 -> 47 pgs (1.22846 -> 1.40824 * mean) 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:oload 110 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 3 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0046 2023-09-30T09:27:09.954 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0050 2023-09-30T09:27:09.955 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 weight 0.6628 -> 0.4328 2023-09-30T09:27:09.955 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 weight 0.7561 -> 0.5810 2023-09-30T09:27:09.955 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 weight 0.7535 -> 0.5885 2023-09-30T09:27:09.965 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:27:10.027 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:27:10.074 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:10.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:27:09 smithi019 conmon[202899]: 2023-09-30T09:27:09.948+0000 7f1d921ed700 -1 mgr.server reply reply (3) No such process no change 2023-09-30T09:27:10.174 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:27:10.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:10.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:10.295 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:10.365 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:10.493 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:10 smithi167 ceph-mon[192880]: pgmap v437: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 801 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 5122/38307 objects degraded (13.371%); 2266/38307 objects misplaced (5.915%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:27:10.501 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:10.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:10.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:10.564 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:27:10.614 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:11.219 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:27:11.220 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:10 smithi019 ceph-mon[216815]: pgmap v437: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 801 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 5122/38307 objects degraded (13.371%); 2266/38307 objects misplaced (5.915%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:27:11.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:10 smithi019 ceph-mon[224043]: pgmap v437: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 801 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 5122/38307 objects degraded (13.371%); 2266/38307 objects misplaced (5.915%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:27:11.322 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:11.472 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:11.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:11.579 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:11.580 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:11 smithi019 ceph-mon[216815]: from='client.44957 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:27:11.581 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:11 smithi019 ceph-mon[224043]: from='client.44957 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:27:11.606 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:11.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:11 smithi167 ceph-mon[192880]: from='client.44957 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:27:12.288 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:12.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:12.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:12.409 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:27:12.471 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:12.536 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:12.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:12 smithi019 ceph-mon[216815]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 712 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5122/34023 objects degraded (15.055%); 2070/34023 objects misplaced (6.084%); 2.8 MiB/s, 45 objects/s recovering 2023-09-30T09:27:12.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:12 smithi019 ceph-mon[224043]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 712 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5122/34023 objects degraded (15.055%); 2070/34023 objects misplaced (6.084%); 2.8 MiB/s, 45 objects/s recovering 2023-09-30T09:27:12.598 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:27:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:12.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:12.706 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:12.773 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:27:12.773 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:27:12.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:12 smithi167 ceph-mon[192880]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 712 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5122/34023 objects degraded (15.055%); 2070/34023 objects misplaced (6.084%); 2.8 MiB/s, 45 objects/s recovering 2023-09-30T09:27:12.779 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:13.004 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:13.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:13.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:13.125 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:13.146 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:27:13.197 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:27:13.208 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:13.228 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:27:13.250 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:27:13.343 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:27:13.374 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:27:13.436 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:27:13.471 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:13.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:13.477 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:27:13.493 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:27:13.493 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:27:13.493 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:27:13.493 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:27:13.493 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:27:13.493 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:27:13.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:27:13.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:27:13.578 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:13.612 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:13.703 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:27:13.711 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:13.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:13.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:13.781 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1544223825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:13.781 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:13 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:13.781 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5122/34023 objects degraded (15.055%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:13.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1544223825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:13.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:13 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:13.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5122/34023 objects degraded (15.055%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:13.812 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1544223825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:13.812 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:13 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:13.812 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5122/34023 objects degraded (15.055%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:13.819 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:13.878 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:14.088 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:27:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:27:13] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.43.0" 2023-09-30T09:27:14.270 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:27:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:14 smithi167 ceph-mon[192880]: pgmap v439: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 661 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 5122/31617 objects degraded (16.200%); 1628/31617 objects misplaced (5.149%); 2.6 MiB/s, 42 objects/s recovering 2023-09-30T09:27:14.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:14 smithi019 ceph-mon[216815]: pgmap v439: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 661 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 5122/31617 objects degraded (16.200%); 1628/31617 objects misplaced (5.149%); 2.6 MiB/s, 42 objects/s recovering 2023-09-30T09:27:14.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:14 smithi019 ceph-mon[224043]: pgmap v439: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 661 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 5122/31617 objects degraded (16.200%); 1628/31617 objects misplaced (5.149%); 2.6 MiB/s, 42 objects/s recovering 2023-09-30T09:27:15.258 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:15.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:15.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:15.366 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:15.395 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:15.682 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:15.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:15.793 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:15.884 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:16.044 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:16.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:16.153 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:16.230 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:16.350 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:101994 5.4 5:25bb573f:::benchmark_data_smithi019_287194_object19216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:44.484482+0000", 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 91.860016759000004, 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.267727196, 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.351 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101994 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484482+0000", 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484482+0000", 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484481+0000", 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484488+0000", 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.815e-06 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484503+0000", 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4628e-05 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484509+0000", 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5810000000000002e-06 2023-09-30T09:27:16.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484641+0000", 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000131416 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484762+0000", 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000121465 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.484833+0000", 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0870000000000004e-05 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.487236+0000", 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024033829999999998 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.489532+0000", 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022956859999999999 2023-09-30T09:27:16.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752080+0000", 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26254790500000003 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752152+0000", 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2052999999999996e-05 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752210+0000", 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7763000000000002e-05 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:101997 5.4 5:22799d3d:::benchmark_data_smithi019_287194_object19219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:44.503206+0000", 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 91.841293258999997, 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.249124978, 2023-09-30T09:27:16.358 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101997 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503206+0000", 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503206+0000", 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503204+0000", 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503212+0000", 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7729999999999999e-06 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503230+0000", 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8150000000000001e-05 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503236+0000", 2023-09-30T09:27:16.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2910000000000004e-06 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503281+0000", 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4691000000000003e-05 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503390+0000", 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108766 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.503484+0000", 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4525999999999996e-05 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.506138+0000", 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026531240000000002 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.506307+0000", 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016954000000000001 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752252+0000", 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.245944732 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752285+0000", 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3049999999999997e-05 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752331+0000", 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6051999999999999e-05 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:102003 5.4 5:2fdd004b:::benchmark_data_smithi019_287194_object19225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:44.527152+0000", 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 91.817346924000006, 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.225322248, 2023-09-30T09:27:16.365 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 102003 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527152+0000", 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527152+0000", 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527151+0000", 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527162+0000", 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0954999999999999e-05 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527180+0000", 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8521000000000001e-05 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527187+0000", 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.905e-06 2023-09-30T09:27:16.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527226+0000", 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9542000000000002e-05 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527330+0000", 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103676 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.527423+0000", 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2621000000000006e-05 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.529911+0000", 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024886769999999999 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.530117+0000", 2023-09-30T09:27:16.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0002051 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752386+0000", 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.222269728 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752438+0000", 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1929999999999999e-05 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.752475+0000", 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6337999999999997e-05 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:102006 5.4 5:225d7786:::benchmark_data_smithi019_287194_object19228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:44.546072+0000", 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 91.798427704000005, 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.212841154, 2023-09-30T09:27:16.372 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 102006 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546072+0000", 2023-09-30T09:27:16.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546072+0000", 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546070+0000", 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546077+0000", 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1999999999999999e-06 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546088+0000", 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1708999999999999e-05 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546093+0000", 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0379999999999999e-06 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546133+0000", 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9230000000000002e-05 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546223+0000", 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0109000000000001e-05 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.546293+0000", 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0440999999999999e-05 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.549215+0000", 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029216979999999999 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.549425+0000", 2023-09-30T09:27:16.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000209755 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.758835+0000", 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20941062299999999 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.758876+0000", 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0744999999999997e-05 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.758913+0000", 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6733999999999998e-05 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:102010 5.4 5:23292fd5:::benchmark_data_smithi019_287194_object19232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:44.554081+0000", 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 91.790418416999998, 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20494014999999999, 2023-09-30T09:27:16.379 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 102010 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554081+0000", 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554081+0000", 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554080+0000", 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554084+0000", 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1439999999999996e-06 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554093+0000", 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1139999999999995e-06 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554099+0000", 2023-09-30T09:27:16.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.852e-06 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554147+0000", 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8133999999999997e-05 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554292+0000", 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014435500000000001 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.554437+0000", 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014490999999999999 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.557158+0000", 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027211179999999998 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.557278+0000", 2023-09-30T09:27:16.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000120565 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.758952+0000", 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20167358699999999 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.758991+0000", 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9505999999999997e-05 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:44.759021+0000", 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9689999999999999e-05 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:106971 5.4 5:22b7e9d2:::benchmark_data_smithi019_287194_object24193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:51.691665+0000", 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 84.652833842000007, 2023-09-30T09:27:16.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21919724199999999, 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106971 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691665+0000", 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691665+0000", 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691664+0000", 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691669+0000", 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8040000000000001e-06 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691679+0000", 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0091e-05 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691684+0000", 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4309999999999996e-06 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691717+0000", 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3362999999999999e-05 2023-09-30T09:27:16.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691797+0000", 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9678000000000005e-05 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691859+0000", 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2315000000000002e-05 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.694440+0000", 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002581046 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.695118+0000", 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00067770700000000005 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.910777+0000", 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21565888599999999 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.910823+0000", 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6001999999999998e-05 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.910863+0000", 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9813e-05 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:135235 5.4 5:217e1b30:::benchmark_data_smithi019_287194_object52457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.094969+0000", 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 42.249529926999998, 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.310590904, 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 135235 2023-09-30T09:27:16.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094969+0000", 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094969+0000", 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094968+0000", 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094975+0000", 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3000000000000004e-06 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094996+0000", 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0446e-05 2023-09-30T09:27:16.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.095008+0000", 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2533e-05 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.095122+0000", 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000113575 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.095322+0000", 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00020006200000000001 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.095404+0000", 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2356999999999998e-05 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.099603+0000", 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041981800000000001 2023-09-30T09:27:16.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.108491+0000", 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088885509999999997 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.405464+0000", 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.29697248999999998 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.405518+0000", 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4491999999999999e-05 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.405560+0000", 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2120999999999999e-05 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:135240 5.9 5:91fa266b:::benchmark_data_smithi019_287194_object52462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.104890+0000", 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 42.239609616000003, 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22504527499999999, 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 135240 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.401 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.104890+0000", 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.104890+0000", 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.104889+0000", 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.104894+0000", 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3190000000000002e-06 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.104903+0000", 2023-09-30T09:27:16.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4450000000000002e-06 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.104908+0000", 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.809e-06 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.104950+0000", 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1418000000000002e-05 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105059+0000", 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109647 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 4,6", 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105155+0000", 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6058999999999996e-05 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109159+0000", 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0040040839999999998 2023-09-30T09:27:16.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329759+0000", 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22059933400000001 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329844+0000", 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5632999999999998e-05 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329903+0000", 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8680000000000001e-05 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329935+0000", 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1807999999999998e-05 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.44600.0:135236 5.d e179/174 5:bc8de907:::benchmark_data_smithi019_287194_object52458:head v 179'8607, mlcod=179'8604)", 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.105402+0000", 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 42.239096922999998, 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22442235399999999, 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105402+0000", 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105402+0000", 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105402+0000", 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105407+0000", 2023-09-30T09:27:16.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.639e-06 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105414+0000", 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5730000000000002e-06 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105418+0000", 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3209999999999998e-06 2023-09-30T09:27:16.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105458+0000", 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9446000000000001e-05 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.105475+0000", 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7176999999999998e-05 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329792+0000", 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.224316555 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329825+0000", 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3062999999999998e-05 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop_reply(client.44600.0:135240 5.9 e179/113 ondisk, result = 0)", 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.109220+0000", 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 42.235279104, 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22054605499999999, 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109220+0000", 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109220+0000", 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109220+0000", 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109225+0000", 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0089999999999998e-06 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109233+0000", 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8349999999999993e-06 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109237+0000", 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5609999999999999e-06 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329706+0000", 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22046874799999999 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329737+0000", 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1352000000000001e-05 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329766+0000", 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9099999999999999e-05 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.44600.0:135241 5.6 e179/113 5:666eac6b:::benchmark_data_smithi019_287194_object52463:head v 179'8271, mlcod=179'8268)", 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.109663+0000", 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 42.234835830999998, 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22442856899999999, 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.417 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109663+0000", 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109663+0000", 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109663+0000", 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.109667+0000", 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.498e-06 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329771+0000", 2023-09-30T09:27:16.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22010391700000001 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329795+0000", 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3986999999999999e-05 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329828+0000", 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2790000000000003e-05 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329845+0000", 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7198999999999999e-05 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334053+0000", 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042076120000000003 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334092+0000", 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9038999999999997e-05 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.44600.0:135244 5.1 e179/113 5:87738a27:::benchmark_data_smithi019_287194_object52466:head v 179'8609, mlcod=179'8607)", 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.114838+0000", 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 42.229661344999997, 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.219380873, 2023-09-30T09:27:16.422 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114838+0000", 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114838+0000", 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114837+0000", 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114842+0000", 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8799999999999999e-06 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329686+0000", 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21484391799999999 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329709+0000", 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3153999999999999e-05 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329815+0000", 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105517 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.329838+0000", 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2946e-05 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334184+0000", 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004345977 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334219+0000", 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5009000000000003e-05 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:137736 5.4 5:2e136433:::benchmark_data_smithi019_287194_object54958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:38.239980+0000", 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 38.104519375000002, 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.232586131, 2023-09-30T09:27:16.427 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 137736 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.239980+0000", 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.239980+0000", 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.239979+0000", 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.239984+0000", 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4419999999999997e-06 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.239994+0000", 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.842e-06 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.239999+0000", 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9389999999999996e-06 2023-09-30T09:27:16.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.240048+0000", 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9549000000000003e-05 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.240187+0000", 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000138565 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.240313+0000", 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012606999999999999 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.244788+0000", 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044753040000000003 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253365+0000", 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0085762849999999995 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.472490+0000", 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21912485400000001 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.472530+0000", 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0074999999999998e-05 2023-09-30T09:27:16.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.472566+0000", 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6403999999999999e-05 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:147185 5.4 5:2bee6658:::benchmark_data_smithi019_287194_object64407:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:27:16.434 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:55.552707+0000", 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.791792417, 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21013853499999999, 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 147185 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.552707+0000", 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.552707+0000", 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.552705+0000", 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.552713+0000", 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2949999999999996e-06 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.552727+0000", 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3909e-05 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.552735+0000", 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1640000000000001e-06 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.552791+0000", 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5788999999999997e-05 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.552908+0000", 2023-09-30T09:27:16.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011794999999999999 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553020+0000", 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000111301 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.557395+0000", 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043748550000000004 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.558424+0000", 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010288840000000001 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.762758+0000", 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20433472599999999 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.762804+0000", 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5785000000000003e-05 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.762845+0000", 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1316000000000002e-05 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.441 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:147196 5.4 5:24deb2f8:::benchmark_data_smithi019_287194_object64418:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:55.562418+0000", 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.782081225999999, 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.208129602, 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 147196 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.442 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562418+0000", 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562418+0000", 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562417+0000", 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562422+0000", 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1109999999999999e-06 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562432+0000", 2023-09-30T09:27:16.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8390000000000002e-06 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562437+0000", 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.826e-06 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562489+0000", 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1755000000000002e-05 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562613+0000", 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000124165 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562691+0000", 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8299000000000004e-05 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.762879+0000", 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20018792999999999 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.762907+0000", 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8277e-05 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.770470+0000", 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075625629999999996 2023-09-30T09:27:16.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.770508+0000", 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8563999999999999e-05 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.770548+0000", 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9243999999999999e-05 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.448 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:147197 5.4 5:2bd07f96:::benchmark_data_smithi019_287194_object64419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:55.562452+0000", 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.782047694999999, 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21591123000000001, 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.449 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 147197 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562452+0000", 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562452+0000", 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562451+0000", 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562454+0000", 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1109999999999999e-06 2023-09-30T09:27:16.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562461+0000", 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8950000000000001e-06 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.562465+0000", 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3039999999999998e-06 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.758882+0000", 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.196417063 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.758986+0000", 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103928 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.759092+0000", 2023-09-30T09:27:16.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000106141 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.770555+0000", 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011462117000000001 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.770758+0000", 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000203482 2023-09-30T09:27:16.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.778262+0000", 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075036499999999997 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.778318+0000", 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6246999999999999e-05 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.778363+0000", 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4814999999999997e-05 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.44600.0:147198 5.6 e187/113 5:64069671:::benchmark_data_smithi019_287194_object64420:head v 187'8994, mlcod=187'8992)", 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:55.568717+0000", 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.775782381999999, 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20108077999999999, 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.568717+0000", 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.568717+0000", 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.568716+0000", 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.568726+0000", 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3999999999999998e-06 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.756804+0000", 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.188078564 2023-09-30T09:27:16.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.756814+0000", 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3829999999999998e-06 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.756869+0000", 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5769000000000001e-05 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.756892+0000", 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2228000000000001e-05 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.769773+0000", 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012881406999999999 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.769798+0000", 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4723999999999999e-05 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.460 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:148084 5.9 5:9b0c38a5:::benchmark_data_smithi019_287194_object65306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:57.456450+0000", 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.888049229, 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20712261800000001, 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 148084 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456450+0000", 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456450+0000", 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456449+0000", 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456454+0000", 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0289999999999996e-06 2023-09-30T09:27:16.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456464+0000", 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0087e-05 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456469+0000", 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7430000000000002e-06 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456507+0000", 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7595000000000002e-05 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456599+0000", 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2810000000000001e-05 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 4,6", 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456674+0000", 2023-09-30T09:27:16.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4349000000000003e-05 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.654546+0000", 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.197871828 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.657725+0000", 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031790999999999998 2023-09-30T09:27:16.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663484+0000", 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057593360000000003 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663529+0000", 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5014e-05 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663573+0000", 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3652999999999997e-05 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:148087 5.9 5:9e38f89b:::benchmark_data_smithi019_287194_object65309:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:57.456481+0000", 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.888018343999999, 2023-09-30T09:27:16.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20725511999999999, 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 148087 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456481+0000", 2023-09-30T09:27:16.469 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456481+0000", 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456481+0000", 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:27:16.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456483+0000", 2023-09-30T09:27:16.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1220000000000002e-06 2023-09-30T09:27:16.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456487+0000", 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0779999999999997e-06 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456489+0000", 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6019999999999998e-06 2023-09-30T09:27:16.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.650181+0000", 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.19369166700000001 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.650287+0000", 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010559499999999999 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 4,6", 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.650400+0000", 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112932 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663474+0000", 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013074153999999999 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:27:16.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663584+0000", 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010997099999999999 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663653+0000", 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8832999999999996e-05 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663694+0000", 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1761000000000002e-05 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663736+0000", 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1730999999999997e-05 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.44600.0:148086 5.d e187/174 5:b7d8be45:::benchmark_data_smithi019_287194_object65308:head v 187'9428, mlcod=187'9426)", 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:57.456962+0000", 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.887537260999999, 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20613295500000001, 2023-09-30T09:27:16.477 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456962+0000", 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456962+0000", 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:27:16.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456961+0000", 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.456967+0000", 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6280000000000002e-06 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.646304+0000", 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.18933688300000001 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.646318+0000", 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4577e-05 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.646363+0000", 2023-09-30T09:27:16.480 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4461e-05 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.646396+0000", 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2648999999999999e-05 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663067+0000", 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016671579999999998 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:27:16.481 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:27:16.482 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.663095+0000", 2023-09-30T09:27:16.482 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7739999999999999e-05 2023-09-30T09:27:16.482 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.482 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.482 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.482 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:27:16.482 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:27:16.482 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:27:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:16.483 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:16.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:16.584 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:16.630 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:16.706 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:16.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:16.814 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:16.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:16 smithi019 ceph-mon[216815]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 519 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5122/24771 objects degraded (20.677%); 1420/24771 objects misplaced (5.733%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:16.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:16 smithi019 ceph-mon[224043]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 519 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5122/24771 objects degraded (20.677%); 1420/24771 objects misplaced (5.733%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:16.844 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:16.923 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:16.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:16.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:16 smithi167 ceph-mon[192880]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 519 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5122/24771 objects degraded (20.677%); 1420/24771 objects misplaced (5.733%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:17.030 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:17.104 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:17.202 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:17.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:17.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:17.257 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:27:17.308 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:17.324 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:27:17.324 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:27:17.325 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-30T09:27:17.325 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:27:17.384 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:17.593 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:17.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:17.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:17.614 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:27:17.667 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2214047798' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:17.667 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:17.667 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2214047798' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:17.667 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:17.712 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:17.718 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:27:17.747 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:17.852 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:17.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:17.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:17.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2214047798' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:17.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:17.962 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:18.017 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:18.083 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:18.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:18.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:18.191 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:18.244 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:18.330 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:18.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:18.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:18.546 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: pgmap v441: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 482 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5122/22992 objects degraded (22.277%); 1088/22992 objects misplaced (4.732%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:27:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: osdmap e192: 8 total, 8 up, 8 in 2023-09-30T09:27:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: failed to encode map e192 with expected crc 2023-09-30T09:27:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: failed to encode map e192 with expected crc 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: failed to encode map e192 with expected crc 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5122/22992 objects degraded (22.277%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: pgmap v441: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 482 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5122/22992 objects degraded (22.277%); 1088/22992 objects misplaced (4.732%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: osdmap e192: 8 total, 8 up, 8 in 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: failed to encode map e192 with expected crc 2023-09-30T09:27:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: failed to encode map e192 with expected crc 2023-09-30T09:27:18.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: failed to encode map e192 with expected crc 2023-09-30T09:27:18.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:18.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:18.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5122/22992 objects degraded (22.277%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: pgmap v441: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 482 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5122/22992 objects degraded (22.277%); 1088/22992 objects misplaced (4.732%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:27:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: osdmap e192: 8 total, 8 up, 8 in 2023-09-30T09:27:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: failed to encode map e192 with expected crc 2023-09-30T09:27:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: failed to encode map e192 with expected crc 2023-09-30T09:27:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: failed to encode map e192 with expected crc 2023-09-30T09:27:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5122/22992 objects degraded (22.277%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:19.424 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:19.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:19.529 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:19.598 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:19.761 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:19.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:19.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:19.769 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:19.793 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:19.859 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:19.864 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:19.896 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:19.973 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:19.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:20.082 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:20.213 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:20.753 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:20.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:20.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:20 smithi019 ceph-mon[216815]: pgmap v443: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 411 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5122/19590 objects degraded (26.146%); 1088/19590 objects misplaced (5.554%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:20.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:20 smithi019 ceph-mon[224043]: pgmap v443: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 411 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5122/19590 objects degraded (26.146%); 1088/19590 objects misplaced (5.554%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:20.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:20.869 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:20.958 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:20 smithi167 ceph-mon[192880]: pgmap v443: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 411 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5122/19590 objects degraded (26.146%); 1088/19590 objects misplaced (5.554%); 2.5 MiB/s, 40 objects/s recovering 2023-09-30T09:27:21.203 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:21.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:21.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:21.311 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:21.422 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:21.520 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:27:21.944 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:21.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:21.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:22.049 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:22.077 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:22.122 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:22.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:22.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:22.229 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:22.268 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:22.369 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:22.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:22.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:22.483 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:22.581 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:22.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:22 smithi019 ceph-mon[216815]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 5122/15693 objects degraded (32.639%); 902/15693 objects misplaced (5.748%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:27:22.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:22 smithi019 ceph-mon[224043]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 5122/15693 objects degraded (32.639%); 902/15693 objects misplaced (5.748%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:27:23.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:22 smithi167 ceph-mon[192880]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 5122/15693 objects degraded (32.639%); 902/15693 objects misplaced (5.748%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:27:23.804 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5122/15693 objects degraded (32.639%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:23.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5122/15693 objects degraded (32.639%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:24.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5122/15693 objects degraded (32.639%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:27:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:27:23] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.43.0" 2023-09-30T09:27:24.099 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:24.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:24.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:24.209 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:24.317 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:24.676 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:24 smithi019 ceph-mon[224043]: pgmap v445: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 282 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5122/13395 objects degraded (38.238%); 508/13395 objects misplaced (3.792%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:27:24.695 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:24.822 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:24.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:24.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:24.871 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:27:24.910 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:27:24.933 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:24.958 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:27:24.958 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:27:24.978 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:24 smithi019 ceph-mon[216815]: pgmap v445: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 282 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5122/13395 objects degraded (38.238%); 508/13395 objects misplaced (3.792%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:27:24.979 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:27:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:24 smithi167 ceph-mon[192880]: pgmap v445: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 282 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5122/13395 objects degraded (38.238%); 508/13395 objects misplaced (3.792%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:27:25.027 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:27:25.027 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:25.058 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:27:25.105 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:27:25.145 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:27:25.184 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:27:25.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:27:25.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:27:25.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:27:25.211 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:27:25.212 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:27:25.212 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:27:25.212 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:27:25.212 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:27:25.402 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:25.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:25.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:25.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:27:25.555 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:25.629 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:27:25.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:25.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:27:25.735 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:25.831 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:27:25.839 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:25.916 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:27:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:25 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2841717420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:25 smithi167 ceph-mon[192880]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 135 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5122/6333 objects degraded (80.878%); 396/6333 objects misplaced (6.253%); 1.9 MiB/s, 41 objects/s recovering 2023-09-30T09:27:26.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:25 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2841717420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:26.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:25 smithi019 ceph-mon[216815]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 135 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5122/6333 objects degraded (80.878%); 396/6333 objects misplaced (6.253%); 1.9 MiB/s, 41 objects/s recovering 2023-09-30T09:27:26.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:25 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2841717420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:26.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:25 smithi019 ceph-mon[224043]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 135 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5122/6333 objects degraded (80.878%); 396/6333 objects misplaced (6.253%); 1.9 MiB/s, 41 objects/s recovering 2023-09-30T09:27:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:26 smithi167 ceph-mon[192880]: from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:27:27.034 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Removed 82775 objects 2023-09-30T09:27:27.034 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Clean up completed and total clean up time :177.97 2023-09-30T09:27:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:26 smithi019 ceph-mon[216815]: from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:27:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:26 smithi019 ceph-mon[224043]: from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:27:27.588 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:27.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:27.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:27.665 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:27:27.694 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:27.737 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:27 smithi019 ceph-mon[216815]: pgmap v447: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 97 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5122/4509 objects degraded (113.595%); 16/4509 objects misplaced (0.355%); 1.9 MiB/s, 41 objects/s recovering 2023-09-30T09:27:27.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:27 smithi019 ceph-mon[224043]: pgmap v447: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 97 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5122/4509 objects degraded (113.595%); 16/4509 objects misplaced (0.355%); 1.9 MiB/s, 41 objects/s recovering 2023-09-30T09:27:27.888 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:27.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:27.993 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:27 smithi167 ceph-mon[192880]: pgmap v447: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 97 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5122/4509 objects degraded (113.595%); 16/4509 objects misplaced (0.355%); 1.9 MiB/s, 41 objects/s recovering 2023-09-30T09:27:28.026 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:28.133 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:28.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:28.237 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:28.266 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5122/4509 objects degraded (113.595%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5122/4509 objects degraded (113.595%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5122/4509 objects degraded (113.595%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:29.969 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:29.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:29 smithi019 ceph-mon[216815]: pgmap v448: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 46 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 5122/2067 objects degraded (247.799%); 16/2067 objects misplaced (0.774%); 1.6 MiB/s, 36 objects/s recovering 2023-09-30T09:27:29.970 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:29 smithi019 ceph-mon[224043]: pgmap v448: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 46 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 5122/2067 objects degraded (247.799%); 16/2067 objects misplaced (0.774%); 1.6 MiB/s, 36 objects/s recovering 2023-09-30T09:27:29.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:29.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:30.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:29 smithi167 ceph-mon[192880]: pgmap v448: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 46 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 5122/2067 objects degraded (247.799%); 16/2067 objects misplaced (0.774%); 1.6 MiB/s, 36 objects/s recovering 2023-09-30T09:27:30.072 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:30.075 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:27:30.101 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:30.156 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:30.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:30.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:30.260 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:30.290 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:30.629 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":446,"stamp":"2023-09-30T09:27:25.092717+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":141236159,"num_objects":2111,"num_object_clones":0,"num_object_copies":6333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":396,"num_objects_unfound":0,"num_objects_dirty":2111,"num_whiteouts":0,"num_read":6443,"num_read_kb":13071,"num_write":164615,"num_write_kb":5312742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24409,"num_bytes_recovered":1462833728,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141123,"ondisk_log_size":141123,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1539644,"kb_used_data":675644,"kb_used_omap":19,"kb_used_meta":863916,"kb_avail":748421572,"statfs":{"total":767960285184,"available":766383689728,"internally_reserved":0,"allocated":691859456,"data_stored":684342759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":884650407},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,0,4,3,9,2,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-402784256,"num_objects":-6146,"num_object_clones":0,"num_object_copies":-18438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1024,"num_objects_unfound":0,"num_objects_dirty":-6146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6146,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":415,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009299"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":169,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498333+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:27:07.498333+0000","last_peered":"2023-09-30T09:27:07.498333+0000","last_clean":"2023-09-30T09:27:07.498333+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:27:07.498333+0000","last_undegraded":"2023-09-30T09:27:07.498333+0000","last_fullsized":"2023-09-30T09:27:07.498333+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.499341+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:27:07.499341+0000","last_peered":"2023-09-30T09:27:07.499341+0000","last_clean":"2023-09-30T09:27:07.499341+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:27:07.499341+0000","last_undegraded":"2023-09-30T09:27:07.499341+0000","last_fullsized":"2023-09-30T09:27:07.499341+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":153,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.503478+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:27:07.503478+0000","last_peered":"2023-09-30T09:27:07.503478+0000","last_clean":"2023-09-30T09:27:07.503478+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:27:07.503478+0000","last_undegraded":"2023-09-30T09:27:07.503478+0000","last_fullsized":"2023-09-30T09:27:07.503478+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":84,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:14.441949+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:27:14.441949+0000","last_peered":"2023-09-30T09:27:14.441949+0000","last_clean":"2023-09-30T09:27:14.441949+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:27:14.441949+0000","last_undegraded":"2023-09-30T09:27:14.441949+0000","last_fullsized":"2023-09-30T09:27:14.441949+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":198,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498710+0000","last_change":"2023-09-30T09:26:51.318844+0000","last_active":"2023-09-30T09:27:07.498710+0000","last_peered":"2023-09-30T09:27:07.498710+0000","last_clean":"2023-09-30T09:27:07.498710+0000","last_became_active":"2023-09-30T09:26:51.316788+0000","last_became_peered":"2023-09-30T09:26:51.316788+0000","last_unstale":"2023-09-30T09:27:07.498710+0000","last_undegraded":"2023-09-30T09:27:07.498710+0000","last_fullsized":"2023-09-30T09:27:07.498710+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:20:35.308469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":151,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.507681+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:27:07.507681+0000","last_peered":"2023-09-30T09:27:07.507681+0000","last_clean":"2023-09-30T09:27:07.507681+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:27:07.507681+0000","last_undegraded":"2023-09-30T09:27:07.507681+0000","last_fullsized":"2023-09-30T09:27:07.507681+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":83,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.494636+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:27:07.494636+0000","last_peered":"2023-09-30T09:27:07.494636+0000","last_clean":"2023-09-30T09:27:07.494636+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:27:07.494636+0000","last_undegraded":"2023-09-30T09:27:07.494636+0000","last_fullsized":"2023-09-30T09:27:07.494636+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498787+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:27:07.498787+0000","last_peered":"2023-09-30T09:27:07.498787+0000","last_clean":"2023-09-30T09:27:07.498787+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:27:07.498787+0000","last_undegraded":"2023-09-30T09:27:07.498787+0000","last_fullsized":"2023-09-30T09:27:07.498787+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":43,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502916+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:27:07.502916+0000","last_peered":"2023-09-30T09:27:07.502916+0000","last_clean":"2023-09-30T09:27:07.502916+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:27:07.502916+0000","last_undegraded":"2023-09-30T09:27:07.502916+0000","last_fullsized":"2023-09-30T09:27:07.502916+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":107,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498980+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:27:07.498980+0000","last_peered":"2023-09-30T09:27:07.498980+0000","last_clean":"2023-09-30T09:27:07.498980+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:27:07.498980+0000","last_undegraded":"2023-09-30T09:27:07.498980+0000","last_fullsized":"2023-09-30T09:27:07.498980+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":155,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509398+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:27:07.509398+0000","last_peered":"2023-09-30T09:27:07.509398+0000","last_clean":"2023-09-30T09:27:07.509398+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:27:07.509398+0000","last_undegraded":"2023-09-30T09:27:07.509398+0000","last_fullsized":"2023-09-30T09:27:07.509398+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:17.617134+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:27:17.617134+0000","last_peered":"2023-09-30T09:27:17.617134+0000","last_clean":"2023-09-30T09:27:17.617134+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:27:17.617134+0000","last_undegraded":"2023-09-30T09:27:17.617134+0000","last_fullsized":"2023-09-30T09:27:17.617134+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498685+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:27:07.498685+0000","last_peered":"2023-09-30T09:27:07.498685+0000","last_clean":"2023-09-30T09:27:07.498685+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:27:07.498685+0000","last_undegraded":"2023-09-30T09:27:07.498685+0000","last_fullsized":"2023-09-30T09:27:07.498685+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498465+0000","last_change":"2023-09-30T09:23:03.986755+0000","last_active":"2023-09-30T09:27:07.498465+0000","last_peered":"2023-09-30T09:27:07.498465+0000","last_clean":"2023-09-30T09:27:07.498465+0000","last_became_active":"2023-09-30T09:23:03.986411+0000","last_became_peered":"2023-09-30T09:23:03.986411+0000","last_unstale":"2023-09-30T09:27:07.498465+0000","last_undegraded":"2023-09-30T09:27:07.498465+0000","last_fullsized":"2023-09-30T09:27:07.498465+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:53:23.627823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":154,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.511205+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:27:07.511205+0000","last_peered":"2023-09-30T09:27:07.511205+0000","last_clean":"2023-09-30T09:27:07.511205+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:27:07.511205+0000","last_undegraded":"2023-09-30T09:27:07.511205+0000","last_fullsized":"2023-09-30T09:27:07.511205+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":170,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.499070+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:27:07.499070+0000","last_peered":"2023-09-30T09:27:07.499070+0000","last_clean":"2023-09-30T09:27:07.499070+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:27:07.499070+0000","last_undegraded":"2023-09-30T09:27:07.499070+0000","last_fullsized":"2023-09-30T09:27:07.499070+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":83,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.496518+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:27:07.496518+0000","last_peered":"2023-09-30T09:27:07.496518+0000","last_clean":"2023-09-30T09:27:07.496518+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:27:07.496518+0000","last_undegraded":"2023-09-30T09:27:07.496518+0000","last_fullsized":"2023-09-30T09:27:07.496518+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509524+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:27:07.509524+0000","last_peered":"2023-09-30T09:27:07.509524+0000","last_clean":"2023-09-30T09:27:07.509524+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:27:07.509524+0000","last_undegraded":"2023-09-30T09:27:07.509524+0000","last_fullsized":"2023-09-30T09:27:07.509524+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":135,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509191+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:27:07.509191+0000","last_peered":"2023-09-30T09:27:07.509191+0000","last_clean":"2023-09-30T09:27:07.509191+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:27:07.509191+0000","last_undegraded":"2023-09-30T09:27:07.509191+0000","last_fullsized":"2023-09-30T09:27:07.509191+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":83,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.494521+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:27:07.494521+0000","last_peered":"2023-09-30T09:27:07.494521+0000","last_clean":"2023-09-30T09:27:07.494521+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:27:07.494521+0000","last_undegraded":"2023-09-30T09:27:07.494521+0000","last_fullsized":"2023-09-30T09:27:07.494521+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":264,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.507000+0000","last_change":"2023-09-30T09:26:51.339701+0000","last_active":"2023-09-30T09:27:07.507000+0000","last_peered":"2023-09-30T09:27:07.507000+0000","last_clean":"2023-09-30T09:27:07.507000+0000","last_became_active":"2023-09-30T09:26:51.324149+0000","last_became_peered":"2023-09-30T09:26:51.324149+0000","last_unstale":"2023-09-30T09:27:07.507000+0000","last_undegraded":"2023-09-30T09:27:07.507000+0000","last_fullsized":"2023-09-30T09:27:07.507000+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:26.372798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":170,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.499471+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:27:07.499471+0000","last_peered":"2023-09-30T09:27:07.499471+0000","last_clean":"2023-09-30T09:27:07.499471+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:27:07.499471+0000","last_undegraded":"2023-09-30T09:27:07.499471+0000","last_fullsized":"2023-09-30T09:27:07.499471+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"192'10205","reported_seq":10292,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:23.936617+0000","last_change":"2023-09-30T09:22:48.114186+0000","last_active":"2023-09-30T09:27:23.936617+0000","last_peered":"2023-09-30T09:27:23.936617+0000","last_clean":"2023-09-30T09:27:23.936617+0000","last_became_active":"2023-09-30T09:22:48.113857+0000","last_became_peered":"2023-09-30T09:22:48.113857+0000","last_unstale":"2023-09-30T09:27:23.936617+0000","last_undegraded":"2023-09-30T09:27:23.936617+0000","last_fullsized":"2023-09-30T09:27:23.936617+0000","mapping_epoch":113,"log_start":"126'2873","ondisk_log_start":"126'2873","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7332,"log_dups_size":0,"ondisk_log_size":7332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:18.973595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7012352,"num_objects":107,"num_object_clones":0,"num_object_copies":321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10205,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":195,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509509+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:27:07.509509+0000","last_peered":"2023-09-30T09:27:07.509509+0000","last_clean":"2023-09-30T09:27:07.509509+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:27:07.509509+0000","last_undegraded":"2023-09-30T09:27:07.509509+0000","last_fullsized":"2023-09-30T09:27:07.509509+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.508991+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:27:07.508991+0000","last_peered":"2023-09-30T09:27:07.508991+0000","last_clean":"2023-09-30T09:27:07.508991+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:27:07.508991+0000","last_undegraded":"2023-09-30T09:27:07.508991+0000","last_fullsized":"2023-09-30T09:27:07.508991+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"192'10398","reported_seq":13584,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:24.952416+0000","last_change":"2023-09-30T09:24:25.563637+0000","last_active":"2023-09-30T09:27:24.952416+0000","last_peered":"2023-09-30T09:27:24.952416+0000","last_clean":"2023-09-30T09:27:24.952416+0000","last_became_active":"2023-09-30T09:24:25.554649+0000","last_became_peered":"2023-09-30T09:24:25.554649+0000","last_unstale":"2023-09-30T09:27:24.952416+0000","last_undegraded":"2023-09-30T09:27:24.952416+0000","last_fullsized":"2023-09-30T09:27:24.952416+0000","mapping_epoch":145,"log_start":"117'300","ondisk_log_start":"117'300","created":113,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":10098,"log_dups_size":0,"ondisk_log_size":10098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:09:56.974492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5767168,"num_objects":88,"num_object_clones":0,"num_object_copies":264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10398,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1538,"num_bytes_recovered":100794368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":196,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509578+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:27:07.509578+0000","last_peered":"2023-09-30T09:27:07.509578+0000","last_clean":"2023-09-30T09:27:07.509578+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:27:07.509578+0000","last_undegraded":"2023-09-30T09:27:07.509578+0000","last_fullsized":"2023-09-30T09:27:07.509578+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498605+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:27:07.498605+0000","last_peered":"2023-09-30T09:27:07.498605+0000","last_clean":"2023-09-30T09:27:07.498605+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:27:07.498605+0000","last_undegraded":"2023-09-30T09:27:07.498605+0000","last_fullsized":"2023-09-30T09:27:07.498605+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"192'10248","reported_seq":17133,"reported_epoch":192,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:27:24.994531+0000","last_change":"2023-09-30T09:26:52.860632+0000","last_active":"2023-09-30T09:27:24.994531+0000","last_peered":"2023-09-30T09:27:24.994531+0000","last_clean":"2023-09-30T09:26:50.291885+0000","last_became_active":"2023-09-30T09:26:52.587133+0000","last_became_peered":"2023-09-30T09:26:52.587133+0000","last_unstale":"2023-09-30T09:27:24.994531+0000","last_undegraded":"2023-09-30T09:26:52.365466+0000","last_fullsized":"2023-09-30T09:26:52.309603+0000","mapping_epoch":183,"log_start":"117'200","ondisk_log_start":"117'200","created":113,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":10048,"log_dups_size":0,"ondisk_log_size":10048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:17.860428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5636096,"num_objects":86,"num_object_clones":0,"num_object_copies":258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":37,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":86,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10248,"num_write_kb":330688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3379,"num_bytes_recovered":189267968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2","objects":49}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":178,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509274+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:27:07.509274+0000","last_peered":"2023-09-30T09:27:07.509274+0000","last_clean":"2023-09-30T09:27:07.509274+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:27:07.509274+0000","last_undegraded":"2023-09-30T09:27:07.509274+0000","last_fullsized":"2023-09-30T09:27:07.509274+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":161,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.507912+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:27:07.507912+0000","last_peered":"2023-09-30T09:27:07.507912+0000","last_clean":"2023-09-30T09:27:07.507912+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:27:07.507912+0000","last_undegraded":"2023-09-30T09:27:07.507912+0000","last_fullsized":"2023-09-30T09:27:07.507912+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"192'10090","reported_seq":11074,"reported_epoch":192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:27:24.913786+0000","last_change":"2023-09-30T09:26:52.801407+0000","last_active":"2023-09-30T09:27:24.913786+0000","last_peered":"2023-09-30T09:27:24.913786+0000","last_clean":"2023-09-30T09:26:50.289677+0000","last_became_active":"2023-09-30T09:26:52.624412+0000","last_became_peered":"2023-09-30T09:26:52.624412+0000","last_unstale":"2023-09-30T09:27:24.913786+0000","last_undegraded":"2023-09-30T09:26:52.490781+0000","last_fullsized":"2023-09-30T09:26:52.309929+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":10090,"log_dups_size":0,"ondisk_log_size":10090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:26:45.772833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5767168,"num_objects":88,"num_object_clones":0,"num_object_copies":264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5085,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10090,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":433,"num_bytes_recovered":9240576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3","objects":-1156},{"shards":"2,3,4","objects":88}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":151,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502719+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:27:07.502719+0000","last_peered":"2023-09-30T09:27:07.502719+0000","last_clean":"2023-09-30T09:27:07.502719+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:27:07.502719+0000","last_undegraded":"2023-09-30T09:27:07.502719+0000","last_fullsized":"2023-09-30T09:27:07.502719+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":80,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.495624+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:27:07.495624+0000","last_peered":"2023-09-30T09:27:07.495624+0000","last_clean":"2023-09-30T09:27:07.495624+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:27:07.495624+0000","last_undegraded":"2023-09-30T09:27:07.495624+0000","last_fullsized":"2023-09-30T09:27:07.495624+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"192'10410","reported_seq":12732,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:24.913685+0000","last_change":"2023-09-30T09:26:11.289797+0000","last_active":"2023-09-30T09:27:24.913685+0000","last_peered":"2023-09-30T09:27:24.913685+0000","last_clean":"2023-09-30T09:27:24.913685+0000","last_became_active":"2023-09-30T09:26:11.276208+0000","last_became_peered":"2023-09-30T09:26:11.276208+0000","last_unstale":"2023-09-30T09:27:24.913685+0000","last_undegraded":"2023-09-30T09:27:24.913685+0000","last_fullsized":"2023-09-30T09:27:24.913685+0000","mapping_epoch":174,"log_start":"117'400","ondisk_log_start":"117'400","created":113,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":10010,"log_dups_size":0,"ondisk_log_size":10010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:35:30.421912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4653080,"num_objects":72,"num_object_clones":0,"num_object_copies":216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10410,"num_write_kb":335361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1104,"num_bytes_recovered":46661656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":190,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498764+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:27:07.498764+0000","last_peered":"2023-09-30T09:27:07.498764+0000","last_clean":"2023-09-30T09:27:07.498764+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:27:07.498764+0000","last_undegraded":"2023-09-30T09:27:07.498764+0000","last_fullsized":"2023-09-30T09:27:07.498764+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":150,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498597+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:27:07.498597+0000","last_peered":"2023-09-30T09:27:07.498597+0000","last_clean":"2023-09-30T09:27:07.498597+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:27:07.498597+0000","last_undegraded":"2023-09-30T09:27:07.498597+0000","last_fullsized":"2023-09-30T09:27:07.498597+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"192'9883","reported_seq":12901,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:21.682571+0000","last_change":"2023-09-30T09:24:20.547592+0000","last_active":"2023-09-30T09:27:21.682571+0000","last_peered":"2023-09-30T09:27:21.682571+0000","last_clean":"2023-09-30T09:27:21.682571+0000","last_became_active":"2023-09-30T09:24:20.539203+0000","last_became_peered":"2023-09-30T09:24:20.539203+0000","last_unstale":"2023-09-30T09:27:21.682571+0000","last_undegraded":"2023-09-30T09:27:21.682571+0000","last_fullsized":"2023-09-30T09:27:21.682571+0000","mapping_epoch":143,"log_start":"125'2526","ondisk_log_start":"125'2526","created":113,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7357,"log_dups_size":0,"ondisk_log_size":7357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:33.219264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13565952,"num_objects":207,"num_object_clones":0,"num_object_copies":621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9883,"num_write_kb":322880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1455,"num_bytes_recovered":95354880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":152,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.507986+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:27:07.507986+0000","last_peered":"2023-09-30T09:27:07.507986+0000","last_clean":"2023-09-30T09:27:07.507986+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:27:07.507986+0000","last_undegraded":"2023-09-30T09:27:07.507986+0000","last_fullsized":"2023-09-30T09:27:07.507986+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.510826+0000","last_change":"2023-09-30T09:26:51.322946+0000","last_active":"2023-09-30T09:27:07.510826+0000","last_peered":"2023-09-30T09:27:07.510826+0000","last_clean":"2023-09-30T09:27:07.510826+0000","last_became_active":"2023-09-30T09:26:51.322618+0000","last_became_peered":"2023-09-30T09:26:51.322618+0000","last_unstale":"2023-09-30T09:27:07.510826+0000","last_undegraded":"2023-09-30T09:27:07.510826+0000","last_fullsized":"2023-09-30T09:27:07.510826+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:23.279885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.f","version":"192'10358","reported_seq":10445,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:23.969526+0000","last_change":"2023-09-30T09:22:48.115158+0000","last_active":"2023-09-30T09:27:23.969526+0000","last_peered":"2023-09-30T09:27:23.969526+0000","last_clean":"2023-09-30T09:27:23.969526+0000","last_became_active":"2023-09-30T09:22:48.114453+0000","last_became_peered":"2023-09-30T09:22:48.114453+0000","last_unstale":"2023-09-30T09:27:23.969526+0000","last_undegraded":"2023-09-30T09:27:23.969526+0000","last_fullsized":"2023-09-30T09:27:23.969526+0000","mapping_epoch":113,"log_start":"117'300","ondisk_log_start":"117'300","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":10058,"log_dups_size":0,"ondisk_log_size":10058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:31:35.934276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8781824,"num_objects":134,"num_object_clones":0,"num_object_copies":402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10358,"num_write_kb":335744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":151,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498455+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:27:07.498455+0000","last_peered":"2023-09-30T09:27:07.498455+0000","last_clean":"2023-09-30T09:27:07.498455+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:27:07.498455+0000","last_undegraded":"2023-09-30T09:27:07.498455+0000","last_fullsized":"2023-09-30T09:27:07.498455+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":150,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498819+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:27:07.498819+0000","last_peered":"2023-09-30T09:27:07.498819+0000","last_clean":"2023-09-30T09:27:07.498819+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:27:07.498819+0000","last_undegraded":"2023-09-30T09:27:07.498819+0000","last_fullsized":"2023-09-30T09:27:07.498819+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"192'10188","reported_seq":16370,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:24.326801+0000","last_change":"2023-09-30T09:27:00.293947+0000","last_active":"2023-09-30T09:27:24.326801+0000","last_peered":"2023-09-30T09:27:24.326801+0000","last_clean":"2023-09-30T09:27:24.326801+0000","last_became_active":"2023-09-30T09:27:00.269642+0000","last_became_peered":"2023-09-30T09:27:00.269642+0000","last_unstale":"2023-09-30T09:27:24.326801+0000","last_undegraded":"2023-09-30T09:27:24.326801+0000","last_fullsized":"2023-09-30T09:27:24.326801+0000","mapping_epoch":189,"log_start":"124'1916","ondisk_log_start":"124'1916","created":113,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":8272,"log_dups_size":0,"ondisk_log_size":8272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:49:24.410554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7077888,"num_objects":108,"num_object_clones":0,"num_object_copies":324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10188,"num_write_kb":329472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3011,"num_bytes_recovered":178192384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":157,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498541+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:27:07.498541+0000","last_peered":"2023-09-30T09:27:07.498541+0000","last_clean":"2023-09-30T09:27:07.498541+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:27:07.498541+0000","last_undegraded":"2023-09-30T09:27:07.498541+0000","last_fullsized":"2023-09-30T09:27:07.498541+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.503011+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:27:07.503011+0000","last_peered":"2023-09-30T09:27:07.503011+0000","last_clean":"2023-09-30T09:27:07.503011+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:27:07.503011+0000","last_undegraded":"2023-09-30T09:27:07.503011+0000","last_fullsized":"2023-09-30T09:27:07.503011+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":535,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.503728+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:27:07.503728+0000","last_peered":"2023-09-30T09:27:07.503728+0000","last_clean":"2023-09-30T09:27:07.503728+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:27:07.503728+0000","last_undegraded":"2023-09-30T09:27:07.503728+0000","last_fullsized":"2023-09-30T09:27:07.503728+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":168,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502790+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:27:07.502790+0000","last_peered":"2023-09-30T09:27:07.502790+0000","last_clean":"2023-09-30T09:27:07.502790+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:27:07.502790+0000","last_undegraded":"2023-09-30T09:27:07.502790+0000","last_fullsized":"2023-09-30T09:27:07.502790+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":150,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502440+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:27:07.502440+0000","last_peered":"2023-09-30T09:27:07.502440+0000","last_clean":"2023-09-30T09:27:07.502440+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:27:07.502440+0000","last_undegraded":"2023-09-30T09:27:07.502440+0000","last_fullsized":"2023-09-30T09:27:07.502440+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"192'10250","reported_seq":14319,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:24.449831+0000","last_change":"2023-09-30T09:26:04.207784+0000","last_active":"2023-09-30T09:27:24.449831+0000","last_peered":"2023-09-30T09:27:24.449831+0000","last_clean":"2023-09-30T09:27:24.449831+0000","last_became_active":"2023-09-30T09:26:04.186571+0000","last_became_peered":"2023-09-30T09:26:04.186571+0000","last_unstale":"2023-09-30T09:27:24.449831+0000","last_undegraded":"2023-09-30T09:27:24.449831+0000","last_fullsized":"2023-09-30T09:27:24.449831+0000","mapping_epoch":170,"log_start":"117'300","ondisk_log_start":"117'300","created":113,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":9950,"log_dups_size":0,"ondisk_log_size":9950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:24.650075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6029312,"num_objects":92,"num_object_clones":0,"num_object_copies":276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10250,"num_write_kb":330944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1972,"num_bytes_recovered":105644032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"192'10133","reported_seq":17491,"reported_epoch":192,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:27:21.679611+0000","last_change":"2023-09-30T09:26:52.865298+0000","last_active":"2023-09-30T09:27:21.679611+0000","last_peered":"2023-09-30T09:27:21.679611+0000","last_clean":"2023-09-30T09:26:50.286528+0000","last_became_active":"2023-09-30T09:26:52.501136+0000","last_became_peered":"2023-09-30T09:26:52.501136+0000","last_unstale":"2023-09-30T09:27:21.679611+0000","last_undegraded":"2023-09-30T09:27:21.679611+0000","last_fullsized":"2023-09-30T09:27:21.679611+0000","mapping_epoch":183,"log_start":"126'2756","ondisk_log_start":"126'2756","created":113,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7377,"log_dups_size":0,"ondisk_log_size":7377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:24:15.839508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14483456,"num_objects":221,"num_object_clones":0,"num_object_copies":663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":396,"num_objects_unfound":0,"num_objects_dirty":221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10133,"num_write_kb":331328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3559,"num_bytes_recovered":213909504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":221}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":338,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502593+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:27:07.502593+0000","last_peered":"2023-09-30T09:27:07.502593+0000","last_clean":"2023-09-30T09:27:07.502593+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:27:07.502593+0000","last_undegraded":"2023-09-30T09:27:07.502593+0000","last_fullsized":"2023-09-30T09:27:07.502593+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":171,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498511+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:27:07.498511+0000","last_peered":"2023-09-30T09:27:07.498511+0000","last_clean":"2023-09-30T09:27:07.498511+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:27:07.498511+0000","last_undegraded":"2023-09-30T09:27:07.498511+0000","last_fullsized":"2023-09-30T09:27:07.498511+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":152,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.510597+0000","last_change":"2023-09-30T09:23:23.238851+0000","last_active":"2023-09-30T09:27:07.510597+0000","last_peered":"2023-09-30T09:27:07.510597+0000","last_clean":"2023-09-30T09:27:07.510597+0000","last_became_active":"2023-09-30T09:23:23.238582+0000","last_became_peered":"2023-09-30T09:23:23.238582+0000","last_unstale":"2023-09-30T09:27:07.510597+0000","last_undegraded":"2023-09-30T09:27:07.510597+0000","last_fullsized":"2023-09-30T09:27:07.510597+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:52.808966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"192'10066","reported_seq":10158,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:24.992775+0000","last_change":"2023-09-30T09:22:48.108513+0000","last_active":"2023-09-30T09:27:24.992775+0000","last_peered":"2023-09-30T09:27:24.992775+0000","last_clean":"2023-09-30T09:27:24.992775+0000","last_became_active":"2023-09-30T09:22:48.108231+0000","last_became_peered":"2023-09-30T09:22:48.108231+0000","last_unstale":"2023-09-30T09:27:24.992775+0000","last_undegraded":"2023-09-30T09:27:24.992775+0000","last_fullsized":"2023-09-30T09:27:24.992775+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:09.723671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6422528,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10066,"num_write_kb":325248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":570,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:09.945638+0000","last_change":"2023-09-30T09:23:11.172147+0000","last_active":"2023-09-30T09:27:09.945638+0000","last_peered":"2023-09-30T09:27:09.945638+0000","last_clean":"2023-09-30T09:27:09.945638+0000","last_became_active":"2023-09-30T09:23:11.171540+0000","last_became_peered":"2023-09-30T09:23:11.171540+0000","last_unstale":"2023-09-30T09:27:09.945638+0000","last_undegraded":"2023-09-30T09:27:09.945638+0000","last_fullsized":"2023-09-30T09:27:09.945638+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:24.074307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502647+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:27:07.502647+0000","last_peered":"2023-09-30T09:27:07.502647+0000","last_clean":"2023-09-30T09:27:07.502647+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:27:07.502647+0000","last_undegraded":"2023-09-30T09:27:07.502647+0000","last_fullsized":"2023-09-30T09:27:07.502647+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":186,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502971+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:27:07.502971+0000","last_peered":"2023-09-30T09:27:07.502971+0000","last_clean":"2023-09-30T09:27:07.502971+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:27:07.502971+0000","last_undegraded":"2023-09-30T09:27:07.502971+0000","last_fullsized":"2023-09-30T09:27:07.502971+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":599,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502610+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:27:07.502610+0000","last_peered":"2023-09-30T09:27:07.502610+0000","last_clean":"2023-09-30T09:27:07.502610+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:27:07.502610+0000","last_undegraded":"2023-09-30T09:27:07.502610+0000","last_fullsized":"2023-09-30T09:27:07.502610+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":611,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509088+0000","last_change":"2023-09-30T09:26:54.259496+0000","last_active":"2023-09-30T09:27:07.509088+0000","last_peered":"2023-09-30T09:27:07.509088+0000","last_clean":"2023-09-30T09:27:07.509088+0000","last_became_active":"2023-09-30T09:26:54.259005+0000","last_became_peered":"2023-09-30T09:26:54.259005+0000","last_unstale":"2023-09-30T09:27:07.509088+0000","last_undegraded":"2023-09-30T09:27:07.509088+0000","last_fullsized":"2023-09-30T09:27:07.509088+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:15:07.442889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":106,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502344+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:27:07.502344+0000","last_peered":"2023-09-30T09:27:07.502344+0000","last_clean":"2023-09-30T09:27:07.502344+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:27:07.502344+0000","last_undegraded":"2023-09-30T09:27:07.502344+0000","last_fullsized":"2023-09-30T09:27:07.502344+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":134,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.507780+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:27:07.507780+0000","last_peered":"2023-09-30T09:27:07.507780+0000","last_clean":"2023-09-30T09:27:07.507780+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:27:07.507780+0000","last_undegraded":"2023-09-30T09:27:07.507780+0000","last_fullsized":"2023-09-30T09:27:07.507780+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"192'10148","reported_seq":10235,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:23.958200+0000","last_change":"2023-09-30T09:22:48.113213+0000","last_active":"2023-09-30T09:27:23.958200+0000","last_peered":"2023-09-30T09:27:23.958200+0000","last_clean":"2023-09-30T09:27:23.958200+0000","last_became_active":"2023-09-30T09:22:48.112904+0000","last_became_peered":"2023-09-30T09:22:48.112904+0000","last_unstale":"2023-09-30T09:27:23.958200+0000","last_undegraded":"2023-09-30T09:27:23.958200+0000","last_fullsized":"2023-09-30T09:27:23.958200+0000","mapping_epoch":113,"log_start":"126'2828","ondisk_log_start":"126'2828","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7320,"log_dups_size":0,"ondisk_log_size":7320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:37:37.122003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8781824,"num_objects":134,"num_object_clones":0,"num_object_copies":402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10148,"num_write_kb":329024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"192'10031","reported_seq":22717,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:21.669457+0000","last_change":"2023-09-30T09:26:59.251597+0000","last_active":"2023-09-30T09:27:21.669457+0000","last_peered":"2023-09-30T09:27:21.669457+0000","last_clean":"2023-09-30T09:27:21.669457+0000","last_became_active":"2023-09-30T09:26:59.226392+0000","last_became_peered":"2023-09-30T09:26:59.226392+0000","last_unstale":"2023-09-30T09:27:21.669457+0000","last_undegraded":"2023-09-30T09:27:21.669457+0000","last_fullsized":"2023-09-30T09:27:21.669457+0000","mapping_epoch":188,"log_start":"125'2639","ondisk_log_start":"125'2639","created":113,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7392,"log_dups_size":0,"ondisk_log_size":7392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:44:06.605358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13697024,"num_objects":209,"num_object_clones":0,"num_object_copies":627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10031,"num_write_kb":327680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6257,"num_bytes_recovered":410058752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":752,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509129+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:27:07.509129+0000","last_peered":"2023-09-30T09:27:07.509129+0000","last_clean":"2023-09-30T09:27:07.509129+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:27:07.509129+0000","last_undegraded":"2023-09-30T09:27:07.509129+0000","last_fullsized":"2023-09-30T09:27:07.509129+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":108,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:17.605641+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:27:17.605641+0000","last_peered":"2023-09-30T09:27:17.605641+0000","last_clean":"2023-09-30T09:27:17.605641+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:27:17.605641+0000","last_undegraded":"2023-09-30T09:27:17.605641+0000","last_fullsized":"2023-09-30T09:27:17.605641+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502726+0000","last_change":"2023-09-30T09:26:51.326488+0000","last_active":"2023-09-30T09:27:07.502726+0000","last_peered":"2023-09-30T09:27:07.502726+0000","last_clean":"2023-09-30T09:27:07.502726+0000","last_became_active":"2023-09-30T09:26:51.325984+0000","last_became_peered":"2023-09-30T09:26:51.325984+0000","last_unstale":"2023-09-30T09:27:07.502726+0000","last_undegraded":"2023-09-30T09:27:07.502726+0000","last_fullsized":"2023-09-30T09:27:07.502726+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:22:23.475972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":882,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.509331+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:27:07.509331+0000","last_peered":"2023-09-30T09:27:07.509331+0000","last_clean":"2023-09-30T09:27:07.509331+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:27:07.509331+0000","last_undegraded":"2023-09-30T09:27:07.509331+0000","last_fullsized":"2023-09-30T09:27:07.509331+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":146,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.503650+0000","last_change":"2023-09-30T09:25:14.359586+0000","last_active":"2023-09-30T09:27:07.503650+0000","last_peered":"2023-09-30T09:27:07.503650+0000","last_clean":"2023-09-30T09:27:07.503650+0000","last_became_active":"2023-09-30T09:25:14.359198+0000","last_became_peered":"2023-09-30T09:25:14.359198+0000","last_unstale":"2023-09-30T09:27:07.503650+0000","last_undegraded":"2023-09-30T09:27:07.503650+0000","last_fullsized":"2023-09-30T09:27:07.503650+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:48.533039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":80,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:17.608896+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:27:17.608896+0000","last_peered":"2023-09-30T09:27:17.608896+0000","last_clean":"2023-09-30T09:27:17.608896+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:27:17.608896+0000","last_undegraded":"2023-09-30T09:27:17.608896+0000","last_fullsized":"2023-09-30T09:27:17.608896+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"192'10566","reported_seq":13781,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:23.871198+0000","last_change":"2023-09-30T09:24:12.672495+0000","last_active":"2023-09-30T09:27:23.871198+0000","last_peered":"2023-09-30T09:27:23.871198+0000","last_clean":"2023-09-30T09:27:23.871198+0000","last_became_active":"2023-09-30T09:24:12.668774+0000","last_became_peered":"2023-09-30T09:24:12.668774+0000","last_unstale":"2023-09-30T09:27:23.871198+0000","last_undegraded":"2023-09-30T09:27:23.871198+0000","last_fullsized":"2023-09-30T09:27:23.871198+0000","mapping_epoch":140,"log_start":"124'2170","ondisk_log_start":"124'2170","created":113,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":8396,"log_dups_size":0,"ondisk_log_size":8396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:01:27.375538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7995392,"num_objects":122,"num_object_clones":0,"num_object_copies":366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10566,"num_write_kb":342016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1553,"num_bytes_recovered":101777408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":178,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.499548+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:27:07.499548+0000","last_peered":"2023-09-30T09:27:07.499548+0000","last_clean":"2023-09-30T09:27:07.499548+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:27:07.499548+0000","last_undegraded":"2023-09-30T09:27:07.499548+0000","last_fullsized":"2023-09-30T09:27:07.499548+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1140,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502778+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:27:07.502778+0000","last_peered":"2023-09-30T09:27:07.502778+0000","last_clean":"2023-09-30T09:27:07.502778+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:27:07.502778+0000","last_undegraded":"2023-09-30T09:27:07.502778+0000","last_fullsized":"2023-09-30T09:27:07.502778+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":175,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498628+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:27:07.498628+0000","last_peered":"2023-09-30T09:27:07.498628+0000","last_clean":"2023-09-30T09:27:07.498628+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:27:07.498628+0000","last_undegraded":"2023-09-30T09:27:07.498628+0000","last_fullsized":"2023-09-30T09:27:07.498628+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"192'10394","reported_seq":10481,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:23.936214+0000","last_change":"2023-09-30T09:22:48.113865+0000","last_active":"2023-09-30T09:27:23.936214+0000","last_peered":"2023-09-30T09:27:23.936214+0000","last_clean":"2023-09-30T09:27:23.936214+0000","last_became_active":"2023-09-30T09:22:48.113578+0000","last_became_peered":"2023-09-30T09:22:48.113578+0000","last_unstale":"2023-09-30T09:27:23.936214+0000","last_undegraded":"2023-09-30T09:27:23.936214+0000","last_fullsized":"2023-09-30T09:27:23.936214+0000","mapping_epoch":113,"log_start":"124'2400","ondisk_log_start":"124'2400","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":7994,"log_dups_size":0,"ondisk_log_size":7994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:17.848647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7340032,"num_objects":112,"num_object_clones":0,"num_object_copies":336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10394,"num_write_kb":336192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":445,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.507603+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:27:07.507603+0000","last_peered":"2023-09-30T09:27:07.507603+0000","last_clean":"2023-09-30T09:27:07.507603+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:27:07.507603+0000","last_undegraded":"2023-09-30T09:27:07.507603+0000","last_fullsized":"2023-09-30T09:27:07.507603+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":68,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502879+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:27:07.502879+0000","last_peered":"2023-09-30T09:27:07.502879+0000","last_clean":"2023-09-30T09:27:07.502879+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:27:07.502879+0000","last_undegraded":"2023-09-30T09:27:07.502879+0000","last_fullsized":"2023-09-30T09:27:07.502879+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":133,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498701+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:27:07.498701+0000","last_peered":"2023-09-30T09:27:07.498701+0000","last_clean":"2023-09-30T09:27:07.498701+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:27:07.498701+0000","last_undegraded":"2023-09-30T09:27:07.498701+0000","last_fullsized":"2023-09-30T09:27:07.498701+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"192'10101","reported_seq":10193,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-30T09:27:22.003846+0000","last_change":"2023-09-30T09:22:48.113933+0000","last_active":"2023-09-30T09:27:22.003846+0000","last_peered":"2023-09-30T09:27:22.003846+0000","last_clean":"2023-09-30T09:27:22.003846+0000","last_became_active":"2023-09-30T09:22:48.113640+0000","last_became_peered":"2023-09-30T09:22:48.113640+0000","last_unstale":"2023-09-30T09:27:22.003846+0000","last_undegraded":"2023-09-30T09:27:22.003846+0000","last_fullsized":"2023-09-30T09:27:22.003846+0000","mapping_epoch":113,"log_start":"124'1930","ondisk_log_start":"124'1930","created":113,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:22:47.546753+0000","last_clean_scrub_stamp":"2023-09-30T09:22:47.546753+0000","objects_scrubbed":0,"log_size":8171,"log_dups_size":0,"ondisk_log_size":8171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:16.995917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13434880,"num_objects":205,"num_object_clones":0,"num_object_copies":615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10101,"num_write_kb":329792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":50,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498556+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:27:07.498556+0000","last_peered":"2023-09-30T09:27:07.498556+0000","last_clean":"2023-09-30T09:27:07.498556+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:27:07.498556+0000","last_undegraded":"2023-09-30T09:27:07.498556+0000","last_fullsized":"2023-09-30T09:27:07.498556+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.503895+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:27:07.503895+0000","last_peered":"2023-09-30T09:27:07.503895+0000","last_clean":"2023-09-30T09:27:07.503895+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:27:07.503895+0000","last_undegraded":"2023-09-30T09:27:07.503895+0000","last_fullsized":"2023-09-30T09:27:07.503895+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":153,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502763+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:27:07.502763+0000","last_peered":"2023-09-30T09:27:07.502763+0000","last_clean":"2023-09-30T09:27:07.502763+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:27:07.502763+0000","last_undegraded":"2023-09-30T09:27:07.502763+0000","last_fullsized":"2023-09-30T09:27:07.502763+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498675+0000","last_change":"2023-09-30T09:26:51.311932+0000","last_active":"2023-09-30T09:27:07.498675+0000","last_peered":"2023-09-30T09:27:07.498675+0000","last_clean":"2023-09-30T09:27:07.498675+0000","last_became_active":"2023-09-30T09:26:51.311569+0000","last_became_peered":"2023-09-30T09:26:51.311569+0000","last_unstale":"2023-09-30T09:27:07.498675+0000","last_undegraded":"2023-09-30T09:27:07.498675+0000","last_fullsized":"2023-09-30T09:27:07.498675+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:49.722161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":136,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502818+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:27:07.502818+0000","last_peered":"2023-09-30T09:27:07.502818+0000","last_clean":"2023-09-30T09:27:07.502818+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:27:07.502818+0000","last_undegraded":"2023-09-30T09:27:07.502818+0000","last_fullsized":"2023-09-30T09:27:07.502818+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.502430+0000","last_change":"2023-09-30T09:26:51.311705+0000","last_active":"2023-09-30T09:27:07.502430+0000","last_peered":"2023-09-30T09:27:07.502430+0000","last_clean":"2023-09-30T09:27:07.502430+0000","last_became_active":"2023-09-30T09:26:51.311360+0000","last_became_peered":"2023-09-30T09:26:51.311360+0000","last_unstale":"2023-09-30T09:27:07.502430+0000","last_undegraded":"2023-09-30T09:27:07.502430+0000","last_fullsized":"2023-09-30T09:27:07.502430+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:08:08.201274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":162,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.499751+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:27:07.499751+0000","last_peered":"2023-09-30T09:27:07.499751+0000","last_clean":"2023-09-30T09:27:07.499751+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:27:07.499751+0000","last_undegraded":"2023-09-30T09:27:07.499751+0000","last_fullsized":"2023-09-30T09:27:07.499751+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":193,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498896+0000","last_change":"2023-09-30T09:23:08.021721+0000","last_active":"2023-09-30T09:27:07.498896+0000","last_peered":"2023-09-30T09:27:07.498896+0000","last_clean":"2023-09-30T09:27:07.498896+0000","last_became_active":"2023-09-30T09:23:08.021401+0000","last_became_peered":"2023-09-30T09:23:08.021401+0000","last_unstale":"2023-09-30T09:27:07.498896+0000","last_undegraded":"2023-09-30T09:27:07.498896+0000","last_fullsized":"2023-09-30T09:27:07.498896+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:33:33.611604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":75,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.503108+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:27:07.503108+0000","last_peered":"2023-09-30T09:27:07.503108+0000","last_clean":"2023-09-30T09:27:07.503108+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:27:07.503108+0000","last_undegraded":"2023-09-30T09:27:07.503108+0000","last_fullsized":"2023-09-30T09:27:07.503108+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":150,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-30T09:27:07.498221+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:27:07.498221+0000","last_peered":"2023-09-30T09:27:07.498221+0000","last_clean":"2023-09-30T09:27:07.498221+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:27:07.498221+0000","last_undegraded":"2023-09-30T09:27:07.498221+0000","last_fullsized":"2023-09-30T09:27:07.498221+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":136445976,"num_objects":2083,"num_object_clones":0,"num_object_copies":6249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":396,"num_objects_unfound":0,"num_objects_dirty":2083,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163469,"num_write_kb":5297601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24261,"num_bytes_recovered":1450901528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":616771584,"data_stored":616759368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139931,"ondisk_log_size":139931,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":1384,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234189,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":127372,"kb_used_data":52548,"kb_used_omap":0,"kb_used_meta":74816,"kb_avail":93617780,"statfs":{"total":95995035648,"available":95864606720,"internally_reserved":0,"allocated":53809152,"data_stored":53009321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":76611584},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7810000000000001,"5min":1.681,"15min":0.97099999999999997},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":18.353000000000002,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.91200000000000003},{"interface":"front","average":{"1min":2.8690000000000002,"5min":1.5049999999999999,"15min":0.92600000000000005},"min":{"1min":0.28399999999999997,"5min":0.245,"15min":0.245},"max":{"1min":18.097999999999999,"5min":18.097999999999999,"15min":18.097999999999999},"last":0.23899999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:26:33 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.7599999999999998,"15min":1.2849999999999999},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.8419999999999996,"5min":82.025999999999996,"15min":82.025999999999996},"last":1.056},{"interface":"front","average":{"1min":2.6339999999999999,"5min":2.8109999999999999,"15min":1.3169999999999999},"min":{"1min":0.308,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.2850000000000001,"5min":82.012,"15min":82.012},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":1.605,"15min":0.93899999999999995},"min":{"1min":0.31900000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.7999999999999998,"5min":17.838000000000001,"15min":17.838000000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.7649999999999999,"15min":0.99399999999999999},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.5199999999999996,"5min":17.805,"15min":17.805},"last":13.999000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7639999999999998,"5min":1.3380000000000001,"15min":0.85999999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.1080000000000005,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":0.379},{"interface":"front","average":{"1min":1.9570000000000001,"5min":1.0169999999999999,"15min":0.76300000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":7.1479999999999997,"5min":7.1479999999999997,"15min":7.1479999999999997},"last":6.3559999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":0.77100000000000002,"15min":0.57099999999999995},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.1289999999999996,"5min":6.1289999999999996,"15min":6.1289999999999996},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.05,"5min":0.79700000000000004,"15min":0.60099999999999998},"min":{"1min":0.32100000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.008,"5min":6.282,"15min":6.282},"last":0.28799999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:26:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3170000000000002,"5min":1.024,"15min":0.68700000000000006},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":9.6790000000000003,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.9259999999999999,"5min":0.98299999999999998,"15min":0.70399999999999996},"min":{"1min":0.35699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.022,"5min":11.022,"15min":11.022},"last":0.33800000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":0.89400000000000002,"15min":0.68100000000000005},"min":{"1min":0.38500000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.2249999999999996,"5min":6.2249999999999996,"15min":6.2249999999999996},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.1040000000000001,"5min":0.84199999999999997,"15min":0.66600000000000004},"min":{"1min":0.375,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.8120000000000003,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":0.38500000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495826,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":119036,"kb_used_data":47796,"kb_used_omap":0,"kb_used_meta":71232,"kb_avail":93626116,"statfs":{"total":95995035648,"available":95873142784,"internally_reserved":0,"allocated":48943104,"data_stored":48159472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":72941568},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:26:31 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.161,"15min":0.81100000000000005},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":7.4109999999999996,"5min":7.4109999999999996,"15min":7.4109999999999996},"last":3.1629999999999998},{"interface":"front","average":{"1min":2.4529999999999998,"5min":1.1870000000000001,"15min":0.83899999999999997},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.3700000000000001,"5min":8.327,"15min":8.327},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:26:31 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.151,"15min":0.80500000000000005},"min":{"1min":0.29799999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.292,"5min":10.292,"15min":10.292},"last":3.2050000000000001},{"interface":"front","average":{"1min":3.1779999999999999,"5min":1.321,"15min":0.86299999999999999},"min":{"1min":0.38400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.327,"5min":10.327,"15min":10.327},"last":0.54000000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:27:04 2023","interfaces":[{"interface":"back","average":{"1min":5.032,"5min":2.6440000000000001,"15min":1.262},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":30.638999999999999,"5min":37.978000000000002,"15min":37.978000000000002},"last":4.2549999999999999},{"interface":"front","average":{"1min":3.488,"5min":2.165,"15min":1.1259999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":29.759,"5min":37.759999999999998,"15min":37.759999999999998},"last":3.2389999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:26:35 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":1.1850000000000001,"15min":0.79200000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.3140000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":3.3140000000000001},{"interface":"front","average":{"1min":3.056,"5min":1.5249999999999999,"15min":0.90100000000000002},"min":{"1min":0.41299999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.5049999999999999,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.47099999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:26:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":1.6830000000000001,"15min":0.89100000000000001},"min":{"1min":0.29699999999999999,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":8.1270000000000007,"5min":73.656000000000006,"15min":73.656000000000006},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.518,"15min":0.85299999999999998},"min":{"1min":0.29499999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.6390000000000002,"5min":73.882999999999996,"15min":73.882999999999996},"last":7.1509999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:26:31 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":0.98599999999999999,"15min":0.66100000000000003},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":10.113,"5min":10.113,"15min":10.113},"last":8.3450000000000006},{"interface":"front","average":{"1min":2.0299999999999998,"5min":0.997,"15min":0.71099999999999997},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.311},"max":{"1min":11.725,"5min":11.725,"15min":11.725},"last":3.2629999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:27:02 2023","interfaces":[{"interface":"back","average":{"1min":3.2370000000000001,"5min":1.327,"15min":0.82999999999999996},"min":{"1min":0.379,"5min":0.32100000000000001,"15min":0.30199999999999999},"max":{"1min":9.5419999999999998,"5min":12.709,"15min":12.709},"last":6.5039999999999996},{"interface":"front","average":{"1min":2.3900000000000001,"5min":1.4279999999999999,"15min":0.88500000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":6.4320000000000004,"5min":10.339,"15min":10.339},"last":3.2879999999999998}]}]},{"osd":1,"up_from":99,"seq":425201762390,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":185488,"kb_used_data":81672,"kb_used_omap":5,"kb_used_meta":103802,"kb_avail":93559664,"statfs":{"total":95995035648,"available":95805095936,"internally_reserved":0,"allocated":83632128,"data_stored":82541218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":106293430},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":11.446,"5min":4.3010000000000002,"15min":1.681},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":150.93899999999999,"5min":150.93899999999999,"15min":150.93899999999999},"last":0.442},{"interface":"front","average":{"1min":11.558,"5min":4.4089999999999998,"15min":1.7410000000000001},"min":{"1min":0.224,"5min":0.218,"15min":0.218},"max":{"1min":156.74199999999999,"5min":156.74199999999999,"15min":156.74199999999999},"last":7.0620000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:27:01 2023","interfaces":[{"interface":"back","average":{"1min":20.456,"5min":6.7880000000000003,"15min":2.5449999999999999},"min":{"1min":0.372,"5min":0.249,"15min":0.249},"max":{"1min":144.48699999999999,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.995},{"interface":"front","average":{"1min":20.797999999999998,"5min":6.6710000000000003,"15min":2.4889999999999999},"min":{"1min":0.29299999999999998,"5min":0.254,"15min":0.254},"max":{"1min":144.62,"5min":144.62,"15min":144.62},"last":7.0899999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:26:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.131,"15min":0.65100000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":18.757999999999999,"5min":18.757999999999999,"15min":18.757999999999999},"last":16.704000000000001},{"interface":"front","average":{"1min":1.702,"5min":0.98299999999999998,"15min":0.58899999999999997},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":7.2759999999999998,"5min":11.06,"15min":11.06},"last":11.173}]},{"osd":4,"last update":"Sat Sep 30 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":0.95199999999999996,"15min":0.747},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.359,"5min":5.6340000000000003,"15min":5.6340000000000003},"last":7.0190000000000001},{"interface":"front","average":{"1min":1.409,"5min":0.95799999999999996,"15min":0.77400000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.7670000000000003,"5min":4.7670000000000003,"15min":4.7670000000000003},"last":3.7240000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":0.92400000000000004,"15min":0.72199999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.3369999999999997,"5min":7.3369999999999997,"15min":7.3369999999999997},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.718,"5min":1.048,"15min":0.78100000000000003},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.8230000000000004,"5min":4.8230000000000004,"15min":4.8230000000000004},"last":3.7869999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":1.075,"15min":0.77300000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":10.003,"5min":10.003,"15min":10.003},"last":14.356999999999999},{"interface":"front","average":{"1min":2.294,"5min":1.2330000000000001,"15min":0.86699999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":15.864000000000001,"5min":15.864000000000001,"15min":15.864000000000001},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":2.081,"5min":1.2450000000000001,"15min":0.86799999999999999},"min":{"1min":0.40500000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.7320000000000002,"5min":7.4820000000000002,"15min":7.4820000000000002},"last":14.403},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.419,"15min":0.93600000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":15.808,"5min":15.808,"15min":15.808},"last":0.36699999999999999}]}]},{"osd":0,"up_from":94,"seq":403726925923,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":292016,"kb_used_data":68968,"kb_used_omap":5,"kb_used_meta":223034,"kb_avail":93453136,"statfs":{"total":95995035648,"available":95696011264,"internally_reserved":0,"allocated":70623232,"data_stored":69505880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":228387069},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:26:37 2023","interfaces":[{"interface":"back","average":{"1min":2.8460000000000001,"5min":1.3129999999999999,"15min":0.67700000000000005},"min":{"1min":0.216,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":18.358000000000001,"5min":18.358000000000001,"15min":18.358000000000001},"last":0.247},{"interface":"front","average":{"1min":2.6749999999999998,"5min":1.383,"15min":0.71699999999999997},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":18.693000000000001,"5min":18.693000000000001,"15min":18.693000000000001},"last":3.5099999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:27:03 2023","interfaces":[{"interface":"back","average":{"1min":3.0750000000000002,"5min":1.6759999999999999,"15min":0.81100000000000005},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":30.359999999999999,"5min":30.359999999999999,"15min":30.359999999999999},"last":2.4300000000000002},{"interface":"front","average":{"1min":3.488,"5min":1.778,"15min":0.88300000000000001},"min":{"1min":0.26600000000000001,"5min":0.245,"15min":0.245},"max":{"1min":31.661000000000001,"5min":31.661000000000001,"15min":31.661000000000001},"last":10.458}]},{"osd":3,"last update":"Sat Sep 30 09:26:37 2023","interfaces":[{"interface":"back","average":{"1min":3.101,"5min":1.5049999999999999,"15min":0.82499999999999996},"min":{"1min":0.28599999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":16.670000000000002,"5min":16.670000000000002,"15min":16.670000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.1299999999999999,"5min":1.3069999999999999,"15min":0.72299999999999998},"min":{"1min":0.35799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.802,"5min":11.802,"15min":11.802},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":0.998,"15min":0.81999999999999995},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.988,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":2.5030000000000001},{"interface":"front","average":{"1min":1.958,"5min":1.2410000000000001,"15min":0.92500000000000004},"min":{"1min":0.34399999999999997,"5min":0.31,"15min":0.31},"max":{"1min":11.119999999999999,"5min":12.99,"15min":12.99},"last":0.313}]},{"osd":5,"last update":"Sat Sep 30 09:26:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.48,"15min":1.034},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36199999999999999},"max":{"1min":11.073,"5min":12.949999999999999,"15min":12.949999999999999},"last":4.6070000000000002},{"interface":"front","average":{"1min":1.772,"5min":1.375,"15min":1.004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.8490000000000002,"5min":12.823,"15min":12.823},"last":4.6970000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":1.0960000000000001,"15min":0.89700000000000002},"min":{"1min":0.41299999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.8220000000000001,"5min":6.9660000000000002,"15min":6.9660000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.2589999999999999,"5min":0.97699999999999998,"15min":0.89700000000000002},"min":{"1min":0.439,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.024,"5min":7.556,"15min":7.556},"last":2.468}]},{"osd":7,"last update":"Sat Sep 30 09:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":1.23,"15min":0.91900000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.7850000000000001,"5min":7.0339999999999998,"15min":7.0339999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.3680000000000001,"5min":1.0029999999999999,"15min":0.89900000000000002},"min":{"1min":0.40400000000000003,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.024,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":2.536}]}]},{"osd":2,"up_from":104,"seq":446676598864,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":151100,"kb_used_data":47284,"kb_used_omap":5,"kb_used_meta":103802,"kb_avail":93594052,"statfs":{"total":95995035648,"available":95840309248,"internally_reserved":0,"allocated":48418816,"data_stored":47345032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":106293901},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,6,2,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.141,"5min":1.823,"15min":0.85099999999999998},"min":{"1min":0.20399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.4580000000000002,"5min":15.843999999999999,"15min":15.843999999999999},"last":2.3439999999999999},{"interface":"front","average":{"1min":2.415,"5min":1.4219999999999999,"15min":0.77600000000000002},"min":{"1min":0.28000000000000003,"5min":0.222,"15min":0.222},"max":{"1min":7.2389999999999999,"5min":10.065,"15min":10.065},"last":3.1629999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.3620000000000001,"15min":0.70999999999999996},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.3789999999999996,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":6.1520000000000001},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.256,"15min":0.69999999999999996},"min":{"1min":0.252,"5min":0.249,"15min":0.249},"max":{"1min":8.3810000000000002,"5min":8.3810000000000002,"15min":8.3810000000000002},"last":3.2480000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.298,"15min":0.72999999999999998},"min":{"1min":0.29599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.3700000000000001,"5min":18.635999999999999,"15min":18.635999999999999},"last":3.129},{"interface":"front","average":{"1min":2.7200000000000002,"5min":1.6160000000000001,"15min":0.874},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.288,"5min":18.026,"15min":18.026},"last":2.375}]},{"osd":4,"last update":"Sat Sep 30 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":1.123,"15min":0.77900000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.282,"5min":7.282,"15min":7.282},"last":3.2210000000000001},{"interface":"front","average":{"1min":3.8170000000000002,"5min":1.901,"15min":1.0640000000000001},"min":{"1min":0.33900000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.5579999999999998,"5min":11.125999999999999,"15min":11.125999999999999},"last":2.415}]},{"osd":5,"last update":"Sat Sep 30 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":2.476,"5min":1.4490000000000001,"15min":0.91700000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.371},"max":{"1min":6.7910000000000004,"5min":8.8640000000000008,"15min":8.8640000000000008},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.498,"15min":0.93700000000000006},"min":{"1min":0.42499999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.7249999999999996,"5min":11.866,"15min":11.866},"last":3.1909999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":1.9790000000000001,"15min":1.1180000000000001},"min":{"1min":0.40400000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":7.6609999999999996,"5min":11.247999999999999,"15min":11.247999999999999},"last":2.4399999999999999},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.571,"15min":0.97499999999999998},"min":{"1min":0.46600000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":7.6269999999999998,"5min":8.9619999999999997,"15min":8.9619999999999997},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.5590000000000002,"5min":1.9850000000000001,"15min":1.115},"min":{"1min":0.57599999999999996,"5min":0.432,"15min":0.432},"max":{"1min":10.066000000000001,"5min":11.209,"15min":11.209},"last":2.4820000000000002},{"interface":"front","average":{"1min":2.9329999999999998,"5min":1.6060000000000001,"15min":1.012},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":7.5110000000000001,"5min":7.5110000000000001,"15min":7.5110000000000001},"last":0.49199999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435337,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":120524,"kb_used_data":52868,"kb_used_omap":2,"kb_used_meta":67645,"kb_avail":93624628,"statfs":{"total":95995035648,"available":95871619072,"internally_reserved":0,"allocated":54136832,"data_stored":53046096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":69268839},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":3.0379999999999998,"15min":1.27},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.377000000000001,"5min":130.946,"15min":130.946},"last":7.7009999999999996},{"interface":"front","average":{"1min":2.6989999999999998,"5min":2.9710000000000001,"15min":1.2769999999999999},"min":{"1min":0.215,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.561999999999999,"5min":130.96899999999999,"15min":130.96899999999999},"last":12.361000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":3.6349999999999998,"5min":1.349,"15min":0.73299999999999998},"min":{"1min":0.27100000000000002,"5min":0.224,"15min":0.224},"max":{"1min":13.039999999999999,"5min":13.039999999999999,"15min":13.039999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":3.0790000000000002,"5min":1.175,"15min":0.66500000000000004},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":11.606999999999999,"5min":11.606999999999999,"15min":11.606999999999999},"last":4.6269999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":5.0469999999999997,"5min":3.7490000000000001,"15min":1.5529999999999999},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":21.074000000000002,"5min":145.75899999999999,"15min":145.75899999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.4409999999999998,"5min":3.3929999999999998,"15min":1.4179999999999999},"min":{"1min":0.314,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":21.143999999999998,"5min":150.358,"15min":150.358},"last":12.712}]},{"osd":4,"last update":"Sat Sep 30 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":0.95999999999999996,"15min":0.72999999999999998},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.312},"max":{"1min":10.301,"5min":10.301,"15min":10.301},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.145,"15min":0.78600000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.9949999999999992,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":4.6710000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":1.238,"15min":0.83399999999999996},"min":{"1min":0.314,"5min":0.308,"15min":0.308},"max":{"1min":10.048999999999999,"5min":10.048999999999999,"15min":10.048999999999999},"last":4.5880000000000001},{"interface":"front","average":{"1min":3.4399999999999999,"5min":1.417,"15min":0.94099999999999995},"min":{"1min":0.38500000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":12.785,"5min":12.785,"15min":12.785},"last":4.1689999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.0349999999999999,"15min":0.79500000000000004},"min":{"1min":0.45300000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":10.348000000000001,"5min":10.348000000000001,"15min":10.348000000000001},"last":2.9609999999999999},{"interface":"front","average":{"1min":2.6480000000000001,"5min":1.2569999999999999,"15min":0.877},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.023999999999999,"5min":10.023999999999999,"15min":10.023999999999999},"last":4.7009999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":3.839,"5min":1.4430000000000001,"15min":0.93300000000000005},"min":{"1min":0.45900000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":12.742000000000001,"5min":12.742000000000001,"15min":12.742000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.0289999999999999,"5min":1.0109999999999999,"15min":0.79300000000000004},"min":{"1min":0.42599999999999999,"5min":0.372,"15min":0.372},"max":{"1min":10.43,"5min":10.43,"15min":10.43},"last":2.7719999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019099,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265148,"kb_used_data":159860,"kb_used_omap":0,"kb_used_meta":105280,"kb_avail":93480004,"statfs":{"total":95995035648,"available":95723524096,"internally_reserved":0,"allocated":163696640,"data_stored":162912579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":107806720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:27:21 2023","interfaces":[{"interface":"back","average":{"1min":7.931,"5min":5.9820000000000002,"15min":2.4100000000000001},"min":{"1min":0.27200000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":45.884999999999998,"5min":128.79300000000001,"15min":128.79300000000001},"last":6.4710000000000001},{"interface":"front","average":{"1min":7.2859999999999996,"5min":5.6900000000000004,"15min":2.3140000000000001},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.248},"max":{"1min":45.871000000000002,"5min":128.38800000000001,"15min":128.38800000000001},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:26:35 2023","interfaces":[{"interface":"back","average":{"1min":2.4489999999999998,"5min":1.4319999999999999,"15min":0.85899999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":12.185,"5min":31.469999999999999,"15min":31.469999999999999},"last":4.2050000000000001},{"interface":"front","average":{"1min":1.825,"5min":1.401,"15min":0.84099999999999997},"min":{"1min":0.38100000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.5659999999999998,"5min":31.962,"15min":31.962},"last":0.26800000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:27:05 2023","interfaces":[{"interface":"back","average":{"1min":5.6029999999999998,"5min":4.5609999999999999,"15min":1.9279999999999999},"min":{"1min":0.39100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":56.106000000000002,"5min":140.601,"15min":140.601},"last":0.629},{"interface":"front","average":{"1min":5.1150000000000002,"5min":4.4640000000000004,"15min":1.885},"min":{"1min":0.36899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":56.045999999999999,"5min":140.54900000000001,"15min":140.54900000000001},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:26:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.9179999999999999,"15min":1.0369999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":9.8279999999999994,"5min":86.620000000000005,"15min":86.620000000000005},"last":0.91900000000000004},{"interface":"front","average":{"1min":2.7010000000000001,"5min":2.0110000000000001,"15min":1.083},"min":{"1min":0.33000000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":12.135,"5min":86.786000000000001,"15min":86.786000000000001},"last":0.93400000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:27:21 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.5509999999999999,"15min":0.84199999999999997},"min":{"1min":0.27600000000000002,"5min":0.252,"15min":0.252},"max":{"1min":6.2949999999999999,"5min":86.691000000000003,"15min":86.691000000000003},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.3769999999999998,"5min":1.8460000000000001,"15min":0.96799999999999997},"min":{"1min":0.30499999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":11.101000000000001,"5min":86.822000000000003,"15min":86.822000000000003},"last":0.30299999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:26:42 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.64100000000000001,"15min":0.55900000000000005},"min":{"1min":0.311,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.6029999999999998,"5min":4.6029999999999998,"15min":4.6029999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.044,"5min":0.878,"15min":0.67200000000000004},"min":{"1min":0.34499999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.327,"5min":6.327,"15min":6.327},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":1.887,"15min":1.016},"min":{"1min":0.38400000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.057,"5min":86.728999999999999,"15min":86.728999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.129,"5min":0.995,"15min":0.72899999999999998},"min":{"1min":0.35499999999999998,"5min":0.314,"15min":0.314},"max":{"1min":6.3609999999999998,"5min":6.3609999999999998,"15min":6.3609999999999998},"last":0.46100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757463,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278960,"kb_used_data":164648,"kb_used_omap":0,"kb_used_meta":114304,"kb_avail":93466192,"statfs":{"total":95995035648,"available":95709380608,"internally_reserved":0,"allocated":168599552,"data_stored":167823161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":117047296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:27:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":4.3879999999999999,"15min":1.8440000000000001},"min":{"1min":0.246,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.0299999999999994,"5min":148.79900000000001,"15min":148.79900000000001},"last":9.0299999999999994},{"interface":"front","average":{"1min":4.9290000000000003,"5min":4.9500000000000002,"15min":2.0590000000000002},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":19.004000000000001,"5min":148.779,"15min":148.779},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":9.8399999999999999,"5min":3.7890000000000001,"15min":1.647},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":166.708,"5min":166.708,"15min":166.708},"last":0.54200000000000004},{"interface":"front","average":{"1min":8.7029999999999994,"5min":3.4860000000000002,"15min":1.593},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":166.607,"5min":166.607,"15min":166.607},"last":4.6180000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:27:06 2023","interfaces":[{"interface":"back","average":{"1min":11.182,"5min":6.3170000000000002,"15min":2.504},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":131.655,"5min":165.768,"15min":165.768},"last":0.51200000000000001},{"interface":"front","average":{"1min":9.3930000000000007,"5min":7.7240000000000002,"15min":3.004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":131.77600000000001,"5min":166.24000000000001,"15min":166.24000000000001},"last":9.0760000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:26:38 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.8109999999999999,"15min":1.0309999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.9049999999999994,"5min":28.015999999999998,"15min":28.015999999999998},"last":1.49},{"interface":"front","average":{"1min":1.968,"5min":1.5549999999999999,"15min":0.94899999999999995},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":8.9380000000000006,"5min":28.039999999999999,"15min":28.039999999999999},"last":1.4550000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:27:19 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":0.874,"15min":0.60899999999999999},"min":{"1min":0.28599999999999998,"5min":0.222,"15min":0.222},"max":{"1min":6.298,"5min":8.5359999999999996,"15min":8.5359999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":2,"5min":1.0309999999999999,"15min":0.69499999999999995},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.9640000000000004,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:26:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":0.77400000000000002,"15min":0.61099999999999999},"min":{"1min":0.32300000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.9760000000000009,"5min":8.9760000000000009,"15min":8.9760000000000009},"last":0.44},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.0489999999999999,"15min":0.75900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":9.0380000000000003,"5min":9.0380000000000003,"15min":9.0380000000000003},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 30 09:26:53 2023","interfaces":[{"interface":"back","average":{"1min":2.8679999999999999,"5min":1.206,"15min":0.79500000000000004},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":19.175999999999998,"5min":19.175999999999998,"15min":19.175999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.8899999999999999,"5min":0.97199999999999998,"15min":0.71099999999999997},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.8810000000000002,"5min":8.8810000000000002,"15min":8.8810000000000002},"last":0.47899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":54984704,"data_stored":54984704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":70123520,"data_stored":70123520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":41881600,"data_stored":41877528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":50200576,"data_stored":50200576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":157351936,"data_stored":157351936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":157552640,"data_stored":157548568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":37814272,"data_stored":37814272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":46862336,"data_stored":46858264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:27:30.637 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:27:30.638 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-30T09:27:30.697 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:30.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:30.741 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:30.773 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:30.808 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:30.847 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:31.007 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:31.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:31.112 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:31.138 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:31.181 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:31.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:31.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:31.289 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:31.316 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:31.360 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:31.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:31.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:31.465 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:31.492 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:31.536 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:31.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:31.644 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:31.677 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:31.779 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:31.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:31.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:31.882 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:32.440 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:32 smithi019 ceph-mon[216815]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 6.4 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 5122/171 objects degraded (2995.322%); 16/171 objects misplaced (9.357%); 1.6 MiB/s, 83 objects/s recovering 2023-09-30T09:27:32.441 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:32 smithi019 ceph-mon[224043]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 6.4 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 5122/171 objects degraded (2995.322%); 16/171 objects misplaced (9.357%); 1.6 MiB/s, 83 objects/s recovering 2023-09-30T09:27:32.473 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:32 smithi167 ceph-mon[192880]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 6.4 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 5122/171 objects degraded (2995.322%); 16/171 objects misplaced (9.357%); 1.6 MiB/s, 83 objects/s recovering 2023-09-30T09:27:33.162 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:27:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:33 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/239768634' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:27:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:33.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:33 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/239768634' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:27:33.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:33 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/239768634' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:27:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:27:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:27:33] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.43.0" 2023-09-30T09:27:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:34 smithi167 ceph-mon[192880]: pgmap v450: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 5122/84 objects degraded (6097.619%); 597 KiB/s, 68 objects/s recovering 2023-09-30T09:27:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:34 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/239768634' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:27:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:34 smithi167 ceph-mon[192880]: osdmap e193: 8 total, 8 up, 8 in 2023-09-30T09:27:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:34 smithi167 ceph-mon[192880]: failed to encode map e193 with expected crc 2023-09-30T09:27:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:34 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5122/84 objects degraded (6097.619%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[216815]: pgmap v450: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 5122/84 objects degraded (6097.619%); 597 KiB/s, 68 objects/s recovering 2023-09-30T09:27:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/239768634' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:27:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[216815]: osdmap e193: 8 total, 8 up, 8 in 2023-09-30T09:27:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[216815]: failed to encode map e193 with expected crc 2023-09-30T09:27:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5122/84 objects degraded (6097.619%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[224043]: pgmap v450: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 5122/84 objects degraded (6097.619%); 597 KiB/s, 68 objects/s recovering 2023-09-30T09:27:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:27:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/239768634' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:27:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[224043]: osdmap e193: 8 total, 8 up, 8 in 2023-09-30T09:27:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[224043]: failed to encode map e193 with expected crc 2023-09-30T09:27:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:34 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5122/84 objects degraded (6097.619%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:27:35.486 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:35 smithi019 ceph-mon[224043]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.486 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:35 smithi019 ceph-mon[224043]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.486 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:35 smithi019 ceph-mon[224043]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:35 smithi019 ceph-mon[216815]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.487 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:35 smithi019 ceph-mon[216815]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.487 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:35 smithi019 ceph-mon[216815]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:35 smithi167 ceph-mon[192880]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:35 smithi167 ceph-mon[192880]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:35 smithi167 ceph-mon[192880]: failed to encode map e193 with expected crc 2023-09-30T09:27:35.531 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:35.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:35.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:35.635 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:27:35.661 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:35.705 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:27:35.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:35.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:27:35.811 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:35.837 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:35.882 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:35.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:35.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:35.988 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:36.039 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:36.158 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:36 smithi019 ceph-mon[216815]: pgmap v452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:36 smithi167 ceph-mon[192880]: pgmap v452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:36 smithi019 ceph-mon[224043]: pgmap v452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:37.820 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:37.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:37.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:37.924 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:37.963 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:38.251 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:38.267 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:38.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:38.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:38.313 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:27:38.334 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:27:38.359 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:27:38.373 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:38.385 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:27:38.407 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:38.407 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:27:38.429 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:27:38.449 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:27:38.468 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:27:38.470 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:27:38.470 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:27:38.470 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:27:38.470 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:27:38.470 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:27:38.470 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:27:38.470 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:27:38.470 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:27:38.495 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:38 smithi019 ceph-mon[224043]: pgmap v453: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:38.495 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:38 smithi019 ceph-mon[216815]: pgmap v453: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:38 smithi167 ceph-mon[192880]: pgmap v453: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:38.610 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:27:38.610 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:27:38.610 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:27:38.611 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:27:38.611 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:27:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2270011552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2270011552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:39.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2270011552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:39.949 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:27:40.160 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:40.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:40.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:40.265 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:40.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[216815]: pgmap v454: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:40.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[216815]: osdmap e194: 8 total, 8 up, 8 in 2023-09-30T09:27:40.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[216815]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[216815]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[216815]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[216815]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[216815]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[224043]: pgmap v454: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:40.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[224043]: osdmap e194: 8 total, 8 up, 8 in 2023-09-30T09:27:40.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[224043]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[224043]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[224043]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[224043]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:40 smithi019 ceph-mon[224043]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.296 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:40 smithi167 ceph-mon[192880]: pgmap v454: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 5122/84 objects degraded (6097.619%); 0 B/s, 191 objects/s recovering 2023-09-30T09:27:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:40 smithi167 ceph-mon[192880]: osdmap e194: 8 total, 8 up, 8 in 2023-09-30T09:27:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:40 smithi167 ceph-mon[192880]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:40 smithi167 ceph-mon[192880]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:40 smithi167 ceph-mon[192880]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:40 smithi167 ceph-mon[192880]: failed to encode map e194 with expected crc 2023-09-30T09:27:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:40 smithi167 ceph-mon[192880]: failed to encode map e194 with expected crc 2023-09-30T09:27:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:41 smithi167 ceph-mon[192880]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:41 smithi167 ceph-mon[192880]: osdmap e195: 8 total, 8 up, 8 in 2023-09-30T09:27:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:41 smithi167 ceph-mon[192880]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:41 smithi167 ceph-mon[192880]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:41 smithi167 ceph-mon[192880]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:41 smithi167 ceph-mon[192880]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:41 smithi167 ceph-mon[192880]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[216815]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[216815]: osdmap e195: 8 total, 8 up, 8 in 2023-09-30T09:27:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[216815]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[216815]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[216815]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[216815]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[216815]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[224043]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[224043]: osdmap e195: 8 total, 8 up, 8 in 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[224043]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[224043]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[224043]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[224043]: failed to encode map e195 with expected crc 2023-09-30T09:27:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:41 smithi019 ceph-mon[224043]: failed to encode map e195 with expected crc 2023-09-30T09:27:42.208 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:42.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:42.277 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:42 smithi019 ceph-mon[216815]: pgmap v457: 89 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 431 objects/s recovering 2023-09-30T09:27:42.277 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:42 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1244/84 objects degraded (1480.952%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:27:42.277 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:42 smithi019 ceph-mon[224043]: pgmap v457: 89 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 431 objects/s recovering 2023-09-30T09:27:42.277 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:42 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1244/84 objects degraded (1480.952%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:27:42.312 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:27:42.356 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:42.403 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:27:42.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:42.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:27:42.443 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:27:42.509 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:42 smithi167 ceph-mon[192880]: pgmap v457: 89 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 431 objects/s recovering 2023-09-30T09:27:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:42 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1244/84 objects degraded (1480.952%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:27:42.546 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:42.602 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:42.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:42.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:42.706 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:27:42.737 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:42.783 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:27:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:42.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:42.891 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:42.920 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:44.068 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:44.070 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:27:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:27:43] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.43.0" 2023-09-30T09:27:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:44.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:44.172 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:44.202 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:44.249 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:44.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:44.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:44.302 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:27:44.330 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:44.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:44 smithi019 ceph-mon[216815]: pgmap v458: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 262 objects/s recovering 2023-09-30T09:27:44.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:44 smithi019 ceph-mon[224043]: pgmap v458: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 262 objects/s recovering 2023-09-30T09:27:44.354 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:44.404 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:44.406 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:44.501 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:44.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:44 smithi167 ceph-mon[192880]: pgmap v458: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 262 objects/s recovering 2023-09-30T09:27:44.605 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:27:44.635 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:44.681 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:27:44.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:44.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:27:44.789 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:27:44.815 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:44.858 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:27:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:44.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:44.964 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:44.990 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:45.030 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:45.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:45.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:45.138 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:45.169 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:45.672 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:45.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:45.777 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:45.806 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:45.854 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:45.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:45.958 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:27:45.989 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:46.038 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:27:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:46.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:27:46.143 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:46.172 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:46.220 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:27:46.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:46.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:27:46.235 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 1 pg_num to 11 2023-09-30T09:27:46.326 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:27:46.356 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:46.401 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:27:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:46.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:27:46.509 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:46 smithi167 ceph-mon[192880]: pgmap v459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 357 objects/s recovering 2023-09-30T09:27:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2595321055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-30T09:27:46.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:46 smithi019 ceph-mon[224043]: pgmap v459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 357 objects/s recovering 2023-09-30T09:27:46.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2595321055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-30T09:27:46.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:46 smithi019 ceph-mon[216815]: pgmap v459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 357 objects/s recovering 2023-09-30T09:27:46.539 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2595321055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-30T09:27:46.541 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:46.599 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:46.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:46.704 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:46.734 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:46.789 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:46.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:46.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:46.892 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:46.922 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2595321055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-30T09:27:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: osdmap e196: 8 total, 8 up, 8 in 2023-09-30T09:27:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:27:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2595321055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: osdmap e196: 8 total, 8 up, 8 in 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2595321055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: osdmap e196: 8 total, 8 up, 8 in 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: failed to encode map e196 with expected crc 2023-09-30T09:27:47.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:47.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-30T09:27:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:48 smithi167 ceph-mon[192880]: failed to encode map e196 with expected crc 2023-09-30T09:27:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:48 smithi167 ceph-mon[192880]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 362 objects/s recovering 2023-09-30T09:27:48.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:48.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-30T09:27:48.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:48 smithi167 ceph-mon[192880]: osdmap e197: 8 total, 8 up, 8 in 2023-09-30T09:27:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[224043]: failed to encode map e196 with expected crc 2023-09-30T09:27:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[224043]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 362 objects/s recovering 2023-09-30T09:27:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-30T09:27:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[224043]: osdmap e197: 8 total, 8 up, 8 in 2023-09-30T09:27:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[216815]: failed to encode map e196 with expected crc 2023-09-30T09:27:48.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[216815]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 362 objects/s recovering 2023-09-30T09:27:48.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:27:48.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-30T09:27:48.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:48 smithi019 ceph-mon[216815]: osdmap e197: 8 total, 8 up, 8 in 2023-09-30T09:27:49.065 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:49.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:49.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:49.169 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:27:49.333 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-30T09:27:49.334 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-30T09:27:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[224043]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[224043]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[224043]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[224043]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[224043]: osdmap e198: 8 total, 8 up, 8 in 2023-09-30T09:27:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[216815]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[216815]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[216815]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[216815]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:49 smithi019 ceph-mon[216815]: osdmap e198: 8 total, 8 up, 8 in 2023-09-30T09:27:49.361 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:49.507 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:27:49.513 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:49 smithi167 ceph-mon[192880]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.513 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:49 smithi167 ceph-mon[192880]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.513 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:49 smithi167 ceph-mon[192880]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.513 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:49 smithi167 ceph-mon[192880]: failed to encode map e197 with expected crc 2023-09-30T09:27:49.513 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:49 smithi167 ceph-mon[192880]: osdmap e198: 8 total, 8 up, 8 in 2023-09-30T09:27:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:49.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:27:49.615 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:49.651 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:49.670 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:49.824 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:49.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:49.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:49.928 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:49.962 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:50.242 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[216815]: failed to encode map e198 with expected crc 2023-09-30T09:27:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:50 smithi167 ceph-mon[192880]: failed to encode map e198 with expected crc 2023-09-30T09:27:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:50 smithi167 ceph-mon[192880]: pgmap v464: 99 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 125 objects/s recovering 2023-09-30T09:27:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:50 smithi167 ceph-mon[192880]: failed to encode map e198 with expected crc 2023-09-30T09:27:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:50 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1244/78 objects degraded (1594.872%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:27:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[216815]: pgmap v464: 99 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 125 objects/s recovering 2023-09-30T09:27:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[216815]: failed to encode map e198 with expected crc 2023-09-30T09:27:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1244/78 objects degraded (1594.872%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:27:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[224043]: failed to encode map e198 with expected crc 2023-09-30T09:27:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[224043]: pgmap v464: 99 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 125 objects/s recovering 2023-09-30T09:27:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[224043]: failed to encode map e198 with expected crc 2023-09-30T09:27:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:27:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:50 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1244/78 objects degraded (1594.872%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:27:51.255 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:27:51.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2013845748' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:51.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:51 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2013845748' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:51 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:51.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2013845748' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:51.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:51 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:27:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:52 smithi167 ceph-mon[192880]: pgmap v465: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 unknown, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 142 objects/s recovering 2023-09-30T09:27:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:52 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:27:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:27:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:52 smithi167 ceph-mon[192880]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:52 smithi167 ceph-mon[192880]: osdmap e199: 8 total, 8 up, 8 in 2023-09-30T09:27:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:52 smithi167 ceph-mon[192880]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:52 smithi167 ceph-mon[192880]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[216815]: pgmap v465: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 unknown, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 142 objects/s recovering 2023-09-30T09:27:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:27:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:27:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[216815]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[216815]: osdmap e199: 8 total, 8 up, 8 in 2023-09-30T09:27:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[216815]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[216815]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[224043]: pgmap v465: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 unknown, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 142 objects/s recovering 2023-09-30T09:27:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:27:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:27:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[224043]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[224043]: osdmap e199: 8 total, 8 up, 8 in 2023-09-30T09:27:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[224043]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:52 smithi019 ceph-mon[224043]: failed to encode map e199 with expected crc 2023-09-30T09:27:52.698 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:52.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:52.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:52.802 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:27:52.836 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:52.886 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:27:52.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:52.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:27:52.992 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:53.641 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:27:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:27:53] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-09-30T09:27:54.334 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:27:54.335 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:27:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:54 smithi167 ceph-mon[192880]: pgmap v467: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 unknown, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 142 objects/s recovering 2023-09-30T09:27:54.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:54 smithi019 ceph-mon[216815]: pgmap v467: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 unknown, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 142 objects/s recovering 2023-09-30T09:27:54.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:54 smithi019 ceph-mon[224043]: pgmap v467: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 unknown, 87 active+clean; 4.1 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1244/78 objects degraded (1594.872%); 0 B/s, 142 objects/s recovering 2023-09-30T09:27:54.607 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:54.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:54.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:54.713 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:27:54.747 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:55.200 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:27:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:55.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:27:55.305 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:27:55.339 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:55 smithi167 ceph-mon[192880]: 1.0 deep-scrub starts 2023-09-30T09:27:55.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:55 smithi167 ceph-mon[192880]: 1.0 deep-scrub ok 2023-09-30T09:27:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:55 smithi019 ceph-mon[216815]: 1.0 deep-scrub starts 2023-09-30T09:27:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:55 smithi019 ceph-mon[216815]: 1.0 deep-scrub ok 2023-09-30T09:27:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:55 smithi019 ceph-mon[224043]: 1.0 deep-scrub starts 2023-09-30T09:27:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:55 smithi019 ceph-mon[224043]: 1.0 deep-scrub ok 2023-09-30T09:27:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:56 smithi167 ceph-mon[192880]: pgmap v468: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 329 objects/s recovering 2023-09-30T09:27:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:56 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1244/84 objects degraded (1480.952%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:27:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:56 smithi019 ceph-mon[216815]: pgmap v468: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 329 objects/s recovering 2023-09-30T09:27:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:56 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1244/84 objects degraded (1480.952%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:27:56.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:56 smithi019 ceph-mon[224043]: pgmap v468: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 1244/84 objects degraded (1480.952%); 0 B/s, 329 objects/s recovering 2023-09-30T09:27:56.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:56 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1244/84 objects degraded (1480.952%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:27:56.968 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:27:57.357 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:27:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:57.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:27:57.465 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:27:57.498 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:27:57.510 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:57 smithi167 ceph-mon[192880]: 1.3 scrub starts 2023-09-30T09:27:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:57 smithi167 ceph-mon[192880]: 1.3 scrub ok 2023-09-30T09:27:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:57 smithi019 ceph-mon[216815]: 1.3 scrub starts 2023-09-30T09:27:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:57 smithi019 ceph-mon[216815]: 1.3 scrub ok 2023-09-30T09:27:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:57 smithi019 ceph-mon[224043]: 1.3 scrub starts 2023-09-30T09:27:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:57 smithi019 ceph-mon[224043]: 1.3 scrub ok 2023-09-30T09:27:57.632 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:27:57.632 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:27:57.633 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:27:57.633 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:27:57.633 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.633 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80438 5.e 5:7ec295bd:::benchmark_data_smithi019_287194_object80437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:27:57.633 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:27:57.633 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 236.006952732, 2023-09-30T09:27:57.633 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5009281310000002, 2023-09-30T09:27:57.633 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80438 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.634 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621660+0000", 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.423e-06 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621684+0000", 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4569000000000001e-05 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621694+0000", 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5210000000000008e-06 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621700+0000", 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6710000000000002e-06 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.622254+0000", 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00055331 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.622263+0000", 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8419999999999994e-06 2023-09-30T09:27:57.638 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603497+0000", 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98123395899999999 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603507+0000", 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0285999999999999e-05 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.078297+0000", 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4747904549999999 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.084057+0000", 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0057597530000000003 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122473+0000", 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.038415673999999997 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122586+0000", 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113668 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.641 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80444 5.5 5:a3c25906:::benchmark_data_smithi019_287194_object80443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 236.006900282, 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.335433417, 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80444 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3099999999999998e-07 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621733+0000", 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1899000000000002e-05 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621740+0000", 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7290000000000003e-06 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621745+0000", 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3489999999999999e-06 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621802+0000", 2023-09-30T09:27:57.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7148000000000002e-05 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621813+0000", 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1136e-05 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.602972+0000", 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98115875399999997 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.602983+0000", 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1294000000000001e-05 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.941598+0000", 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.338614129 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.941805+0000", 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020759399999999999 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.942011+0000", 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020628199999999999 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949084+0000", 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0070728550000000003 2023-09-30T09:27:57.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954359+0000", 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0052750540000000004 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954690+0000", 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033033200000000002 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954747+0000", 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6943e-05 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957144+0000", 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023974880000000001 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80452 5.5 5:a7eeee83:::benchmark_data_smithi019_287194_object80451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 235.996112044, 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3299363909999999, 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80452 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.651 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632500+0000", 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.576e-06 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632525+0000", 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4213999999999999e-05 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632543+0000", 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8309e-05 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632552+0000", 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5860000000000005e-06 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.654 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632615+0000", 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3384000000000002e-05 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632628+0000", 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2727e-05 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603007+0000", 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97037908100000003 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603015+0000", 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7049999999999999e-06 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947003+0000", 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.343988728 2023-09-30T09:27:57.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947114+0000", 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000110746 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947240+0000", 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000126136 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954457+0000", 2023-09-30T09:27:57.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0072169809999999999 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957159+0000", 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00270178 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.960528+0000", 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033696120000000001 2023-09-30T09:27:57.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.960588+0000", 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9361000000000003e-05 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962435+0000", 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001847465 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.659 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80456 5.5 5:a29acd63:::benchmark_data_smithi019_287194_object80455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 235.98918930900001, 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3250314799999998, 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.660 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80456 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.661 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639424+0000", 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0729999999999999e-06 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639449+0000", 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5324999999999999e-05 2023-09-30T09:27:57.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639467+0000", 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7923000000000001e-05 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639477+0000", 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6809999999999993e-06 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639552+0000", 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5191000000000006e-05 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639566+0000", 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3654e-05 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603040+0000", 2023-09-30T09:27:57.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96347411699999996 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603047+0000", 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.126e-06 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949179+0000", 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3461320999999999 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949348+0000", 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016921899999999999 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949468+0000", 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011944599999999999 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954573+0000", 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005105371 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957167+0000", 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025936269999999998 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962500+0000", 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0053332429999999997 2023-09-30T09:27:57.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962562+0000", 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2016999999999999e-05 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.964453+0000", 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018913669999999999 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.668 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80465 5.e 5:72d43b9b:::benchmark_data_smithi019_287194_object80464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 235.96550871900001, 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5017385250000004, 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.669 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80465 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:27:57.670 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663104+0000", 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9649999999999998e-06 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663128+0000", 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3878000000000001e-05 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.671 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663145+0000", 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.649e-05 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663153+0000", 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5750000000000001e-06 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663245+0000", 2023-09-30T09:27:57.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2249000000000004e-05 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663259+0000", 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3422e-05 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603534+0000", 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94027541000000003 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.673 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603543+0000", 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4120000000000001e-06 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122620+0000", 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5190771290000002 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.674 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122769+0000", 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014931999999999999 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122900+0000", 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013077799999999999 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.131893+0000", 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089926810000000006 2023-09-30T09:27:57.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135017+0000", 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031246440000000002 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.162244+0000", 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027226509999999999 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.162314+0000", 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0116000000000004e-05 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.164841+0000", 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002526946 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80467 5.e 5:7e7fcfbf:::benchmark_data_smithi019_287194_object80466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 233.88988842699999, 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4284252850000003, 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80467 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.678 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.679 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738724+0000", 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.823e-06 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738752+0000", 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7413e-05 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.740661+0000", 2023-09-30T09:27:57.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001908795 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.740676+0000", 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5333999999999999e-05 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125570+0000", 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3848938300000002 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125759+0000", 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018891200000000001 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125968+0000", 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020954400000000001 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.682 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.134957+0000", 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089886380000000002 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135084+0000", 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012721999999999999 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.164930+0000", 2023-09-30T09:27:57.683 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.02984645 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.165000+0000", 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9052000000000002e-05 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167148+0000", 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021482739999999999 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.684 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80472 5.e 5:7c0728d2:::benchmark_data_smithi019_287194_object80471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 233.69408306899999, 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2349960290000004, 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.685 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80472 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.686 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934529+0000", 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4750000000000001e-06 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934554+0000", 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4953999999999999e-05 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934572+0000", 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7317e-05 2023-09-30T09:27:57.687 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934580+0000", 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5080000000000005e-06 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.128799+0000", 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1942186069999998 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.128998+0000", 2023-09-30T09:27:57.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000199447 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.129303+0000", 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00030457599999999998 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141779+0000", 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012476716000000001 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.690 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.690 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141802+0000", 2023-09-30T09:27:57.690 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2903000000000001e-05 2023-09-30T09:27:57.691 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.691 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.691 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.691 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167214+0000", 2023-09-30T09:27:57.691 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.025411309 2023-09-30T09:27:57.691 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.691 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.691 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167261+0000", 2023-09-30T09:27:57.692 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7747999999999998e-05 2023-09-30T09:27:57.692 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.692 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.692 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169524+0000", 2023-09-30T09:27:57.692 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002262469 2023-09-30T09:27:57.692 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.693 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.693 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.693 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.693 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.693 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80493 5.e 5:7e14abc9:::benchmark_data_smithi019_287194_object80492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:27:57.693 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:27:57.693 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 233.59257217800001, 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1357754939999998, 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80493 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.694 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.695 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036040+0000", 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5269999999999999e-06 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036063+0000", 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3034999999999999e-05 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036076+0000", 2023-09-30T09:27:57.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2979999999999999e-05 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036085+0000", 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6340000000000007e-06 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.131929+0000", 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.095844402 2023-09-30T09:27:57.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.698 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.698 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.132099+0000", 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016993999999999999 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.132318+0000", 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021850799999999999 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141810+0000", 2023-09-30T09:27:57.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0094919229999999993 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141869+0000", 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9252000000000001e-05 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169591+0000", 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027721632999999999 2023-09-30T09:27:57.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169673+0000", 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2330999999999995e-05 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171814+0000", 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002141329 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.701 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80495 5.e 5:72ad1293:::benchmark_data_smithi019_287194_object80494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 233.589023142, 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1346890580000002, 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.702 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80495 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.703 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039590+0000", 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0020000000000001e-06 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039617+0000", 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6701000000000001e-05 2023-09-30T09:27:57.704 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039628+0000", 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1557e-05 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039635+0000", 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1829999999999997e-06 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.705 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135122+0000", 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0954865739999997 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135319+0000", 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019667600000000001 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135541+0000", 2023-09-30T09:27:57.706 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022274 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141816+0000", 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062751109999999999 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151681+0000", 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0098647409999999998 2023-09-30T09:27:57.707 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171874+0000", 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020192412999999999 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171931+0000", 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7716000000000001e-05 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.708 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174277+0000", 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002345644 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80498 5.e 5:7631c3ad:::benchmark_data_smithi019_287194_object80497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:27:57.709 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 233.58088033499999, 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1287919449999997, 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80498 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.710 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047732+0000", 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.832e-06 2023-09-30T09:27:57.711 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047759+0000", 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6628999999999999e-05 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047779+0000", 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9468000000000001e-05 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.712 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047789+0000", 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0864e-05 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138222+0000", 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0904321699999997 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138406+0000", 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018471300000000001 2023-09-30T09:27:57.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138613+0000", 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000206596 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.148131+0000", 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095180230000000005 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151779+0000", 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036479849999999999 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174332+0000", 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022552744 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174400+0000", 2023-09-30T09:27:57.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7886999999999996e-05 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176523+0000", 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021230340000000002 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.716 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80506 5.e 5:7cef17fd:::benchmark_data_smithi019_287194_object80505:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 233.54469639800001, 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0963020300000004, 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80506 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.717 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.718 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083916+0000", 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9199999999999999e-07 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083934+0000", 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8879e-05 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.719 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083943+0000", 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0620000000000007e-06 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083950+0000", 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1519999999999997e-06 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.720 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141906+0000", 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0579564440000002 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.142127+0000", 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000220447 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.142337+0000", 2023-09-30T09:27:57.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000210867 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.148142+0000", 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058050389999999997 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.153209+0000", 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050662160000000001 2023-09-30T09:27:57.722 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176580+0000", 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.02337171 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176646+0000", 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5775e-05 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.180217+0000", 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003570447 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80509 5.e 5:7a2f2184:::benchmark_data_smithi019_287194_object80508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 233.52203434899999, 2023-09-30T09:27:57.724 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0801843260000004, 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80509 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.725 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106578+0000", 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6199999999999999e-06 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.726 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106608+0000", 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9841000000000001e-05 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106623+0000", 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4628e-05 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.727 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106632+0000", 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6560000000000004e-06 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145019+0000", 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0383863680000003 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145216+0000", 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000196942 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145433+0000", 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000217436 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151985+0000", 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0065521779999999996 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.153262+0000", 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00127673 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.184424+0000", 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031162419 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.184490+0000", 2023-09-30T09:27:57.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5228999999999995e-05 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.186761+0000", 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022712790000000002 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.731 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:82845 5.3 5:c557c714:::benchmark_data_smithi019_287194_object67:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 208.43909081000001, 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.525600517, 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 82845 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.732 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189521+0000", 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9499999999999998e-07 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189523+0000", 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1469999999999999e-06 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189529+0000", 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0719999999999996e-06 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189534+0000", 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4090000000000002e-06 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.459994+0000", 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.270459909 2023-09-30T09:27:57.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.466180+0000", 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061857609999999997 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.331107+0000", 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86492722099999997 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.339884+0000", 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087769049999999998 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.642343+0000", 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.302458745 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.647436+0000", 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050929809999999999 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.656592+0000", 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091567330000000002 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.695178+0000", 2023-09-30T09:27:57.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.038585174999999999 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.697589+0000", 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024115120000000002 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.698724+0000", 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001134456 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.701606+0000", 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028820909999999998 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.715121+0000", 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013515005 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:83209 5.3 5:c8e07ed0:::benchmark_data_smithi019_287194_object431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 207.143882066, 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8400215319999997, 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 83209 2023-09-30T09:27:57.741 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.742 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484730+0000", 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3769999999999999e-06 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484736+0000", 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2889999999999997e-06 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484750+0000", 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4929999999999999e-05 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484758+0000", 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1910000000000003e-06 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.744 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.512794+0000", 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028036607000000002 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.515536+0000", 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027418659999999999 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.902937+0000", 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3874005220000001 2023-09-30T09:27:57.745 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.909306+0000", 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063690129999999998 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.219767+0000", 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3104613839999999 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.746 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.229321+0000", 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095544229999999994 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.239410+0000", 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010088311000000001 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.268110+0000", 2023-09-30T09:27:57.747 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028700013 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.272145+0000", 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040355829999999997 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.298154+0000", 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026008963999999999 2023-09-30T09:27:57.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.302424+0000", 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042697239999999999 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.324750+0000", 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022326334999999999 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.749 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:85292 5.3 5:c22e0b20:::benchmark_data_smithi019_287194_object2514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 191.522743773, 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0057864740000002, 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.750 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 85292 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.751 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105869+0000", 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6419999999999999e-06 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.752 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105875+0000", 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6390000000000003e-06 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105892+0000", 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6092000000000001e-05 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.753 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105902+0000", 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0723000000000001e-05 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.151432+0000", 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.045529575000000003 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.157840+0000", 2023-09-30T09:27:57.754 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0064082469999999997 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:47.639845+0000", 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4820053099999999 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:47.646225+0000", 2023-09-30T09:27:57.755 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063791960000000002 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.022311+0000", 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.376086213 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.027327+0000", 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050160719999999999 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.756 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.035994+0000", 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086666720000000003 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.078087+0000", 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.042092989999999997 2023-09-30T09:27:57.757 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.079289+0000", 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001202172 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.081249+0000", 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019603310000000001 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.758 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.085070+0000", 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038209870000000001 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.111654+0000", 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026583612999999999 2023-09-30T09:27:57.759 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87684 5.3 5:c18d78e7:::benchmark_data_smithi019_287194_object4906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 172.83220734099999, 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1651827400000001, 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.760 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87684 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.761 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796405+0000", 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.406e-06 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.762 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796411+0000", 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9139999999999996e-06 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796422+0000", 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1493e-05 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.763 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796429+0000", 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.567e-06 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.070700+0000", 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.274271406 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:27:57.764 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.077044+0000", 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063433329999999996 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.143755+0000", 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.066710900000000004 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.150381+0000", 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0066263609999999999 2023-09-30T09:27:57.765 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.882648+0000", 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.732266774 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.888309+0000", 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005660867 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.766 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.897037+0000", 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087287739999999999 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.939441+0000", 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.042403771999999999 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.767 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.941115+0000", 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001673316 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.943881+0000", 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027661420000000001 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.947752+0000", 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038716940000000002 2023-09-30T09:27:57.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.961586+0000", 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013834021 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87688 5.3 5:cc6b6e52:::benchmark_data_smithi019_287194_object4910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:27:57.769 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 172.82645428199999, 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1848223249999998, 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87688 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.770 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802159+0000", 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.322e-06 2023-09-30T09:27:57.771 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802168+0000", 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8470000000000002e-06 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802199+0000", 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1343000000000001e-05 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.772 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802212+0000", 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2368000000000001e-05 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.077878+0000", 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27566626799999999 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.087824+0000", 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0099457680000000007 2023-09-30T09:27:57.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.196502+0000", 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.10867827200000001 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.211705+0000", 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015202983 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.919095+0000", 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.707390357 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.924017+0000", 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049221409999999997 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:27:57.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.932712+0000", 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086950120000000002 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.942222+0000", 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095098939999999996 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.963866+0000", 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.021643335999999999 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.966772+0000", 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029066550000000002 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.971211+0000", 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0044391040000000001 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.986979+0000", 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015767654999999998 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87754 5.3 5:cecacda3:::benchmark_data_smithi019_287194_object4976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 172.41640584000001, 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1694036080000001, 2023-09-30T09:27:57.778 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87754 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212206+0000", 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2869999999999999e-06 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212211+0000", 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6789999999999996e-06 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212224+0000", 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2931999999999999e-05 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212233+0000", 2023-09-30T09:27:57.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0289999999999994e-06 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.299820+0000", 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.087586586999999994 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.303276+0000", 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034559970000000001 2023-09-30T09:27:57.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.858386+0000", 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5551105619999999 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.864253+0000", 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058669530000000003 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.276710+0000", 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.412457187 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.287222+0000", 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010511855000000001 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:27:57.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.300806+0000", 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013584003000000001 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.358020+0000", 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.057214108999999999 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.359098+0000", 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0010772889999999999 2023-09-30T09:27:57.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.361377+0000", 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022794439999999998 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.367047+0000", 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0056699250000000001 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.381609+0000", 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.01456177 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145688 5.e 5:72fbdf89:::benchmark_data_smithi019_287194_object62910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.263921639000003, 2023-09-30T09:27:57.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.140027033, 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145688 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364691+0000", 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3039999999999999e-06 2023-09-30T09:27:57.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364696+0000", 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5860000000000004e-06 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364709+0000", 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2938999999999999e-05 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364716+0000", 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7009999999999999e-06 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364773+0000", 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7463999999999997e-05 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364784+0000", 2023-09-30T09:27:57.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.077e-05 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305414+0000", 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94062986800000004 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305424+0000", 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6339999999999996e-06 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.474904+0000", 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.169480576 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.475014+0000", 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011030400000000001 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4,5", 2023-09-30T09:27:57.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.475093+0000", 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8994999999999998e-05 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478115+0000", 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030214360000000002 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.481419+0000", 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033041759999999998 2023-09-30T09:27:57.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.487360+0000", 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0059407549999999998 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504511+0000", 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017150822 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504581+0000", 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0104000000000005e-05 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504716+0000", 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000135599 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145692 5.e 5:778ecb27:::benchmark_data_smithi019_287194_object62914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.257476104000006, 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1338203870000001, 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145692 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.797 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371136+0000", 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4499999999999996e-07 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371141+0000", 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9710000000000003e-06 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:27:57.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371151+0000", 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0669000000000001e-05 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371157+0000", 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7219999999999996e-06 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371209+0000", 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1801000000000001e-05 2023-09-30T09:27:57.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371218+0000", 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7860000000000002e-06 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305986+0000", 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.934768932 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305996+0000", 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7399999999999999e-06 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478142+0000", 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172145298 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:27:57.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478283+0000", 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000141037 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4,5", 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478364+0000", 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1600999999999993e-05 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.481503+0000", 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031385779999999999 2023-09-30T09:27:57.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.487053+0000", 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055504949999999999 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.495859+0000", 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088061919999999991 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504770+0000", 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089107639999999998 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504831+0000", 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1209000000000003e-05 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:27:57.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504955+0000", 2023-09-30T09:27:57.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012384700000000001 2023-09-30T09:27:57.806 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.806 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.806 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.806 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:27:57.806 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:27:57.806 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:27:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:58 smithi167 ceph-mon[192880]: pgmap v469: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 274 KiB/s wr, 84 op/s; 1244/84 objects degraded (1480.952%); 393 KiB/s, 290 objects/s recovering 2023-09-30T09:27:58.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:58 smithi019 ceph-mon[216815]: pgmap v469: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 274 KiB/s wr, 84 op/s; 1244/84 objects degraded (1480.952%); 393 KiB/s, 290 objects/s recovering 2023-09-30T09:27:58.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:58 smithi019 ceph-mon[224043]: pgmap v469: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 274 KiB/s wr, 84 op/s; 1244/84 objects degraded (1480.952%); 393 KiB/s, 290 objects/s recovering 2023-09-30T09:27:59.002 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:27:59.002 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:27:59.002 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:27:59.002 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:27:59.002 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:27:59.002 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:27:59.003 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:27:59.004 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:27:59.485 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:27:59.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:59.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:27:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:59 smithi167 ceph-mon[192880]: 1.8 scrub starts 2023-09-30T09:27:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:59 smithi167 ceph-mon[192880]: 1.8 scrub ok 2023-09-30T09:27:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:27:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2996512041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:59.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:59 smithi019 ceph-mon[216815]: 1.8 scrub starts 2023-09-30T09:27:59.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:59 smithi019 ceph-mon[216815]: 1.8 scrub ok 2023-09-30T09:27:59.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:27:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2996512041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:59 smithi019 ceph-mon[224043]: 1.8 scrub starts 2023-09-30T09:27:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:59 smithi019 ceph-mon[224043]: 1.8 scrub ok 2023-09-30T09:27:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:27:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2996512041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:27:59.591 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:27:59.623 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:59.673 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:27:59.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:27:59.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:27:59.782 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:27:59.820 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:27:59.886 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:28:00.123 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:00.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:00.226 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:00.255 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:00 smithi167 ceph-mon[192880]: pgmap v470: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1013 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 154 KiB/s wr, 47 op/s; 1244/87 objects degraded (1429.885%); 220 KiB/s, 231 objects/s recovering 2023-09-30T09:28:00.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:00 smithi019 ceph-mon[216815]: pgmap v470: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1013 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 154 KiB/s wr, 47 op/s; 1244/87 objects degraded (1429.885%); 220 KiB/s, 231 objects/s recovering 2023-09-30T09:28:00.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:00 smithi019 ceph-mon[224043]: pgmap v470: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1013 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 154 KiB/s wr, 47 op/s; 1244/87 objects degraded (1429.885%); 220 KiB/s, 231 objects/s recovering 2023-09-30T09:28:01.008 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:01.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:01.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:01.111 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:01.144 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:01.192 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:01.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:01.297 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:01.324 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:01.368 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:01.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:01.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:01.473 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:01.505 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:01.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:01 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 629/87 objects degraded (722.989%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:28:01.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:01 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 629/87 objects degraded (722.989%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:28:01.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:01 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 629/87 objects degraded (722.989%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:28:01.620 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:01.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:01.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:01.701 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:01.728 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:01.728 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:01.769 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:01.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:01.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:02.033 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:02.067 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:02.371 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:02.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:02.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:02.478 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:02.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:02 smithi019 ceph-mon[216815]: pgmap v471: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 154 KiB/s wr, 47 op/s; 629/87 objects degraded (722.989%); 220 KiB/s, 263 objects/s recovering 2023-09-30T09:28:02.497 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:02 smithi019 ceph-mon[216815]: 1.1 scrub starts 2023-09-30T09:28:02.497 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:02 smithi019 ceph-mon[216815]: 1.1 scrub ok 2023-09-30T09:28:02.497 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:02 smithi019 ceph-mon[224043]: pgmap v471: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 154 KiB/s wr, 47 op/s; 629/87 objects degraded (722.989%); 220 KiB/s, 263 objects/s recovering 2023-09-30T09:28:02.497 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:02 smithi019 ceph-mon[224043]: 1.1 scrub starts 2023-09-30T09:28:02.497 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:02 smithi019 ceph-mon[224043]: 1.1 scrub ok 2023-09-30T09:28:02.511 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:02.560 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:02.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:02.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:02.665 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:02.691 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:02 smithi167 ceph-mon[192880]: pgmap v471: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 154 KiB/s wr, 47 op/s; 629/87 objects degraded (722.989%); 220 KiB/s, 263 objects/s recovering 2023-09-30T09:28:02.726 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:02 smithi167 ceph-mon[192880]: 1.1 scrub starts 2023-09-30T09:28:02.726 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:02 smithi167 ceph-mon[192880]: 1.1 scrub ok 2023-09-30T09:28:02.727 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:02.739 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:02.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:02.845 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:02.875 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:03.248 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_0' removed 2023-09-30T09:28:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3837791291' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:28:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:28:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:28:03.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[216815]: osdmap e200: 8 total, 8 up, 8 in 2023-09-30T09:28:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3837791291' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:28:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:28:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:28:03.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:03 smithi019 ceph-mon[224043]: osdmap e200: 8 total, 8 up, 8 in 2023-09-30T09:28:03.614 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:28:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3837791291' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:28:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:28:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:28:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:03 smithi167 ceph-mon[192880]: osdmap e200: 8 total, 8 up, 8 in 2023-09-30T09:28:03.901 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:03.903 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:28:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:28:03] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2023-09-30T09:28:03.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:03.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:04.005 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:04.032 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:04.076 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:04.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:04.183 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:04.208 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:04.250 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:04.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:04.355 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:04.381 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:04.422 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:04.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:04.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:04.528 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:04.554 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:04.595 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:04.602 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:04 smithi167 ceph-mon[192880]: pgmap v472: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 130 KiB/s wr, 39 op/s; 629/87 objects degraded (722.989%); 186 KiB/s, 221 objects/s recovering 2023-09-30T09:28:04.602 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:04 smithi167 ceph-mon[192880]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.602 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:04 smithi167 ceph-mon[192880]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.602 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:04 smithi167 ceph-mon[192880]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.602 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:04 smithi167 ceph-mon[192880]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:04.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:04.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[216815]: pgmap v472: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 130 KiB/s wr, 39 op/s; 629/87 objects degraded (722.989%); 186 KiB/s, 221 objects/s recovering 2023-09-30T09:28:04.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[216815]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[216815]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[216815]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[216815]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.610 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[224043]: pgmap v472: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 130 KiB/s wr, 39 op/s; 629/87 objects degraded (722.989%); 186 KiB/s, 221 objects/s recovering 2023-09-30T09:28:04.610 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[224043]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.611 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[224043]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.611 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[224043]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.611 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:04 smithi019 ceph-mon[224043]: failed to encode map e200 with expected crc 2023-09-30T09:28:04.704 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:04.730 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:04.771 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:04.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:04.876 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:04.902 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:04.943 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:04.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:05.048 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:05.075 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:05.117 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:05.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:05.171 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-30T09:28:05.172 INFO:tasks.radosbench:Beginning radosbench... 2023-09-30T09:28:05.172 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-30T09:28:05.172 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-30T09:28:05.222 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:05.252 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:05.254 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:05.361 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:28:05.419 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-30T09:28:05.420 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:05 smithi019 ceph-mon[216815]: 1.7 scrub starts 2023-09-30T09:28:05.420 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:05 smithi019 ceph-mon[216815]: 1.7 scrub ok 2023-09-30T09:28:05.420 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1063570398' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:05.420 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-30T09:28:05.420 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:05 smithi019 ceph-mon[224043]: 1.7 scrub starts 2023-09-30T09:28:05.420 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:05 smithi019 ceph-mon[224043]: 1.7 scrub ok 2023-09-30T09:28:05.421 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1063570398' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-30T09:28:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:05 smithi167 ceph-mon[192880]: 1.7 scrub starts 2023-09-30T09:28:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:05 smithi167 ceph-mon[192880]: 1.7 scrub ok 2023-09-30T09:28:05.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1063570398' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:06.529 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:06.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:06.553 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:28:06.553 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:28:06.615 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: pgmap v474: 83 pgs: 83 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 629/87 objects degraded (722.989%), 1 pg degraded, 1 pg undersized) 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1063570398' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: osdmap e201: 8 total, 8 up, 8 in 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.616 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.617 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[216815]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: pgmap v474: 83 pgs: 83 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:06.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 629/87 objects degraded (722.989%), 1 pg degraded, 1 pg undersized) 2023-09-30T09:28:06.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:28:06.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:06.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-30T09:28:06.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1063570398' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:28:06.617 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: osdmap e201: 8 total, 8 up, 8 in 2023-09-30T09:28:06.618 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.618 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.618 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.618 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.618 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.618 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:06 smithi019 ceph-mon[224043]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.634 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:06.673 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: pgmap v474: 83 pgs: 83 active+clean; 4.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 629/87 objects degraded (722.989%), 1 pg degraded, 1 pg undersized) 2023-09-30T09:28:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:28:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-30T09:28:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1063570398' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:28:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: osdmap e201: 8 total, 8 up, 8 in 2023-09-30T09:28:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:06 smithi167 ceph-mon[192880]: failed to encode map e201 with expected crc 2023-09-30T09:28:06.920 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:06.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:06.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:06.989 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:07.027 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:07.058 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:07.106 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:07.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:07.211 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:07.239 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:07.284 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:07.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:07.365 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:07 smithi167 ceph-mon[192880]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.365 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:07 smithi167 ceph-mon[192880]: osdmap e202: 8 total, 8 up, 8 in 2023-09-30T09:28:07.365 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:07 smithi167 ceph-mon[192880]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.365 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:07 smithi167 ceph-mon[192880]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.390 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:07.418 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[216815]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[216815]: osdmap e202: 8 total, 8 up, 8 in 2023-09-30T09:28:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[216815]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[216815]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[216815]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[216815]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[224043]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[224043]: osdmap e202: 8 total, 8 up, 8 in 2023-09-30T09:28:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[224043]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[224043]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[224043]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:07 smithi019 ceph-mon[224043]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:07 smithi167 ceph-mon[192880]: failed to encode map e202 with expected crc 2023-09-30T09:28:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:07 smithi167 ceph-mon[192880]: failed to encode map e202 with expected crc 2023-09-30T09:28:08.293 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:28:08.522 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:08.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: pgmap v477: 83 pgs: 6 remapped+peering, 77 active+clean; 4.5 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 14 op/s 2023-09-30T09:28:08.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-30T09:28:08.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: osdmap e203: 8 total, 8 up, 8 in 2023-09-30T09:28:08.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: osdmap e204: 8 total, 8 up, 8 in 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: pgmap v477: 83 pgs: 6 remapped+peering, 77 active+clean; 4.5 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 14 op/s 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: osdmap e203: 8 total, 8 up, 8 in 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: osdmap e204: 8 total, 8 up, 8 in 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:08 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:08.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:08.629 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:08.644 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:28:08.666 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:08.723 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:08.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: pgmap v477: 83 pgs: 6 remapped+peering, 77 active+clean; 4.5 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 14 op/s 2023-09-30T09:28:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-30T09:28:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: osdmap e203: 8 total, 8 up, 8 in 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e203 with expected crc 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: osdmap e204: 8 total, 8 up, 8 in 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:08 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:08.828 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:08.856 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:09.658 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:28:09.659 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:28:09.659 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:28:09.659 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:28:09.659 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:28:09.659 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:28:09.679 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:28:09.698 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:28:09.699 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:28:09.699 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:28:09.699 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:28:09.699 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:28:09.700 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:28:09.700 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:28:09.700 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:28:09.700 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:28:09.715 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:09.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:09.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:09.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.986 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.986 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.986 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.986 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.986 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:09.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:09.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:28:09.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:28:09.988 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:09.988 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: osdmap e205: 8 total, 8 up, 8 in 2023-09-30T09:28:09.988 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:09.988 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:09.988 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:09.988 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:09.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:09.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e204 with expected crc 2023-09-30T09:28:09.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:09.991 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:10.006 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:28:10.006 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:28:10.006 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:10.006 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: osdmap e205: 8 total, 8 up, 8 in 2023-09-30T09:28:10.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:09 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.008 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:10.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:10.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:10.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:10.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:10.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e204 with expected crc 2023-09-30T09:28:10.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: osdmap e205: 8 total, 8 up, 8 in 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:09 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.039 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:10.089 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:10.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:10.196 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:10.226 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:10.271 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:10.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:10.376 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:10.411 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:10.457 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:10.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:10.561 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:10.589 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:10.633 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:10.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:10.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:10.739 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:10.765 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:10.766 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: pgmap v480: 83 pgs: 6 peering, 4 remapped+peering, 73 active+clean; 4.5 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 165 KiB/s wr, 50 op/s; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.767 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/605113449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: pgmap v480: 83 pgs: 6 peering, 4 remapped+peering, 73 active+clean; 4.5 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 165 KiB/s wr, 50 op/s; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: failed to encode map e205 with expected crc 2023-09-30T09:28:10.769 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/605113449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:10.770 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:10.815 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:10.829 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:10.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:10.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:10.933 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:10.962 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: pgmap v480: 83 pgs: 6 peering, 4 remapped+peering, 73 active+clean; 4.5 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 165 KiB/s wr, 50 op/s; 79 KiB/s, 0 objects/s recovering 2023-09-30T09:28:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: failed to encode map e205 with expected crc 2023-09-30T09:28:11.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/605113449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:11.086 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:11.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:11.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:11.189 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:11.218 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:11.263 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:11.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:11.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:11.368 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:11.394 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:11.437 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:11.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:11.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:11.542 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:11.551 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_1' created 2023-09-30T09:28:11.570 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:11.611 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:11.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:11.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:11.717 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:11.743 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2177762919' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-30T09:28:11.743 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:11 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-30T09:28:11.743 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2177762919' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-30T09:28:11.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:11 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-30T09:28:11.746 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:11.844 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:11.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:11.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:11.948 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:11.975 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:12.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2177762919' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-30T09:28:12.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:11 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-30T09:28:12.272 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:12.930 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:28:12.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:12.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:12.932 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: pgmap v482: 83 pgs: 6 peering, 2 remapped+peering, 75 active+clean; 4.5 MiB data, 1007 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 139 KiB/s wr, 42 op/s; 120 KiB/s, 0 objects/s recovering 2023-09-30T09:28:12.932 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-30T09:28:12.932 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.932 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.932 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.932 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: osdmap e206: 8 total, 8 up, 8 in 2023-09-30T09:28:12.933 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.933 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.933 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.933 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[216815]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.933 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: pgmap v482: 83 pgs: 6 peering, 2 remapped+peering, 75 active+clean; 4.5 MiB data, 1007 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 139 KiB/s wr, 42 op/s; 120 KiB/s, 0 objects/s recovering 2023-09-30T09:28:12.933 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-30T09:28:12.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: osdmap e206: 8 total, 8 up, 8 in 2023-09-30T09:28:12.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.935 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:12 smithi019 ceph-mon[224043]: failed to encode map e206 with expected crc 2023-09-30T09:28:12.936 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-30T09:28:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: pgmap v482: 83 pgs: 6 peering, 2 remapped+peering, 75 active+clean; 4.5 MiB data, 1007 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 139 KiB/s wr, 42 op/s; 120 KiB/s, 0 objects/s recovering 2023-09-30T09:28:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-30T09:28:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: failed to encode map e206 with expected crc 2023-09-30T09:28:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: failed to encode map e206 with expected crc 2023-09-30T09:28:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: failed to encode map e206 with expected crc 2023-09-30T09:28:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: osdmap e206: 8 total, 8 up, 8 in 2023-09-30T09:28:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: failed to encode map e206 with expected crc 2023-09-30T09:28:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: failed to encode map e206 with expected crc 2023-09-30T09:28:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: failed to encode map e206 with expected crc 2023-09-30T09:28:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:12 smithi167 ceph-mon[192880]: failed to encode map e206 with expected crc 2023-09-30T09:28:13.032 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:13.061 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:13.267 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:13.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:13.372 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:13.403 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:13.451 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:13.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:13.472 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:13.556 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:13.595 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:13 smithi167 ceph-mon[192880]: osdmap e207: 8 total, 8 up, 8 in 2023-09-30T09:28:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:13 smithi167 ceph-mon[192880]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:13 smithi167 ceph-mon[192880]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:13 smithi167 ceph-mon[192880]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:13 smithi167 ceph-mon[192880]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-30T09:28:13.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[216815]: osdmap e207: 8 total, 8 up, 8 in 2023-09-30T09:28:13.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[216815]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[216815]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[216815]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[216815]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-30T09:28:13.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[224043]: osdmap e207: 8 total, 8 up, 8 in 2023-09-30T09:28:13.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[224043]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[224043]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[224043]: failed to encode map e207 with expected crc 2023-09-30T09:28:13.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:13 smithi019 ceph-mon[224043]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-30T09:28:13.955 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:13.995 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:13.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:14.019 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:28:14.020 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:28:14.020 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:28:14.020 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:28:14.020 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:28:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:28:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:28:13] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2023-09-30T09:28:14.099 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:14.132 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:14.569 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:28:15.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:14 smithi167 ceph-mon[192880]: pgmap v485: 99 pgs: 6 creating+peering, 10 unknown, 6 peering, 77 active+clean; 4.5 MiB data, 1007 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-30T09:28:15.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:14 smithi167 ceph-mon[192880]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:28:15.688 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:14 smithi167 ceph-mon[192880]: osdmap e208: 8 total, 8 up, 8 in 2023-09-30T09:28:15.688 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3738484850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:28:15.688 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:14 smithi167 ceph-mon[192880]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.688 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[216815]: pgmap v485: 99 pgs: 6 creating+peering, 10 unknown, 6 peering, 77 active+clean; 4.5 MiB data, 1007 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-30T09:28:15.688 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[216815]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:28:15.688 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[216815]: osdmap e208: 8 total, 8 up, 8 in 2023-09-30T09:28:15.688 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3738484850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:28:15.688 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[216815]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[224043]: pgmap v485: 99 pgs: 6 creating+peering, 10 unknown, 6 peering, 77 active+clean; 4.5 MiB data, 1007 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-30T09:28:15.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[224043]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:28:15.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[224043]: osdmap e208: 8 total, 8 up, 8 in 2023-09-30T09:28:15.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3738484850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:28:15.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:14 smithi019 ceph-mon[224043]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[224043]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[224043]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[224043]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.690 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:15.695 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3738484850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:28:15.695 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[224043]: failed to encode map e209 with expected crc 2023-09-30T09:28:15.695 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[224043]: osdmap e209: 8 total, 8 up, 8 in 2023-09-30T09:28:15.765 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:28:15.824 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[216815]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[216815]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[216815]: failed to encode map e208 with expected crc 2023-09-30T09:28:15.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:15.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3738484850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:28:15.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[216815]: failed to encode map e209 with expected crc 2023-09-30T09:28:15.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:15 smithi019 ceph-mon[216815]: osdmap e209: 8 total, 8 up, 8 in 2023-09-30T09:28:16.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:15 smithi167 ceph-mon[192880]: failed to encode map e208 with expected crc 2023-09-30T09:28:16.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:15 smithi167 ceph-mon[192880]: failed to encode map e208 with expected crc 2023-09-30T09:28:16.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:15 smithi167 ceph-mon[192880]: failed to encode map e208 with expected crc 2023-09-30T09:28:16.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:15 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:16.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3738484850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:28:16.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:15 smithi167 ceph-mon[192880]: failed to encode map e209 with expected crc 2023-09-30T09:28:16.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:15 smithi167 ceph-mon[192880]: osdmap e209: 8 total, 8 up, 8 in 2023-09-30T09:28:16.374 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:16.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:16.479 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:16.507 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:16.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:16 smithi019 ceph-mon[216815]: pgmap v488: 99 pgs: 6 creating+peering, 93 active+clean; 4.5 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2023-09-30T09:28:16.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:16 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-30T09:28:16.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:16 smithi019 ceph-mon[216815]: failed to encode map e209 with expected crc 2023-09-30T09:28:16.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:16 smithi019 ceph-mon[216815]: failed to encode map e209 with expected crc 2023-09-30T09:28:16.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:16 smithi019 ceph-mon[224043]: pgmap v488: 99 pgs: 6 creating+peering, 93 active+clean; 4.5 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2023-09-30T09:28:16.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:16 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-30T09:28:16.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:16 smithi019 ceph-mon[224043]: failed to encode map e209 with expected crc 2023-09-30T09:28:16.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:16 smithi019 ceph-mon[224043]: failed to encode map e209 with expected crc 2023-09-30T09:28:16.966 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:16.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:16 smithi167 ceph-mon[192880]: pgmap v488: 99 pgs: 6 creating+peering, 93 active+clean; 4.5 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2023-09-30T09:28:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:16 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-30T09:28:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:16 smithi167 ceph-mon[192880]: failed to encode map e209 with expected crc 2023-09-30T09:28:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:16 smithi167 ceph-mon[192880]: failed to encode map e209 with expected crc 2023-09-30T09:28:17.070 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:17.101 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:17.210 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:17.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:17.315 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:17.345 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:17.495 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:17.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:17.600 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:17.620 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:17.628 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:17 smithi019 ceph-mon[216815]: failed to encode map e209 with expected crc 2023-09-30T09:28:17.628 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:17 smithi019 ceph-mon[224043]: failed to encode map e209 with expected crc 2023-09-30T09:28:17.632 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:17.687 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:17.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:17.728 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:28:17.748 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:28:17.768 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:28:17.788 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:28:17.791 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:17.807 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:28:17.822 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:17 smithi167 ceph-mon[192880]: failed to encode map e209 with expected crc 2023-09-30T09:28:17.826 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:17.826 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:28:17.846 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:28:17.867 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:28:17.869 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:28:17.869 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:28:17.869 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:28:17.869 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:28:17.869 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:28:17.869 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:28:17.869 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:28:17.869 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:28:17.877 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:17.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:17.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:17.983 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:18.028 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:18.077 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:18.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:18.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:18.142 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:28:18.181 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:18.214 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:18.579 INFO:teuthology.orchestra.run.smithi019.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-30T09:28:18.664 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:18.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:18.769 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:18.796 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[216815]: pgmap v489: 99 pgs: 4 creating+peering, 95 active+clean; 4.5 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-30T09:28:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3005084984' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:28:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:28:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1834642725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:18.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[224043]: pgmap v489: 99 pgs: 4 creating+peering, 95 active+clean; 4.5 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-30T09:28:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3005084984' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:28:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:28:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1834642725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:18.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:18 smithi167 ceph-mon[192880]: pgmap v489: 99 pgs: 4 creating+peering, 95 active+clean; 4.5 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-30T09:28:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3005084984' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:28:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:18 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:28:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1834642725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:19.345 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:19.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:19.450 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:19.482 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:19.529 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:19.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:19.638 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:19.666 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:19 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:28:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:19 smithi167 ceph-mon[192880]: failed to encode map e210 with expected crc 2023-09-30T09:28:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:19 smithi167 ceph-mon[192880]: osdmap e210: 8 total, 8 up, 8 in 2023-09-30T09:28:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:19 smithi167 ceph-mon[192880]: failed to encode map e210 with expected crc 2023-09-30T09:28:19.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:19 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:28:19.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:19 smithi019 ceph-mon[216815]: failed to encode map e210 with expected crc 2023-09-30T09:28:19.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:19 smithi019 ceph-mon[216815]: osdmap e210: 8 total, 8 up, 8 in 2023-09-30T09:28:19.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:19 smithi019 ceph-mon[216815]: failed to encode map e210 with expected crc 2023-09-30T09:28:19.830 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:19 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:28:19.830 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:19 smithi019 ceph-mon[224043]: failed to encode map e210 with expected crc 2023-09-30T09:28:19.830 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:19 smithi019 ceph-mon[224043]: osdmap e210: 8 total, 8 up, 8 in 2023-09-30T09:28:19.830 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:19 smithi019 ceph-mon[224043]: failed to encode map e210 with expected crc 2023-09-30T09:28:19.891 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:19.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:19.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:19.955 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:19.998 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:20.032 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:20.084 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:20.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:20.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:20.188 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:20.219 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:20.267 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:20.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:20.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:20.370 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:20.402 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:20.503 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:20.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:20.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:20.534 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:28:20.534 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:28:20.607 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:20.639 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:20.687 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:20.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:20.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:20.737 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:28:20.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:20 smithi019 ceph-mon[216815]: pgmap v491: 99 pgs: 99 active+clean; 4.5 MiB data, 976 MiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:20.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:20 smithi019 ceph-mon[216815]: failed to encode map e210 with expected crc 2023-09-30T09:28:20.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:20 smithi019 ceph-mon[216815]: failed to encode map e210 with expected crc 2023-09-30T09:28:20.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:20 smithi019 ceph-mon[216815]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:28:20.764 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:20 smithi019 ceph-mon[224043]: pgmap v491: 99 pgs: 99 active+clean; 4.5 MiB data, 976 MiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:20.764 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:20 smithi019 ceph-mon[224043]: failed to encode map e210 with expected crc 2023-09-30T09:28:20.764 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:20 smithi019 ceph-mon[224043]: failed to encode map e210 with expected crc 2023-09-30T09:28:20.764 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:20 smithi019 ceph-mon[224043]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:28:20.793 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:20.821 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:20.846 DEBUG:teuthology.orchestra.run.smithi019:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-30T09:28:20.847 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-09-30T09:28:20.847 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:28:20.849 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-30T09:28:20.849 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-30T09:28:20.867 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:20.871 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:20 smithi167 ceph-mon[192880]: pgmap v491: 99 pgs: 99 active+clean; 4.5 MiB data, 976 MiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:20.872 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:20 smithi167 ceph-mon[192880]: failed to encode map e210 with expected crc 2023-09-30T09:28:20.872 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:20 smithi167 ceph-mon[192880]: failed to encode map e210 with expected crc 2023-09-30T09:28:20.872 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:20 smithi167 ceph-mon[192880]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:28:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:20.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:20.900 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:hints = 1 2023-09-30T09:28:20.900 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-30T09:28:20.900 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object prefix: benchmark_data_smithi019_355417 2023-09-30T09:28:20.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:28:20.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 0 0 0 0 0 0 - 0 2023-09-30T09:28:20.973 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:21.034 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:21.610 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:21.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:21.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:21.714 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:21.739 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:21.781 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:21.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:21.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:21.887 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:21.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 1 16 2462 2446 152.861 152.875 0.00213663 0.00530505 2023-09-30T09:28:21.913 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:21.955 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:21.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:22.060 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:22.116 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:22.149 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:28:22.313 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:22.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:22.419 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:22.478 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:22.510 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:22.543 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:22.642 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:22.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:22.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:22.751 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:22.789 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:22.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:22 smithi019 ceph-mon[216815]: pgmap v492: 99 pgs: 99 active+clean; 4.5 MiB data, 976 MiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:22.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:22 smithi019 ceph-mon[224043]: pgmap v492: 99 pgs: 99 active+clean; 4.5 MiB data, 976 MiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:22.861 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:22.865 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:22 smithi167 ceph-mon[192880]: pgmap v492: 99 pgs: 99 active+clean; 4.5 MiB data, 976 MiB used, 714 GiB / 715 GiB avail 2023-09-30T09:28:22.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:22.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:22.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 2 15 4856 4841 151.268 149.688 0.00324392 0.00659722 2023-09-30T09:28:22.966 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:23.038 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:23.431 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:23.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:23.537 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:23.558 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:23.610 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:23.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 3 16 7412 7396 154.069 159.688 0.00343456 0.0064825 2023-09-30T09:28:24.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:28:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:28:23] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.43.0" 2023-09-30T09:28:24.248 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:28:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:24 smithi167 ceph-mon[192880]: pgmap v493: 99 pgs: 99 active+clean; 73 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s wr, 127 op/s 2023-09-30T09:28:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1225167069' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:28:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:24 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:28:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:24 smithi019 ceph-mon[216815]: pgmap v493: 99 pgs: 99 active+clean; 73 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s wr, 127 op/s 2023-09-30T09:28:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1225167069' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:28:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:24 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:28:24.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:24 smithi019 ceph-mon[224043]: pgmap v493: 99 pgs: 99 active+clean; 73 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s wr, 127 op/s 2023-09-30T09:28:24.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1225167069' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:28:24.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:24 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:28:24.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 4 16 10474 10458 163.39 191.375 0.00195077 0.00610369 2023-09-30T09:28:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: osdmap e211: 8 total, 8 up, 8 in 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: 1.3 scrub starts 2023-09-30T09:28:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:25 smithi167 ceph-mon[192880]: 1.3 scrub ok 2023-09-30T09:28:25.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:25.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:28:25.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: osdmap e211: 8 total, 8 up, 8 in 2023-09-30T09:28:25.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: 1.3 scrub starts 2023-09-30T09:28:25.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[216815]: 1.3 scrub ok 2023-09-30T09:28:25.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:25.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:28:25.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: osdmap e211: 8 total, 8 up, 8 in 2023-09-30T09:28:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: failed to encode map e211 with expected crc 2023-09-30T09:28:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: 1.3 scrub starts 2023-09-30T09:28:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:25 smithi019 ceph-mon[224043]: 1.3 scrub ok 2023-09-30T09:28:25.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 5 16 12881 12865 160.796 150.438 0.00428907 0.0062114 2023-09-30T09:28:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:26 smithi167 ceph-mon[192880]: 1.9 scrub starts 2023-09-30T09:28:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:26 smithi167 ceph-mon[192880]: 1.9 scrub ok 2023-09-30T09:28:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:26 smithi167 ceph-mon[192880]: pgmap v495: 99 pgs: 99 active+clean; 390 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s wr, 771 op/s 2023-09-30T09:28:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:26 smithi019 ceph-mon[224043]: 1.9 scrub starts 2023-09-30T09:28:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:26 smithi019 ceph-mon[224043]: 1.9 scrub ok 2023-09-30T09:28:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:26 smithi019 ceph-mon[224043]: pgmap v495: 99 pgs: 99 active+clean; 390 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s wr, 771 op/s 2023-09-30T09:28:26.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:26 smithi019 ceph-mon[216815]: 1.9 scrub starts 2023-09-30T09:28:26.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:26 smithi019 ceph-mon[216815]: 1.9 scrub ok 2023-09-30T09:28:26.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:26 smithi019 ceph-mon[216815]: pgmap v495: 99 pgs: 99 active+clean; 390 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s wr, 771 op/s 2023-09-30T09:28:26.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 6 16 14779 14763 153.766 118.625 0.00420606 0.0064978 2023-09-30T09:28:27.290 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:27.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:27.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:27.396 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:27.481 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:27.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:27 smithi019 ceph-mon[216815]: 1.2 scrub starts 2023-09-30T09:28:27.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:27 smithi019 ceph-mon[216815]: 1.2 scrub ok 2023-09-30T09:28:27.504 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:27 smithi019 ceph-mon[224043]: 1.2 scrub starts 2023-09-30T09:28:27.504 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:27 smithi019 ceph-mon[224043]: 1.2 scrub ok 2023-09-30T09:28:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:27 smithi167 ceph-mon[192880]: 1.2 scrub starts 2023-09-30T09:28:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:27 smithi167 ceph-mon[192880]: 1.2 scrub ok 2023-09-30T09:28:27.613 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:27.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:27.719 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:27.761 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:27.826 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:27.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:27.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:27.902 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 7 16 17100 17084 152.511 145.062 0.00649988 0.00654943 2023-09-30T09:28:27.933 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:27.975 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:28.046 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:28.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:28.155 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:28.200 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:28.265 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:28.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:28.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:28.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[216815]: 1.4 scrub starts 2023-09-30T09:28:28.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[216815]: 1.4 scrub ok 2023-09-30T09:28:28.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[216815]: 1.8 scrub starts 2023-09-30T09:28:28.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[216815]: 1.8 scrub ok 2023-09-30T09:28:28.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[216815]: pgmap v496: 99 pgs: 99 active+clean; 486 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-09-30T09:28:28.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[216815]: 1.1 scrub starts 2023-09-30T09:28:28.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[216815]: 1.1 scrub ok 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[224043]: 1.4 scrub starts 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[224043]: 1.4 scrub ok 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[224043]: 1.8 scrub starts 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[224043]: 1.8 scrub ok 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[224043]: pgmap v496: 99 pgs: 99 active+clean; 486 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[224043]: 1.1 scrub starts 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[224043]: 1.1 scrub ok 2023-09-30T09:28:28.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:28 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:28.371 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:28.436 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:28 smithi167 ceph-mon[192880]: 1.4 scrub starts 2023-09-30T09:28:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:28 smithi167 ceph-mon[192880]: 1.4 scrub ok 2023-09-30T09:28:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:28 smithi167 ceph-mon[192880]: 1.8 scrub starts 2023-09-30T09:28:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:28 smithi167 ceph-mon[192880]: 1.8 scrub ok 2023-09-30T09:28:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:28 smithi167 ceph-mon[192880]: pgmap v496: 99 pgs: 99 active+clean; 486 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-09-30T09:28:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:28 smithi167 ceph-mon[192880]: 1.1 scrub starts 2023-09-30T09:28:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:28 smithi167 ceph-mon[192880]: 1.1 scrub ok 2023-09-30T09:28:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:28 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:28.903 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 8 16 19549 19533 152.557 153.062 0.0051499 0.00654788 2023-09-30T09:28:29.335 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:28:29.445 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:29.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:29 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:28:29.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:29 smithi019 ceph-mon[224043]: 1.a scrub ok 2023-09-30T09:28:29.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:29 smithi019 ceph-mon[224043]: 1.6 scrub starts 2023-09-30T09:28:29.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:29 smithi019 ceph-mon[224043]: 1.6 scrub ok 2023-09-30T09:28:29.455 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:29 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:28:29.455 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:29 smithi019 ceph-mon[216815]: 1.a scrub ok 2023-09-30T09:28:29.455 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:29 smithi019 ceph-mon[216815]: 1.6 scrub starts 2023-09-30T09:28:29.455 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:29 smithi019 ceph-mon[216815]: 1.6 scrub ok 2023-09-30T09:28:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:29.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:29.557 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:29.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:29 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:28:29.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:29 smithi167 ceph-mon[192880]: 1.a scrub ok 2023-09-30T09:28:29.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:29 smithi167 ceph-mon[192880]: 1.6 scrub starts 2023-09-30T09:28:29.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:29 smithi167 ceph-mon[192880]: 1.6 scrub ok 2023-09-30T09:28:29.591 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:29.646 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:29.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:29.750 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:29.785 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:29.843 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:29.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:29.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:29.907 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 9 15 21546 21531 149.415 124.875 0.00661364 0.00668508 2023-09-30T09:28:29.948 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:29.981 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:30.001 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:28:30.044 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:30.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:30.049 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:28:30.069 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:28:30.087 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:28:30.098 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:28:30.149 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:28:30.150 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:30.190 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:28:30.233 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:30.238 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:28:30.282 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:28:30.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:30 smithi019 ceph-mon[224043]: pgmap v497: 99 pgs: 99 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-30T09:28:30.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2414688659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:30.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:30 smithi019 ceph-mon[216815]: pgmap v497: 99 pgs: 99 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-30T09:28:30.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2414688659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:30.338 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:28:30.350 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:28:30.350 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:28:30.350 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:28:30.350 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:28:30.350 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:28:30.350 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:28:30.350 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:28:30.350 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35130.0:22409 6.1 e211/206 6:8f20f64d:::benchmark_data_smithi019_355417_object22408:head v 211'1480, mlcod=211'1478)", 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:30.422220+0000", 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.00086504500000000005, 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00089601999999999998, 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:28:30.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422220+0000", 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422220+0000", 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:28:30.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422221+0000", 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1900000000000001e-07 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422248+0000", 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7750999999999999e-05 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422266+0000", 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7365999999999999e-05 2023-09-30T09:28:30.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422274+0000", 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1489999999999994e-06 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422339+0000", 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4710999999999998e-05 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:28:30.428 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422362+0000", 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3649e-05 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35130.0:22413 6.1 e211/206)", 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:30.422296+0000", 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.00078864600000000005, 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00088229400000000005, 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:28:30.429 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "queued for pg", 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422296+0000", 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422296+0000", 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.430 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422296+0000", 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8999999999999998e-07 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422330+0000", 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4041999999999997e-05 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422338+0000", 2023-09-30T09:28:30.431 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3370000000000002e-06 2023-09-30T09:28:30.432 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.432 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.432 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422343+0000", 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.451e-06 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35130.0:22419 6.1 e211/206)", 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:30.422352+0000", 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.00073278299999999998, 2023-09-30T09:28:30.433 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000879289, 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "queued for pg", 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422352+0000", 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422352+0000", 2023-09-30T09:28:30.434 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422352+0000", 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6800000000000002e-07 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422382+0000", 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9788e-05 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.436 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:28:30.730 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422389+0000", 2023-09-30T09:28:30.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6359999999999997e-06 2023-09-30T09:28:30.730 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.730 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.730 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:28:30.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422393+0000", 2023-09-30T09:28:30.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7849999999999999e-06 2023-09-30T09:28:30.734 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.734 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:28:30.734 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35130.0:22420 6.1 e211/206)", 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:30.422400+0000", 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.00068439900000000003, 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00087638199999999999, 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "queued for pg", 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422400+0000", 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422400+0000", 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422400+0000", 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1500000000000001e-07 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422422+0000", 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1152999999999999e-05 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422427+0000", 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6540000000000004e-06 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422432+0000", 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0300000000000001e-06 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35130.0:22418 6.2 e211/206 6:450ab0aa:::benchmark_data_smithi019_355417_object22417:head v 211'1369, mlcod=211'1366)", 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:30.422455+0000", 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.00062930899999999999, 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00087852500000000001, 2023-09-30T09:28:30.738 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422455+0000", 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422455+0000", 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422456+0000", 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3000000000000002e-07 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:28:30.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422478+0000", 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2654999999999999e-05 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422485+0000", 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0190000000000001e-06 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422490+0000", 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7759999999999997e-06 2023-09-30T09:28:30.741 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422546+0000", 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5714000000000002e-05 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:30.422565+0000", 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9457e-05 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.742 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:28:30.743 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.743 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:28:30.743 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2023-09-30T09:28:30.743 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 5 2023-09-30T09:28:30.743 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:28:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:30 smithi167 ceph-mon[192880]: pgmap v497: 99 pgs: 99 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-30T09:28:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2414688659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:30.909 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 10 16 23463 23447 146.427 119.75 0.00775122 0.00682314 2023-09-30T09:28:31.375 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:31.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:31.481 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:31.533 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:31 smithi019 ceph-mon[224043]: from='client.35142 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:28:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:31 smithi019 ceph-mon[224043]: 1.7 scrub starts 2023-09-30T09:28:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:31 smithi019 ceph-mon[224043]: 1.7 scrub ok 2023-09-30T09:28:31.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:31 smithi019 ceph-mon[216815]: from='client.35142 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:28:31.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:31 smithi019 ceph-mon[216815]: 1.7 scrub starts 2023-09-30T09:28:31.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:31 smithi019 ceph-mon[216815]: 1.7 scrub ok 2023-09-30T09:28:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:31 smithi167 ceph-mon[192880]: from='client.35142 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:28:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:31 smithi167 ceph-mon[192880]: 1.7 scrub starts 2023-09-30T09:28:31.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:31 smithi167 ceph-mon[192880]: 1.7 scrub ok 2023-09-30T09:28:31.909 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 11 16 25474 25458 144.542 125.688 0.00184218 0.0068969 2023-09-30T09:28:32.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:32 smithi019 ceph-mon[216815]: pgmap v498: 99 pgs: 99 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.80k op/s 2023-09-30T09:28:32.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:32 smithi019 ceph-mon[224043]: pgmap v498: 99 pgs: 99 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.80k op/s 2023-09-30T09:28:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:32 smithi167 ceph-mon[192880]: pgmap v498: 99 pgs: 99 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.80k op/s 2023-09-30T09:28:32.778 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":497,"stamp":"2023-09-30T09:28:29.110022+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1105091614,"num_objects":16817,"num_object_clones":0,"num_object_copies":50451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16817,"num_whiteouts":0,"num_read":6122,"num_read_kb":12481,"num_write":17784,"num_write_kb":1088163,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":140,"num_bytes_recovered":9618736,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":19592,"ondisk_log_size":19592,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3963020,"kb_used_data":2997128,"kb_used_omap":19,"kb_used_meta":965804,"kb_avail":745998196,"statfs":{"total":767960285184,"available":763902152704,"internally_reserved":0,"allocated":3069059072,"data_stored":3061221191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20057,"internal_metadata":988983719},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1100355100,"num_objects":16789,"num_object_clones":0,"num_object_copies":50367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16789,"num_whiteouts":0,"num_read":-273,"num_read_kb":-503,"num_write":16659,"num_write_kb":1073241,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-7,"num_bytes_recovered":-1937775,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002737"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":188,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574860+0000","last_change":"2023-09-30T09:20:56.405763+0000","last_active":"2023-09-30T09:28:18.574860+0000","last_peered":"2023-09-30T09:28:18.574860+0000","last_clean":"2023-09-30T09:28:18.574860+0000","last_became_active":"2023-09-30T09:20:56.405410+0000","last_became_peered":"2023-09-30T09:20:56.405410+0000","last_unstale":"2023-09-30T09:28:18.574860+0000","last_undegraded":"2023-09-30T09:28:18.574860+0000","last_fullsized":"2023-09-30T09:28:18.574860+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:03.613509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":175,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.572673+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:28:18.572673+0000","last_peered":"2023-09-30T09:28:18.572673+0000","last_clean":"2023-09-30T09:28:18.572673+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:28:18.572673+0000","last_undegraded":"2023-09-30T09:28:18.572673+0000","last_fullsized":"2023-09-30T09:28:18.572673+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":173,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216466+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:28:19.216466+0000","last_peered":"2023-09-30T09:28:19.216466+0000","last_clean":"2023-09-30T09:28:19.216466+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:28:19.216466+0000","last_undegraded":"2023-09-30T09:28:19.216466+0000","last_fullsized":"2023-09-30T09:28:19.216466+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":103,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.575003+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:28:18.575003+0000","last_peered":"2023-09-30T09:28:18.575003+0000","last_clean":"2023-09-30T09:28:18.575003+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:28:18.575003+0000","last_undegraded":"2023-09-30T09:28:18.575003+0000","last_fullsized":"2023-09-30T09:28:18.575003+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":217,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.572531+0000","last_change":"2023-09-30T09:26:51.318844+0000","last_active":"2023-09-30T09:28:18.572531+0000","last_peered":"2023-09-30T09:28:18.572531+0000","last_clean":"2023-09-30T09:28:18.572531+0000","last_became_active":"2023-09-30T09:26:51.316788+0000","last_became_peered":"2023-09-30T09:26:51.316788+0000","last_unstale":"2023-09-30T09:28:18.572531+0000","last_undegraded":"2023-09-30T09:28:18.572531+0000","last_fullsized":"2023-09-30T09:28:18.572531+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:20:35.308469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":170,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219793+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:28:19.219793+0000","last_peered":"2023-09-30T09:28:19.219793+0000","last_clean":"2023-09-30T09:28:19.219793+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:28:19.219793+0000","last_undegraded":"2023-09-30T09:28:19.219793+0000","last_fullsized":"2023-09-30T09:28:19.219793+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":103,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574760+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:28:18.574760+0000","last_peered":"2023-09-30T09:28:18.574760+0000","last_clean":"2023-09-30T09:28:18.574760+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:28:18.574760+0000","last_undegraded":"2023-09-30T09:28:18.574760+0000","last_fullsized":"2023-09-30T09:28:18.574760+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":176,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:24.237840+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:28:24.237840+0000","last_peered":"2023-09-30T09:28:24.237840+0000","last_clean":"2023-09-30T09:28:24.237840+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:28:24.237840+0000","last_undegraded":"2023-09-30T09:28:24.237840+0000","last_fullsized":"2023-09-30T09:28:24.237840+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":62,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.218092+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:28:19.218092+0000","last_peered":"2023-09-30T09:28:19.218092+0000","last_clean":"2023-09-30T09:28:19.218092+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:28:19.218092+0000","last_undegraded":"2023-09-30T09:28:19.218092+0000","last_fullsized":"2023-09-30T09:28:19.218092+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":127,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:24.236603+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:28:24.236603+0000","last_peered":"2023-09-30T09:28:24.236603+0000","last_clean":"2023-09-30T09:28:24.236603+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:28:24.236603+0000","last_undegraded":"2023-09-30T09:28:24.236603+0000","last_fullsized":"2023-09-30T09:28:24.236603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":174,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.220322+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:28:19.220322+0000","last_peered":"2023-09-30T09:28:19.220322+0000","last_clean":"2023-09-30T09:28:19.220322+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:28:19.220322+0000","last_undegraded":"2023-09-30T09:28:19.220322+0000","last_fullsized":"2023-09-30T09:28:19.220322+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":150,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216596+0000","last_change":"2023-09-30T09:20:56.401030+0000","last_active":"2023-09-30T09:28:19.216596+0000","last_peered":"2023-09-30T09:28:19.216596+0000","last_clean":"2023-09-30T09:28:19.216596+0000","last_became_active":"2023-09-30T09:20:56.400701+0000","last_became_peered":"2023-09-30T09:20:56.400701+0000","last_unstale":"2023-09-30T09:28:19.216596+0000","last_undegraded":"2023-09-30T09:28:19.216596+0000","last_fullsized":"2023-09-30T09:28:19.216596+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:58.703511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":175,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.575151+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:28:18.575151+0000","last_peered":"2023-09-30T09:28:18.575151+0000","last_clean":"2023-09-30T09:28:18.575151+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:28:18.575151+0000","last_undegraded":"2023-09-30T09:28:18.575151+0000","last_fullsized":"2023-09-30T09:28:18.575151+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":179,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.575012+0000","last_change":"2023-09-30T09:23:03.986755+0000","last_active":"2023-09-30T09:28:18.575012+0000","last_peered":"2023-09-30T09:28:18.575012+0000","last_clean":"2023-09-30T09:28:18.575012+0000","last_became_active":"2023-09-30T09:23:03.986411+0000","last_became_peered":"2023-09-30T09:23:03.986411+0000","last_unstale":"2023-09-30T09:28:18.575012+0000","last_undegraded":"2023-09-30T09:28:18.575012+0000","last_fullsized":"2023-09-30T09:28:18.575012+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:53:23.627823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":173,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219489+0000","last_change":"2023-09-30T09:20:56.404351+0000","last_active":"2023-09-30T09:28:19.219489+0000","last_peered":"2023-09-30T09:28:19.219489+0000","last_clean":"2023-09-30T09:28:19.219489+0000","last_became_active":"2023-09-30T09:20:56.403911+0000","last_became_peered":"2023-09-30T09:20:56.403911+0000","last_unstale":"2023-09-30T09:28:19.219489+0000","last_undegraded":"2023-09-30T09:28:19.219489+0000","last_fullsized":"2023-09-30T09:28:19.219489+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:06.235879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":190,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:24.241808+0000","last_change":"2023-09-30T09:21:29.092287+0000","last_active":"2023-09-30T09:28:24.241808+0000","last_peered":"2023-09-30T09:28:24.241808+0000","last_clean":"2023-09-30T09:28:24.241808+0000","last_became_active":"2023-09-30T09:21:29.091959+0000","last_became_peered":"2023-09-30T09:21:29.091959+0000","last_unstale":"2023-09-30T09:28:24.241808+0000","last_undegraded":"2023-09-30T09:28:24.241808+0000","last_fullsized":"2023-09-30T09:28:24.241808+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:00.016133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":103,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574729+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:28:18.574729+0000","last_peered":"2023-09-30T09:28:18.574729+0000","last_clean":"2023-09-30T09:28:18.574729+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:28:18.574729+0000","last_undegraded":"2023-09-30T09:28:18.574729+0000","last_fullsized":"2023-09-30T09:28:18.574729+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":132,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.220398+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:28:19.220398+0000","last_peered":"2023-09-30T09:28:19.220398+0000","last_clean":"2023-09-30T09:28:19.220398+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:28:19.220398+0000","last_undegraded":"2023-09-30T09:28:19.220398+0000","last_fullsized":"2023-09-30T09:28:19.220398+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219620+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:28:19.219620+0000","last_peered":"2023-09-30T09:28:19.219620+0000","last_clean":"2023-09-30T09:28:19.219620+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:28:19.219620+0000","last_undegraded":"2023-09-30T09:28:19.219620+0000","last_fullsized":"2023-09-30T09:28:19.219620+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":103,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574938+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:28:18.574938+0000","last_peered":"2023-09-30T09:28:18.574938+0000","last_clean":"2023-09-30T09:28:18.574938+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:28:18.574938+0000","last_undegraded":"2023-09-30T09:28:18.574938+0000","last_fullsized":"2023-09-30T09:28:18.574938+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":283,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.218658+0000","last_change":"2023-09-30T09:26:51.339701+0000","last_active":"2023-09-30T09:28:19.218658+0000","last_peered":"2023-09-30T09:28:19.218658+0000","last_clean":"2023-09-30T09:28:19.218658+0000","last_became_active":"2023-09-30T09:26:51.324149+0000","last_became_peered":"2023-09-30T09:26:51.324149+0000","last_unstale":"2023-09-30T09:28:19.218658+0000","last_undegraded":"2023-09-30T09:28:19.218658+0000","last_fullsized":"2023-09-30T09:28:19.218658+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:26.372798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":189,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.573494+0000","last_change":"2023-09-30T09:21:29.092140+0000","last_active":"2023-09-30T09:28:18.573494+0000","last_peered":"2023-09-30T09:28:18.573494+0000","last_clean":"2023-09-30T09:28:18.573494+0000","last_became_active":"2023-09-30T09:21:29.091808+0000","last_became_peered":"2023-09-30T09:21:29.091808+0000","last_unstale":"2023-09-30T09:28:18.573494+0000","last_undegraded":"2023-09-30T09:28:18.573494+0000","last_fullsized":"2023-09-30T09:28:18.573494+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:27.653462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"211'1305","reported_seq":1318,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:28.982708+0000","last_change":"2023-09-30T09:28:12.718364+0000","last_active":"2023-09-30T09:28:28.982708+0000","last_peered":"2023-09-30T09:28:28.982708+0000","last_clean":"2023-09-30T09:28:28.982708+0000","last_became_active":"2023-09-30T09:28:12.717931+0000","last_became_peered":"2023-09-30T09:28:12.717931+0000","last_unstale":"2023-09-30T09:28:28.982708+0000","last_undegraded":"2023-09-30T09:28:28.982708+0000","last_fullsized":"2023-09-30T09:28:28.982708+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1305,"log_dups_size":0,"ondisk_log_size":1305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:12.025351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85524480,"num_objects":1305,"num_object_clones":0,"num_object_copies":3915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1305,"num_write_kb":83520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":214,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219790+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:28:19.219790+0000","last_peered":"2023-09-30T09:28:19.219790+0000","last_clean":"2023-09-30T09:28:19.219790+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:28:19.219790+0000","last_undegraded":"2023-09-30T09:28:19.219790+0000","last_fullsized":"2023-09-30T09:28:19.219790+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":74,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219116+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:28:19.219116+0000","last_peered":"2023-09-30T09:28:19.219116+0000","last_clean":"2023-09-30T09:28:19.219116+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:28:19.219116+0000","last_undegraded":"2023-09-30T09:28:19.219116+0000","last_fullsized":"2023-09-30T09:28:19.219116+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"211'936","reported_seq":952,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:27.200678+0000","last_change":"2023-09-30T09:28:12.563756+0000","last_active":"2023-09-30T09:28:27.200678+0000","last_peered":"2023-09-30T09:28:27.200678+0000","last_clean":"2023-09-30T09:28:27.200678+0000","last_became_active":"2023-09-30T09:28:12.563460+0000","last_became_peered":"2023-09-30T09:28:12.563460+0000","last_unstale":"2023-09-30T09:28:27.200678+0000","last_undegraded":"2023-09-30T09:28:27.200678+0000","last_fullsized":"2023-09-30T09:28:27.200678+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":936,"log_dups_size":0,"ondisk_log_size":936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:44.011638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61341696,"num_objects":936,"num_object_clones":0,"num_object_copies":2808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":936,"num_write_kb":59904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":215,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219910+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:28:19.219910+0000","last_peered":"2023-09-30T09:28:19.219910+0000","last_clean":"2023-09-30T09:28:19.219910+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:28:19.219910+0000","last_undegraded":"2023-09-30T09:28:19.219910+0000","last_fullsized":"2023-09-30T09:28:19.219910+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":186,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574909+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:28:18.574909+0000","last_peered":"2023-09-30T09:28:18.574909+0000","last_clean":"2023-09-30T09:28:18.574909+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:28:18.574909+0000","last_undegraded":"2023-09-30T09:28:18.574909+0000","last_fullsized":"2023-09-30T09:28:18.574909+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"211'1164","reported_seq":1177,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:29.009131+0000","last_change":"2023-09-30T09:28:12.563362+0000","last_active":"2023-09-30T09:28:29.009131+0000","last_peered":"2023-09-30T09:28:29.009131+0000","last_clean":"2023-09-30T09:28:29.009131+0000","last_became_active":"2023-09-30T09:28:12.563039+0000","last_became_peered":"2023-09-30T09:28:12.563039+0000","last_unstale":"2023-09-30T09:28:29.009131+0000","last_undegraded":"2023-09-30T09:28:29.009131+0000","last_fullsized":"2023-09-30T09:28:29.009131+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1164,"log_dups_size":0,"ondisk_log_size":1164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:45:08.601296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76283904,"num_objects":1164,"num_object_clones":0,"num_object_copies":3492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1164,"num_write_kb":74496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":197,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.220243+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:28:19.220243+0000","last_peered":"2023-09-30T09:28:19.220243+0000","last_clean":"2023-09-30T09:28:19.220243+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:28:19.220243+0000","last_undegraded":"2023-09-30T09:28:19.220243+0000","last_fullsized":"2023-09-30T09:28:19.220243+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":180,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.220055+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:28:19.220055+0000","last_peered":"2023-09-30T09:28:19.220055+0000","last_clean":"2023-09-30T09:28:19.220055+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:28:19.220055+0000","last_undegraded":"2023-09-30T09:28:19.220055+0000","last_fullsized":"2023-09-30T09:28:19.220055+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"211'1224","reported_seq":1237,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:29.009164+0000","last_change":"2023-09-30T09:28:12.563770+0000","last_active":"2023-09-30T09:28:29.009164+0000","last_peered":"2023-09-30T09:28:29.009164+0000","last_clean":"2023-09-30T09:28:29.009164+0000","last_became_active":"2023-09-30T09:28:12.563370+0000","last_became_peered":"2023-09-30T09:28:12.563370+0000","last_unstale":"2023-09-30T09:28:29.009164+0000","last_undegraded":"2023-09-30T09:28:29.009164+0000","last_fullsized":"2023-09-30T09:28:29.009164+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1224,"log_dups_size":0,"ondisk_log_size":1224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:12.893457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80216064,"num_objects":1224,"num_object_clones":0,"num_object_copies":3672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1224,"num_write_kb":78336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":170,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.217848+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:28:19.217848+0000","last_peered":"2023-09-30T09:28:19.217848+0000","last_clean":"2023-09-30T09:28:19.217848+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:28:19.217848+0000","last_undegraded":"2023-09-30T09:28:19.217848+0000","last_fullsized":"2023-09-30T09:28:19.217848+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":100,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574646+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:28:18.574646+0000","last_peered":"2023-09-30T09:28:18.574646+0000","last_clean":"2023-09-30T09:28:18.574646+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:28:18.574646+0000","last_undegraded":"2023-09-30T09:28:18.574646+0000","last_fullsized":"2023-09-30T09:28:18.574646+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"211'1199","reported_seq":1211,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:28.964351+0000","last_change":"2023-09-30T09:28:12.561527+0000","last_active":"2023-09-30T09:28:28.964351+0000","last_peered":"2023-09-30T09:28:28.964351+0000","last_clean":"2023-09-30T09:28:28.964351+0000","last_became_active":"2023-09-30T09:28:12.561191+0000","last_became_peered":"2023-09-30T09:28:12.561191+0000","last_unstale":"2023-09-30T09:28:28.964351+0000","last_undegraded":"2023-09-30T09:28:28.964351+0000","last_fullsized":"2023-09-30T09:28:28.964351+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1199,"log_dups_size":0,"ondisk_log_size":1199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:22.516207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78577664,"num_objects":1199,"num_object_clones":0,"num_object_copies":3597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1199,"num_write_kb":76736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":670,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:25.074419+0000","last_change":"2023-09-30T09:28:25.074370+0000","last_active":"2023-09-30T09:28:25.074419+0000","last_peered":"2023-09-30T09:28:25.074419+0000","last_clean":"2023-09-30T09:28:25.074419+0000","last_became_active":"2023-09-30T09:28:09.225545+0000","last_became_peered":"2023-09-30T09:28:09.225545+0000","last_unstale":"2023-09-30T09:28:25.074419+0000","last_undegraded":"2023-09-30T09:28:25.074419+0000","last_fullsized":"2023-09-30T09:28:25.074419+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:18.476747+0000","scrub_duration":0.0079898150000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":210,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.573212+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:28:18.573212+0000","last_peered":"2023-09-30T09:28:18.573212+0000","last_clean":"2023-09-30T09:28:18.573212+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:28:18.573212+0000","last_undegraded":"2023-09-30T09:28:18.573212+0000","last_fullsized":"2023-09-30T09:28:18.573212+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":169,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.575113+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:28:18.575113+0000","last_peered":"2023-09-30T09:28:18.575113+0000","last_clean":"2023-09-30T09:28:18.575113+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:28:18.575113+0000","last_undegraded":"2023-09-30T09:28:18.575113+0000","last_fullsized":"2023-09-30T09:28:18.575113+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"211'666","reported_seq":682,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:25.023407+0000","last_change":"2023-09-30T09:28:12.560255+0000","last_active":"2023-09-30T09:28:25.023407+0000","last_peered":"2023-09-30T09:28:25.023407+0000","last_clean":"2023-09-30T09:28:25.023407+0000","last_became_active":"2023-09-30T09:28:12.559959+0000","last_became_peered":"2023-09-30T09:28:12.559959+0000","last_unstale":"2023-09-30T09:28:25.023407+0000","last_undegraded":"2023-09-30T09:28:25.023407+0000","last_fullsized":"2023-09-30T09:28:25.023407+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":666,"log_dups_size":0,"ondisk_log_size":666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:25:12.708305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43646976,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":666,"num_write_kb":42624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":602,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.215615+0000","last_change":"2023-09-30T09:28:09.222290+0000","last_active":"2023-09-30T09:28:19.215615+0000","last_peered":"2023-09-30T09:28:19.215615+0000","last_clean":"2023-09-30T09:28:19.215615+0000","last_became_active":"2023-09-30T09:28:09.221790+0000","last_became_peered":"2023-09-30T09:28:09.221790+0000","last_unstale":"2023-09-30T09:28:19.215615+0000","last_undegraded":"2023-09-30T09:28:19.215615+0000","last_fullsized":"2023-09-30T09:28:19.215615+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:58.188734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:27:58.188734+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":531037,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":171,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219917+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:28:19.219917+0000","last_peered":"2023-09-30T09:28:19.219917+0000","last_clean":"2023-09-30T09:28:19.219917+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:28:19.219917+0000","last_undegraded":"2023-09-30T09:28:19.219917+0000","last_fullsized":"2023-09-30T09:28:19.219917+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":139,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216248+0000","last_change":"2023-09-30T09:26:51.322946+0000","last_active":"2023-09-30T09:28:19.216248+0000","last_peered":"2023-09-30T09:28:19.216248+0000","last_clean":"2023-09-30T09:28:19.216248+0000","last_became_active":"2023-09-30T09:26:51.322618+0000","last_became_peered":"2023-09-30T09:26:51.322618+0000","last_unstale":"2023-09-30T09:28:19.216248+0000","last_undegraded":"2023-09-30T09:28:19.216248+0000","last_fullsized":"2023-09-30T09:28:19.216248+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:23.279885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"211'973","reported_seq":989,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:27.201001+0000","last_change":"2023-09-30T09:28:12.718177+0000","last_active":"2023-09-30T09:28:27.201001+0000","last_peered":"2023-09-30T09:28:27.201001+0000","last_clean":"2023-09-30T09:28:27.201001+0000","last_became_active":"2023-09-30T09:28:12.717830+0000","last_became_peered":"2023-09-30T09:28:12.717830+0000","last_unstale":"2023-09-30T09:28:27.201001+0000","last_undegraded":"2023-09-30T09:28:27.201001+0000","last_fullsized":"2023-09-30T09:28:27.201001+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":973,"log_dups_size":0,"ondisk_log_size":973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:24:22.843748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63766528,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":973,"num_write_kb":62272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":170,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574865+0000","last_change":"2023-09-30T09:20:56.405731+0000","last_active":"2023-09-30T09:28:18.574865+0000","last_peered":"2023-09-30T09:28:18.574865+0000","last_clean":"2023-09-30T09:28:18.574865+0000","last_became_active":"2023-09-30T09:20:56.405354+0000","last_became_peered":"2023-09-30T09:20:56.405354+0000","last_unstale":"2023-09-30T09:28:18.574865+0000","last_undegraded":"2023-09-30T09:28:18.574865+0000","last_fullsized":"2023-09-30T09:28:18.574865+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:28.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":169,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.575411+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:28:18.575411+0000","last_peered":"2023-09-30T09:28:18.575411+0000","last_clean":"2023-09-30T09:28:18.575411+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:28:18.575411+0000","last_undegraded":"2023-09-30T09:28:18.575411+0000","last_fullsized":"2023-09-30T09:28:18.575411+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"211'1210","reported_seq":1223,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:28.974794+0000","last_change":"2023-09-30T09:28:12.561778+0000","last_active":"2023-09-30T09:28:28.974794+0000","last_peered":"2023-09-30T09:28:28.974794+0000","last_clean":"2023-09-30T09:28:28.974794+0000","last_became_active":"2023-09-30T09:28:12.561409+0000","last_became_peered":"2023-09-30T09:28:12.561409+0000","last_unstale":"2023-09-30T09:28:28.974794+0000","last_undegraded":"2023-09-30T09:28:28.974794+0000","last_fullsized":"2023-09-30T09:28:28.974794+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1210,"log_dups_size":0,"ondisk_log_size":1210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:56.660361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79298560,"num_objects":1210,"num_object_clones":0,"num_object_copies":3630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1210,"num_write_kb":77440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":602,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216889+0000","last_change":"2023-09-30T09:28:09.226575+0000","last_active":"2023-09-30T09:28:19.216889+0000","last_peered":"2023-09-30T09:28:19.216889+0000","last_clean":"2023-09-30T09:28:19.216889+0000","last_became_active":"2023-09-30T09:28:09.226163+0000","last_became_peered":"2023-09-30T09:28:09.226163+0000","last_unstale":"2023-09-30T09:28:19.216889+0000","last_undegraded":"2023-09-30T09:28:19.216889+0000","last_fullsized":"2023-09-30T09:28:19.216889+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":177,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.572356+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:28:18.572356+0000","last_peered":"2023-09-30T09:28:18.572356+0000","last_clean":"2023-09-30T09:28:18.572356+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:28:18.572356+0000","last_undegraded":"2023-09-30T09:28:18.572356+0000","last_fullsized":"2023-09-30T09:28:18.572356+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":150,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216850+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:28:19.216850+0000","last_peered":"2023-09-30T09:28:19.216850+0000","last_clean":"2023-09-30T09:28:19.216850+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:28:19.216850+0000","last_undegraded":"2023-09-30T09:28:19.216850+0000","last_fullsized":"2023-09-30T09:28:19.216850+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"211'604","reported_seq":620,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:24.458268+0000","last_change":"2023-09-30T09:28:12.553531+0000","last_active":"2023-09-30T09:28:24.458268+0000","last_peered":"2023-09-30T09:28:24.458268+0000","last_clean":"2023-09-30T09:28:24.458268+0000","last_became_active":"2023-09-30T09:28:12.553212+0000","last_became_peered":"2023-09-30T09:28:12.553212+0000","last_unstale":"2023-09-30T09:28:24.458268+0000","last_undegraded":"2023-09-30T09:28:24.458268+0000","last_fullsized":"2023-09-30T09:28:24.458268+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":604,"log_dups_size":0,"ondisk_log_size":604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:19:10.408778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39583744,"num_objects":604,"num_object_clones":0,"num_object_copies":1812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":604,"num_write_kb":38656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":445,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219708+0000","last_change":"2023-09-30T09:28:09.228614+0000","last_active":"2023-09-30T09:28:19.219708+0000","last_peered":"2023-09-30T09:28:19.219708+0000","last_clean":"2023-09-30T09:28:19.219708+0000","last_became_active":"2023-09-30T09:28:09.228266+0000","last_became_peered":"2023-09-30T09:28:09.228266+0000","last_unstale":"2023-09-30T09:28:19.219708+0000","last_undegraded":"2023-09-30T09:28:19.219708+0000","last_fullsized":"2023-09-30T09:28:19.219708+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:05.190268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:05.190268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":554,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.217098+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:28:19.217098+0000","last_peered":"2023-09-30T09:28:19.217098+0000","last_clean":"2023-09-30T09:28:19.217098+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:28:19.217098+0000","last_undegraded":"2023-09-30T09:28:19.217098+0000","last_fullsized":"2023-09-30T09:28:19.217098+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":187,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.217914+0000","last_change":"2023-09-30T09:21:29.092844+0000","last_active":"2023-09-30T09:28:19.217914+0000","last_peered":"2023-09-30T09:28:19.217914+0000","last_clean":"2023-09-30T09:28:19.217914+0000","last_became_active":"2023-09-30T09:21:29.092485+0000","last_became_peered":"2023-09-30T09:21:29.092485+0000","last_unstale":"2023-09-30T09:28:19.217914+0000","last_undegraded":"2023-09-30T09:28:19.217914+0000","last_fullsized":"2023-09-30T09:28:19.217914+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:46.903936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":169,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216605+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:28:19.216605+0000","last_peered":"2023-09-30T09:28:19.216605+0000","last_clean":"2023-09-30T09:28:19.216605+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:28:19.216605+0000","last_undegraded":"2023-09-30T09:28:19.216605+0000","last_fullsized":"2023-09-30T09:28:19.216605+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"211'1025","reported_seq":1041,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:27.200667+0000","last_change":"2023-09-30T09:28:12.563904+0000","last_active":"2023-09-30T09:28:27.200667+0000","last_peered":"2023-09-30T09:28:27.200667+0000","last_clean":"2023-09-30T09:28:27.200667+0000","last_became_active":"2023-09-30T09:28:12.563604+0000","last_became_peered":"2023-09-30T09:28:12.563604+0000","last_unstale":"2023-09-30T09:28:27.200667+0000","last_undegraded":"2023-09-30T09:28:27.200667+0000","last_fullsized":"2023-09-30T09:28:27.200667+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1025,"log_dups_size":0,"ondisk_log_size":1025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:14:35.904733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67174400,"num_objects":1025,"num_object_clones":0,"num_object_copies":3075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1025,"num_write_kb":65600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":613,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:27.323557+0000","last_change":"2023-09-30T09:28:27.323519+0000","last_active":"2023-09-30T09:28:27.323557+0000","last_peered":"2023-09-30T09:28:27.323557+0000","last_clean":"2023-09-30T09:28:27.323557+0000","last_became_active":"2023-09-30T09:28:09.219302+0000","last_became_peered":"2023-09-30T09:28:09.219302+0000","last_unstale":"2023-09-30T09:28:27.323557+0000","last_undegraded":"2023-09-30T09:28:27.323557+0000","last_fullsized":"2023-09-30T09:28:27.323557+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:27.441689+0000","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":358,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216743+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:28:19.216743+0000","last_peered":"2023-09-30T09:28:19.216743+0000","last_clean":"2023-09-30T09:28:19.216743+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:28:19.216743+0000","last_undegraded":"2023-09-30T09:28:19.216743+0000","last_fullsized":"2023-09-30T09:28:19.216743+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":190,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.575102+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:28:18.575102+0000","last_peered":"2023-09-30T09:28:18.575102+0000","last_clean":"2023-09-30T09:28:18.575102+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:28:18.575102+0000","last_undegraded":"2023-09-30T09:28:18.575102+0000","last_fullsized":"2023-09-30T09:28:18.575102+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":172,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.215921+0000","last_change":"2023-09-30T09:23:23.238851+0000","last_active":"2023-09-30T09:28:19.215921+0000","last_peered":"2023-09-30T09:28:19.215921+0000","last_clean":"2023-09-30T09:28:19.215921+0000","last_became_active":"2023-09-30T09:23:23.238582+0000","last_became_peered":"2023-09-30T09:23:23.238582+0000","last_unstale":"2023-09-30T09:28:19.215921+0000","last_undegraded":"2023-09-30T09:28:19.215921+0000","last_fullsized":"2023-09-30T09:28:19.215921+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:52.808966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"211'1251","reported_seq":1264,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:29.003888+0000","last_change":"2023-09-30T09:28:12.718838+0000","last_active":"2023-09-30T09:28:29.003888+0000","last_peered":"2023-09-30T09:28:29.003888+0000","last_clean":"2023-09-30T09:28:29.003888+0000","last_became_active":"2023-09-30T09:28:12.718534+0000","last_became_peered":"2023-09-30T09:28:12.718534+0000","last_unstale":"2023-09-30T09:28:29.003888+0000","last_undegraded":"2023-09-30T09:28:29.003888+0000","last_fullsized":"2023-09-30T09:28:29.003888+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1251,"log_dups_size":0,"ondisk_log_size":1251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:46:57.286309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81985536,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1251,"num_write_kb":80064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":670,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:26.036508+0000","last_change":"2023-09-30T09:28:26.036455+0000","last_active":"2023-09-30T09:28:26.036508+0000","last_peered":"2023-09-30T09:28:26.036508+0000","last_clean":"2023-09-30T09:28:26.036508+0000","last_became_active":"2023-09-30T09:28:09.226703+0000","last_became_peered":"2023-09-30T09:28:09.226703+0000","last_unstale":"2023-09-30T09:28:26.036508+0000","last_undegraded":"2023-09-30T09:28:26.036508+0000","last_fullsized":"2023-09-30T09:28:26.036508+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:43:27.284704+0000","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":589,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216094+0000","last_change":"2023-09-30T09:23:11.172147+0000","last_active":"2023-09-30T09:28:19.216094+0000","last_peered":"2023-09-30T09:28:19.216094+0000","last_clean":"2023-09-30T09:28:19.216094+0000","last_became_active":"2023-09-30T09:23:11.171540+0000","last_became_peered":"2023-09-30T09:23:11.171540+0000","last_unstale":"2023-09-30T09:28:19.216094+0000","last_undegraded":"2023-09-30T09:28:19.216094+0000","last_fullsized":"2023-09-30T09:28:19.216094+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:24.074307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":150,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:21.727311+0000","last_change":"2023-09-30T09:20:56.400928+0000","last_active":"2023-09-30T09:28:21.727311+0000","last_peered":"2023-09-30T09:28:21.727311+0000","last_clean":"2023-09-30T09:28:21.727311+0000","last_became_active":"2023-09-30T09:20:56.400625+0000","last_became_peered":"2023-09-30T09:20:56.400625+0000","last_unstale":"2023-09-30T09:28:21.727311+0000","last_undegraded":"2023-09-30T09:28:21.727311+0000","last_fullsized":"2023-09-30T09:28:21.727311+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:23:33.927726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":205,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216829+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:28:19.216829+0000","last_peered":"2023-09-30T09:28:19.216829+0000","last_clean":"2023-09-30T09:28:19.216829+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:28:19.216829+0000","last_undegraded":"2023-09-30T09:28:19.216829+0000","last_fullsized":"2023-09-30T09:28:19.216829+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"211'1219","reported_seq":1232,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:28.976703+0000","last_change":"2023-09-30T09:28:12.562340+0000","last_active":"2023-09-30T09:28:28.976703+0000","last_peered":"2023-09-30T09:28:28.976703+0000","last_clean":"2023-09-30T09:28:28.976703+0000","last_became_active":"2023-09-30T09:28:12.562049+0000","last_became_peered":"2023-09-30T09:28:12.562049+0000","last_unstale":"2023-09-30T09:28:28.976703+0000","last_undegraded":"2023-09-30T09:28:28.976703+0000","last_fullsized":"2023-09-30T09:28:28.976703+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1219,"log_dups_size":0,"ondisk_log_size":1219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:00:16.000443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79888384,"num_objects":1219,"num_object_clones":0,"num_object_copies":3657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1219,"num_write_kb":78016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":618,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.217553+0000","last_change":"2023-09-30T09:20:56.403258+0000","last_active":"2023-09-30T09:28:19.217553+0000","last_peered":"2023-09-30T09:28:19.217553+0000","last_clean":"2023-09-30T09:28:19.217553+0000","last_became_active":"2023-09-30T09:20:56.402777+0000","last_became_peered":"2023-09-30T09:20:56.402777+0000","last_unstale":"2023-09-30T09:28:19.217553+0000","last_undegraded":"2023-09-30T09:28:19.217553+0000","last_fullsized":"2023-09-30T09:28:19.217553+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:49.477515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":651,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.218895+0000","last_change":"2023-09-30T09:27:54.119268+0000","last_active":"2023-09-30T09:28:19.218895+0000","last_peered":"2023-09-30T09:28:19.218895+0000","last_clean":"2023-09-30T09:28:19.218895+0000","last_became_active":"2023-09-30T09:27:49.106373+0000","last_became_peered":"2023-09-30T09:27:49.106373+0000","last_unstale":"2023-09-30T09:28:19.218895+0000","last_undegraded":"2023-09-30T09:28:19.218895+0000","last_fullsized":"2023-09-30T09:28:19.218895+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:28.856049+0000","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":126,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.572720+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:28:18.572720+0000","last_peered":"2023-09-30T09:28:18.572720+0000","last_clean":"2023-09-30T09:28:18.572720+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:28:18.572720+0000","last_undegraded":"2023-09-30T09:28:18.572720+0000","last_fullsized":"2023-09-30T09:28:18.572720+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":153,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219600+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:28:19.219600+0000","last_peered":"2023-09-30T09:28:19.219600+0000","last_clean":"2023-09-30T09:28:19.219600+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:28:19.219600+0000","last_undegraded":"2023-09-30T09:28:19.219600+0000","last_fullsized":"2023-09-30T09:28:19.219600+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"211'893","reported_seq":907,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:26.762837+0000","last_change":"2023-09-30T09:28:12.562729+0000","last_active":"2023-09-30T09:28:26.762837+0000","last_peered":"2023-09-30T09:28:26.762837+0000","last_clean":"2023-09-30T09:28:26.762837+0000","last_became_active":"2023-09-30T09:28:12.562371+0000","last_became_peered":"2023-09-30T09:28:12.562371+0000","last_unstale":"2023-09-30T09:28:26.762837+0000","last_undegraded":"2023-09-30T09:28:26.762837+0000","last_fullsized":"2023-09-30T09:28:26.762837+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":893,"log_dups_size":0,"ondisk_log_size":893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:40:03.859186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58523648,"num_objects":893,"num_object_clones":0,"num_object_copies":2679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":893,"num_write_kb":57152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":615,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:24.276606+0000","last_change":"2023-09-30T09:28:24.276678+0000","last_active":"2023-09-30T09:28:24.276606+0000","last_peered":"2023-09-30T09:28:24.276606+0000","last_clean":"2023-09-30T09:28:24.276606+0000","last_became_active":"2023-09-30T09:28:09.226029+0000","last_became_peered":"2023-09-30T09:28:09.226029+0000","last_unstale":"2023-09-30T09:28:24.276606+0000","last_undegraded":"2023-09-30T09:28:24.276606+0000","last_fullsized":"2023-09-30T09:28:24.276606+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:03:49.533925+0000","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":771,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.218795+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:28:19.218795+0000","last_peered":"2023-09-30T09:28:19.218795+0000","last_clean":"2023-09-30T09:28:19.218795+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:28:19.218795+0000","last_undegraded":"2023-09-30T09:28:19.218795+0000","last_fullsized":"2023-09-30T09:28:19.218795+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574431+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:28:18.574431+0000","last_peered":"2023-09-30T09:28:18.574431+0000","last_clean":"2023-09-30T09:28:18.574431+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:28:18.574431+0000","last_undegraded":"2023-09-30T09:28:18.574431+0000","last_fullsized":"2023-09-30T09:28:18.574431+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":107,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216794+0000","last_change":"2023-09-30T09:26:51.326488+0000","last_active":"2023-09-30T09:28:19.216794+0000","last_peered":"2023-09-30T09:28:19.216794+0000","last_clean":"2023-09-30T09:28:19.216794+0000","last_became_active":"2023-09-30T09:26:51.325984+0000","last_became_peered":"2023-09-30T09:26:51.325984+0000","last_unstale":"2023-09-30T09:28:19.216794+0000","last_undegraded":"2023-09-30T09:28:19.216794+0000","last_fullsized":"2023-09-30T09:28:19.216794+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:22:23.475972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"211'957","reported_seq":973,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:26.630375+0000","last_change":"2023-09-30T09:28:12.718023+0000","last_active":"2023-09-30T09:28:26.630375+0000","last_peered":"2023-09-30T09:28:26.630375+0000","last_clean":"2023-09-30T09:28:26.630375+0000","last_became_active":"2023-09-30T09:28:12.717695+0000","last_became_peered":"2023-09-30T09:28:12.717695+0000","last_unstale":"2023-09-30T09:28:26.630375+0000","last_undegraded":"2023-09-30T09:28:26.630375+0000","last_fullsized":"2023-09-30T09:28:26.630375+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":957,"log_dups_size":0,"ondisk_log_size":957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:05:58.846726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62717952,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":957,"num_write_kb":61248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":455,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:28.453813+0000","last_change":"2023-09-30T09:28:28.453752+0000","last_active":"2023-09-30T09:28:28.453813+0000","last_peered":"2023-09-30T09:28:28.453813+0000","last_clean":"2023-09-30T09:28:28.453813+0000","last_became_active":"2023-09-30T09:28:09.228162+0000","last_became_peered":"2023-09-30T09:28:09.228162+0000","last_unstale":"2023-09-30T09:28:28.453813+0000","last_undegraded":"2023-09-30T09:28:28.453813+0000","last_fullsized":"2023-09-30T09:28:28.453813+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:43:23.683089+0000","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":901,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.220281+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:28:19.220281+0000","last_peered":"2023-09-30T09:28:19.220281+0000","last_clean":"2023-09-30T09:28:19.220281+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:28:19.220281+0000","last_undegraded":"2023-09-30T09:28:19.220281+0000","last_fullsized":"2023-09-30T09:28:19.220281+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":165,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.217052+0000","last_change":"2023-09-30T09:25:14.359586+0000","last_active":"2023-09-30T09:28:19.217052+0000","last_peered":"2023-09-30T09:28:19.217052+0000","last_clean":"2023-09-30T09:28:19.217052+0000","last_became_active":"2023-09-30T09:25:14.359198+0000","last_became_peered":"2023-09-30T09:25:14.359198+0000","last_unstale":"2023-09-30T09:28:19.217052+0000","last_undegraded":"2023-09-30T09:28:19.217052+0000","last_fullsized":"2023-09-30T09:28:19.217052+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:48.533039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":99,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574724+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:28:18.574724+0000","last_peered":"2023-09-30T09:28:18.574724+0000","last_clean":"2023-09-30T09:28:18.574724+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:28:18.574724+0000","last_undegraded":"2023-09-30T09:28:18.574724+0000","last_fullsized":"2023-09-30T09:28:18.574724+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"211'949","reported_seq":965,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:27.209558+0000","last_change":"2023-09-30T09:28:12.718312+0000","last_active":"2023-09-30T09:28:27.209558+0000","last_peered":"2023-09-30T09:28:27.209558+0000","last_clean":"2023-09-30T09:28:27.209558+0000","last_became_active":"2023-09-30T09:28:12.717839+0000","last_became_peered":"2023-09-30T09:28:12.717839+0000","last_unstale":"2023-09-30T09:28:27.209558+0000","last_undegraded":"2023-09-30T09:28:27.209558+0000","last_fullsized":"2023-09-30T09:28:27.209558+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":949,"log_dups_size":0,"ondisk_log_size":949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:09:02.377213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62193664,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":949,"num_write_kb":60736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":603,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.575104+0000","last_change":"2023-09-30T09:28:09.223206+0000","last_active":"2023-09-30T09:28:18.575104+0000","last_peered":"2023-09-30T09:28:18.575104+0000","last_clean":"2023-09-30T09:28:18.575104+0000","last_became_active":"2023-09-30T09:28:09.222798+0000","last_became_peered":"2023-09-30T09:28:09.222798+0000","last_unstale":"2023-09-30T09:28:18.575104+0000","last_undegraded":"2023-09-30T09:28:18.575104+0000","last_fullsized":"2023-09-30T09:28:18.575104+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:08:36.918738+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":198,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.573409+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:28:18.573409+0000","last_peered":"2023-09-30T09:28:18.573409+0000","last_clean":"2023-09-30T09:28:18.573409+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:28:18.573409+0000","last_undegraded":"2023-09-30T09:28:18.573409+0000","last_fullsized":"2023-09-30T09:28:18.573409+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1159,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216888+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:28:19.216888+0000","last_peered":"2023-09-30T09:28:19.216888+0000","last_clean":"2023-09-30T09:28:19.216888+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:28:19.216888+0000","last_undegraded":"2023-09-30T09:28:19.216888+0000","last_fullsized":"2023-09-30T09:28:19.216888+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":195,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.572370+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:28:18.572370+0000","last_peered":"2023-09-30T09:28:18.572370+0000","last_clean":"2023-09-30T09:28:18.572370+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:28:18.572370+0000","last_undegraded":"2023-09-30T09:28:18.572370+0000","last_fullsized":"2023-09-30T09:28:18.572370+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"211'1215","reported_seq":1228,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:28.999043+0000","last_change":"2023-09-30T09:28:12.718351+0000","last_active":"2023-09-30T09:28:28.999043+0000","last_peered":"2023-09-30T09:28:28.999043+0000","last_clean":"2023-09-30T09:28:28.999043+0000","last_became_active":"2023-09-30T09:28:12.718006+0000","last_became_peered":"2023-09-30T09:28:12.718006+0000","last_unstale":"2023-09-30T09:28:28.999043+0000","last_undegraded":"2023-09-30T09:28:28.999043+0000","last_fullsized":"2023-09-30T09:28:28.999043+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":1215,"log_dups_size":0,"ondisk_log_size":1215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:03.170613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79626240,"num_objects":1215,"num_object_clones":0,"num_object_copies":3645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1215,"num_write_kb":77760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":610,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-30T09:28:26.894243+0000","last_change":"2023-09-30T09:28:26.894367+0000","last_active":"2023-09-30T09:28:26.894243+0000","last_peered":"2023-09-30T09:28:26.894243+0000","last_clean":"2023-09-30T09:28:26.894243+0000","last_became_active":"2023-09-30T09:28:09.227189+0000","last_became_peered":"2023-09-30T09:28:09.227189+0000","last_unstale":"2023-09-30T09:28:26.894243+0000","last_undegraded":"2023-09-30T09:28:26.894243+0000","last_fullsized":"2023-09-30T09:28:26.894243+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:17.282328+0000","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":464,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.219610+0000","last_change":"2023-09-30T09:20:56.404621+0000","last_active":"2023-09-30T09:28:19.219610+0000","last_peered":"2023-09-30T09:28:19.219610+0000","last_clean":"2023-09-30T09:28:19.219610+0000","last_became_active":"2023-09-30T09:20:56.404156+0000","last_became_peered":"2023-09-30T09:20:56.404156+0000","last_unstale":"2023-09-30T09:28:19.219610+0000","last_undegraded":"2023-09-30T09:28:19.219610+0000","last_fullsized":"2023-09-30T09:28:19.219610+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:51.521326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":87,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.216956+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:28:19.216956+0000","last_peered":"2023-09-30T09:28:19.216956+0000","last_clean":"2023-09-30T09:28:19.216956+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:28:19.216956+0000","last_undegraded":"2023-09-30T09:28:19.216956+0000","last_fullsized":"2023-09-30T09:28:19.216956+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574792+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:28:18.574792+0000","last_peered":"2023-09-30T09:28:18.574792+0000","last_clean":"2023-09-30T09:28:18.574792+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:28:18.574792+0000","last_undegraded":"2023-09-30T09:28:18.574792+0000","last_fullsized":"2023-09-30T09:28:18.574792+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":69,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.575164+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:28:18.575164+0000","last_peered":"2023-09-30T09:28:18.575164+0000","last_clean":"2023-09-30T09:28:18.575164+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:28:18.575164+0000","last_undegraded":"2023-09-30T09:28:18.575164+0000","last_fullsized":"2023-09-30T09:28:18.575164+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":121,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.217177+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:28:19.217177+0000","last_peered":"2023-09-30T09:28:19.217177+0000","last_clean":"2023-09-30T09:28:19.217177+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:28:19.217177+0000","last_undegraded":"2023-09-30T09:28:19.217177+0000","last_fullsized":"2023-09-30T09:28:19.217177+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":172,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.217979+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:28:19.217979+0000","last_peered":"2023-09-30T09:28:19.217979+0000","last_clean":"2023-09-30T09:28:19.217979+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:28:19.217979+0000","last_undegraded":"2023-09-30T09:28:19.217979+0000","last_fullsized":"2023-09-30T09:28:19.217979+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":144,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.573113+0000","last_change":"2023-09-30T09:26:51.311932+0000","last_active":"2023-09-30T09:28:18.573113+0000","last_peered":"2023-09-30T09:28:18.573113+0000","last_clean":"2023-09-30T09:28:18.573113+0000","last_became_active":"2023-09-30T09:26:51.311569+0000","last_became_peered":"2023-09-30T09:26:51.311569+0000","last_unstale":"2023-09-30T09:28:18.573113+0000","last_undegraded":"2023-09-30T09:28:18.573113+0000","last_fullsized":"2023-09-30T09:28:18.573113+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:49.722161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":155,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.217967+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:28:19.217967+0000","last_peered":"2023-09-30T09:28:19.217967+0000","last_clean":"2023-09-30T09:28:19.217967+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:28:19.217967+0000","last_undegraded":"2023-09-30T09:28:19.217967+0000","last_fullsized":"2023-09-30T09:28:19.217967+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":175,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.572807+0000","last_change":"2023-09-30T09:26:51.311705+0000","last_active":"2023-09-30T09:28:18.572807+0000","last_peered":"2023-09-30T09:28:18.572807+0000","last_clean":"2023-09-30T09:28:18.572807+0000","last_became_active":"2023-09-30T09:26:51.311360+0000","last_became_peered":"2023-09-30T09:26:51.311360+0000","last_unstale":"2023-09-30T09:28:18.572807+0000","last_undegraded":"2023-09-30T09:28:18.572807+0000","last_fullsized":"2023-09-30T09:28:18.572807+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:08:08.201274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":182,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.572962+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:28:18.572962+0000","last_peered":"2023-09-30T09:28:18.572962+0000","last_clean":"2023-09-30T09:28:18.572962+0000","last_became_active":"2023-09-30T09:21:29.104314+0000","last_became_peered":"2023-09-30T09:21:29.104314+0000","last_unstale":"2023-09-30T09:28:18.572962+0000","last_undegraded":"2023-09-30T09:28:18.572962+0000","last_fullsized":"2023-09-30T09:28:18.572962+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:36:39.374204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":212,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.573405+0000","last_change":"2023-09-30T09:23:08.021721+0000","last_active":"2023-09-30T09:28:18.573405+0000","last_peered":"2023-09-30T09:28:18.573405+0000","last_clean":"2023-09-30T09:28:18.573405+0000","last_became_active":"2023-09-30T09:23:08.021401+0000","last_became_peered":"2023-09-30T09:23:08.021401+0000","last_unstale":"2023-09-30T09:28:18.573405+0000","last_undegraded":"2023-09-30T09:28:18.573405+0000","last_fullsized":"2023-09-30T09:28:18.573405+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:33:33.611604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":94,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:19.218185+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:28:19.218185+0000","last_peered":"2023-09-30T09:28:19.218185+0000","last_clean":"2023-09-30T09:28:19.218185+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:28:19.218185+0000","last_undegraded":"2023-09-30T09:28:19.218185+0000","last_fullsized":"2023-09-30T09:28:19.218185+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":169,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-30T09:28:18.574757+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:28:18.574757+0000","last_peered":"2023-09-30T09:28:18.574757+0000","last_clean":"2023-09-30T09:28:18.574757+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:28:18.574757+0000","last_undegraded":"2023-09-30T09:28:18.574757+0000","last_fullsized":"2023-09-30T09:28:18.574757+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1100349440,"num_objects":16790,"num_object_clones":0,"num_object_copies":50370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16790,"num_write_kb":1074560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3162439680,"data_stored":3162439680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16790,"ondisk_log_size":16790,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":1384,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":542359,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":180,"num_read_kb":335,"num_write":87,"num_write_kb":878,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1228744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234202,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":562532,"kb_used_data":476188,"kb_used_omap":0,"kb_used_meta":86336,"kb_avail":93182620,"statfs":{"total":95995035648,"available":95419002880,"internally_reserved":0,"allocated":487616512,"data_stored":486814509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":88408064},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.8,"15min":1.0029999999999999},"min":{"1min":0.253,"5min":0.224,"15min":0.224},"max":{"1min":11.710000000000001,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.518,"15min":0.92000000000000004},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.6589999999999998,"5min":18.097999999999999,"15min":18.097999999999999},"last":0.45500000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:27:36 2023","interfaces":[{"interface":"back","average":{"1min":4.9139999999999997,"5min":3.6280000000000001,"15min":1.5760000000000001},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":51.639000000000003,"5min":82.025999999999996,"15min":82.025999999999996},"last":38.017000000000003},{"interface":"front","average":{"1min":4.2729999999999997,"5min":3.5489999999999999,"15min":1.5640000000000001},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":51.570999999999998,"5min":82.012,"15min":82.012},"last":37.999000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.8129999999999999,"15min":1.004},"min":{"1min":0.32900000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.6389999999999993,"5min":17.838000000000001,"15min":17.838000000000001},"last":39.445},{"interface":"front","average":{"1min":1.885,"5min":2.0350000000000001,"15min":1.079},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":13.999000000000001,"5min":17.805,"15min":17.805},"last":40.988999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:27:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":1.641,"15min":0.96099999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.9730000000000008,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":38.167000000000002},{"interface":"front","average":{"1min":1.819,"5min":1.2529999999999999,"15min":0.84099999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.3559999999999999,"5min":7.1479999999999997,"15min":7.1479999999999997},"last":39.084000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.0009999999999999,"15min":0.64800000000000002},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.9329999999999998,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.081,"5min":0.90400000000000003,"15min":0.63900000000000001},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.0800000000000001,"5min":6.282,"15min":6.282},"last":0.28999999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:27:59 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":1.0680000000000001,"15min":0.69999999999999996},"min":{"1min":0.308,"5min":0.248,"15min":0.248},"max":{"1min":3.5979999999999999,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.5620000000000001,"5min":1.1819999999999999,"15min":0.77000000000000002},"min":{"1min":0.33800000000000002,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":11.015000000000001,"5min":11.022,"15min":11.022},"last":0.33500000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":1.1020000000000001,"15min":0.755},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":12.701000000000001,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.141,"5min":0.94299999999999995,"15min":0.70499999999999996},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.1550000000000002,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":0.32400000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495839,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525660,"kb_used_data":448916,"kb_used_omap":0,"kb_used_meta":76736,"kb_avail":93219492,"statfs":{"total":95995035648,"available":95456759808,"internally_reserved":0,"allocated":459689984,"data_stored":458896556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":78577664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2730000000000001,"5min":1.704,"15min":0.98699999999999999},"min":{"1min":0.252,"5min":0.222,"15min":0.222},"max":{"1min":35.280999999999999,"5min":35.280999999999999,"15min":35.280999999999999},"last":21.611000000000001},{"interface":"front","average":{"1min":4.548,"5min":1.994,"15min":1.099},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":35.426000000000002,"5min":35.426000000000002,"15min":35.426000000000002},"last":21.393000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8940000000000001,"5min":1.8129999999999999,"15min":1.0229999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":44.384999999999998,"5min":44.384999999999998,"15min":44.384999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":4.7919999999999998,"5min":2.1640000000000001,"15min":1.141},"min":{"1min":0.36299999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":44.363999999999997,"5min":44.363999999999997,"15min":44.363999999999997},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":2.6850000000000001,"15min":1.2789999999999999},"min":{"1min":0.316,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.2549999999999999,"5min":37.978000000000002,"15min":37.978000000000002},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":2.2349999999999999,"15min":1.147},"min":{"1min":0.35899999999999999,"5min":0.31,"15min":0.31},"max":{"1min":3.6459999999999999,"5min":37.759999999999998,"15min":37.759999999999998},"last":0.39400000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2280000000000002,"5min":1.712,"15min":0.96799999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":20.818999999999999,"5min":20.818999999999999,"15min":20.818999999999999},"last":20.408999999999999},{"interface":"front","average":{"1min":3.3759999999999999,"5min":2.0819999999999999,"15min":1.0860000000000001},"min":{"1min":0.35199999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":21.015000000000001,"5min":21.015000000000001,"15min":21.015000000000001},"last":19.940999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":5.3739999999999997,"5min":2.6579999999999999,"15min":1.2170000000000001},"min":{"1min":0.34699999999999998,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":49.286999999999999,"5min":73.656000000000006,"15min":73.656000000000006},"last":0.28399999999999997},{"interface":"front","average":{"1min":4.3490000000000002,"5min":2.27,"15min":1.109},"min":{"1min":0.314,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":49.252000000000002,"5min":73.882999999999996,"15min":73.882999999999996},"last":0.436}]},{"osd":5,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":4.4900000000000002,"5min":1.7809999999999999,"15min":0.92900000000000005},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":47.408999999999999,"5min":47.408999999999999,"15min":47.408999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":3.8639999999999999,"5min":1.647,"15min":0.93100000000000005},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.311},"max":{"1min":47.344000000000001,"5min":47.344000000000001,"15min":47.344000000000001},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:28:03 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":2.0169999999999999,"15min":1.0600000000000001},"min":{"1min":0.36099999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":53.927999999999997,"5min":53.927999999999997,"15min":53.927999999999997},"last":0.35699999999999998},{"interface":"front","average":{"1min":3.8780000000000001,"5min":2.0760000000000001,"15min":1.103},"min":{"1min":0.35599999999999998,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":51.886000000000003,"5min":51.886000000000003,"15min":51.886000000000003},"last":0.47199999999999998}]}]},{"osd":1,"up_from":99,"seq":425201762403,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":379124,"kb_used_data":275304,"kb_used_omap":5,"kb_used_meta":103802,"kb_avail":93366028,"statfs":{"total":95995035648,"available":95606812672,"internally_reserved":0,"allocated":281911296,"data_stored":280754083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":106293430},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":7.1360000000000001,"5min":5.6589999999999998,"15min":2.1320000000000001},"min":{"1min":0.20399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":88.045000000000002,"5min":150.93899999999999,"15min":150.93899999999999},"last":8.9160000000000004},{"interface":"front","average":{"1min":6.7709999999999999,"5min":5.7000000000000002,"15min":2.165},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":91.021000000000001,"5min":156.74199999999999,"15min":156.74199999999999},"last":9.6699999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:28:03 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":7.2300000000000004,"15min":2.6890000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":22.93,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.6840000000000002,"5min":7.1379999999999999,"15min":2.641},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":22.489000000000001,"5min":144.62,"15min":144.62},"last":0.28799999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:27:39 2023","interfaces":[{"interface":"back","average":{"1min":2.641,"5min":1.577,"15min":0.79900000000000004},"min":{"1min":0.33000000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":16.704000000000001,"5min":18.757999999999999,"15min":18.757999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.0710000000000002,"5min":1.3180000000000001,"15min":0.70099999999999996},"min":{"1min":0.29199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":11.173,"5min":11.173,"15min":11.173},"last":0.33600000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.319,"15min":0.86699999999999999},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.173,"5min":10.173,"15min":10.173},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.411,"5min":1.3169999999999999,"15min":0.88900000000000001},"min":{"1min":0.36799999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.9299999999999997,"5min":8.9299999999999997,"15min":8.9299999999999997},"last":0.48199999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9650000000000001,"5min":1.202,"15min":0.81100000000000005},"min":{"1min":0.308,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":10.26,"5min":10.26,"15min":10.26},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.3769999999999998,"5min":1.397,"15min":0.89600000000000002},"min":{"1min":0.35499999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.3729999999999993,"5min":9.3729999999999993,"15min":9.3729999999999993},"last":0.53300000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":1.4630000000000001,"15min":0.90100000000000002},"min":{"1min":0.38100000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":14.356999999999999,"5min":14.356999999999999,"15min":14.356999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.125,"5min":1.5349999999999999,"15min":0.96299999999999997},"min":{"1min":0.375,"5min":0.313,"15min":0.313},"max":{"1min":9.6660000000000004,"5min":15.864000000000001,"15min":15.864000000000001},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:27:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3860000000000001,"5min":1.802,"15min":1.048},"min":{"1min":0.436,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":14.403,"5min":14.403,"15min":14.403},"last":0.44900000000000001},{"interface":"front","average":{"1min":8.609,"5min":3.012,"15min":1.464},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":122.56699999999999,"5min":122.56699999999999,"15min":122.56699999999999},"last":0.50900000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925935,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":295568,"kb_used_data":72520,"kb_used_omap":5,"kb_used_meta":223034,"kb_avail":93449584,"statfs":{"total":95995035648,"available":95692374016,"internally_reserved":0,"allocated":74260480,"data_stored":73069606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":228387069},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:27:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.665,"15min":0.79300000000000004},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":25.920000000000002,"5min":25.920000000000002,"15min":25.920000000000002},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.778,"5min":1.6100000000000001,"15min":0.81000000000000005},"min":{"1min":0.248,"5min":0.192,"15min":0.192},"max":{"1min":5.5780000000000003,"5min":18.693000000000001,"15min":18.693000000000001},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.8400000000000001,"15min":0.86699999999999999},"min":{"1min":0.29099999999999998,"5min":0.218,"15min":0.218},"max":{"1min":6.5999999999999996,"5min":30.359999999999999,"15min":30.359999999999999},"last":0.495},{"interface":"front","average":{"1min":1.4159999999999999,"5min":1.9710000000000001,"15min":0.94799999999999995},"min":{"1min":0.308,"5min":0.245,"15min":0.245},"max":{"1min":10.458,"5min":31.661000000000001,"15min":31.661000000000001},"last":0.38500000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:27:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":1.954,"15min":0.97499999999999998},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":15.204000000000001,"5min":16.670000000000002,"15min":16.670000000000002},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.7829999999999999,"5min":1.778,"15min":0.88},"min":{"1min":0.29999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":25.876000000000001,"5min":25.876000000000001,"15min":25.876000000000001},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:27:28 2023","interfaces":[{"interface":"back","average":{"1min":8.6050000000000004,"5min":2.6040000000000001,"15min":1.3440000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":121.486,"5min":121.486,"15min":121.486},"last":0.44700000000000001},{"interface":"front","average":{"1min":10.138,"5min":3.1360000000000001,"15min":1.5489999999999999},"min":{"1min":0.313,"5min":0.31,"15min":0.31},"max":{"1min":121.44499999999999,"5min":121.44499999999999,"15min":121.44499999999999},"last":0.33500000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:27:28 2023","interfaces":[{"interface":"back","average":{"1min":3.048,"5min":1.952,"15min":1.181},"min":{"1min":0.47699999999999998,"5min":0.38400000000000001,"15min":0.36199999999999999},"max":{"1min":15.162000000000001,"5min":15.162000000000001,"15min":15.162000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.843,"5min":1.8169999999999999,"15min":1.1379999999999999},"min":{"1min":0.45200000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":13.766,"5min":13.766,"15min":13.766},"last":0.65100000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:27:28 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.47,"15min":1.0089999999999999},"min":{"1min":0.42399999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.811999999999999,"5min":13.811999999999999,"15min":13.811999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":2.2559999999999998,"5min":1.3029999999999999,"15min":0.98799999999999999},"min":{"1min":0.35399999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.407,"5min":7.556,"15min":7.556},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:27:28 2023","interfaces":[{"interface":"back","average":{"1min":9.4649999999999999,"5min":2.9980000000000002,"15min":1.498},"min":{"1min":0.47699999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":127.199,"5min":127.199,"15min":127.199},"last":0.73999999999999999},{"interface":"front","average":{"1min":8.8399999999999999,"5min":2.6509999999999998,"15min":1.4299999999999999},"min":{"1min":0.505,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":118.551,"5min":118.551,"15min":118.551},"last":0.55900000000000005}]}]},{"osd":2,"up_from":104,"seq":446676598876,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":504484,"kb_used_data":378460,"kb_used_omap":5,"kb_used_meta":126010,"kb_avail":93240668,"statfs":{"total":95995035648,"available":95478444032,"internally_reserved":0,"allocated":387543040,"data_stored":386396198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":129034893},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.877,"15min":0.874},"min":{"1min":0.23300000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":5.077,"5min":15.843999999999999,"15min":15.843999999999999},"last":2.7160000000000002},{"interface":"front","average":{"1min":0.89000000000000001,"5min":1.526,"15min":0.80500000000000005},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.298,"5min":10.065,"15min":10.065},"last":2.4950000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.538,"15min":0.76600000000000001},"min":{"1min":0.28000000000000003,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.532,"5min":10.532,"15min":10.532},"last":1.4570000000000001},{"interface":"front","average":{"1min":0.997,"5min":1.371,"15min":0.73899999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.532,"5min":8.532,"15min":8.532},"last":1.972}]},{"osd":3,"last update":"Sat Sep 30 09:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.538,"15min":0.81000000000000005},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.3650000000000002,"5min":18.635999999999999,"15min":18.635999999999999},"last":2.0670000000000002},{"interface":"front","average":{"1min":2.5030000000000001,"5min":2.016,"15min":1.0069999999999999},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.0730000000000004,"5min":18.026,"15min":18.026},"last":3.7090000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.2250000000000001,"15min":0.81100000000000005},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.4139999999999997,"5min":7.4139999999999997,"15min":7.4139999999999997},"last":2.1360000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.97,"15min":1.089},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.3710000000000004,"5min":11.125999999999999,"15min":11.125999999999999},"last":3.4780000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.554,"15min":0.94799999999999995},"min":{"1min":0.40699999999999997,"5min":0.374,"15min":0.371},"max":{"1min":10.821999999999999,"5min":10.821999999999999,"15min":10.821999999999999},"last":1.536},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.5920000000000001,"15min":0.96899999999999997},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.4459999999999997,"5min":11.866,"15min":11.866},"last":2.101}]},{"osd":6,"last update":"Sat Sep 30 09:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":2.04,"15min":1.139},"min":{"1min":0.437,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.4180000000000001,"5min":11.247999999999999,"15min":11.247999999999999},"last":3.5299999999999998},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.6850000000000001,"15min":1.0089999999999999},"min":{"1min":0.39700000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.884,"5min":10.884,"15min":10.884},"last":1.498}]},{"osd":7,"last update":"Sat Sep 30 09:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":2.04,"15min":1.1399999999999999},"min":{"1min":0.51400000000000001,"5min":0.432,"15min":0.432},"max":{"1min":5.4530000000000003,"5min":11.209,"15min":11.209},"last":3.6699999999999999},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.726,"15min":1.0469999999999999},"min":{"1min":0.443,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":10.855,"5min":10.855,"15min":10.855},"last":1.5609999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435350,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":758396,"kb_used_data":646324,"kb_used_omap":2,"kb_used_meta":112061,"kb_avail":92986756,"statfs":{"total":95995035648,"available":95218438144,"internally_reserved":0,"allocated":661835776,"data_stored":660664227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":114750823},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:27:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":3.4529999999999998,"15min":1.4079999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":10.856,"5min":130.946,"15min":130.946},"last":1.3240000000000001},{"interface":"front","average":{"1min":2.1509999999999998,"5min":3.3149999999999999,"15min":1.391},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":12.361000000000001,"5min":130.96899999999999,"15min":130.96899999999999},"last":0.58999999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:27:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.6279999999999999,"15min":0.82599999999999996},"min":{"1min":0.23699999999999999,"5min":0.23200000000000001,"15min":0.224},"max":{"1min":12.131,"5min":13.039999999999999,"15min":13.039999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.7429999999999999,"5min":1.641,"15min":0.81999999999999995},"min":{"1min":0.249,"5min":0.221,"15min":0.221},"max":{"1min":14.057,"5min":14.057,"15min":14.057},"last":1.2909999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:27:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6520000000000001,"5min":4.1879999999999997,"15min":1.7},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.308999999999999,"5min":145.75899999999999,"15min":145.75899999999999},"last":0.437},{"interface":"front","average":{"1min":3.3849999999999998,"5min":3.984,"15min":1.615},"min":{"1min":0.29399999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.429,"5min":150.358,"15min":150.358},"last":1.909}]},{"osd":4,"last update":"Sat Sep 30 09:27:42 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.042,"15min":0.75800000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.3920000000000003,"5min":10.301,"15min":10.301},"last":1.363},{"interface":"front","average":{"1min":2.0329999999999999,"5min":1.4299999999999999,"15min":0.88100000000000001},"min":{"1min":0.38500000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.2560000000000002,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.624}]},{"osd":5,"last update":"Sat Sep 30 09:27:42 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":1.5720000000000001,"15min":0.94499999999999995},"min":{"1min":0.41899999999999998,"5min":0.308,"15min":0.308},"max":{"1min":7.8380000000000001,"5min":10.048999999999999,"15min":10.048999999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.5640000000000001,"5min":1.7889999999999999,"15min":1.0649999999999999},"min":{"1min":0.40999999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.231999999999999,"5min":12.785,"15min":12.785},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:27:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.2649999999999999,"15min":0.872},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":10.308999999999999,"5min":10.348000000000001,"15min":10.348000000000001},"last":1.4770000000000001},{"interface":"front","average":{"1min":2.9039999999999999,"5min":1.7010000000000001,"15min":1.0249999999999999},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.69,"5min":14.69,"15min":14.69},"last":0.65400000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:27:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":1.637,"15min":0.997},"min":{"1min":0.35899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.8680000000000003,"5min":12.742000000000001,"15min":12.742000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.54,"5min":1.1819999999999999,"15min":0.84999999999999998},"min":{"1min":0.377,"5min":0.372,"15min":0.372},"max":{"1min":5.4619999999999997,"5min":10.43,"15min":10.43},"last":1.3919999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019112,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":449052,"kb_used_data":331412,"kb_used_omap":0,"kb_used_meta":117632,"kb_avail":93296100,"statfs":{"total":95995035648,"available":95535206400,"internally_reserved":0,"allocated":339365888,"data_stored":338572031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":120455168},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":6.2649999999999997,"15min":2.4990000000000001},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":20.542000000000002,"5min":128.79300000000001,"15min":128.79300000000001},"last":0.375},{"interface":"front","average":{"1min":1.593,"5min":5.8819999999999997,"15min":2.3769999999999998},"min":{"1min":0.30099999999999999,"5min":0.248,"15min":0.248},"max":{"1min":20.469999999999999,"5min":128.38800000000001,"15min":128.38800000000001},"last":0.40100000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:27:36 2023","interfaces":[{"interface":"back","average":{"1min":8.8030000000000008,"5min":3.0840000000000001,"15min":1.4079999999999999},"min":{"1min":0.28399999999999997,"5min":0.245,"15min":0.245},"max":{"1min":93.244,"5min":93.244,"15min":93.244},"last":0.496},{"interface":"front","average":{"1min":9.984,"5min":3.294,"15min":1.47},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":93.465999999999994,"5min":93.465999999999994,"15min":93.465999999999994},"last":0.433}]},{"osd":2,"last update":"Sat Sep 30 09:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":4.7939999999999996,"15min":2.004},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.816000000000001,"5min":140.601,"15min":140.601},"last":77.620000000000005},{"interface":"front","average":{"1min":1.5669999999999999,"5min":4.6600000000000001,"15min":1.95},"min":{"1min":0.36799999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.774,"5min":140.54900000000001,"15min":140.54900000000001},"last":77.638999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:27:36 2023","interfaces":[{"interface":"back","average":{"1min":2.456,"5min":2.29,"15min":1.161},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.0350000000000001,"5min":86.620000000000005,"15min":86.620000000000005},"last":75.593999999999994},{"interface":"front","average":{"1min":1.6140000000000001,"5min":2.2090000000000001,"15min":1.149},"min":{"1min":0.36599999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.8540000000000001,"5min":86.786000000000001,"15min":86.786000000000001},"last":75.510999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:28:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.5649999999999999,"15min":0.84699999999999998},"min":{"1min":0.35799999999999998,"5min":0.252,"15min":0.252},"max":{"1min":2.109,"5min":86.691000000000003,"15min":86.691000000000003},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.8360000000000001,"15min":0.96599999999999997},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.80200000000000005,"5min":86.822000000000003,"15min":86.822000000000003},"last":0.28499999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:27:46 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":0.82299999999999995,"15min":0.625},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6310000000000002,"5min":5.6310000000000002,"15min":5.6310000000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.93300000000000005,"15min":0.69599999999999995},"min":{"1min":0.35899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.335,"5min":6.327,"15min":6.327},"last":0.30399999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":2.0470000000000002,"15min":1.0780000000000001},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.9800000000000004,"5min":86.728999999999999,"15min":86.728999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.1850000000000001,"15min":0.79500000000000004},"min":{"1min":0.46100000000000002,"5min":0.314,"15min":0.314},"max":{"1min":8.5869999999999997,"5min":8.5869999999999997,"15min":8.5869999999999997},"last":0.35499999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757476,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":488204,"kb_used_data":368004,"kb_used_omap":0,"kb_used_meta":120192,"kb_avail":93256948,"statfs":{"total":95995035648,"available":95495114752,"internally_reserved":0,"allocated":376836096,"data_stored":376053981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":123076608},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":4.4160000000000004,"15min":1.8460000000000001},"min":{"1min":0.252,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.7879999999999998,"5min":148.79900000000001,"15min":148.79900000000001},"last":2.2469999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":4.9560000000000004,"15min":2.0590000000000002},"min":{"1min":0.26200000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.9100000000000001,"5min":148.779,"15min":148.779},"last":0.253}]},{"osd":1,"last update":"Sat Sep 30 09:27:36 2023","interfaces":[{"interface":"back","average":{"1min":7.5199999999999996,"5min":5.1849999999999996,"15min":2.1099999999999999},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.251},"max":{"1min":114.20999999999999,"5min":166.708,"15min":166.708},"last":0.29199999999999998},{"interface":"front","average":{"1min":7.9279999999999999,"5min":4.9539999999999997,"15min":2.0779999999999998},"min":{"1min":0.35799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":114.051,"5min":166.607,"15min":166.607},"last":1.675}]},{"osd":2,"last update":"Sat Sep 30 09:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":6.4459999999999997,"15min":2.5409999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.5119999999999996,"5min":165.768,"15min":165.768},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.429,"5min":7.899,"15min":3.0569999999999999},"min":{"1min":0.313,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.0760000000000005,"5min":166.24000000000001,"15min":166.24000000000001},"last":2.1259999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:27:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":2.1779999999999999,"15min":1.1539999999999999},"min":{"1min":0.35499999999999998,"5min":0.311,"15min":0.311},"max":{"1min":7.6289999999999996,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.756,"15min":1.016},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":4.2320000000000002,"5min":28.039999999999999,"15min":28.039999999999999},"last":1.6339999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.88600000000000001,"15min":0.61299999999999999},"min":{"1min":0.30199999999999999,"5min":0.222,"15min":0.222},"max":{"1min":1.5469999999999999,"5min":8.5359999999999996,"15min":8.5359999999999996},"last":2.093},{"interface":"front","average":{"1min":0.63,"5min":1.054,"15min":0.69999999999999996},"min":{"1min":0.313,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.653,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":2.1779999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:27:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":1.153,"15min":0.74299999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":14.23,"5min":14.23,"15min":14.23},"last":1.885},{"interface":"front","average":{"1min":2.8180000000000001,"5min":1.48,"15min":0.90800000000000003},"min":{"1min":0.34300000000000003,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":18.960000000000001,"5min":18.960000000000001,"15min":18.960000000000001},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:27:54 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.3340000000000001,"15min":0.84099999999999997},"min":{"1min":0.38200000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.3719999999999999,"5min":19.175999999999998,"15min":19.175999999999998},"last":0.372},{"interface":"front","average":{"1min":1.012,"5min":1.048,"15min":0.74199999999999999},"min":{"1min":0.40000000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.7429999999999999,"5min":8.8810000000000002,"15min":8.8810000000000002},"last":2.5099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":78839808,"data_stored":78839808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":293601280,"data_stored":293601280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":386924544,"data_stored":386924544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":698023936,"data_stored":698023936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":325255168,"data_stored":325255168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":406585344,"data_stored":406585344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":490864640,"data_stored":490864640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":482344960,"data_stored":482344960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:28:32.787 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:28:32.787 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pg_num 26 2023-09-30T09:28:32.813 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:32.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:32.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:32.913 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 12 16 26934 26918 140.098 91.25 0.00827519 0.00713189 2023-09-30T09:28:32.923 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:32.962 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:33.027 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:33.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:33.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:33.135 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:33.165 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:33.207 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:33.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:33.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:33.312 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:33.378 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:33.594 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:33.611 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:33.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:33.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:33.632 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:28:33.720 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:33.759 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:33.779 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:33.827 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:33.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:33.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:33.910 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 13 16 28452 28436 136.611 94.875 0.00817798 0.00731364 2023-09-30T09:28:33.922 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:28:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:28:33] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-30T09:28:33.934 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:33.968 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:34.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:34.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:33 smithi167 ceph-mon[192880]: pgmap v499: 99 pgs: 99 active+clean; 1.4 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-09-30T09:28:34.026 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:34.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:34.134 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:34.178 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:34.241 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:34.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:34.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:34.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:34.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:34.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:33 smithi019 ceph-mon[216815]: pgmap v499: 99 pgs: 99 active+clean; 1.4 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-09-30T09:28:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:33 smithi019 ceph-mon[224043]: pgmap v499: 99 pgs: 99 active+clean; 1.4 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-09-30T09:28:34.347 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:34.381 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:34.457 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:34.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:34.565 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:34.599 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:34.653 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:34.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:34.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:34.725 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:28:34.758 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:34.798 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:34.910 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 14 16 29798 29782 132.864 84.125 0.00168288 0.00743979 2023-09-30T09:28:35.910 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 15 16 31120 31104 129.516 82.625 0.00869256 0.00771654 2023-09-30T09:28:35.925 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:35.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:36.032 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:36.058 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:36.111 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:36.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:36.217 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:36.243 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:36.284 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:36.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:36.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:36 smithi019 ceph-mon[216815]: pgmap v500: 99 pgs: 99 active+clean; 1.7 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.06k op/s 2023-09-30T09:28:36.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:36 smithi019 ceph-mon[224043]: pgmap v500: 99 pgs: 99 active+clean; 1.7 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.06k op/s 2023-09-30T09:28:36.388 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:36.873 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:36 smithi167 ceph-mon[192880]: pgmap v500: 99 pgs: 99 active+clean; 1.7 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.06k op/s 2023-09-30T09:28:36.873 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:36.909 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:36.949 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 16 16 32492 32476 126.772 85.75 0.0103794 0.0078816 2023-09-30T09:28:36.971 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:36.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:36.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:37.075 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:37.113 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:37.141 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:37.174 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:37.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:37.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:37.283 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:37.361 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:37.601 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:37.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:37.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:37.716 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:37.804 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:37.912 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 17 16 33956 33940 124.697 91.5 0.00561493 0.00801513 2023-09-30T09:28:38.237 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 6 pg_num to 26 2023-09-30T09:28:38.237 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:28:38.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:38 smithi019 ceph-mon[216815]: pgmap v501: 99 pgs: 99 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.83k op/s 2023-09-30T09:28:38.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2824336996' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:28:38.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/134568311' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:28:38.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:38 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:28:38.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:38 smithi019 ceph-mon[224043]: pgmap v501: 99 pgs: 99 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.83k op/s 2023-09-30T09:28:38.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2824336996' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:28:38.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/134568311' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:28:38.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:38 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:28:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:38 smithi167 ceph-mon[192880]: pgmap v501: 99 pgs: 99 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.83k op/s 2023-09-30T09:28:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2824336996' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:28:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/134568311' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:28:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:38 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:28:38.912 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 18 16 35532 35516 123.242 98.5 0.00594864 0.00810875 2023-09-30T09:28:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:28:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:28:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2824336996' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:28:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-30T09:28:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: osdmap e212: 8 total, 8 up, 8 in 2023-09-30T09:28:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:39 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-30T09:28:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:28:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:28:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2824336996' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:28:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-30T09:28:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: osdmap e212: 8 total, 8 up, 8 in 2023-09-30T09:28:39.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2824336996' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:28:39.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-30T09:28:39.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: osdmap e212: 8 total, 8 up, 8 in 2023-09-30T09:28:39.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: failed to encode map e212 with expected crc 2023-09-30T09:28:39.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:39 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-30T09:28:39.912 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 19 16 35954 35938 118.146 26.375 0.152024 0.00818771 2023-09-30T09:28:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:40 smithi167 ceph-mon[192880]: pgmap v503: 99 pgs: 99 active+clean; 2.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:28:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:40 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-30T09:28:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:40 smithi167 ceph-mon[192880]: osdmap e213: 8 total, 8 up, 8 in 2023-09-30T09:28:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:40 smithi167 ceph-mon[192880]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:40 smithi167 ceph-mon[192880]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:40 smithi167 ceph-mon[192880]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:40 smithi167 ceph-mon[192880]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[224043]: pgmap v503: 99 pgs: 99 active+clean; 2.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:28:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-30T09:28:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[224043]: osdmap e213: 8 total, 8 up, 8 in 2023-09-30T09:28:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[224043]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[224043]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[224043]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[224043]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[216815]: pgmap v503: 99 pgs: 99 active+clean; 2.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:28:40.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-30T09:28:40.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[216815]: osdmap e213: 8 total, 8 up, 8 in 2023-09-30T09:28:40.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[216815]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[216815]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[216815]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:40 smithi019 ceph-mon[216815]: failed to encode map e213 with expected crc 2023-09-30T09:28:40.913 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:28:40.912371+0000 min lat: 0.00113601 max lat: 1.04444 avg lat: 0.00866405 2023-09-30T09:28:40.914 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:28:40.914 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 20 16 36953 36937 115.362 62.4375 0.00251307 0.00866405 2023-09-30T09:28:41.196 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:41.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:41.326 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:41.365 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:41.433 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:41.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:41.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:41.486 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: osdmap e214: 8 total, 8 up, 8 in 2023-09-30T09:28:41.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: 1.5 scrub starts 2023-09-30T09:28:41.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[224043]: 1.5 scrub ok 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: osdmap e214: 8 total, 8 up, 8 in 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: 1.5 scrub starts 2023-09-30T09:28:41.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:41 smithi019 ceph-mon[216815]: 1.5 scrub ok 2023-09-30T09:28:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: osdmap e214: 8 total, 8 up, 8 in 2023-09-30T09:28:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: failed to encode map e214 with expected crc 2023-09-30T09:28:41.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: 1.5 scrub starts 2023-09-30T09:28:41.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:41 smithi167 ceph-mon[192880]: 1.5 scrub ok 2023-09-30T09:28:41.542 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:41.623 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:41.850 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:41.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:41.910 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:28:41.912 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 21 16 38718 38702 115.121 110.312 0.0172163 0.00868092 2023-09-30T09:28:41.960 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:41.971 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:28:41.980 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:28:41.980 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:28:41.980 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:28:41.980 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:28:41.980 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:28:42.018 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:28:42.039 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:42.058 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:28:42.095 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:28:42.135 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:28:42.174 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:28:42.209 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:28:42.213 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:28:42.213 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:28:42.213 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:28:42.213 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:28:42.213 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:28:42.213 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:28:42.213 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:28:42.213 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:28:42.250 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:42.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:42.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:42.357 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:42.390 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:42.501 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:42.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:42.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:42.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:42 smithi019 ceph-mon[216815]: pgmap v506: 109 pgs: 1 peering, 10 unknown, 98 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-30T09:28:42.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:42 smithi019 ceph-mon[216815]: 6.1 scrub starts 2023-09-30T09:28:42.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:42 smithi019 ceph-mon[216815]: 6.1 scrub ok 2023-09-30T09:28:42.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1820502645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:42.508 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:42 smithi019 ceph-mon[224043]: pgmap v506: 109 pgs: 1 peering, 10 unknown, 98 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-30T09:28:42.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:42 smithi019 ceph-mon[224043]: 6.1 scrub starts 2023-09-30T09:28:42.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:42 smithi019 ceph-mon[224043]: 6.1 scrub ok 2023-09-30T09:28:42.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1820502645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:42 smithi167 ceph-mon[192880]: pgmap v506: 109 pgs: 1 peering, 10 unknown, 98 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-30T09:28:42.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:42 smithi167 ceph-mon[192880]: 6.1 scrub starts 2023-09-30T09:28:42.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:42 smithi167 ceph-mon[192880]: 6.1 scrub ok 2023-09-30T09:28:42.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1820502645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:42.608 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:42.650 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:42.740 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:42.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:42.844 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:42.911 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:42.912 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 22 16 40428 40412 114.746 106.875 0.0186421 0.00870829 2023-09-30T09:28:43.221 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:28:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:43 smithi019 ceph-mon[216815]: 6.6 scrub starts 2023-09-30T09:28:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:43 smithi019 ceph-mon[216815]: 6.6 scrub ok 2023-09-30T09:28:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:43 smithi019 ceph-mon[216815]: 6.7 scrub starts 2023-09-30T09:28:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:43 smithi019 ceph-mon[216815]: 6.7 scrub ok 2023-09-30T09:28:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:43 smithi019 ceph-mon[224043]: 6.6 scrub starts 2023-09-30T09:28:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:43 smithi019 ceph-mon[224043]: 6.6 scrub ok 2023-09-30T09:28:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:43 smithi019 ceph-mon[224043]: 6.7 scrub starts 2023-09-30T09:28:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:43 smithi019 ceph-mon[224043]: 6.7 scrub ok 2023-09-30T09:28:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:43 smithi167 ceph-mon[192880]: 6.6 scrub starts 2023-09-30T09:28:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:43 smithi167 ceph-mon[192880]: 6.6 scrub ok 2023-09-30T09:28:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:43 smithi167 ceph-mon[192880]: 6.7 scrub starts 2023-09-30T09:28:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:43 smithi167 ceph-mon[192880]: 6.7 scrub ok 2023-09-30T09:28:43.912 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 23 16 42125 42109 114.368 106.062 0.0133246 0.00873839 2023-09-30T09:28:44.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:28:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:28:43] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-30T09:28:44.217 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:44.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:44.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:44.321 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:44.361 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:44.430 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:44.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:44.539 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:44.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:44 smithi019 ceph-mon[216815]: pgmap v507: 109 pgs: 1 active+clean+scrubbing, 1 peering, 6 unknown, 101 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-30T09:28:44.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:44 smithi019 ceph-mon[216815]: 6.4 scrub starts 2023-09-30T09:28:44.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:44 smithi019 ceph-mon[216815]: 6.4 scrub ok 2023-09-30T09:28:44.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:44 smithi019 ceph-mon[224043]: pgmap v507: 109 pgs: 1 active+clean+scrubbing, 1 peering, 6 unknown, 101 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-30T09:28:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:44 smithi019 ceph-mon[224043]: 6.4 scrub starts 2023-09-30T09:28:44.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:44 smithi019 ceph-mon[224043]: 6.4 scrub ok 2023-09-30T09:28:44.603 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:44.694 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:28:44.769 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:44.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:44 smithi167 ceph-mon[192880]: pgmap v507: 109 pgs: 1 active+clean+scrubbing, 1 peering, 6 unknown, 101 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-30T09:28:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:44 smithi167 ceph-mon[192880]: 6.4 scrub starts 2023-09-30T09:28:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:44 smithi167 ceph-mon[192880]: 6.4 scrub ok 2023-09-30T09:28:44.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:44.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:44.879 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:44.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 24 16 43726 43710 113.746 100.062 0.00765153 0.00878611 2023-09-30T09:28:44.942 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:44.975 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:45.091 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:45.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:45.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:45.210 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:45.303 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:45.383 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:45.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:45.497 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:45.533 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:45 smithi019 ceph-mon[216815]: 6.14 scrub starts 2023-09-30T09:28:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:45 smithi019 ceph-mon[216815]: 6.14 scrub ok 2023-09-30T09:28:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:45 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-30T09:28:45.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:45 smithi019 ceph-mon[224043]: 6.14 scrub starts 2023-09-30T09:28:45.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:45 smithi019 ceph-mon[224043]: 6.14 scrub ok 2023-09-30T09:28:45.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:45 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-30T09:28:45.592 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:45.594 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:45 smithi167 ceph-mon[192880]: 6.14 scrub starts 2023-09-30T09:28:45.594 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:45 smithi167 ceph-mon[192880]: 6.14 scrub ok 2023-09-30T09:28:45.594 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:45 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-30T09:28:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:45.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:45.696 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:45.731 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:45.787 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:45.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:45.861 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:28:45.861 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:28:45.862 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-30T09:28:45.862 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:28:45.862 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:28:45.862 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-30T09:28:45.890 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:45.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 25 16 44754 44738 111.767 64.25 0.00204931 0.00879797 2023-09-30T09:28:45.919 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:45.969 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:45.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:45.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:46.073 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:46.100 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:46.197 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:46.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:46.256 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:28:46.300 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:46.328 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:46.380 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:46.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:46.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:46.472 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: 6.0 scrub starts 2023-09-30T09:28:46.472 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: pgmap v508: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: osdmap e215: 8 total, 8 up, 8 in 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: 6.17 scrub starts 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: 6.11 scrub starts 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: 6.0 scrub ok 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: 6.11 scrub ok 2023-09-30T09:28:46.473 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: 6.17 scrub ok 2023-09-30T09:28:46.474 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:46 smithi167 ceph-mon[192880]: osdmap e216: 8 total, 8 up, 8 in 2023-09-30T09:28:46.483 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:46.512 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:46.555 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:46.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:46.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: 6.0 scrub starts 2023-09-30T09:28:46.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: pgmap v508: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-30T09:28:46.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-30T09:28:46.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: osdmap e215: 8 total, 8 up, 8 in 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: 6.17 scrub starts 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: 6.11 scrub starts 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: 6.0 scrub ok 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: 6.11 scrub ok 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: 6.17 scrub ok 2023-09-30T09:28:46.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[216815]: osdmap e216: 8 total, 8 up, 8 in 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: 6.0 scrub starts 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: pgmap v508: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: osdmap e215: 8 total, 8 up, 8 in 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: failed to encode map e215 with expected crc 2023-09-30T09:28:46.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: 6.17 scrub starts 2023-09-30T09:28:46.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: 6.11 scrub starts 2023-09-30T09:28:46.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: 6.0 scrub ok 2023-09-30T09:28:46.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: 6.11 scrub ok 2023-09-30T09:28:46.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: 6.17 scrub ok 2023-09-30T09:28:46.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:46 smithi019 ceph-mon[224043]: osdmap e216: 8 total, 8 up, 8 in 2023-09-30T09:28:46.658 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:46.693 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:46.744 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:46.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:46.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:46.758 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:28:46.848 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:46.875 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:46.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 26 16 44754 44738 107.471 0 - 0.00879797 2023-09-30T09:28:46.975 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:46.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:47.079 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:47.115 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:47.181 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:47.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:47.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:47.222 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:47.287 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:47.318 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:47.359 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:47.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:47.362 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:47.461 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:47.497 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:47.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: 6.12 scrub starts 2023-09-30T09:28:47.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: 6.5 scrub starts 2023-09-30T09:28:47.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: 6.12 scrub ok 2023-09-30T09:28:47.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-30T09:28:47.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-30T09:28:47.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[216815]: osdmap e217: 8 total, 8 up, 8 in 2023-09-30T09:28:47.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: 6.12 scrub starts 2023-09-30T09:28:47.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: 6.5 scrub starts 2023-09-30T09:28:47.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: 6.12 scrub ok 2023-09-30T09:28:47.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-30T09:28:47.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-30T09:28:47.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:47 smithi019 ceph-mon[224043]: osdmap e217: 8 total, 8 up, 8 in 2023-09-30T09:28:47.554 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:47.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:47.659 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:47.683 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: 6.12 scrub starts 2023-09-30T09:28:47.683 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: 6.5 scrub starts 2023-09-30T09:28:47.684 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: 6.12 scrub ok 2023-09-30T09:28:47.684 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.684 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.684 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: failed to encode map e216 with expected crc 2023-09-30T09:28:47.684 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-30T09:28:47.684 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-30T09:28:47.684 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:47 smithi167 ceph-mon[192880]: osdmap e217: 8 total, 8 up, 8 in 2023-09-30T09:28:47.686 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:47.733 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:47.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:47.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:47.837 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:47.863 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:47.905 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:47.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:47.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:47.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 27 16 45164 45148 104.441 12.8125 0.00387629 0.0094277 2023-09-30T09:28:48.008 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:28:48.034 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:48.078 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:28:48.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:48.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:28:48.181 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:48.207 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:48.251 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:48.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:48.354 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:48.381 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:48.424 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:48.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:48.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:48.528 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:48.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: failed to encode map e216 with expected crc 2023-09-30T09:28:48.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: 6.9 scrub starts 2023-09-30T09:28:48.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: pgmap v511: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 7.1 MiB/s wr, 110 op/s 2023-09-30T09:28:48.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: 6.9 scrub ok 2023-09-30T09:28:48.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:48.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:48.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: osdmap e218: 8 total, 8 up, 8 in 2023-09-30T09:28:48.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1555277715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:48.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:48.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: failed to encode map e216 with expected crc 2023-09-30T09:28:48.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: 6.9 scrub starts 2023-09-30T09:28:48.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: pgmap v511: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 7.1 MiB/s wr, 110 op/s 2023-09-30T09:28:48.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: 6.9 scrub ok 2023-09-30T09:28:48.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.564 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.564 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.564 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:48.564 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:48.564 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: osdmap e218: 8 total, 8 up, 8 in 2023-09-30T09:28:48.564 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1555277715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:48.565 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:48.565 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: failed to encode map e216 with expected crc 2023-09-30T09:28:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: 6.9 scrub starts 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: pgmap v511: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 7.1 MiB/s wr, 110 op/s 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: 6.9 scrub ok 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: failed to encode map e217 with expected crc 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: osdmap e218: 8 total, 8 up, 8 in 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1555277715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:28:48.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 28 16 45164 45148 100.713 0 - 0.0094277 2023-09-30T09:28:49.067 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:49.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:49.172 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:49.202 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:49.249 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:28:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: failed to encode map e217 with expected crc 2023-09-30T09:28:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: failed to encode map e217 with expected crc 2023-09-30T09:28:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: 6.2 scrub starts 2023-09-30T09:28:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: 6.2 scrub ok 2023-09-30T09:28:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-30T09:28:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:28:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-30T09:28:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[224043]: osdmap e219: 8 total, 8 up, 8 in 2023-09-30T09:28:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: failed to encode map e217 with expected crc 2023-09-30T09:28:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: failed to encode map e217 with expected crc 2023-09-30T09:28:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: 6.2 scrub starts 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: 6.2 scrub ok 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-30T09:28:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:49 smithi019 ceph-mon[216815]: osdmap e219: 8 total, 8 up, 8 in 2023-09-30T09:28:49.366 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:49.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:49.472 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:49.528 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:49.615 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:49.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:49.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:49.718 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:49.746 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: failed to encode map e217 with expected crc 2023-09-30T09:28:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: failed to encode map e217 with expected crc 2023-09-30T09:28:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: 6.2 scrub starts 2023-09-30T09:28:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: 6.2 scrub ok 2023-09-30T09:28:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: failed to encode map e218 with expected crc 2023-09-30T09:28:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-30T09:28:49.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:28:49.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:28:49.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-30T09:28:49.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:49 smithi167 ceph-mon[192880]: osdmap e219: 8 total, 8 up, 8 in 2023-09-30T09:28:49.794 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:49.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:49.898 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:49.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 29 16 45618 45602 98.2198 14.1875 0.00198119 0.0100836 2023-09-30T09:28:49.941 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:50.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:50 smithi167 ceph-mon[192880]: failed to encode map e218 with expected crc 2023-09-30T09:28:50.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:50 smithi167 ceph-mon[192880]: failed to encode map e218 with expected crc 2023-09-30T09:28:50.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:50 smithi167 ceph-mon[192880]: 6.3 scrub starts 2023-09-30T09:28:50.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:50 smithi167 ceph-mon[192880]: pgmap v514: 109 pgs: 109 active+clean; 2.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-09-30T09:28:50.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:50 smithi167 ceph-mon[192880]: 6.3 scrub ok 2023-09-30T09:28:50.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:50 smithi167 ceph-mon[192880]: failed to encode map e219 with expected crc 2023-09-30T09:28:50.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:50 smithi167 ceph-mon[192880]: failed to encode map e219 with expected crc 2023-09-30T09:28:50.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:50 smithi167 ceph-mon[192880]: osdmap e220: 8 total, 8 up, 8 in 2023-09-30T09:28:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[216815]: failed to encode map e218 with expected crc 2023-09-30T09:28:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[216815]: failed to encode map e218 with expected crc 2023-09-30T09:28:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[216815]: 6.3 scrub starts 2023-09-30T09:28:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[216815]: pgmap v514: 109 pgs: 109 active+clean; 2.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-09-30T09:28:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[216815]: 6.3 scrub ok 2023-09-30T09:28:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[216815]: failed to encode map e219 with expected crc 2023-09-30T09:28:50.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[216815]: failed to encode map e219 with expected crc 2023-09-30T09:28:50.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[216815]: osdmap e220: 8 total, 8 up, 8 in 2023-09-30T09:28:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[224043]: failed to encode map e218 with expected crc 2023-09-30T09:28:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[224043]: failed to encode map e218 with expected crc 2023-09-30T09:28:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[224043]: 6.3 scrub starts 2023-09-30T09:28:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[224043]: pgmap v514: 109 pgs: 109 active+clean; 2.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-09-30T09:28:50.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[224043]: 6.3 scrub ok 2023-09-30T09:28:50.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[224043]: failed to encode map e219 with expected crc 2023-09-30T09:28:50.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[224043]: failed to encode map e219 with expected crc 2023-09-30T09:28:50.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:50 smithi019 ceph-mon[224043]: osdmap e220: 8 total, 8 up, 8 in 2023-09-30T09:28:50.644 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:28:50.644 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:28:50.645 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:28:50.919 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 30 16 45764 45748 95.2515 9.125 0.00236861 0.0100662 2023-09-30T09:28:51.554 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:51.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[216815]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[216815]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[216815]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-30T09:28:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-30T09:28:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[216815]: osdmap e221: 8 total, 8 up, 8 in 2023-09-30T09:28:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[224043]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[224043]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[224043]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-30T09:28:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-30T09:28:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:51 smithi019 ceph-mon[224043]: osdmap e221: 8 total, 8 up, 8 in 2023-09-30T09:28:51.667 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:28:51.716 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:51 smithi167 ceph-mon[192880]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:51 smithi167 ceph-mon[192880]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:51 smithi167 ceph-mon[192880]: failed to encode map e220 with expected crc 2023-09-30T09:28:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-30T09:28:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-30T09:28:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:51 smithi167 ceph-mon[192880]: osdmap e221: 8 total, 8 up, 8 in 2023-09-30T09:28:51.904 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:28:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:51.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:28:51.919 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 31 16 46179 46163 93.0165 25.9375 0.00316018 0.0106072 2023-09-30T09:28:52.008 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:52.047 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:52.057 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:28:52.057 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:28:52.115 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:52.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:52.219 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:52.259 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[216815]: pgmap v517: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 1998/135207 objects misplaced (1.478%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:28:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[216815]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[216815]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[216815]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[216815]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[216815]: osdmap e222: 8 total, 8 up, 8 in 2023-09-30T09:28:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[224043]: pgmap v517: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 1998/135207 objects misplaced (1.478%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:28:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[224043]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[224043]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[224043]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[224043]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:52 smithi019 ceph-mon[224043]: osdmap e222: 8 total, 8 up, 8 in 2023-09-30T09:28:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:52 smithi167 ceph-mon[192880]: pgmap v517: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 1998/135207 objects misplaced (1.478%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:28:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:52 smithi167 ceph-mon[192880]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:52 smithi167 ceph-mon[192880]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:52 smithi167 ceph-mon[192880]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:52 smithi167 ceph-mon[192880]: failed to encode map e221 with expected crc 2023-09-30T09:28:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:52 smithi167 ceph-mon[192880]: osdmap e222: 8 total, 8 up, 8 in 2023-09-30T09:28:52.826 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:28:52.826 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:28:52.826 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:28:52.826 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:28:52.826 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:28:52.838 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:28:52.862 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:28:52.872 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:52.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:52.889 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:28:52.890 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:28:52.890 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:28:52.890 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:28:52.891 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:28:52.891 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:28:52.891 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:28:52.891 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:28:52.891 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:28:52.919 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 32 16 46179 46163 90.1111 0 - 0.0106072 2023-09-30T09:28:52.976 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:53.007 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:53.056 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:53.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:53.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:53.163 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:28:53.196 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:53.567 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:28:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[224043]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[224043]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[224043]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[224043]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3307560306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[224043]: osdmap e223: 8 total, 8 up, 8 in 2023-09-30T09:28:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[216815]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[216815]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[216815]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[216815]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3307560306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:53 smithi019 ceph-mon[216815]: osdmap e223: 8 total, 8 up, 8 in 2023-09-30T09:28:53.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:53 smithi167 ceph-mon[192880]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:53 smithi167 ceph-mon[192880]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:53 smithi167 ceph-mon[192880]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:53 smithi167 ceph-mon[192880]: failed to encode map e222 with expected crc 2023-09-30T09:28:53.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3307560306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:28:53.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:28:53.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:53 smithi167 ceph-mon[192880]: osdmap e223: 8 total, 8 up, 8 in 2023-09-30T09:28:53.919 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 33 16 46939 46923 88.8204 23.75 0.00590387 0.0112126 2023-09-30T09:28:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:28:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:28:53] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.43.0" 2023-09-30T09:28:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:54 smithi167 ceph-mon[192880]: pgmap v520: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 1417/136824 objects degraded (1.036%); 3415/136824 objects misplaced (2.496%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:28:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:54 smithi167 ceph-mon[192880]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:54 smithi167 ceph-mon[192880]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:54 smithi167 ceph-mon[192880]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:54 smithi167 ceph-mon[192880]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:54 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 1417/136824 objects degraded (1.036%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:28:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[216815]: pgmap v520: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 1417/136824 objects degraded (1.036%); 3415/136824 objects misplaced (2.496%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:28:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[216815]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[216815]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[216815]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[216815]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 1417/136824 objects degraded (1.036%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:28:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[224043]: pgmap v520: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 1417/136824 objects degraded (1.036%); 3415/136824 objects misplaced (2.496%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:28:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[224043]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[224043]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[224043]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[224043]: failed to encode map e223 with expected crc 2023-09-30T09:28:54.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:54 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 1417/136824 objects degraded (1.036%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:28:54.923 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 34 16 48177 48161 88.473 77.375 0.0135824 0.0112971 2023-09-30T09:28:55.560 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:28:55.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:55.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:28:55.671 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:55.703 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:55.757 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:55.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:55.860 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:55.891 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:55.923 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 35 16 49218 49202 87.8043 65.0625 0.00256128 0.0112568 2023-09-30T09:28:55.945 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:55.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:55.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:56.049 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:56.077 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:56.121 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:56.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:56.227 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:56.254 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:56.300 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:56.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:56.407 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:56.458 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:28:56.458 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:28:56.458 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 2, 3, 4, 5, 6, 7, 0] 2023-09-30T09:28:56.458 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-30T09:28:56.477 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:56.573 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:28:56.660 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:56.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:56.677 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[224043]: pgmap v522: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2810/140970 objects degraded (1.993%); 3211/140970 objects misplaced (2.278%); 19 MiB/s, 296 objects/s recovering 2023-09-30T09:28:56.677 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[224043]: osdmap e224: 8 total, 8 up, 8 in 2023-09-30T09:28:56.677 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[224043]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.677 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[224043]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.677 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[224043]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[216815]: pgmap v522: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2810/140970 objects degraded (1.993%); 3211/140970 objects misplaced (2.278%); 19 MiB/s, 296 objects/s recovering 2023-09-30T09:28:56.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[216815]: osdmap e224: 8 total, 8 up, 8 in 2023-09-30T09:28:56.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[216815]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[216815]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.679 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:56 smithi019 ceph-mon[216815]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:56 smithi167 ceph-mon[192880]: pgmap v522: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2810/140970 objects degraded (1.993%); 3211/140970 objects misplaced (2.278%); 19 MiB/s, 296 objects/s recovering 2023-09-30T09:28:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:56 smithi167 ceph-mon[192880]: osdmap e224: 8 total, 8 up, 8 in 2023-09-30T09:28:56.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:56 smithi167 ceph-mon[192880]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:56 smithi167 ceph-mon[192880]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:56 smithi167 ceph-mon[192880]: failed to encode map e224 with expected crc 2023-09-30T09:28:56.777 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:28:56.807 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:56.831 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:56.866 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:28:56.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:56.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:28:56.923 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 36 16 49780 49764 86.3417 35.125 0.010642 0.0115762 2023-09-30T09:28:56.972 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:57.081 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:57.271 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:28:57.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:57.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:28:57.383 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:28:57.461 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:58.078 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:57 smithi167 ceph-mon[192880]: osdmap e225: 8 total, 8 up, 8 in 2023-09-30T09:28:58.088 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:57 smithi167 ceph-mon[192880]: failed to encode map e225 with expected crc 2023-09-30T09:28:58.088 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:57 smithi167 ceph-mon[192880]: failed to encode map e225 with expected crc 2023-09-30T09:28:58.088 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:28:58.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:57 smithi019 ceph-mon[216815]: osdmap e225: 8 total, 8 up, 8 in 2023-09-30T09:28:58.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:57 smithi019 ceph-mon[216815]: failed to encode map e225 with expected crc 2023-09-30T09:28:58.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:57 smithi019 ceph-mon[216815]: failed to encode map e225 with expected crc 2023-09-30T09:28:58.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:57 smithi019 ceph-mon[224043]: osdmap e225: 8 total, 8 up, 8 in 2023-09-30T09:28:58.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:57 smithi019 ceph-mon[224043]: failed to encode map e225 with expected crc 2023-09-30T09:28:58.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:57 smithi019 ceph-mon[224043]: failed to encode map e225 with expected crc 2023-09-30T09:28:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:58.090 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:28:58.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:28:58.091 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 37 16 51100 51084 86.2377 82.5 0.0134785 0.0115906 2023-09-30T09:28:58.191 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:28:58.266 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:58.335 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:28:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:58.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:28:58.442 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:58.511 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:58.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:58 smithi167 ceph-mon[192880]: pgmap v525: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1393/142227 objects degraded (0.979%); 4160/142227 objects misplaced (2.925%); 22 MiB/s, 344 objects/s recovering 2023-09-30T09:28:58.580 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:58.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:58.623 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:28:58.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:58 smithi019 ceph-mon[216815]: pgmap v525: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1393/142227 objects degraded (0.979%); 4160/142227 objects misplaced (2.925%); 22 MiB/s, 344 objects/s recovering 2023-09-30T09:28:58.676 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:58 smithi019 ceph-mon[224043]: pgmap v525: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1393/142227 objects degraded (0.979%); 4160/142227 objects misplaced (2.925%); 22 MiB/s, 344 objects/s recovering 2023-09-30T09:28:58.685 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:28:58.724 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:58.793 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:28:58.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:28:58.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:28:58.899 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:28:58.923 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 38 16 52602 52586 86.4384 93.875 0.0105451 0.011563 2023-09-30T09:28:58.992 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:28:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:28:59 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1393/153078 objects degraded (0.910%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:28:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:28:59 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1393/153078 objects degraded (0.910%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:28:59.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:28:59 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1393/153078 objects degraded (0.910%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:28:59.925 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 39 16 54153 54137 86.7032 96.9375 0.00994291 0.0115287 2023-09-30T09:29:00.235 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:00.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:00.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:00.292 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:00.342 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:00.460 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:00.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:00 smithi019 ceph-mon[216815]: pgmap v526: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s; 1393/153078 objects degraded (0.910%); 4160/153078 objects misplaced (2.718%); 18 MiB/s, 280 objects/s recovering 2023-09-30T09:29:00.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:00 smithi019 ceph-mon[224043]: pgmap v526: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s; 1393/153078 objects degraded (0.910%); 4160/153078 objects misplaced (2.718%); 18 MiB/s, 280 objects/s recovering 2023-09-30T09:29:00.695 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:00.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:00.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:00.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:00 smithi167 ceph-mon[192880]: pgmap v526: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s; 1393/153078 objects degraded (0.910%); 4160/153078 objects misplaced (2.718%); 18 MiB/s, 280 objects/s recovering 2023-09-30T09:29:00.804 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:00.842 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:00.924 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:00.925 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:29:00.925570+0000 min lat: 0.00113601 max lat: 2.08892 avg lat: 0.0115562 2023-09-30T09:29:00.925 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:29:00.925 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 40 16 55403 55387 86.4887 78.125 0.00535128 0.0115562 2023-09-30T09:29:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:00.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:01.030 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:01.067 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:01.135 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:01.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:01.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:01.242 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:01.280 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:01.347 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:01.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:01.454 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:01.533 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:01.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 41 16 56867 56851 86.6106 91.5 0.0125504 0.0115409 2023-09-30T09:29:02.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:02 smithi167 ceph-mon[192880]: pgmap v527: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s; 1393/156306 objects degraded (0.891%); 2872/156306 objects misplaced (1.837%); 18 MiB/s, 295 objects/s recovering 2023-09-30T09:29:02.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:02 smithi019 ceph-mon[224043]: pgmap v527: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s; 1393/156306 objects degraded (0.891%); 2872/156306 objects misplaced (1.837%); 18 MiB/s, 295 objects/s recovering 2023-09-30T09:29:02.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:02 smithi019 ceph-mon[216815]: pgmap v527: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s; 1393/156306 objects degraded (0.891%); 2872/156306 objects misplaced (1.837%); 18 MiB/s, 295 objects/s recovering 2023-09-30T09:29:02.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 42 16 58228 58212 86.5736 85.0625 0.00220789 0.0114857 2023-09-30T09:29:03.438 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:03.501 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: osdmap e226: 8 total, 8 up, 8 in 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:03.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3451205410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:03.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:03.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: osdmap e226: 8 total, 8 up, 8 in 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:03.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3451205410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: osdmap e226: 8 total, 8 up, 8 in 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: failed to encode map e226 with expected crc 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:03.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:03.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3451205410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:03.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:03.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:03.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 43 16 58751 58735 85.321 32.6875 0.00644921 0.0117152 2023-09-30T09:29:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:29:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:29:03] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2023-09-30T09:29:04.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:04 smithi167 ceph-mon[192880]: pgmap v529: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 1393/163734 objects degraded (0.851%); 2178/163734 objects misplaced (1.330%); 9.9 MiB/s, 158 objects/s recovering 2023-09-30T09:29:04.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:04 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:04.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:04 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:04.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:04 smithi167 ceph-mon[192880]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:04 smithi167 ceph-mon[192880]: osdmap e227: 8 total, 8 up, 8 in 2023-09-30T09:29:04.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:04 smithi167 ceph-mon[192880]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:04 smithi167 ceph-mon[192880]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.812 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[224043]: pgmap v529: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 1393/163734 objects degraded (0.851%); 2178/163734 objects misplaced (1.330%); 9.9 MiB/s, 158 objects/s recovering 2023-09-30T09:29:04.812 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:04.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:04.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[224043]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[224043]: osdmap e227: 8 total, 8 up, 8 in 2023-09-30T09:29:04.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[224043]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[224043]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.814 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[216815]: pgmap v529: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 1393/163734 objects degraded (0.851%); 2178/163734 objects misplaced (1.330%); 9.9 MiB/s, 158 objects/s recovering 2023-09-30T09:29:04.814 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:04.814 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:04.814 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[216815]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.815 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[216815]: osdmap e227: 8 total, 8 up, 8 in 2023-09-30T09:29:04.815 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[216815]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.815 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:04 smithi019 ceph-mon[216815]: failed to encode map e227 with expected crc 2023-09-30T09:29:04.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 44 16 60088 60072 85.2809 83.5625 0.00960631 0.0117209 2023-09-30T09:29:05.176 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:05.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:05.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:05.283 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:05.314 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:05.607 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:05.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:05.727 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:05.796 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:05 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1393/175605 objects degraded (0.793%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:29:05.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:05 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1393/175605 objects degraded (0.793%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:29:05.803 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:05.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 45 16 61248 61232 84.9968 72.5 0.0108631 0.0117604 2023-09-30T09:29:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:05 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1393/175605 objects degraded (0.793%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:29:06.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:06 smithi019 ceph-mon[216815]: pgmap v531: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 1393/175605 objects degraded (0.793%); 1672/175605 objects misplaced (0.952%); 9.7 MiB/s, 155 objects/s recovering 2023-09-30T09:29:06.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:06 smithi019 ceph-mon[224043]: pgmap v531: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 1393/175605 objects degraded (0.793%); 1672/175605 objects misplaced (0.952%); 9.7 MiB/s, 155 objects/s recovering 2023-09-30T09:29:06.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 46 16 62589 62573 84.9709 83.8125 0.0101492 0.0117634 2023-09-30T09:29:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:06 smithi167 ceph-mon[192880]: pgmap v531: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 1393/175605 objects degraded (0.793%); 1672/175605 objects misplaced (0.952%); 9.7 MiB/s, 155 objects/s recovering 2023-09-30T09:29:07.131 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:07.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:07.243 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:07.281 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:07.348 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:07.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:07.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:07.550 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:07.692 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:07.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:07.765 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:29:07.765 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:29:07.765 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:63632 6.b 6:dbffd936:::benchmark_data_smithi019_355417_object63631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:29:07.747252+0000", 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.012070866, 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012105467, 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:07.766 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 63632 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.747252+0000", 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:07.767 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.747252+0000", 2023-09-30T09:29:07.768 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:07.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.768 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.768 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:07.768 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.747252+0000", 2023-09-30T09:29:07.768 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3000000000000001e-07 2023-09-30T09:29:07.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.768 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.769 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:07.769 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.747277+0000", 2023-09-30T09:29:07.769 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4456000000000001e-05 2023-09-30T09:29:07.769 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.769 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.770 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:07.770 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.747290+0000", 2023-09-30T09:29:07.770 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3281999999999999e-05 2023-09-30T09:29:07.770 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.770 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.747298+0000", 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4359999999999996e-06 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.751009+0000", 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0037117690000000002 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.771 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.751141+0000", 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000131457 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.751301+0000", 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000160335 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.755416+0000", 2023-09-30T09:29:07.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041151879999999997 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.755664+0000", 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000247758 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.773 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:63636 6.b 6:d4d330b1:::benchmark_data_smithi019_355417_object63635:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:29:07.756463+0000", 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0028597380000000001, 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002992696, 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 63636 2023-09-30T09:29:07.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.775 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:07.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:07.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.756463+0000", 2023-09-30T09:29:07.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:07.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.756463+0000", 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.756465+0000", 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.409e-06 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:07.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.756497+0000", 2023-09-30T09:29:07.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2580000000000003e-05 2023-09-30T09:29:07.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:07.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.756515+0000", 2023-09-30T09:29:07.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7439999999999999e-05 2023-09-30T09:29:07.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.778 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:07.778 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.756524+0000", 2023-09-30T09:29:07.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1700000000000003e-06 2023-09-30T09:29:07.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:07.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.756603+0000", 2023-09-30T09:29:07.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9240999999999996e-05 2023-09-30T09:29:07.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:07.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.758124+0000", 2023-09-30T09:29:07.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015213729999999999 2023-09-30T09:29:07.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:29:07.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.758280+0000", 2023-09-30T09:29:07.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015576399999999999 2023-09-30T09:29:07.781 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:07.781 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:07.781 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:07.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.781 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:63637 6.b 6.804c7d5b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-30T09:29:07.782 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:29:07.757252+0000", 2023-09-30T09:29:07.782 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0020713039999999999, 2023-09-30T09:29:07.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022770690000000001, 2023-09-30T09:29:07.782 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:07.782 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "queued for pg", 2023-09-30T09:29:07.783 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:07.783 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:07.783 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:07.783 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 63637 2023-09-30T09:29:07.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.783 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:07.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:07.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.757252+0000", 2023-09-30T09:29:07.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:07.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:07.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.757252+0000", 2023-09-30T09:29:07.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:07.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:07.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.757253+0000", 2023-09-30T09:29:07.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0300000000000001e-06 2023-09-30T09:29:07.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:07.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.757275+0000", 2023-09-30T09:29:07.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2405999999999999e-05 2023-09-30T09:29:07.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:07.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.757286+0000", 2023-09-30T09:29:07.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1333e-05 2023-09-30T09:29:07.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:07.757294+0000", 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9419999999999995e-06 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 3 2023-09-30T09:29:07.788 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:29:07.806 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:07.885 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:07.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 47 16 63828 63812 84.8106 77.4375 0.0130835 0.0117848 2023-09-30T09:29:08.007 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:29:08.159 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:08.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:08.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:08 smithi019 conmon[216791]: 2023-09-30T09:29:08.012+0000 7faf52971700 -1 mon.a@0(leader).osd e227 definitely_dead 0 2023-09-30T09:29:08.274 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:08.304 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:29:08.352 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:29:08.396 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:29:08.424 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:08.446 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:29:08.483 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:29:08.525 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:29:08.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[216815]: pgmap v532: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 1393/177723 objects degraded (0.784%); 718/177723 objects misplaced (0.404%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:29:08.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2172539972' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:29:08.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:29:08.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[216815]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-30T09:29:08.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3764908747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:08.584 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:29:08.585 INFO:teuthology.orchestra.run.smithi019.stderr:marked out osd.2. 2023-09-30T09:29:08.613 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:29:08.614 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:29:08.614 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:29:08.614 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:29:08.614 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:29:08.615 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:29:08.615 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:29:08.615 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:29:08.615 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:29:08.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[224043]: pgmap v532: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 1393/177723 objects degraded (0.784%); 718/177723 objects misplaced (0.404%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:29:08.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2172539972' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:29:08.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:29:08.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[224043]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-30T09:29:08.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3764908747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:08.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 48 16 64560 64544 83.9971 45.75 0.00297209 0.0118142 2023-09-30T09:29:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:08 smithi167 ceph-mon[192880]: pgmap v532: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 1393/177723 objects degraded (0.784%); 718/177723 objects misplaced (0.404%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:29:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2172539972' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:29:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:08 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:29:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:08 smithi167 ceph-mon[192880]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-30T09:29:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3764908747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:09.508 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:09.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:09.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:09.612 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:09.645 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:09.741 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:09.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:09.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-30T09:29:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: osdmap e228: 8 total, 8 up, 7 in 2023-09-30T09:29:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: osdmap e228: 8 total, 8 up, 7 in 2023-09-30T09:29:09.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:29:09.846 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: osdmap e228: 8 total, 8 up, 7 in 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.870 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.871 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.871 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.871 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: failed to encode map e228 with expected crc 2023-09-30T09:29:09.871 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:29:09.872 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:09.922 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:09.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:09.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:09.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 49 16 64560 64544 82.2836 0 - 0.0118142 2023-09-30T09:29:10.026 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:10.053 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:10.095 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:10.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:10.200 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:10.227 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:10.273 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:10.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:10.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:10.380 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:10.407 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:10.511 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:10.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:10.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:10.555 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:29:10.614 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:10.723 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: pgmap v534: 109 pgs: 21 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 82 MiB/s wr, 1.32k op/s; 1393/189894 objects degraded (0.734%); 718/189894 objects misplaced (0.378%); 6.9 MiB/s, 109 objects/s recovering 2023-09-30T09:29:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:29:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: osdmap e229: 8 total, 8 up, 7 in 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:10 smithi167 ceph-mon[192880]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: pgmap v534: 109 pgs: 21 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 82 MiB/s wr, 1.32k op/s; 1393/189894 objects degraded (0.734%); 718/189894 objects misplaced (0.378%); 6.9 MiB/s, 109 objects/s recovering 2023-09-30T09:29:10.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:29:10.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: osdmap e229: 8 total, 8 up, 7 in 2023-09-30T09:29:10.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[216815]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: pgmap v534: 109 pgs: 21 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 82 MiB/s wr, 1.32k op/s; 1393/189894 objects degraded (0.734%); 718/189894 objects misplaced (0.378%); 6.9 MiB/s, 109 objects/s recovering 2023-09-30T09:29:10.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:29:10.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.800 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.801 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.801 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.801 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.801 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.801 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: osdmap e229: 8 total, 8 up, 7 in 2023-09-30T09:29:10.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.802 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:10 smithi019 ceph-mon[224043]: failed to encode map e229 with expected crc 2023-09-30T09:29:10.856 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:10.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:10.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 50 16 64605 64589 80.6948 1.40625 0.00506067 0.0122369 2023-09-30T09:29:10.932 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:10.983 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:11.012 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:11.061 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:11.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:11.167 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:11.227 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:11.505 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:11.747 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:11.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[224043]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[224043]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[224043]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[224043]: osdmap e230: 8 total, 8 up, 7 in 2023-09-30T09:29:11.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[224043]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[224043]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[224043]: 6.12 starting backfill to osd.3 from (0'0,217'2510] MIN to 227'3074 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[216815]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[216815]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[216815]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[216815]: osdmap e230: 8 total, 8 up, 7 in 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[216815]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[216815]: failed to encode map e230 with expected crc 2023-09-30T09:29:11.750 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:11 smithi019 ceph-mon[216815]: 6.12 starting backfill to osd.3 from (0'0,217'2510] MIN to 227'3074 2023-09-30T09:29:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:11.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:11.852 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:12.004 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 51 16 64617 64601 79.1279 0.75 0.00655842 0.0122356 2023-09-30T09:29:12.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:11 smithi167 ceph-mon[192880]: failed to encode map e230 with expected crc 2023-09-30T09:29:12.067 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:11 smithi167 ceph-mon[192880]: failed to encode map e230 with expected crc 2023-09-30T09:29:12.067 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:11 smithi167 ceph-mon[192880]: failed to encode map e230 with expected crc 2023-09-30T09:29:12.067 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:11 smithi167 ceph-mon[192880]: osdmap e230: 8 total, 8 up, 7 in 2023-09-30T09:29:12.067 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:11 smithi167 ceph-mon[192880]: failed to encode map e230 with expected crc 2023-09-30T09:29:12.067 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:11 smithi167 ceph-mon[192880]: failed to encode map e230 with expected crc 2023-09-30T09:29:12.067 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:11 smithi167 ceph-mon[192880]: 6.12 starting backfill to osd.3 from (0'0,217'2510] MIN to 227'3074 2023-09-30T09:29:12.068 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:12.182 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:12.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:12.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:12.288 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:12.321 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:12.685 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:12.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:12.746 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:29:12.789 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:12.830 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:12.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: pgmap v537: 109 pgs: 22 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 14 KiB/s rd, 53 MiB/s wr, 856 op/s; 1393/190830 objects degraded (0.730%); 718/190830 objects misplaced (0.376%); 5.0 MiB/s, 79 objects/s recovering 2023-09-30T09:29:12.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1393/190830 objects degraded (0.730%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:29:12.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: osdmap e231: 8 total, 8 up, 7 in 2023-09-30T09:29:12.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[216815]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: pgmap v537: 109 pgs: 22 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 14 KiB/s rd, 53 MiB/s wr, 856 op/s; 1393/190830 objects degraded (0.730%); 718/190830 objects misplaced (0.376%); 5.0 MiB/s, 79 objects/s recovering 2023-09-30T09:29:12.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1393/190830 objects degraded (0.730%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:29:12.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: osdmap e231: 8 total, 8 up, 7 in 2023-09-30T09:29:12.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.898 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.898 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:12 smithi019 ceph-mon[224043]: failed to encode map e231 with expected crc 2023-09-30T09:29:12.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 52 16 64624 64608 77.6153 0.4375 1.62995 0.0123374 2023-09-30T09:29:12.952 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:12.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:12.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: pgmap v537: 109 pgs: 22 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 14 KiB/s rd, 53 MiB/s wr, 856 op/s; 1393/190830 objects degraded (0.730%); 718/190830 objects misplaced (0.376%); 5.0 MiB/s, 79 objects/s recovering 2023-09-30T09:29:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1393/190830 objects degraded (0.730%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:29:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: failed to encode map e231 with expected crc 2023-09-30T09:29:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: failed to encode map e231 with expected crc 2023-09-30T09:29:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: failed to encode map e231 with expected crc 2023-09-30T09:29:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: failed to encode map e231 with expected crc 2023-09-30T09:29:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: osdmap e231: 8 total, 8 up, 7 in 2023-09-30T09:29:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: failed to encode map e231 with expected crc 2023-09-30T09:29:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: failed to encode map e231 with expected crc 2023-09-30T09:29:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:12 smithi167 ceph-mon[192880]: failed to encode map e231 with expected crc 2023-09-30T09:29:13.059 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:13.093 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:13.500 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:13.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:13.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:13.604 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:13.633 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:13.678 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:13.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:13.783 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:13.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: osdmap e232: 8 total, 8 up, 7 in 2023-09-30T09:29:13.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: osdmap e232: 8 total, 8 up, 7 in 2023-09-30T09:29:13.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:13 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.810 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:13.856 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:13.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:13.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:13.927 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 53 16 64624 64608 76.1514 0 - 0.0123374 2023-09-30T09:29:13.961 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:13.985 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.985 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: osdmap e232: 8 total, 8 up, 7 in 2023-09-30T09:29:13.985 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.986 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.986 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.986 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: failed to encode map e232 with expected crc 2023-09-30T09:29:13.986 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.986 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.986 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:13 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:13.987 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:14.031 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:14.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:14.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:14.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:29:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:29:13] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2023-09-30T09:29:14.139 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:14.175 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:14.215 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:14.238 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:14.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:14.342 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:14.368 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:14.411 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:14.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:14.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:14.516 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:14.556 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:14.709 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:29:14.709 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:29:14.717 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:29:14.927 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 54 16 64624 64608 74.7418 0 - 0.0123374 2023-09-30T09:29:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:14 smithi167 ceph-mon[192880]: pgmap v540: 109 pgs: 22 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9.2 MiB/s wr, 147 op/s; 3903/191670 objects degraded (2.036%); 617/191670 objects misplaced (0.322%); 2.3 MiB/s, 36 objects/s recovering 2023-09-30T09:29:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2067360825' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:29:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:14 smithi019 ceph-mon[216815]: pgmap v540: 109 pgs: 22 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9.2 MiB/s wr, 147 op/s; 3903/191670 objects degraded (2.036%); 617/191670 objects misplaced (0.322%); 2.3 MiB/s, 36 objects/s recovering 2023-09-30T09:29:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2067360825' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:29:15.076 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:14 smithi019 ceph-mon[224043]: pgmap v540: 109 pgs: 22 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9.2 MiB/s wr, 147 op/s; 3903/191670 objects degraded (2.036%); 617/191670 objects misplaced (0.322%); 2.3 MiB/s, 36 objects/s recovering 2023-09-30T09:29:15.076 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2067360825' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:29:15.639 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:29:15.639 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:29:15.640 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-09-30T09:29:15.640 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:29:15.820 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:15.823 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:15 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:15.823 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2067360825' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:29:15.823 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:15 smithi019 ceph-mon[216815]: failed to encode map e233 with expected crc 2023-09-30T09:29:15.824 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:15 smithi019 ceph-mon[216815]: osdmap e233: 8 total, 8 up, 7 in 2023-09-30T09:29:15.824 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:15 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:15.824 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2067360825' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:29:15.824 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:15 smithi019 ceph-mon[224043]: failed to encode map e233 with expected crc 2023-09-30T09:29:15.824 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:15 smithi019 ceph-mon[224043]: osdmap e233: 8 total, 8 up, 7 in 2023-09-30T09:29:15.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:15.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:15.926 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:15.927 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 55 16 64624 64608 73.3834 0 - 0.0123374 2023-09-30T09:29:15.951 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:15 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:15.951 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2067360825' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:29:15.951 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:15 smithi167 ceph-mon[192880]: failed to encode map e233 with expected crc 2023-09-30T09:29:15.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:15 smithi167 ceph-mon[192880]: osdmap e233: 8 total, 8 up, 7 in 2023-09-30T09:29:15.955 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:16.003 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:16.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:16.108 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:16.140 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:16.187 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:29:16.258 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:16.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:16.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:16.362 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:16.406 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:16.926 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:16.927 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 56 16 64624 64608 72.0735 0 - 0.0123374 2023-09-30T09:29:16.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:16.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:16 smithi167 ceph-mon[192880]: pgmap v542: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4.3 MiB/s wr, 69 op/s; 22058/191775 objects degraded (11.502%); 3921/191775 objects misplaced (2.045%); 11 MiB/s, 2 keys/s, 171 objects/s recovering 2023-09-30T09:29:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:16 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:29:17.030 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:17.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:16 smithi019 ceph-mon[224043]: pgmap v542: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4.3 MiB/s wr, 69 op/s; 22058/191775 objects degraded (11.502%); 3921/191775 objects misplaced (2.045%); 11 MiB/s, 2 keys/s, 171 objects/s recovering 2023-09-30T09:29:17.065 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:16 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:29:17.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:16 smithi019 ceph-mon[216815]: pgmap v542: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4.3 MiB/s wr, 69 op/s; 22058/191775 objects degraded (11.502%); 3921/191775 objects misplaced (2.045%); 11 MiB/s, 2 keys/s, 171 objects/s recovering 2023-09-30T09:29:17.066 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:16 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:29:17.070 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:17.103 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:17.144 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:17.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:17.151 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:17.168 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:29:17.206 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:29:17.231 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:29:17.250 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:17.283 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:17.585 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:17.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:17.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:17.689 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:17.716 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:17.762 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:17.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:17.780 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1538475410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:17.780 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1538475410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:17.866 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:17.904 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:17.927 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 57 16 64624 64608 70.8095 0 - 0.0123374 2023-09-30T09:29:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1538475410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:18.280 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:29:18.314 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:29:18.338 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:29:18.365 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:29:18.388 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:29:18.389 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:29:18.389 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:29:18.389 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:29:18.390 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:29:18.390 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:29:18.390 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:29:18.390 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:29:18.390 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:29:18.784 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:29:19.389 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:18 smithi167 ceph-mon[192880]: pgmap v543: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s; 22058/191775 objects degraded (11.502%); 3921/191775 objects misplaced (2.045%); 8.0 MiB/s, 1 keys/s, 129 objects/s recovering 2023-09-30T09:29:19.458 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:19.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22058/191775 objects degraded (11.502%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:19.459 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 58 16 65142 65126 70.1465 5.39583 0.0119597 0.014251 2023-09-30T09:29:19.459 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:18 smithi019 ceph-mon[216815]: pgmap v543: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s; 22058/191775 objects degraded (11.502%); 3921/191775 objects misplaced (2.045%); 8.0 MiB/s, 1 keys/s, 129 objects/s recovering 2023-09-30T09:29:19.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:19.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22058/191775 objects degraded (11.502%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:19.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:18 smithi019 ceph-mon[224043]: pgmap v543: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s; 22058/191775 objects degraded (11.502%); 3921/191775 objects misplaced (2.045%); 8.0 MiB/s, 1 keys/s, 129 objects/s recovering 2023-09-30T09:29:19.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:19.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22058/191775 objects degraded (11.502%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:19.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 59 16 65710 65694 69.5594 35.5 0.0018846 0.0142866 2023-09-30T09:29:20.199 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:20.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:20.303 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:20.378 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[216815]: pgmap v544: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 196 op/s; 30084/195048 objects degraded (15.424%); 2555/195048 objects misplaced (1.310%); 18 MiB/s, 1 keys/s, 285 objects/s recovering 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[216815]: osdmap e234: 8 total, 8 up, 7 in 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[216815]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[216815]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[216815]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[224043]: pgmap v544: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 196 op/s; 30084/195048 objects degraded (15.424%); 2555/195048 objects misplaced (1.310%); 18 MiB/s, 1 keys/s, 285 objects/s recovering 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[224043]: osdmap e234: 8 total, 8 up, 7 in 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[224043]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[224043]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.379 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:20 smithi019 ceph-mon[224043]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.391 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:20.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:20 smithi167 ceph-mon[192880]: pgmap v544: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 196 op/s; 30084/195048 objects degraded (15.424%); 2555/195048 objects misplaced (1.310%); 18 MiB/s, 1 keys/s, 285 objects/s recovering 2023-09-30T09:29:20.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:20 smithi167 ceph-mon[192880]: osdmap e234: 8 total, 8 up, 7 in 2023-09-30T09:29:20.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:20 smithi167 ceph-mon[192880]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:20 smithi167 ceph-mon[192880]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:20 smithi167 ceph-mon[192880]: failed to encode map e234 with expected crc 2023-09-30T09:29:20.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:29:20.927805+0000 min lat: 0.00113601 max lat: 9.94718 avg lat: 0.0144611 2023-09-30T09:29:20.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:29:20.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 60 16 66398 66382 69.1168 43 0.00820936 0.0144611 2023-09-30T09:29:20.973 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:21.030 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:21.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:21.131 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:29:21.131 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:29:21.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:29:21.131 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:29:21.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.132 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:29783 6.c 6:3a555814:::benchmark_data_smithi019_355417_object29782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-30T09:29:21.132 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:34.755341+0000", 2023-09-30T09:29:21.132 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 46.369682314999999, 2023-09-30T09:29:21.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22802413899999999, 2023-09-30T09:29:21.132 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.133 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.133 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.133 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.133 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.133 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 29783 2023-09-30T09:29:21.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.133 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755341+0000", 2023-09-30T09:29:21.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755341+0000", 2023-09-30T09:29:21.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755342+0000", 2023-09-30T09:29:21.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.091e-06 2023-09-30T09:29:21.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755366+0000", 2023-09-30T09:29:21.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3669999999999999e-05 2023-09-30T09:29:21.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755378+0000", 2023-09-30T09:29:21.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2368000000000001e-05 2023-09-30T09:29:21.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755386+0000", 2023-09-30T09:29:21.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1070000000000005e-06 2023-09-30T09:29:21.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.138 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755456+0000", 2023-09-30T09:29:21.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9789000000000006e-05 2023-09-30T09:29:21.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755608+0000", 2023-09-30T09:29:21.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015203300000000001 2023-09-30T09:29:21.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.755754+0000", 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000145539 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.757642+0000", 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018885169999999999 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.758394+0000", 2023-09-30T09:29:21.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00075221299999999995 2023-09-30T09:29:21.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.983276+0000", 2023-09-30T09:29:21.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22488128499999999 2023-09-30T09:29:21.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.983328+0000", 2023-09-30T09:29:21.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1898999999999999e-05 2023-09-30T09:29:21.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.983365+0000", 2023-09-30T09:29:21.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7628e-05 2023-09-30T09:29:21.143 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.143 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.144 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.144 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:29793 6.c 6:3ad7ce81:::benchmark_data_smithi019_355417_object29792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-30T09:29:21.144 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:34.762983+0000", 2023-09-30T09:29:21.144 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 46.362040358999998, 2023-09-30T09:29:21.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22051864800000001, 2023-09-30T09:29:21.145 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.145 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.145 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.145 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.145 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.145 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 29793 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.762983+0000", 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.762983+0000", 2023-09-30T09:29:21.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.762984+0000", 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3650000000000001e-06 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.763016+0000", 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1279999999999999e-05 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.763030+0000", 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4606e-05 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.763039+0000", 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4649999999999994e-06 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.763097+0000", 2023-09-30T09:29:21.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8511999999999998e-05 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.763233+0000", 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000135556 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.763333+0000", 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000100554 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.764532+0000", 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011986 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.764670+0000", 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013851799999999999 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.983425+0000", 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.218754684 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.983470+0000", 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5225000000000002e-05 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.983502+0000", 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1282999999999998e-05 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.151 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:29798 6.c 6:3cc42497:::benchmark_data_smithi019_355417_object29797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:34.768809+0000", 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 46.356214794000003, 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.216050616, 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.152 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.153 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.153 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 29798 2023-09-30T09:29:21.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.153 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.768809+0000", 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.768809+0000", 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.154 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.768810+0000", 2023-09-30T09:29:21.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.235e-06 2023-09-30T09:29:21.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.768828+0000", 2023-09-30T09:29:21.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8666000000000001e-05 2023-09-30T09:29:21.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.768841+0000", 2023-09-30T09:29:21.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2645e-05 2023-09-30T09:29:21.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.157 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.768847+0000", 2023-09-30T09:29:21.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9109999999999998e-06 2023-09-30T09:29:21.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.157 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.157 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.768900+0000", 2023-09-30T09:29:21.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3057999999999999e-05 2023-09-30T09:29:21.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.158 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.158 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.158 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.769024+0000", 2023-09-30T09:29:21.158 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012408100000000001 2023-09-30T09:29:21.158 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.158 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.158 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:29:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.769132+0000", 2023-09-30T09:29:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010804800000000001 2023-09-30T09:29:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.770175+0000", 2023-09-30T09:29:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0010423330000000001 2023-09-30T09:29:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.770389+0000", 2023-09-30T09:29:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021477200000000001 2023-09-30T09:29:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.984765+0000", 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21437558900000001 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.984821+0000", 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6436e-05 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.162 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:34.984859+0000", 2023-09-30T09:29:21.162 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7842000000000001e-05 2023-09-30T09:29:21.162 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.162 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.162 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.162 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.162 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.163 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35934 6.16 6:6a0d4ce1:::benchmark_data_smithi019_355417_object35933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:29:21.163 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:29:21.163 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 41.865125396000003, 2023-09-30T09:29:21.163 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.022920797, 2023-09-30T09:29:21.163 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.163 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:29:21.163 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.164 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.164 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.164 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35934 2023-09-30T09:29:21.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.164 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:29:21.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.165 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.165 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:29:21.166 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.166 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.166 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259900+0000", 2023-09-30T09:29:21.166 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.573e-06 2023-09-30T09:29:21.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.167 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.167 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.167 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259928+0000", 2023-09-30T09:29:21.167 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8756999999999999e-05 2023-09-30T09:29:21.167 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.167 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.167 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.168 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259941+0000", 2023-09-30T09:29:21.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3066e-05 2023-09-30T09:29:21.168 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.168 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.168 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.168 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259951+0000", 2023-09-30T09:29:21.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8050000000000001e-06 2023-09-30T09:29:21.168 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311357+0000", 2023-09-30T09:29:21.169 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.051405992999999997 2023-09-30T09:29:21.169 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:29:21.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311380+0000", 2023-09-30T09:29:21.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.283e-05 2023-09-30T09:29:21.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.170 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.170 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.282746+0000", 2023-09-30T09:29:21.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97136601 2023-09-30T09:29:21.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.171 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.171 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.282819+0000", 2023-09-30T09:29:21.171 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2762999999999994e-05 2023-09-30T09:29:21.171 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.171 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.171 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.171 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.172 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.172 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35936 6.2 6:47063778:::benchmark_data_smithi019_355417_object35935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:29:21.172 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:29:21.172 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 41.865057413999999, 2023-09-30T09:29:21.172 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0116794739999999, 2023-09-30T09:29:21.172 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.172 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:29:21.172 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.173 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.173 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.173 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35936 2023-09-30T09:29:21.173 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.173 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.173 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.174 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:29:21.174 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.174 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.174 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.174 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.174 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:29:21.174 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.174 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.175 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.175 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.175 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:29:21.175 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0800000000000005e-07 2023-09-30T09:29:21.175 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.175 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259989+0000", 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2908000000000002e-05 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259996+0000", 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8650000000000003e-06 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260001+0000", 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2310000000000004e-06 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.310543+0000", 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.050541517000000001 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:29:21.178 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.310552+0000", 2023-09-30T09:29:21.178 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2320000000000007e-06 2023-09-30T09:29:21.178 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.178 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.178 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.178 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271569+0000", 2023-09-30T09:29:21.178 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96101677600000002 2023-09-30T09:29:21.179 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.179 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.179 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271645+0000", 2023-09-30T09:29:21.179 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6437e-05 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35941 6.6 6:62f338d5:::benchmark_data_smithi019_355417_object35940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 41.865012344999997, 2023-09-30T09:29:21.180 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0179260409999999, 2023-09-30T09:29:21.181 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.181 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.181 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.181 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.181 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.181 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35941 2023-09-30T09:29:21.181 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.181 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.182 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:29:21.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.182 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.182 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.183 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:29:21.183 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.183 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.183 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.183 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.183 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:29:21.183 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8299999999999998e-07 2023-09-30T09:29:21.183 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.184 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260036+0000", 2023-09-30T09:29:21.184 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4766999999999999e-05 2023-09-30T09:29:21.184 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.184 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260043+0000", 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.387e-06 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260047+0000", 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9030000000000003e-06 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311366+0000", 2023-09-30T09:29:21.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.051318726000000002 2023-09-30T09:29:21.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.186 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:29:21.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311377+0000", 2023-09-30T09:29:21.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0621999999999999e-05 2023-09-30T09:29:21.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268227+0000", 2023-09-30T09:29:21.187 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95684991799999997 2023-09-30T09:29:21.187 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268372+0000", 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014511 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268552+0000", 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017976400000000001 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271663+0000", 2023-09-30T09:29:21.188 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031116619999999998 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271672+0000", 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3180000000000007e-06 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277851+0000", 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061798599999999997 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277902+0000", 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0611e-05 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277937+0000", 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5009999999999999e-05 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35951 6.6 6:612781fc:::benchmark_data_smithi019_355417_object35950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 41.793725019999997, 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94995752200000005, 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35951 2023-09-30T09:29:21.191 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.192 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331301+0000", 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9739999999999998e-06 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331348+0000", 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7113000000000001e-05 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.193 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331384+0000", 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5253000000000003e-05 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331402+0000", 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8756000000000001e-05 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331491+0000", 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8213000000000004e-05 2023-09-30T09:29:21.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331502+0000", 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1328999999999999e-05 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270400+0000", 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.93889793200000005 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270562+0000", 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016258000000000001 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270732+0000", 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016932199999999999 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.197 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271969+0000", 2023-09-30T09:29:21.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00123719 2023-09-30T09:29:21.197 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.197 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.197 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274748+0000", 2023-09-30T09:29:21.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002778689 2023-09-30T09:29:21.198 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.198 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.198 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.198 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281163+0000", 2023-09-30T09:29:21.198 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0064157759999999998 2023-09-30T09:29:21.198 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.198 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281223+0000", 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9302000000000002e-05 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281256+0000", 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3093000000000003e-05 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.199 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35953 6.12 6:49e3e35d:::benchmark_data_smithi019_355417_object35952:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.462435+0000", 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 41.662588415000002, 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.81799533899999999, 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35953 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.200 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.462435+0000", 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.462435+0000", 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.462438+0000", 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0290000000000001e-06 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.462484+0000", 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6443999999999998e-05 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.506488+0000", 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.044003674 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.506506+0000", 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7856e-05 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.508162+0000", 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00165642 2023-09-30T09:29:21.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.508180+0000", 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7660999999999999e-05 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274328+0000", 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.76614779300000002 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274488+0000", 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016043900000000001 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274669+0000", 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000180977 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.276589+0000", 2023-09-30T09:29:21.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019196059999999999 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.279949+0000", 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033604860000000002 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.280333+0000", 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00038411400000000002 2023-09-30T09:29:21.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.280390+0000", 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6273999999999997e-05 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.280430+0000", 2023-09-30T09:29:21.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0565999999999998e-05 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:46912 6.c 6:3e9d6e4a:::benchmark_data_smithi019_355417_object46911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.781949+0000", 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.343074316999999, 2023-09-30T09:29:21.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.3366576, 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 46912 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.781949+0000", 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.781949+0000", 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.781950+0000", 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0979999999999999e-06 2023-09-30T09:29:21.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.781980+0000", 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.004e-05 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.781992+0000", 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1980999999999999e-05 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.782000+0000", 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7749999999999993e-06 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.783937+0000", 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001937516 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.784076+0000", 2023-09-30T09:29:21.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013808399999999999 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.784241+0000", 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016556200000000001 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.786276+0000", 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020351779999999999 2023-09-30T09:29:21.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790401+0000", 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041247519999999998 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118496+0000", 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32809468000000003 2023-09-30T09:29:21.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118563+0000", 2023-09-30T09:29:21.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6894e-05 2023-09-30T09:29:21.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118607+0000", 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4039999999999998e-05 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35130.0:46918 6.f e223/206 6:f7531793:::benchmark_data_smithi019_355417_object46917:head v 223'2944, mlcod=223'2942)", 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.788105+0000", 2023-09-30T09:29:21.216 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.336918439000002, 2023-09-30T09:29:21.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33100858300000002, 2023-09-30T09:29:21.217 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.217 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.217 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.217 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.217 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.217 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788105+0000", 2023-09-30T09:29:21.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.218 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.218 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.218 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.218 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788105+0000", 2023-09-30T09:29:21.218 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.218 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.219 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.219 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.219 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788106+0000", 2023-09-30T09:29:21.219 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5499999999999996e-07 2023-09-30T09:29:21.219 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.219 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.219 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.219 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788132+0000", 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5718999999999999e-05 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788145+0000", 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3745e-05 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.220 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.221 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788154+0000", 2023-09-30T09:29:21.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7790000000000001e-06 2023-09-30T09:29:21.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.221 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.221 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.221 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788251+0000", 2023-09-30T09:29:21.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6451000000000002e-05 2023-09-30T09:29:21.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788276+0000", 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5712e-05 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.793334+0000", 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050580210000000002 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119114+0000", 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32577920100000002 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:46913 6.b 6:d8760e11:::benchmark_data_smithi019_355417_object46912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.788219+0000", 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.336804767, 2023-09-30T09:29:21.223 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33233470999999998, 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 46913 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.224 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788219+0000", 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788219+0000", 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788220+0000", 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6670000000000001e-06 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788246+0000", 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5432999999999999e-05 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.226 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788258+0000", 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2565000000000001e-05 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788266+0000", 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7430000000000002e-06 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788480+0000", 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000214157 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788610+0000", 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012949399999999999 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.228 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788753+0000", 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000143206 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118794+0000", 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33004094 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.120419+0000", 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0016254559999999999 2023-09-30T09:29:21.229 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.120459+0000", 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9419999999999999e-05 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.120514+0000", 2023-09-30T09:29:21.230 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5331999999999998e-05 2023-09-30T09:29:21.231 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.231 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.231 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.231 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.120553+0000", 2023-09-30T09:29:21.231 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9297e-05 2023-09-30T09:29:21.231 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.232 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.232 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.232 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.232 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.232 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:46915 6.6 6:61278229:::benchmark_data_smithi019_355417_object46914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.788290+0000", 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.336732988000001, 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33077383199999999, 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.233 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 46915 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788290+0000", 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.234 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788290+0000", 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788291+0000", 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1099999999999996e-07 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788313+0000", 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1744999999999999e-05 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.235 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788322+0000", 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1169999999999993e-06 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788327+0000", 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5470000000000004e-06 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.236 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.789925+0000", 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001597705 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790050+0000", 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012517100000000001 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790182+0000", 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013138700000000001 2023-09-30T09:29:21.237 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.791888+0000", 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0017063709999999999 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118969+0000", 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32708147199999998 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.238 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118982+0000", 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2227e-05 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119027+0000", 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5323e-05 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.239 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119064+0000", 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7255999999999997e-05 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:46917 6.c 6:3c3a6fe1:::benchmark_data_smithi019_355417_object46916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.788344+0000", 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.336679858, 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.332865205, 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.240 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.241 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.241 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.241 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.241 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 46917 2023-09-30T09:29:21.241 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.241 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.241 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.241 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.242 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788344+0000", 2023-09-30T09:29:21.242 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.242 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.242 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.242 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.242 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788344+0000", 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788344+0000", 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2899999999999999e-07 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788365+0000", 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1483000000000001e-05 2023-09-30T09:29:21.243 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788372+0000", 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7000000000000002e-06 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788377+0000", 2023-09-30T09:29:21.244 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.118e-06 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790434+0000", 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020564929999999999 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790561+0000", 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000127248 2023-09-30T09:29:21.245 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790704+0000", 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000142681 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118649+0000", 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32794527499999998 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.246 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118661+0000", 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1939999999999999e-05 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.121113+0000", 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024524909999999998 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.247 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.121162+0000", 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8804000000000002e-05 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.121209+0000", 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6542999999999998e-05 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.248 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:46922 6.2 6:41b29443:::benchmark_data_smithi019_355417_object46921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.788393+0000", 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.336630731, 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33374691699999998, 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 46922 2023-09-30T09:29:21.249 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788393+0000", 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788393+0000", 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.250 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788393+0000", 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.15e-07 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788414+0000", 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1327000000000001e-05 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788421+0000", 2023-09-30T09:29:21.251 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.691e-06 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.788426+0000", 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1089999999999997e-06 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790251+0000", 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001824862 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.252 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790381+0000", 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000130123 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790522+0000", 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014069700000000001 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119349+0000", 2023-09-30T09:29:21.253 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32882678999999998 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119406+0000", 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7670999999999997e-05 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.122040+0000", 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026335180000000001 2023-09-30T09:29:21.254 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.122099+0000", 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9630000000000003e-05 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.122140+0000", 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0083999999999998e-05 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.255 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop_reply(client.35130.0:46912 6.c e223/206 ondisk, result = 0)", 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.790446+0000", 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.334577637999999, 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.328168395, 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.256 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790446+0000", 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790446+0000", 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790446+0000", 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0500000000000003e-07 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.257 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790451+0000", 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4150000000000001e-06 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790459+0000", 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7350000000000002e-06 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.258 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.790466+0000", 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1970000000000001e-06 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118470+0000", 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.328004254 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118488+0000", 2023-09-30T09:29:21.259 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7906000000000001e-05 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118614+0000", 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012628300000000001 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop_reply(client.35130.0:46913 6.b e223/206 ondisk, result = 0)", 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.792578+0000", 2023-09-30T09:29:21.260 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.332445599, 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32784967500000001, 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.792578+0000", 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.261 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.792578+0000", 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.792578+0000", 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2900000000000003e-07 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.792582+0000", 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0679999999999998e-06 2023-09-30T09:29:21.262 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118403+0000", 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32582087500000001 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118409+0000", 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0850000000000002e-06 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.263 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.120405+0000", 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001995737 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.120413+0000", 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8590000000000005e-06 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.120427+0000", 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4422e-05 2023-09-30T09:29:21.264 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop_reply(client.35130.0:46922 6.2 e223/213 ondisk, result = 0)", 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.792608+0000", 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.332415281999999, 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.326748174, 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.265 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.792608+0000", 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.792608+0000", 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.266 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.792609+0000", 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5999999999999999e-07 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.792612+0000", 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9639999999999999e-06 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118307+0000", 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32569595000000001 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.267 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118322+0000", 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5038000000000001e-05 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119336+0000", 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001013353 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.268 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119343+0000", 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.063e-06 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119356+0000", 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3346000000000001e-05 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.269 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop_reply(client.35130.0:46915 6.6 e223/213 ondisk, result = 0)", 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:53.794505+0000", 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.330518313999999, 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32456241899999999, 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.794505+0000", 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.794505+0000", 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.794506+0000", 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3900000000000004e-07 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:53.794511+0000", 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.741e-06 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118718+0000", 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32420710600000002 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118728+0000", 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9839999999999996e-06 2023-09-30T09:29:21.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118956+0000", 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000228273 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.118964+0000", 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6899999999999992e-06 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:54.119067+0000", 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000103886 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:64651 6.f 6:f1713880:::benchmark_data_smithi019_355417_object64650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:29:18.294049+0000", 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.83097486, 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.29963257399999998, 2023-09-30T09:29:21.274 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 64651 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.294049+0000", 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.275 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.294049+0000", 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.294050+0000", 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4619999999999999e-06 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.276 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.294074+0000", 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4000999999999999e-05 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.294092+0000", 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7669e-05 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.294101+0000", 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7680000000000003e-06 2023-09-30T09:29:21.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.544544+0000", 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25044291400000002 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.544642+0000", 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8107000000000001e-05 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6", 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.544793+0000", 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015011399999999999 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.548924+0000", 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041313519999999996 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.554156+0000", 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0052318130000000001 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.591785+0000", 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.037629322 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.591850+0000", 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5257000000000002e-05 2023-09-30T09:29:21.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.593681+0000", 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018307950000000001 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:64678 6.f 6:ff355fa0:::benchmark_data_smithi019_355417_object64677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:29:18.371455+0000", 2023-09-30T09:29:21.281 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.753568215, 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.224308384, 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 64678 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.282 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:29:21.283 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.371455+0000", 2023-09-30T09:29:21.283 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.283 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.283 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.283 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:29:21.283 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.371455+0000", 2023-09-30T09:29:21.283 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:29:21.283 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.371457+0000", 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.581e-06 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.371482+0000", 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4779000000000001e-05 2023-09-30T09:29:21.284 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.371495+0000", 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3858e-05 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.371504+0000", 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0189999999999995e-06 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.546315+0000", 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.174810245 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.546418+0000", 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010317100000000001 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6", 2023-09-30T09:29:21.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.546525+0000", 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000107129 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.550896+0000", 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043709990000000004 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.554220+0000", 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033240829999999998 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.593758+0000", 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.039537561999999998 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.593824+0000", 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6216000000000004e-05 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:29:21.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:29:21.289 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:18.595764+0000", 2023-09-30T09:29:21.289 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019397419999999999 2023-09-30T09:29:21.289 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.289 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.289 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.289 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:29:21.289 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:29:21.289 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:29:21.289 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:21 smithi019 ceph-mon[224043]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:21 smithi019 ceph-mon[224043]: osdmap e235: 8 total, 8 up, 7 in 2023-09-30T09:29:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:21 smithi019 ceph-mon[224043]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:21 smithi019 ceph-mon[224043]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:21 smithi019 ceph-mon[216815]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:21 smithi019 ceph-mon[216815]: osdmap e235: 8 total, 8 up, 7 in 2023-09-30T09:29:21.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:21 smithi019 ceph-mon[216815]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:21 smithi019 ceph-mon[216815]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.338 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:21.420 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:21.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:21.462 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:29:21.475 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:29:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:21 smithi167 ceph-mon[192880]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:21 smithi167 ceph-mon[192880]: osdmap e235: 8 total, 8 up, 7 in 2023-09-30T09:29:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:21 smithi167 ceph-mon[192880]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:21 smithi167 ceph-mon[192880]: failed to encode map e235 with expected crc 2023-09-30T09:29:21.532 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:21.585 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:21.626 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:21.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:21.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:21.732 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:21.789 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:21.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 61 16 67360 67344 68.9694 60.125 0.0153531 0.0144945 2023-09-30T09:29:22.204 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:29:22.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:22 smithi167 ceph-mon[192880]: pgmap v547: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 189 op/s; 30084/195231 objects degraded (15.409%); 2555/195231 objects misplaced (1.309%); 14 MiB/s, 213 objects/s recovering 2023-09-30T09:29:22.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:22 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:22.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:22 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:22.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:22 smithi167 ceph-mon[192880]: from='client.35193 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:22 smithi019 ceph-mon[216815]: pgmap v547: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 189 op/s; 30084/195231 objects degraded (15.409%); 2555/195231 objects misplaced (1.309%); 14 MiB/s, 213 objects/s recovering 2023-09-30T09:29:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:22 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:22 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:22.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:22 smithi019 ceph-mon[216815]: from='client.35193 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:22 smithi019 ceph-mon[224043]: pgmap v547: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 189 op/s; 30084/195231 objects degraded (15.409%); 2555/195231 objects misplaced (1.309%); 14 MiB/s, 213 objects/s recovering 2023-09-30T09:29:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:22 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:22 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:22 smithi019 ceph-mon[224043]: from='client.35193 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:22.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 62 16 68406 68390 68.9113 65.375 0.0089368 0.0144893 2023-09-30T09:29:23.559 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:23.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 30084/195231 objects degraded (15.409%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:23.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:23 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:23.562 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 30084/195231 objects degraded (15.409%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:23.562 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:23 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:23.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:23.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:23.664 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:23.709 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:23.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 30084/195231 objects degraded (15.409%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:23.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:23 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:23.837 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:29:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:29:23] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-09-30T09:29:23.841 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:29:23.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 63 16 69256 69240 68.6608 53.125 0.0164503 0.014559 2023-09-30T09:29:24.151 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":547,"stamp":"2023-09-30T09:29:21.134118+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4267686098,"num_objects":65077,"num_object_clones":0,"num_object_copies":195231,"num_objects_missing_on_primary":1395,"num_objects_missing":1395,"num_objects_degraded":30084,"num_objects_misplaced":2555,"num_objects_unfound":0,"num_objects_dirty":65077,"num_whiteouts":0,"num_read":6031,"num_read_kb":12313,"num_write":49823,"num_write_kb":3140971,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6015,"num_bytes_recovered":394761432,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90288,"ondisk_log_size":90288,"up":327,"acting":312,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11773660,"kb_used_data":10741020,"kb_used_omap":14,"kb_used_meta":1032561,"kb_avail":644442404,"statfs":{"total":671965249536,"available":659909021696,"internally_reserved":0,"allocated":10998804480,"data_stored":10991477840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14566,"internal_metadata":1057343258},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":75353990,"num_objects":1152,"num_object_clones":0,"num_object_copies":3456,"num_objects_missing_on_primary":1395,"num_objects_missing":1395,"num_objects_degraded":8026,"num_objects_misplaced":-1366,"num_objects_unfound":0,"num_objects_dirty":1152,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":1175,"num_write_kb":74011,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1370,"num_bytes_recovered":91495005,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.424010"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":220,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624238+0000","last_change":"2023-09-30T09:29:09.598494+0000","last_active":"2023-09-30T09:29:12.624238+0000","last_peered":"2023-09-30T09:29:12.624238+0000","last_clean":"2023-09-30T09:29:12.624238+0000","last_became_active":"2023-09-30T09:29:09.598158+0000","last_became_peered":"2023-09-30T09:29:09.598158+0000","last_unstale":"2023-09-30T09:29:12.624238+0000","last_undegraded":"2023-09-30T09:29:12.624238+0000","last_fullsized":"2023-09-30T09:29:12.624238+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:03.225682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":198,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.627978+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:29:12.627978+0000","last_peered":"2023-09-30T09:29:12.627978+0000","last_clean":"2023-09-30T09:29:12.627978+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:29:12.627978+0000","last_undegraded":"2023-09-30T09:29:12.627978+0000","last_fullsized":"2023-09-30T09:29:12.627978+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":195,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.635042+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:29:12.635042+0000","last_peered":"2023-09-30T09:29:12.635042+0000","last_clean":"2023-09-30T09:29:12.635042+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:29:12.635042+0000","last_undegraded":"2023-09-30T09:29:12.635042+0000","last_fullsized":"2023-09-30T09:29:12.635042+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":127,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.712023+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:29:14.712023+0000","last_peered":"2023-09-30T09:29:14.712023+0000","last_clean":"2023-09-30T09:29:14.712023+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:29:14.712023+0000","last_undegraded":"2023-09-30T09:29:14.712023+0000","last_fullsized":"2023-09-30T09:29:14.712023+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":248,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626083+0000","last_change":"2023-09-30T09:29:09.606288+0000","last_active":"2023-09-30T09:29:12.626083+0000","last_peered":"2023-09-30T09:29:12.626083+0000","last_clean":"2023-09-30T09:29:12.626083+0000","last_became_active":"2023-09-30T09:29:09.605776+0000","last_became_peered":"2023-09-30T09:29:09.605776+0000","last_unstale":"2023-09-30T09:29:12.626083+0000","last_undegraded":"2023-09-30T09:29:12.626083+0000","last_fullsized":"2023-09-30T09:29:12.626083+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:07.935982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":192,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.629373+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:29:12.629373+0000","last_peered":"2023-09-30T09:29:12.629373+0000","last_clean":"2023-09-30T09:29:12.629373+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:29:12.629373+0000","last_undegraded":"2023-09-30T09:29:12.629373+0000","last_fullsized":"2023-09-30T09:29:12.629373+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"233'3101","reported_seq":7240,"reported_epoch":233,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:29:18.978634+0000","last_change":"2023-09-30T09:29:10.766595+0000","last_active":"2023-09-30T09:29:18.978634+0000","last_peered":"2023-09-30T09:29:18.978634+0000","last_clean":"2023-09-30T09:29:08.564047+0000","last_became_active":"2023-09-30T09:29:10.763967+0000","last_became_peered":"2023-09-30T09:29:10.763967+0000","last_unstale":"2023-09-30T09:29:18.978634+0000","last_undegraded":"2023-09-30T09:29:18.978634+0000","last_fullsized":"2023-09-30T09:29:18.978634+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2497","last_scrub_stamp":"2023-09-30T09:28:46.026181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:46.026181+0000","objects_scrubbed":1334,"log_size":3101,"log_dups_size":0,"ondisk_log_size":3101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:02.322482+0000","scrub_duration":0.15695645899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127008768,"num_objects":1938,"num_object_clones":0,"num_object_copies":5814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1938,"num_objects_unfound":0,"num_objects_dirty":1938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":604,"num_write_kb":38656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1347,"num_bytes_recovered":88276992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,2],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,4,7","objects":1938}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":126,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.711528+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:29:14.711528+0000","last_peered":"2023-09-30T09:29:14.711528+0000","last_clean":"2023-09-30T09:29:14.711528+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:29:14.711528+0000","last_undegraded":"2023-09-30T09:29:14.711528+0000","last_fullsized":"2023-09-30T09:29:14.711528+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":198,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626124+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:29:12.626124+0000","last_peered":"2023-09-30T09:29:12.626124+0000","last_clean":"2023-09-30T09:29:12.626124+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:29:12.626124+0000","last_undegraded":"2023-09-30T09:29:12.626124+0000","last_fullsized":"2023-09-30T09:29:12.626124+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"231'3118","reported_seq":3173,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:14.718453+0000","last_change":"2023-09-30T09:29:12.517047+0000","last_active":"2023-09-30T09:29:14.718453+0000","last_peered":"2023-09-30T09:29:14.718453+0000","last_clean":"2023-09-30T09:28:50.280641+0000","last_became_active":"2023-09-30T09:29:12.506944+0000","last_became_peered":"2023-09-30T09:29:12.506944+0000","last_unstale":"2023-09-30T09:29:14.718453+0000","last_undegraded":"2023-09-30T09:29:12.074374+0000","last_fullsized":"2023-09-30T09:29:10.583109+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3118,"log_dups_size":0,"ondisk_log_size":3118,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131989504,"num_objects":2014,"num_object_clones":0,"num_object_copies":6042,"num_objects_missing_on_primary":1393,"num_objects_missing":1393,"num_objects_degraded":3375,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2014,"num_write_kb":128896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,3,6","objects":589},{"shards":"2,6,7","objects":1393},{"shards":"3,6","objects":32}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":84,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633348+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:29:12.633348+0000","last_peered":"2023-09-30T09:29:12.633348+0000","last_clean":"2023-09-30T09:29:12.633348+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:29:12.633348+0000","last_undegraded":"2023-09-30T09:29:12.633348+0000","last_fullsized":"2023-09-30T09:29:12.633348+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":149,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626315+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:29:12.626315+0000","last_peered":"2023-09-30T09:29:12.626315+0000","last_clean":"2023-09-30T09:29:12.626315+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:29:12.626315+0000","last_undegraded":"2023-09-30T09:29:12.626315+0000","last_fullsized":"2023-09-30T09:29:12.626315+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"234'3114","reported_seq":7151,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-30T09:29:19.334363+0000","last_change":"2023-09-30T09:28:56.382198+0000","last_active":"2023-09-30T09:29:19.334363+0000","last_peered":"2023-09-30T09:29:19.334363+0000","last_clean":"2023-09-30T09:29:19.334363+0000","last_became_active":"2023-09-30T09:28:56.380752+0000","last_became_peered":"2023-09-30T09:28:56.380752+0000","last_unstale":"2023-09-30T09:29:19.334363+0000","last_undegraded":"2023-09-30T09:29:19.334363+0000","last_fullsized":"2023-09-30T09:29:19.334363+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3114,"log_dups_size":0,"ondisk_log_size":3114,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130777088,"num_objects":1996,"num_object_clones":0,"num_object_copies":5988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1996,"num_write_kb":127712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1324,"num_bytes_recovered":86769664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":196,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626820+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:29:12.626820+0000","last_peered":"2023-09-30T09:29:12.626820+0000","last_clean":"2023-09-30T09:29:12.626820+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:29:12.626820+0000","last_undegraded":"2023-09-30T09:29:12.626820+0000","last_fullsized":"2023-09-30T09:29:12.626820+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":180,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633167+0000","last_change":"2023-09-30T09:29:09.620809+0000","last_active":"2023-09-30T09:29:12.633167+0000","last_peered":"2023-09-30T09:29:12.633167+0000","last_clean":"2023-09-30T09:29:12.633167+0000","last_became_active":"2023-09-30T09:29:09.620497+0000","last_became_peered":"2023-09-30T09:29:09.620497+0000","last_unstale":"2023-09-30T09:29:12.633167+0000","last_undegraded":"2023-09-30T09:29:12.633167+0000","last_fullsized":"2023-09-30T09:29:12.633167+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:48.932180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"230'3232","reported_seq":6934,"reported_epoch":232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:12.634500+0000","last_change":"2023-09-30T09:29:10.834441+0000","last_active":"2023-09-30T09:29:12.634500+0000","last_peered":"2023-09-30T09:29:12.634500+0000","last_clean":"2023-09-30T09:28:46.291516+0000","last_became_active":"2023-09-30T09:29:10.829695+0000","last_became_peered":"2023-09-30T09:29:10.829695+0000","last_unstale":"2023-09-30T09:29:12.634500+0000","last_undegraded":"2023-09-30T09:29:10.732734+0000","last_fullsized":"2023-09-30T09:29:10.605437+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2608","last_scrub_stamp":"2023-09-30T09:28:45.597110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.597110+0000","objects_scrubbed":1407,"log_size":3232,"log_dups_size":0,"ondisk_log_size":3232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:02:58.457132+0000","scrub_duration":0.26278548800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133496832,"num_objects":2037,"num_object_clones":0,"num_object_copies":6111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":423,"num_objects_misplaced":617,"num_objects_unfound":0,"num_objects_dirty":2037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":630,"num_write_kb":40320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1205,"num_bytes_recovered":78970880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[1,5],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,3,5","objects":617},{"shards":"1,2,5,6","objects":1},{"shards":"1,2,6","objects":211},{"shards":"1,5","objects":1208}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":197,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624249+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:29:12.624249+0000","last_peered":"2023-09-30T09:29:12.624249+0000","last_clean":"2023-09-30T09:29:12.624249+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:29:12.624249+0000","last_undegraded":"2023-09-30T09:29:12.624249+0000","last_fullsized":"2023-09-30T09:29:12.624249+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":211,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624373+0000","last_change":"2023-09-30T09:29:09.612466+0000","last_active":"2023-09-30T09:29:12.624373+0000","last_peered":"2023-09-30T09:29:12.624373+0000","last_clean":"2023-09-30T09:29:12.624373+0000","last_became_active":"2023-09-30T09:29:09.612173+0000","last_became_peered":"2023-09-30T09:29:09.612173+0000","last_unstale":"2023-09-30T09:29:12.624373+0000","last_undegraded":"2023-09-30T09:29:12.624373+0000","last_fullsized":"2023-09-30T09:29:12.624373+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:15.457876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"230'3195","reported_seq":3246,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.627496+0000","last_change":"2023-09-30T09:28:40.282669+0000","last_active":"2023-09-30T09:29:12.627496+0000","last_peered":"2023-09-30T09:29:12.627496+0000","last_clean":"2023-09-30T09:29:12.627496+0000","last_became_active":"2023-09-30T09:28:40.281544+0000","last_became_peered":"2023-09-30T09:28:40.281544+0000","last_unstale":"2023-09-30T09:29:12.627496+0000","last_undegraded":"2023-09-30T09:29:12.627496+0000","last_fullsized":"2023-09-30T09:29:12.627496+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3195,"log_dups_size":0,"ondisk_log_size":3195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134840320,"num_objects":2058,"num_object_clones":0,"num_object_copies":6174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2058,"num_write_kb":131680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":211,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.623280+0000","last_change":"2023-09-30T09:29:09.626205+0000","last_active":"2023-09-30T09:29:12.623280+0000","last_peered":"2023-09-30T09:29:12.623280+0000","last_clean":"2023-09-30T09:29:12.623280+0000","last_became_active":"2023-09-30T09:29:09.618293+0000","last_became_peered":"2023-09-30T09:29:09.618293+0000","last_unstale":"2023-09-30T09:29:12.623280+0000","last_undegraded":"2023-09-30T09:29:12.623280+0000","last_fullsized":"2023-09-30T09:29:12.623280+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:04.337293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":220,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.627073+0000","last_change":"2023-09-30T09:29:09.606072+0000","last_active":"2023-09-30T09:29:12.627073+0000","last_peered":"2023-09-30T09:29:12.627073+0000","last_clean":"2023-09-30T09:29:12.627073+0000","last_became_active":"2023-09-30T09:29:09.605716+0000","last_became_peered":"2023-09-30T09:29:09.605716+0000","last_unstale":"2023-09-30T09:29:12.627073+0000","last_undegraded":"2023-09-30T09:29:12.627073+0000","last_fullsized":"2023-09-30T09:29:12.627073+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:50.972640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"233'3162","reported_seq":3274,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:18.979569+0000","last_change":"2023-09-30T09:28:45.602319+0000","last_active":"2023-09-30T09:29:18.979569+0000","last_peered":"2023-09-30T09:29:18.979569+0000","last_clean":"2023-09-30T09:29:18.979569+0000","last_became_active":"2023-09-30T09:28:40.272792+0000","last_became_peered":"2023-09-30T09:28:40.272792+0000","last_unstale":"2023-09-30T09:29:18.979569+0000","last_undegraded":"2023-09-30T09:29:18.979569+0000","last_fullsized":"2023-09-30T09:29:18.979569+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2523","last_scrub_stamp":"2023-09-30T09:28:45.602248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.602248+0000","objects_scrubbed":1419,"log_size":3162,"log_dups_size":0,"ondisk_log_size":3162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:31:54.788871+0000","scrub_duration":0.27069680099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135331840,"num_objects":2065,"num_object_clones":0,"num_object_copies":6195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":646,"num_write_kb":41344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":126,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.712392+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:29:14.712392+0000","last_peered":"2023-09-30T09:29:14.712392+0000","last_clean":"2023-09-30T09:29:14.712392+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:29:14.712392+0000","last_undegraded":"2023-09-30T09:29:14.712392+0000","last_fullsized":"2023-09-30T09:29:14.712392+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":154,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626947+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:29:12.626947+0000","last_peered":"2023-09-30T09:29:12.626947+0000","last_clean":"2023-09-30T09:29:12.626947+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:29:12.626947+0000","last_undegraded":"2023-09-30T09:29:12.626947+0000","last_fullsized":"2023-09-30T09:29:12.626947+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"230'3169","reported_seq":3243,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.634073+0000","last_change":"2023-09-30T09:28:44.596036+0000","last_active":"2023-09-30T09:29:12.634073+0000","last_peered":"2023-09-30T09:29:12.634073+0000","last_clean":"2023-09-30T09:29:12.634073+0000","last_became_active":"2023-09-30T09:28:40.272740+0000","last_became_peered":"2023-09-30T09:28:40.272740+0000","last_unstale":"2023-09-30T09:29:12.634073+0000","last_undegraded":"2023-09-30T09:29:12.634073+0000","last_fullsized":"2023-09-30T09:29:12.634073+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"214'2497","last_scrub_stamp":"2023-09-30T09:28:44.595809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:44.595809+0000","objects_scrubbed":1358,"log_size":3169,"log_dups_size":0,"ondisk_log_size":3169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:26:50.541441+0000","scrub_duration":0.31195125499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133562368,"num_objects":2038,"num_object_clones":0,"num_object_copies":6114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":680,"num_write_kb":43520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":176,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.625140+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:29:12.625140+0000","last_peered":"2023-09-30T09:29:12.625140+0000","last_clean":"2023-09-30T09:29:12.625140+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:29:12.625140+0000","last_undegraded":"2023-09-30T09:29:12.625140+0000","last_fullsized":"2023-09-30T09:29:12.625140+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":126,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.711956+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:29:14.711956+0000","last_peered":"2023-09-30T09:29:14.711956+0000","last_clean":"2023-09-30T09:29:14.711956+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:29:14.711956+0000","last_undegraded":"2023-09-30T09:29:14.711956+0000","last_fullsized":"2023-09-30T09:29:14.711956+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"233'3219","reported_seq":3263,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:18.957447+0000","last_change":"2023-09-30T09:28:40.288346+0000","last_active":"2023-09-30T09:29:18.957447+0000","last_peered":"2023-09-30T09:29:18.957447+0000","last_clean":"2023-09-30T09:29:18.957447+0000","last_became_active":"2023-09-30T09:28:40.286899+0000","last_became_peered":"2023-09-30T09:28:40.286899+0000","last_unstale":"2023-09-30T09:29:18.957447+0000","last_undegraded":"2023-09-30T09:29:18.957447+0000","last_fullsized":"2023-09-30T09:29:18.957447+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3219,"log_dups_size":0,"ondisk_log_size":3219,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136806400,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2088,"num_write_kb":133600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":325,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.628853+0000","last_change":"2023-09-30T09:29:09.634829+0000","last_active":"2023-09-30T09:29:12.628853+0000","last_peered":"2023-09-30T09:29:12.628853+0000","last_clean":"2023-09-30T09:29:12.628853+0000","last_became_active":"2023-09-30T09:29:09.612183+0000","last_became_peered":"2023-09-30T09:29:09.612183+0000","last_unstale":"2023-09-30T09:29:12.628853+0000","last_undegraded":"2023-09-30T09:29:12.628853+0000","last_fullsized":"2023-09-30T09:29:12.628853+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:28.144436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":218,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-30T09:29:10.599898+0000","last_change":"2023-09-30T09:29:09.605918+0000","last_active":"2023-09-30T09:29:10.599898+0000","last_peered":"2023-09-30T09:29:10.599898+0000","last_clean":"2023-09-30T09:29:10.599898+0000","last_became_active":"2023-09-30T09:29:09.605521+0000","last_became_peered":"2023-09-30T09:29:09.605521+0000","last_unstale":"2023-09-30T09:29:10.599898+0000","last_undegraded":"2023-09-30T09:29:10.599898+0000","last_fullsized":"2023-09-30T09:29:10.599898+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:43.287337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"233'4081","reported_seq":4115,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:18.980168+0000","last_change":"2023-09-30T09:28:12.718364+0000","last_active":"2023-09-30T09:29:18.980168+0000","last_peered":"2023-09-30T09:29:18.980168+0000","last_clean":"2023-09-30T09:29:18.980168+0000","last_became_active":"2023-09-30T09:28:12.717931+0000","last_became_peered":"2023-09-30T09:28:12.717931+0000","last_unstale":"2023-09-30T09:29:18.980168+0000","last_undegraded":"2023-09-30T09:29:18.980168+0000","last_fullsized":"2023-09-30T09:29:18.980168+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4081,"log_dups_size":0,"ondisk_log_size":4081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:12.025351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267452416,"num_objects":4081,"num_object_clones":0,"num_object_copies":12243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4081,"num_write_kb":261184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":236,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.625791+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:29:12.625791+0000","last_peered":"2023-09-30T09:29:12.625791+0000","last_clean":"2023-09-30T09:29:12.625791+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:29:12.625791+0000","last_undegraded":"2023-09-30T09:29:12.625791+0000","last_fullsized":"2023-09-30T09:29:12.625791+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":96,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626284+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:29:12.626284+0000","last_peered":"2023-09-30T09:29:12.626284+0000","last_clean":"2023-09-30T09:29:12.626284+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:29:12.626284+0000","last_undegraded":"2023-09-30T09:29:12.626284+0000","last_fullsized":"2023-09-30T09:29:12.626284+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"231'4051","reported_seq":4095,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626673+0000","last_change":"2023-09-30T09:28:12.563756+0000","last_active":"2023-09-30T09:29:12.626673+0000","last_peered":"2023-09-30T09:29:12.626673+0000","last_clean":"2023-09-30T09:29:12.626673+0000","last_became_active":"2023-09-30T09:28:12.563460+0000","last_became_peered":"2023-09-30T09:28:12.563460+0000","last_unstale":"2023-09-30T09:29:12.626673+0000","last_undegraded":"2023-09-30T09:29:12.626673+0000","last_fullsized":"2023-09-30T09:29:12.626673+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4051,"log_dups_size":0,"ondisk_log_size":4051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:40:48.509761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265486336,"num_objects":4051,"num_object_clones":0,"num_object_copies":12153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4051,"num_write_kb":259264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":237,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626272+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:29:12.626272+0000","last_peered":"2023-09-30T09:29:12.626272+0000","last_clean":"2023-09-30T09:29:12.626272+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:29:12.626272+0000","last_undegraded":"2023-09-30T09:29:12.626272+0000","last_fullsized":"2023-09-30T09:29:12.626272+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":208,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624557+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:29:12.624557+0000","last_peered":"2023-09-30T09:29:12.624557+0000","last_clean":"2023-09-30T09:29:12.624557+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:29:12.624557+0000","last_undegraded":"2023-09-30T09:29:12.624557+0000","last_fullsized":"2023-09-30T09:29:12.624557+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"233'3144","reported_seq":3200,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:18.961733+0000","last_change":"2023-09-30T09:29:10.874488+0000","last_active":"2023-09-30T09:29:18.961733+0000","last_peered":"2023-09-30T09:29:18.961733+0000","last_clean":"2023-09-30T09:29:08.490081+0000","last_became_active":"2023-09-30T09:29:10.870401+0000","last_became_peered":"2023-09-30T09:29:10.870401+0000","last_unstale":"2023-09-30T09:29:18.961733+0000","last_undegraded":"2023-09-30T09:29:10.657673+0000","last_fullsized":"2023-09-30T09:29:10.593223+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3144,"log_dups_size":0,"ondisk_log_size":3144,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133070848,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1975,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2030,"num_write_kb":129952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1975},{"shards":"3,7","objects":55}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":219,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626653+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:29:12.626653+0000","last_peered":"2023-09-30T09:29:12.626653+0000","last_clean":"2023-09-30T09:29:12.626653+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:29:12.626653+0000","last_undegraded":"2023-09-30T09:29:12.626653+0000","last_fullsized":"2023-09-30T09:29:12.626653+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":202,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.629857+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:29:12.629857+0000","last_peered":"2023-09-30T09:29:12.629857+0000","last_clean":"2023-09-30T09:29:12.629857+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:29:12.629857+0000","last_undegraded":"2023-09-30T09:29:12.629857+0000","last_fullsized":"2023-09-30T09:29:12.629857+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"233'3119","reported_seq":3238,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:18.962576+0000","last_change":"2023-09-30T09:29:10.893105+0000","last_active":"2023-09-30T09:29:18.962576+0000","last_peered":"2023-09-30T09:29:18.962576+0000","last_clean":"2023-09-30T09:29:08.514351+0000","last_became_active":"2023-09-30T09:29:10.883964+0000","last_became_peered":"2023-09-30T09:29:10.883964+0000","last_unstale":"2023-09-30T09:29:18.962576+0000","last_undegraded":"2023-09-30T09:29:10.674378+0000","last_fullsized":"2023-09-30T09:29:10.596450+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2471","last_scrub_stamp":"2023-09-30T09:28:47.207706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:47.207706+0000","objects_scrubbed":1369,"log_size":3119,"log_dups_size":0,"ondisk_log_size":3119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:51.408261+0000","scrub_duration":0.14908584899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132186112,"num_objects":2017,"num_object_clones":0,"num_object_copies":6051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":648,"num_write_kb":41472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1997},{"shards":"3,7","objects":20}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":192,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633453+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:29:12.633453+0000","last_peered":"2023-09-30T09:29:12.633453+0000","last_clean":"2023-09-30T09:29:12.633453+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:29:12.633453+0000","last_undegraded":"2023-09-30T09:29:12.633453+0000","last_fullsized":"2023-09-30T09:29:12.633453+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":123,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.711729+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:29:14.711729+0000","last_peered":"2023-09-30T09:29:14.711729+0000","last_clean":"2023-09-30T09:29:14.711729+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:29:14.711729+0000","last_undegraded":"2023-09-30T09:29:14.711729+0000","last_fullsized":"2023-09-30T09:29:14.711729+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"233'3936","reported_seq":3983,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:18.949887+0000","last_change":"2023-09-30T09:29:10.873749+0000","last_active":"2023-09-30T09:29:18.949887+0000","last_peered":"2023-09-30T09:29:18.949887+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:10.861516+0000","last_became_peered":"2023-09-30T09:29:10.861516+0000","last_unstale":"2023-09-30T09:29:18.949887+0000","last_undegraded":"2023-09-30T09:29:10.748218+0000","last_fullsized":"2023-09-30T09:29:10.601616+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3936,"log_dups_size":0,"ondisk_log_size":3936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:01:49.093253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257949696,"num_objects":3936,"num_object_clones":0,"num_object_copies":11808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3898,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3936,"num_write_kb":251904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":3898},{"shards":"1,4","objects":38}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":712,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.639023+0000","last_change":"2023-09-30T09:29:12.637524+0000","last_active":"2023-09-30T09:29:12.639023+0000","last_peered":"2023-09-30T09:29:12.639023+0000","last_clean":"2023-09-30T09:29:12.639023+0000","last_became_active":"2023-09-30T09:29:12.637171+0000","last_became_peered":"2023-09-30T09:29:12.637171+0000","last_unstale":"2023-09-30T09:29:12.639023+0000","last_undegraded":"2023-09-30T09:29:12.639023+0000","last_fullsized":"2023-09-30T09:29:12.639023+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:14:09.735059+0000","scrub_duration":0.0079898150000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":232,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626883+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:29:12.626883+0000","last_peered":"2023-09-30T09:29:12.626883+0000","last_clean":"2023-09-30T09:29:12.626883+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:29:12.626883+0000","last_undegraded":"2023-09-30T09:29:12.626883+0000","last_fullsized":"2023-09-30T09:29:12.626883+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":191,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624164+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:29:12.624164+0000","last_peered":"2023-09-30T09:29:12.624164+0000","last_clean":"2023-09-30T09:29:12.624164+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:29:12.624164+0000","last_undegraded":"2023-09-30T09:29:12.624164+0000","last_fullsized":"2023-09-30T09:29:12.624164+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"227'4035","reported_seq":4076,"reported_epoch":233,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:18.228985+0000","last_change":"2023-09-30T09:29:18.218552+0000","last_active":"2023-09-30T09:29:10.594659+0000","last_peered":"2023-09-30T09:29:08.549482+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:28:12.559959+0000","last_became_peered":"2023-09-30T09:28:12.559959+0000","last_unstale":"2023-09-30T09:29:18.228985+0000","last_undegraded":"2023-09-30T09:29:18.200399+0000","last_fullsized":"2023-09-30T09:29:10.594659+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4035,"log_dups_size":0,"ondisk_log_size":4035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:12:17.537385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264437760,"num_objects":4035,"num_object_clones":0,"num_object_copies":12105,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":4037,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4035,"num_write_kb":258240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4","objects":4033},{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":609,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.713256+0000","last_change":"2023-09-30T09:29:12.631999+0000","last_active":"2023-09-30T09:29:14.713256+0000","last_peered":"2023-09-30T09:29:14.713256+0000","last_clean":"2023-09-30T09:29:14.713256+0000","last_became_active":"2023-09-30T09:29:12.631620+0000","last_became_peered":"2023-09-30T09:29:12.631620+0000","last_unstale":"2023-09-30T09:29:14.713256+0000","last_undegraded":"2023-09-30T09:29:14.713256+0000","last_fullsized":"2023-09-30T09:29:14.713256+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.999760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.999760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":531037,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":193,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.629785+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:29:12.629785+0000","last_peered":"2023-09-30T09:29:12.629785+0000","last_clean":"2023-09-30T09:29:12.629785+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:29:12.629785+0000","last_undegraded":"2023-09-30T09:29:12.629785+0000","last_fullsized":"2023-09-30T09:29:12.629785+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":117,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.634215+0000","last_change":"2023-09-30T09:29:09.612070+0000","last_active":"2023-09-30T09:29:12.634215+0000","last_peered":"2023-09-30T09:29:12.634215+0000","last_clean":"2023-09-30T09:29:12.634215+0000","last_became_active":"2023-09-30T09:29:09.611725+0000","last_became_peered":"2023-09-30T09:29:09.611725+0000","last_unstale":"2023-09-30T09:29:12.634215+0000","last_undegraded":"2023-09-30T09:29:12.634215+0000","last_fullsized":"2023-09-30T09:29:12.634215+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:55.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"230'3983","reported_seq":4027,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626997+0000","last_change":"2023-09-30T09:28:12.718177+0000","last_active":"2023-09-30T09:29:12.626997+0000","last_peered":"2023-09-30T09:29:12.626997+0000","last_clean":"2023-09-30T09:29:12.626997+0000","last_became_active":"2023-09-30T09:28:12.717830+0000","last_became_peered":"2023-09-30T09:28:12.717830+0000","last_unstale":"2023-09-30T09:29:12.626997+0000","last_undegraded":"2023-09-30T09:29:12.626997+0000","last_fullsized":"2023-09-30T09:29:12.626997+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3983,"log_dups_size":0,"ondisk_log_size":3983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:11.556122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261029888,"num_objects":3983,"num_object_clones":0,"num_object_copies":11949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3983,"num_write_kb":254912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":202,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624419+0000","last_change":"2023-09-30T09:29:09.607621+0000","last_active":"2023-09-30T09:29:12.624419+0000","last_peered":"2023-09-30T09:29:12.624419+0000","last_clean":"2023-09-30T09:29:12.624419+0000","last_became_active":"2023-09-30T09:29:09.607245+0000","last_became_peered":"2023-09-30T09:29:09.607245+0000","last_unstale":"2023-09-30T09:29:12.624419+0000","last_undegraded":"2023-09-30T09:29:12.624419+0000","last_fullsized":"2023-09-30T09:29:12.624419+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:42:42.429421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":191,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626916+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:29:12.626916+0000","last_peered":"2023-09-30T09:29:12.626916+0000","last_clean":"2023-09-30T09:29:12.626916+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:29:12.626916+0000","last_undegraded":"2023-09-30T09:29:12.626916+0000","last_fullsized":"2023-09-30T09:29:12.626916+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"233'4146","reported_seq":4190,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:18.995193+0000","last_change":"2023-09-30T09:29:10.875912+0000","last_active":"2023-09-30T09:29:18.995193+0000","last_peered":"2023-09-30T09:29:18.995193+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:10.862473+0000","last_became_peered":"2023-09-30T09:29:10.862473+0000","last_unstale":"2023-09-30T09:29:18.995193+0000","last_undegraded":"2023-09-30T09:29:10.620322+0000","last_fullsized":"2023-09-30T09:29:10.593116+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4146,"log_dups_size":0,"ondisk_log_size":4146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:45:54.616461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271712256,"num_objects":4146,"num_object_clones":0,"num_object_copies":12438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4112,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4146,"num_write_kb":265344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4112},{"shards":"1,5","objects":34}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":440,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.635302+0000","last_change":"2023-09-30T09:29:12.635044+0000","last_active":"2023-09-30T09:29:12.635302+0000","last_peered":"2023-09-30T09:29:12.635302+0000","last_clean":"2023-09-30T09:29:12.635302+0000","last_became_active":"2023-09-30T09:29:12.634703+0000","last_became_peered":"2023-09-30T09:29:12.634703+0000","last_unstale":"2023-09-30T09:29:12.635302+0000","last_undegraded":"2023-09-30T09:29:12.635302+0000","last_fullsized":"2023-09-30T09:29:12.635302+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.980385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.980385+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":199,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.625949+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:29:12.625949+0000","last_peered":"2023-09-30T09:29:12.625949+0000","last_clean":"2023-09-30T09:29:12.625949+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:29:12.625949+0000","last_undegraded":"2023-09-30T09:29:12.625949+0000","last_fullsized":"2023-09-30T09:29:12.625949+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":172,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.634585+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:29:12.634585+0000","last_peered":"2023-09-30T09:29:12.634585+0000","last_clean":"2023-09-30T09:29:12.634585+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:29:12.634585+0000","last_undegraded":"2023-09-30T09:29:12.634585+0000","last_fullsized":"2023-09-30T09:29:12.634585+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"234'3209","reported_seq":3293,"reported_epoch":234,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:19.317389+0000","last_change":"2023-09-30T09:29:10.765709+0000","last_active":"2023-09-30T09:29:19.317389+0000","last_peered":"2023-09-30T09:29:19.317389+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:10.760103+0000","last_became_peered":"2023-09-30T09:29:10.760103+0000","last_unstale":"2023-09-30T09:29:19.317389+0000","last_undegraded":"2023-09-30T09:29:10.667642+0000","last_fullsized":"2023-09-30T09:29:10.579710+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":3209,"log_dups_size":0,"ondisk_log_size":3209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:30:24.960932+0000","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139657216,"num_objects":2131,"num_object_clones":0,"num_object_copies":6393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2099,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":718,"num_write_kb":45952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2099},{"shards":"0,3","objects":32}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":477,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.629361+0000","last_change":"2023-09-30T09:28:30.495699+0000","last_active":"2023-09-30T09:29:12.629361+0000","last_peered":"2023-09-30T09:29:12.629361+0000","last_clean":"2023-09-30T09:29:12.629361+0000","last_became_active":"2023-09-30T09:28:09.228266+0000","last_became_peered":"2023-09-30T09:28:09.228266+0000","last_unstale":"2023-09-30T09:29:12.629361+0000","last_undegraded":"2023-09-30T09:29:12.629361+0000","last_fullsized":"2023-09-30T09:29:12.629361+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:10:19.705285+0000","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":577,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633735+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:29:12.633735+0000","last_peered":"2023-09-30T09:29:12.633735+0000","last_clean":"2023-09-30T09:29:12.633735+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:29:12.633735+0000","last_undegraded":"2023-09-30T09:29:12.633735+0000","last_fullsized":"2023-09-30T09:29:12.633735+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":219,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.632904+0000","last_change":"2023-09-30T09:29:09.611292+0000","last_active":"2023-09-30T09:29:12.632904+0000","last_peered":"2023-09-30T09:29:12.632904+0000","last_clean":"2023-09-30T09:29:12.632904+0000","last_became_active":"2023-09-30T09:29:09.611015+0000","last_became_peered":"2023-09-30T09:29:09.611015+0000","last_unstale":"2023-09-30T09:29:12.632904+0000","last_undegraded":"2023-09-30T09:29:12.632904+0000","last_fullsized":"2023-09-30T09:29:12.632904+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:48.124378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":192,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633281+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:29:12.633281+0000","last_peered":"2023-09-30T09:29:12.633281+0000","last_clean":"2023-09-30T09:29:12.633281+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:29:12.633281+0000","last_undegraded":"2023-09-30T09:29:12.633281+0000","last_fullsized":"2023-09-30T09:29:12.633281+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"230'3178","reported_seq":3251,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.627577+0000","last_change":"2023-09-30T09:28:42.250412+0000","last_active":"2023-09-30T09:29:12.627577+0000","last_peered":"2023-09-30T09:29:12.627577+0000","last_clean":"2023-09-30T09:29:12.627577+0000","last_became_active":"2023-09-30T09:28:40.266645+0000","last_became_peered":"2023-09-30T09:28:40.266645+0000","last_unstale":"2023-09-30T09:29:12.627577+0000","last_undegraded":"2023-09-30T09:29:12.627577+0000","last_fullsized":"2023-09-30T09:29:12.627577+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2368","last_scrub_stamp":"2023-09-30T09:28:42.250204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.250204+0000","objects_scrubbed":1224,"log_size":3178,"log_dups_size":0,"ondisk_log_size":3178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:25.503159+0000","scrub_duration":0.32304901400000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133693440,"num_objects":2040,"num_object_clones":0,"num_object_copies":6120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":816,"num_write_kb":52224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":655,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.640565+0000","last_change":"2023-09-30T09:29:12.640269+0000","last_active":"2023-09-30T09:29:12.640565+0000","last_peered":"2023-09-30T09:29:12.640565+0000","last_clean":"2023-09-30T09:29:12.640565+0000","last_became_active":"2023-09-30T09:29:12.639925+0000","last_became_peered":"2023-09-30T09:29:12.639925+0000","last_unstale":"2023-09-30T09:29:12.640565+0000","last_undegraded":"2023-09-30T09:29:12.640565+0000","last_fullsized":"2023-09-30T09:29:12.640565+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:45:10.404382+0000","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":380,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633442+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:29:12.633442+0000","last_peered":"2023-09-30T09:29:12.633442+0000","last_clean":"2023-09-30T09:29:12.633442+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:29:12.633442+0000","last_undegraded":"2023-09-30T09:29:12.633442+0000","last_fullsized":"2023-09-30T09:29:12.633442+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":212,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624438+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:29:12.624438+0000","last_peered":"2023-09-30T09:29:12.624438+0000","last_clean":"2023-09-30T09:29:12.624438+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:29:12.624438+0000","last_undegraded":"2023-09-30T09:29:12.624438+0000","last_fullsized":"2023-09-30T09:29:12.624438+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.714237+0000","last_change":"2023-09-30T09:29:09.611507+0000","last_active":"2023-09-30T09:29:14.714237+0000","last_peered":"2023-09-30T09:29:14.714237+0000","last_clean":"2023-09-30T09:29:14.714237+0000","last_became_active":"2023-09-30T09:29:09.611120+0000","last_became_peered":"2023-09-30T09:29:09.611120+0000","last_unstale":"2023-09-30T09:29:14.714237+0000","last_undegraded":"2023-09-30T09:29:14.714237+0000","last_fullsized":"2023-09-30T09:29:14.714237+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:31.622647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.5","version":"233'3204","reported_seq":3256,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:18.997720+0000","last_change":"2023-09-30T09:28:46.022537+0000","last_active":"2023-09-30T09:29:18.997720+0000","last_peered":"2023-09-30T09:29:18.997720+0000","last_clean":"2023-09-30T09:29:18.997720+0000","last_became_active":"2023-09-30T09:28:40.276165+0000","last_became_peered":"2023-09-30T09:28:40.276165+0000","last_unstale":"2023-09-30T09:29:18.997720+0000","last_undegraded":"2023-09-30T09:29:18.997720+0000","last_fullsized":"2023-09-30T09:29:18.997720+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3204,"log_dups_size":0,"ondisk_log_size":3204,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135823360,"num_objects":2072,"num_object_clones":0,"num_object_copies":6216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2072,"num_write_kb":132640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":712,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.639023+0000","last_change":"2023-09-30T09:29:12.637491+0000","last_active":"2023-09-30T09:29:12.639023+0000","last_peered":"2023-09-30T09:29:12.639023+0000","last_clean":"2023-09-30T09:29:12.639023+0000","last_became_active":"2023-09-30T09:29:12.637133+0000","last_became_peered":"2023-09-30T09:29:12.637133+0000","last_unstale":"2023-09-30T09:29:12.639023+0000","last_undegraded":"2023-09-30T09:29:12.639023+0000","last_fullsized":"2023-09-30T09:29:12.639023+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:54:24.000552+0000","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":376,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:14.713935+0000","last_change":"2023-09-30T09:29:10.780197+0000","last_active":"2023-09-30T09:29:14.713935+0000","last_peered":"2023-09-30T09:29:14.713935+0000","last_clean":"2023-09-30T09:13:44.323910+0000","last_became_active":"2023-09-30T09:29:10.779703+0000","last_became_peered":"2023-09-30T09:29:10.779703+0000","last_unstale":"2023-09-30T09:29:14.713935+0000","last_undegraded":"2023-09-30T09:29:10.583371+0000","last_fullsized":"2023-09-30T09:29:10.582761+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:40:26.826296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":180,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633275+0000","last_change":"2023-09-30T09:29:09.607111+0000","last_active":"2023-09-30T09:29:12.633275+0000","last_peered":"2023-09-30T09:29:12.633275+0000","last_clean":"2023-09-30T09:29:12.633275+0000","last_became_active":"2023-09-30T09:29:09.606807+0000","last_became_peered":"2023-09-30T09:29:09.606807+0000","last_unstale":"2023-09-30T09:29:12.633275+0000","last_undegraded":"2023-09-30T09:29:12.633275+0000","last_fullsized":"2023-09-30T09:29:12.633275+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:49.380239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":228,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.634513+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:29:12.634513+0000","last_peered":"2023-09-30T09:29:12.634513+0000","last_clean":"2023-09-30T09:29:12.634513+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:29:12.634513+0000","last_undegraded":"2023-09-30T09:29:12.634513+0000","last_fullsized":"2023-09-30T09:29:12.634513+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"233'3184","reported_seq":3289,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:18.943274+0000","last_change":"2023-09-30T09:28:42.687918+0000","last_active":"2023-09-30T09:29:18.943274+0000","last_peered":"2023-09-30T09:29:18.943274+0000","last_clean":"2023-09-30T09:29:18.943274+0000","last_became_active":"2023-09-30T09:28:40.262566+0000","last_became_peered":"2023-09-30T09:28:40.262566+0000","last_unstale":"2023-09-30T09:29:18.943274+0000","last_undegraded":"2023-09-30T09:29:18.943274+0000","last_fullsized":"2023-09-30T09:29:18.943274+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2390","last_scrub_stamp":"2023-09-30T09:28:42.687821+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.687821+0000","objects_scrubbed":1235,"log_size":3184,"log_dups_size":0,"ondisk_log_size":3184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:07:56.313767+0000","scrub_duration":0.36697584500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133693440,"num_objects":2040,"num_object_clones":0,"num_object_copies":6120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":805,"num_write_kb":51520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":650,"reported_epoch":232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:12.631596+0000","last_change":"2023-09-30T09:29:12.096468+0000","last_active":"2023-09-30T09:29:12.631596+0000","last_peered":"2023-09-30T09:29:12.631596+0000","last_clean":"2023-09-30T09:29:03.507390+0000","last_became_active":"2023-09-30T09:29:12.096051+0000","last_became_peered":"2023-09-30T09:29:12.096051+0000","last_unstale":"2023-09-30T09:29:12.631596+0000","last_undegraded":"2023-09-30T09:29:10.593445+0000","last_fullsized":"2023-09-30T09:29:10.592749+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:49.279327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":673,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626125+0000","last_change":"2023-09-30T09:27:54.119268+0000","last_active":"2023-09-30T09:29:12.626125+0000","last_peered":"2023-09-30T09:29:12.626125+0000","last_clean":"2023-09-30T09:29:12.626125+0000","last_became_active":"2023-09-30T09:27:49.106373+0000","last_became_peered":"2023-09-30T09:27:49.106373+0000","last_unstale":"2023-09-30T09:29:12.626125+0000","last_undegraded":"2023-09-30T09:29:12.626125+0000","last_fullsized":"2023-09-30T09:29:12.626125+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:28.856049+0000","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":148,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626594+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:29:12.626594+0000","last_peered":"2023-09-30T09:29:12.626594+0000","last_clean":"2023-09-30T09:29:12.626594+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:29:12.626594+0000","last_undegraded":"2023-09-30T09:29:12.626594+0000","last_fullsized":"2023-09-30T09:29:12.626594+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":175,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.629353+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:29:12.629353+0000","last_peered":"2023-09-30T09:29:12.629353+0000","last_clean":"2023-09-30T09:29:12.629353+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:29:12.629353+0000","last_undegraded":"2023-09-30T09:29:12.629353+0000","last_fullsized":"2023-09-30T09:29:12.629353+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"230'3220","reported_seq":3292,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633994+0000","last_change":"2023-09-30T09:28:43.582313+0000","last_active":"2023-09-30T09:29:12.633994+0000","last_peered":"2023-09-30T09:29:12.633994+0000","last_clean":"2023-09-30T09:29:12.633994+0000","last_became_active":"2023-09-30T09:28:40.264204+0000","last_became_peered":"2023-09-30T09:28:40.264204+0000","last_unstale":"2023-09-30T09:29:12.633994+0000","last_undegraded":"2023-09-30T09:29:12.633994+0000","last_fullsized":"2023-09-30T09:29:12.633994+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2440","last_scrub_stamp":"2023-09-30T09:28:43.582001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:43.582001+0000","objects_scrubbed":1291,"log_size":3220,"log_dups_size":0,"ondisk_log_size":3220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:36.958560+0000","scrub_duration":0.30691791200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136380416,"num_objects":2081,"num_object_clones":0,"num_object_copies":6243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":790,"num_write_kb":50560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":637,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633642+0000","last_change":"2023-09-30T09:28:24.276678+0000","last_active":"2023-09-30T09:29:12.633642+0000","last_peered":"2023-09-30T09:29:12.633642+0000","last_clean":"2023-09-30T09:29:12.633642+0000","last_became_active":"2023-09-30T09:28:09.226029+0000","last_became_peered":"2023-09-30T09:28:09.226029+0000","last_unstale":"2023-09-30T09:29:12.633642+0000","last_undegraded":"2023-09-30T09:29:12.633642+0000","last_fullsized":"2023-09-30T09:29:12.633642+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:03:49.533925+0000","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":793,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626044+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:29:12.626044+0000","last_peered":"2023-09-30T09:29:12.626044+0000","last_clean":"2023-09-30T09:29:12.626044+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:29:12.626044+0000","last_undegraded":"2023-09-30T09:29:12.626044+0000","last_fullsized":"2023-09-30T09:29:12.626044+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":150,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.711342+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:29:14.711342+0000","last_peered":"2023-09-30T09:29:14.711342+0000","last_clean":"2023-09-30T09:29:14.711342+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:29:14.711342+0000","last_undegraded":"2023-09-30T09:29:14.711342+0000","last_fullsized":"2023-09-30T09:29:14.711342+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":138,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633665+0000","last_change":"2023-09-30T09:29:09.611376+0000","last_active":"2023-09-30T09:29:12.633665+0000","last_peered":"2023-09-30T09:29:12.633665+0000","last_clean":"2023-09-30T09:29:12.633665+0000","last_became_active":"2023-09-30T09:29:09.611090+0000","last_became_peered":"2023-09-30T09:29:09.611090+0000","last_unstale":"2023-09-30T09:29:12.633665+0000","last_undegraded":"2023-09-30T09:29:12.633665+0000","last_fullsized":"2023-09-30T09:29:12.633665+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:07:15.039397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"230'3241","reported_seq":3318,"reported_epoch":232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:12.634768+0000","last_change":"2023-09-30T09:29:10.894114+0000","last_active":"2023-09-30T09:29:12.634768+0000","last_peered":"2023-09-30T09:29:12.634768+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:10.890131+0000","last_became_peered":"2023-09-30T09:29:10.890131+0000","last_unstale":"2023-09-30T09:29:12.634768+0000","last_undegraded":"2023-09-30T09:29:10.649144+0000","last_fullsized":"2023-09-30T09:29:10.605362+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":3241,"log_dups_size":0,"ondisk_log_size":3241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:45:19.157218+0000","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136970240,"num_objects":2090,"num_object_clones":0,"num_object_copies":6270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2087,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":828,"num_write_kb":52992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2086},{"shards":"1,6","objects":3}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":477,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.629102+0000","last_change":"2023-09-30T09:28:28.453752+0000","last_active":"2023-09-30T09:29:12.629102+0000","last_peered":"2023-09-30T09:29:12.629102+0000","last_clean":"2023-09-30T09:29:12.629102+0000","last_became_active":"2023-09-30T09:28:09.228162+0000","last_became_peered":"2023-09-30T09:28:09.228162+0000","last_unstale":"2023-09-30T09:29:12.629102+0000","last_undegraded":"2023-09-30T09:29:12.629102+0000","last_fullsized":"2023-09-30T09:29:12.629102+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:43:23.683089+0000","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":923,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626736+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:29:12.626736+0000","last_peered":"2023-09-30T09:29:12.626736+0000","last_clean":"2023-09-30T09:29:12.626736+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:29:12.626736+0000","last_undegraded":"2023-09-30T09:29:12.626736+0000","last_fullsized":"2023-09-30T09:29:12.626736+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":196,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633902+0000","last_change":"2023-09-30T09:29:09.611808+0000","last_active":"2023-09-30T09:29:12.633902+0000","last_peered":"2023-09-30T09:29:12.633902+0000","last_clean":"2023-09-30T09:29:12.633902+0000","last_became_active":"2023-09-30T09:29:09.611484+0000","last_became_peered":"2023-09-30T09:29:09.611484+0000","last_unstale":"2023-09-30T09:29:12.633902+0000","last_undegraded":"2023-09-30T09:29:12.633902+0000","last_fullsized":"2023-09-30T09:29:12.633902+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:19.562461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":122,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.711816+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:29:14.711816+0000","last_peered":"2023-09-30T09:29:14.711816+0000","last_clean":"2023-09-30T09:29:14.711816+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:29:14.711816+0000","last_undegraded":"2023-09-30T09:29:14.711816+0000","last_fullsized":"2023-09-30T09:29:14.711816+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"230'3073","reported_seq":3152,"reported_epoch":232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:12.627691+0000","last_change":"2023-09-30T09:29:10.778434+0000","last_active":"2023-09-30T09:29:12.627691+0000","last_peered":"2023-09-30T09:29:12.627691+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:10.774906+0000","last_became_peered":"2023-09-30T09:29:10.774906+0000","last_unstale":"2023-09-30T09:29:12.627691+0000","last_undegraded":"2023-09-30T09:29:10.598440+0000","last_fullsized":"2023-09-30T09:29:10.583509+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":3073,"log_dups_size":0,"ondisk_log_size":3073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:48.578501+0000","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131006464,"num_objects":1999,"num_object_clones":0,"num_object_copies":5997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":548,"num_write_kb":35072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":1997},{"shards":"3,6","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":655,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.635546+0000","last_change":"2023-09-30T09:29:12.634588+0000","last_active":"2023-09-30T09:29:12.635546+0000","last_peered":"2023-09-30T09:29:12.635546+0000","last_clean":"2023-09-30T09:29:12.635546+0000","last_became_active":"2023-09-30T09:29:12.634225+0000","last_became_peered":"2023-09-30T09:29:12.634225+0000","last_unstale":"2023-09-30T09:29:12.635546+0000","last_undegraded":"2023-09-30T09:29:12.635546+0000","last_fullsized":"2023-09-30T09:29:12.635546+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:22.392203+0000","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":218,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-30T09:29:10.599815+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:29:10.599815+0000","last_peered":"2023-09-30T09:29:10.599815+0000","last_clean":"2023-09-30T09:29:10.599815+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:29:10.599815+0000","last_undegraded":"2023-09-30T09:29:10.599815+0000","last_fullsized":"2023-09-30T09:29:10.599815+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1182,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633748+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:29:12.633748+0000","last_peered":"2023-09-30T09:29:12.633748+0000","last_clean":"2023-09-30T09:29:12.633748+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:29:12.633748+0000","last_undegraded":"2023-09-30T09:29:12.633748+0000","last_fullsized":"2023-09-30T09:29:12.633748+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":217,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626025+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:29:12.626025+0000","last_peered":"2023-09-30T09:29:12.626025+0000","last_clean":"2023-09-30T09:29:12.626025+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:29:12.626025+0000","last_undegraded":"2023-09-30T09:29:12.626025+0000","last_fullsized":"2023-09-30T09:29:12.626025+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"233'3088","reported_seq":7221,"reported_epoch":233,"state":"active+undersized+remapped","last_fresh":"2023-09-30T09:29:18.746455+0000","last_change":"2023-09-30T09:29:18.353971+0000","last_active":"2023-09-30T09:29:18.746455+0000","last_peered":"2023-09-30T09:29:18.746455+0000","last_clean":"2023-09-30T09:29:08.497534+0000","last_became_active":"2023-09-30T09:29:10.746787+0000","last_became_peered":"2023-09-30T09:29:10.746787+0000","last_unstale":"2023-09-30T09:29:18.746455+0000","last_undegraded":"2023-09-30T09:29:18.746455+0000","last_fullsized":"2023-09-30T09:29:10.594304+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":3088,"log_dups_size":0,"ondisk_log_size":3088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:03:26.665151+0000","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131989504,"num_objects":2014,"num_object_clones":0,"num_object_copies":6042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":600,"num_write_kb":38400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2005,"num_bytes_recovered":131399680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","1","2","6"],"object_location_counts":[{"shards":"6,7","objects":2014}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":632,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.627394+0000","last_change":"2023-09-30T09:28:26.894367+0000","last_active":"2023-09-30T09:29:12.627394+0000","last_peered":"2023-09-30T09:29:12.627394+0000","last_clean":"2023-09-30T09:29:12.627394+0000","last_became_active":"2023-09-30T09:28:09.227189+0000","last_became_peered":"2023-09-30T09:28:09.227189+0000","last_unstale":"2023-09-30T09:29:12.627394+0000","last_undegraded":"2023-09-30T09:29:12.627394+0000","last_fullsized":"2023-09-30T09:29:12.627394+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:17.282328+0000","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":510,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.629296+0000","last_change":"2023-09-30T09:29:09.643760+0000","last_active":"2023-09-30T09:29:12.629296+0000","last_peered":"2023-09-30T09:29:12.629296+0000","last_clean":"2023-09-30T09:29:12.629296+0000","last_became_active":"2023-09-30T09:29:09.620803+0000","last_became_peered":"2023-09-30T09:29:09.620803+0000","last_unstale":"2023-09-30T09:29:12.629296+0000","last_undegraded":"2023-09-30T09:29:12.629296+0000","last_fullsized":"2023-09-30T09:29:12.629296+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:54:49.883760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":110,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633897+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:29:12.633897+0000","last_peered":"2023-09-30T09:29:12.633897+0000","last_clean":"2023-09-30T09:29:12.633897+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:29:12.633897+0000","last_undegraded":"2023-09-30T09:29:12.633897+0000","last_fullsized":"2023-09-30T09:29:12.633897+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":174,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624503+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:29:12.624503+0000","last_peered":"2023-09-30T09:29:12.624503+0000","last_clean":"2023-09-30T09:29:12.624503+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:29:12.624503+0000","last_undegraded":"2023-09-30T09:29:12.624503+0000","last_fullsized":"2023-09-30T09:29:12.624503+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":91,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.624496+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:29:12.624496+0000","last_peered":"2023-09-30T09:29:12.624496+0000","last_clean":"2023-09-30T09:29:12.624496+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:29:12.624496+0000","last_undegraded":"2023-09-30T09:29:12.624496+0000","last_fullsized":"2023-09-30T09:29:12.624496+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":143,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633819+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:29:12.633819+0000","last_peered":"2023-09-30T09:29:12.633819+0000","last_clean":"2023-09-30T09:29:12.633819+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:29:12.633819+0000","last_undegraded":"2023-09-30T09:29:12.633819+0000","last_fullsized":"2023-09-30T09:29:12.633819+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"233'3129","reported_seq":3186,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:19.001013+0000","last_change":"2023-09-30T09:29:10.893253+0000","last_active":"2023-09-30T09:29:19.001013+0000","last_peered":"2023-09-30T09:29:19.001013+0000","last_clean":"2023-09-30T09:29:08.494599+0000","last_became_active":"2023-09-30T09:29:10.886737+0000","last_became_peered":"2023-09-30T09:29:10.886737+0000","last_unstale":"2023-09-30T09:29:19.001013+0000","last_undegraded":"2023-09-30T09:29:10.766629+0000","last_fullsized":"2023-09-30T09:29:10.598246+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3129,"log_dups_size":0,"ondisk_log_size":3129,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132907008,"num_objects":2028,"num_object_clones":0,"num_object_copies":6084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2001,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2028,"num_write_kb":129792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":2001},{"shards":"3,7","objects":27}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":194,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633324+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:29:12.633324+0000","last_peered":"2023-09-30T09:29:12.633324+0000","last_clean":"2023-09-30T09:29:12.633324+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:29:12.633324+0000","last_undegraded":"2023-09-30T09:29:12.633324+0000","last_fullsized":"2023-09-30T09:29:12.633324+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":174,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626765+0000","last_change":"2023-09-30T09:29:09.609752+0000","last_active":"2023-09-30T09:29:12.626765+0000","last_peered":"2023-09-30T09:29:12.626765+0000","last_clean":"2023-09-30T09:29:12.626765+0000","last_became_active":"2023-09-30T09:29:09.609242+0000","last_became_peered":"2023-09-30T09:29:09.609242+0000","last_unstale":"2023-09-30T09:29:12.626765+0000","last_undegraded":"2023-09-30T09:29:12.626765+0000","last_fullsized":"2023-09-30T09:29:12.626765+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:31.366509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"233'3155","reported_seq":3210,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:18.934468+0000","last_change":"2023-09-30T09:29:10.864651+0000","last_active":"2023-09-30T09:29:18.934468+0000","last_peered":"2023-09-30T09:29:18.934468+0000","last_clean":"2023-09-30T09:29:08.532983+0000","last_became_active":"2023-09-30T09:29:10.860470+0000","last_became_peered":"2023-09-30T09:29:10.860470+0000","last_unstale":"2023-09-30T09:29:18.934468+0000","last_undegraded":"2023-09-30T09:29:10.638243+0000","last_fullsized":"2023-09-30T09:29:10.593564+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3155,"log_dups_size":0,"ondisk_log_size":3155,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133791744,"num_objects":2042,"num_object_clones":0,"num_object_copies":6126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2042,"num_write_kb":130656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":2057},{"shards":"3,7","objects":-15}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":177,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.633048+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:29:12.633048+0000","last_peered":"2023-09-30T09:29:12.633048+0000","last_clean":"2023-09-30T09:29:12.633048+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:29:12.633048+0000","last_undegraded":"2023-09-30T09:29:12.633048+0000","last_fullsized":"2023-09-30T09:29:12.633048+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":206,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-30T09:29:14.716444+0000","last_change":"2023-09-30T09:29:09.621218+0000","last_active":"2023-09-30T09:29:14.716444+0000","last_peered":"2023-09-30T09:29:14.716444+0000","last_clean":"2023-09-30T09:29:14.716444+0000","last_became_active":"2023-09-30T09:29:09.620972+0000","last_became_peered":"2023-09-30T09:29:09.620972+0000","last_unstale":"2023-09-30T09:29:14.716444+0000","last_undegraded":"2023-09-30T09:29:14.716444+0000","last_fullsized":"2023-09-30T09:29:14.716444+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:25.640288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":212,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.627745+0000","last_change":"2023-09-30T09:29:09.615385+0000","last_active":"2023-09-30T09:29:12.627745+0000","last_peered":"2023-09-30T09:29:12.627745+0000","last_clean":"2023-09-30T09:29:12.627745+0000","last_became_active":"2023-09-30T09:29:09.614479+0000","last_became_peered":"2023-09-30T09:29:09.614479+0000","last_unstale":"2023-09-30T09:29:12.627745+0000","last_undegraded":"2023-09-30T09:29:12.627745+0000","last_fullsized":"2023-09-30T09:29:12.627745+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:14.671222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":243,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.626251+0000","last_change":"2023-09-30T09:29:09.600457+0000","last_active":"2023-09-30T09:29:12.626251+0000","last_peered":"2023-09-30T09:29:12.626251+0000","last_clean":"2023-09-30T09:29:12.626251+0000","last_became_active":"2023-09-30T09:29:09.600080+0000","last_became_peered":"2023-09-30T09:29:09.600080+0000","last_unstale":"2023-09-30T09:29:12.626251+0000","last_undegraded":"2023-09-30T09:29:12.626251+0000","last_fullsized":"2023-09-30T09:29:12.626251+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:10:02.250349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":116,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.632494+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:29:12.632494+0000","last_peered":"2023-09-30T09:29:12.632494+0000","last_clean":"2023-09-30T09:29:12.632494+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:29:12.632494+0000","last_undegraded":"2023-09-30T09:29:12.632494+0000","last_fullsized":"2023-09-30T09:29:12.632494+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":191,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-30T09:29:12.623940+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:29:12.623940+0000","last_peered":"2023-09-30T09:29:12.623940+0000","last_clean":"2023-09-30T09:29:12.623940+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:29:12.623940+0000","last_undegraded":"2023-09-30T09:29:12.623940+0000","last_fullsized":"2023-09-30T09:29:12.623940+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":4263051264,"num_objects":65050,"num_object_clones":0,"num_object_copies":195150,"num_objects_missing_on_primary":1395,"num_objects_missing":1395,"num_objects_degraded":30058,"num_objects_misplaced":2555,"num_objects_unfound":0,"num_objects_dirty":65050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48873,"num_write_kb":3127808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5885,"num_bytes_recovered":385679360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13335789568,"data_stored":13335789568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87486,"ondisk_log_size":87486,"up":78,"acting":65,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":1418,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":435019,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":43,"num_write_kb":438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234212,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761424,"kb_used_data":1647176,"kb_used_omap":0,"kb_used_meta":114240,"kb_avail":91983728,"statfs":{"total":95995035648,"available":94191337472,"internally_reserved":0,"allocated":1686708224,"data_stored":1685847179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":116981760},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.8,"15min":1.0029999999999999},"min":{"1min":0.253,"5min":0.224,"15min":0.224},"max":{"1min":11.710000000000001,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.518,"15min":0.92000000000000004},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.6589999999999998,"5min":18.097999999999999,"15min":18.097999999999999},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":10.125,"5min":4.8070000000000004,"15min":2.214},"min":{"1min":0.38200000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":70.566999999999993,"5min":82.025999999999996,"15min":82.025999999999996},"last":0.50700000000000001},{"interface":"front","average":{"1min":10.115,"5min":4.7279999999999998,"15min":2.2000000000000002},"min":{"1min":0.36599999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":70.626000000000005,"5min":82.012,"15min":82.012},"last":1.915}]},{"osd":2,"last update":"Sat Sep 30 09:29:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7200000000000002,"5min":2.0009999999999999,"15min":1.145},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":39.445,"5min":39.445,"15min":39.445},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.9870000000000001,"15min":1.2190000000000001},"min":{"1min":0.39700000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":40.988999999999997,"5min":40.988999999999997,"15min":40.988999999999997},"last":3.008}]},{"osd":3,"last update":"Sat Sep 30 09:28:42 2023","interfaces":[{"interface":"back","average":{"1min":6.5199999999999996,"5min":2.6920000000000002,"15min":1.3540000000000001},"min":{"1min":0.38800000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":57.000999999999998,"5min":57.000999999999998,"15min":57.000999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":6.8689999999999998,"5min":2.4289999999999998,"15min":1.2569999999999999},"min":{"1min":0.377,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":56.515000000000001,"5min":56.515000000000001,"15min":56.515000000000001},"last":2.9260000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.95699999999999996,"15min":0.64900000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.78900000000000003,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.379},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.81000000000000005,"15min":0.63700000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.92400000000000004,"5min":6.282,"15min":6.282},"last":0.34599999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.97699999999999998,"15min":0.70999999999999996},"min":{"1min":0.28100000000000003,"5min":0.248,"15min":0.248},"max":{"1min":2.4470000000000001,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.121,"15min":0.77600000000000002},"min":{"1min":0.30199999999999999,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":2.4670000000000001,"5min":11.022,"15min":11.022},"last":0.59899999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.052,"15min":0.75700000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33000000000000002},"max":{"1min":1.121,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.879,"15min":0.70299999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.88900000000000001,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":0.44900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495849,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1569548,"kb_used_data":1468868,"kb_used_omap":0,"kb_used_meta":100672,"kb_avail":92175604,"statfs":{"total":95995035648,"available":94387818496,"internally_reserved":0,"allocated":1504120832,"data_stored":1503264300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":103088128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":4.5540000000000003,"5min":2.46,"15min":1.248},"min":{"1min":0.25700000000000001,"5min":0.222,"15min":0.222},"max":{"1min":45.927999999999997,"5min":45.927999999999997,"15min":45.927999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":4.5359999999999996,"5min":2.7719999999999998,"15min":1.357},"min":{"1min":0.315,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":45.316000000000003,"5min":45.316000000000003,"15min":45.316000000000003},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":4.665,"5min":2.5459999999999998,"15min":1.2909999999999999},"min":{"1min":0.40899999999999997,"5min":0.249,"15min":0.249},"max":{"1min":69.150000000000006,"5min":69.150000000000006,"15min":69.150000000000006},"last":0.39500000000000002},{"interface":"front","average":{"1min":4.6120000000000001,"5min":2.8919999999999999,"15min":1.4059999999999999},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":69.132000000000005,"5min":69.132000000000005,"15min":69.132000000000005},"last":0.41199999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":2.4780000000000002,"15min":1.294},"min":{"1min":0.34999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.4399999999999999,"5min":37.978000000000002,"15min":37.978000000000002},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.9630000000000001,"15min":1.167},"min":{"1min":0.39400000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.9049999999999998,"5min":37.759999999999998,"15min":37.759999999999998},"last":0.51700000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":1.903,"15min":1.0449999999999999},"min":{"1min":0.34599999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":20.408999999999999,"5min":20.818999999999999,"15min":20.818999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.778,"5min":2.0819999999999999,"15min":1.1659999999999999},"min":{"1min":0.40400000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":19.940999999999999,"5min":21.015000000000001,"15min":21.015000000000001},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":2.665,"15min":1.2190000000000001},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":1.423,"5min":73.656000000000006,"15min":73.656000000000006},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":2.2719999999999998,"15min":1.113},"min":{"1min":0.32400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.4790000000000001,"5min":73.882999999999996,"15min":73.882999999999996},"last":0.35299999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":1.77,"15min":0.93200000000000005},"min":{"1min":0.33500000000000002,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":1.304,"5min":47.408999999999999,"15min":47.408999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.64500000000000002,"5min":1.641,"15min":0.93500000000000005},"min":{"1min":0.38,"5min":0.35099999999999998,"15min":0.311},"max":{"1min":1.5009999999999999,"5min":47.344000000000001,"15min":47.344000000000001},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.897,"15min":1.0529999999999999},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.466,"5min":53.927999999999997,"15min":53.927999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.628,"5min":1.8340000000000001,"15min":1.1000000000000001},"min":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":1.454,"5min":51.886000000000003,"15min":51.886000000000003},"last":0.46899999999999997}]}]},{"osd":1,"up_from":99,"seq":425201762413,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1655464,"kb_used_data":1515552,"kb_used_omap":5,"kb_used_meta":139898,"kb_avail":92089688,"statfs":{"total":95995035648,"available":94299840512,"internally_reserved":0,"allocated":1551925248,"data_stored":1550642394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":143255734},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":5.7160000000000002,"15min":2.1829999999999998},"min":{"1min":0.217,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.9160000000000004,"5min":150.93899999999999,"15min":150.93899999999999},"last":0.26500000000000001},{"interface":"front","average":{"1min":1.496,"5min":5.7619999999999996,"15min":2.2370000000000001},"min":{"1min":0.224,"5min":0.216,"15min":0.216},"max":{"1min":9.6699999999999999,"5min":156.74199999999999,"15min":156.74199999999999},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:29:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":7.2290000000000001,"15min":2.7360000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.3159999999999998,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":7.0300000000000002,"15min":2.6909999999999998},"min":{"1min":0.28799999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.7560000000000002,"5min":144.62,"15min":144.62},"last":0.56399999999999995}]},{"osd":3,"last update":"Sat Sep 30 09:28:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.5760000000000001,"15min":0.82899999999999996},"min":{"1min":0.27500000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.8300000000000001,"5min":18.757999999999999,"15min":18.757999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.2250000000000001,"15min":0.74199999999999999},"min":{"1min":0.33600000000000002,"5min":0.214,"15min":0.214},"max":{"1min":7.8769999999999998,"5min":11.173,"15min":11.173},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.242,"15min":0.86299999999999999},"min":{"1min":0.41499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.94799999999999995,"5min":10.173,"15min":10.173},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.29,"15min":0.88300000000000001},"min":{"1min":0.42399999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79300000000000004,"5min":8.9299999999999997,"15min":8.9299999999999997},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.1080000000000001,"15min":0.81100000000000005},"min":{"1min":0.38500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5349999999999999,"5min":10.26,"15min":10.26},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.3420000000000001,"15min":0.89300000000000002},"min":{"1min":0.44700000000000001,"5min":0.35499999999999998,"15min":0.33300000000000002},"max":{"1min":0.76400000000000001,"5min":9.3729999999999993,"15min":9.3729999999999993},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.409,"15min":0.89600000000000002},"min":{"1min":0.40999999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73899999999999999,"5min":14.356999999999999,"15min":14.356999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.508,"15min":0.97999999999999998},"min":{"1min":0.377,"5min":0.313,"15min":0.313},"max":{"1min":2.855,"5min":15.864000000000001,"15min":15.864000000000001},"last":0.61199999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.7869999999999999,"15min":1.048},"min":{"1min":0.42499999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.7929999999999999,"5min":14.403,"15min":14.403},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.95299999999999996,"5min":2.9860000000000002,"15min":1.4810000000000001},"min":{"1min":0.50900000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.8799999999999999,"5min":122.56699999999999,"15min":122.56699999999999},"last":0.58099999999999996}]}]},{"osd":0,"up_from":94,"seq":403726925946,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":760608,"kb_used_data":527832,"kb_used_omap":5,"kb_used_meta":232762,"kb_avail":92984544,"statfs":{"total":95995035648,"available":95216173056,"internally_reserved":0,"allocated":540499968,"data_stored":539168488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":238348541},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:28:41 2023","interfaces":[{"interface":"back","average":{"1min":7.0549999999999997,"5min":2.915,"15min":1.2390000000000001},"min":{"1min":0.191,"5min":0.188,"15min":0.188},"max":{"1min":125.024,"5min":125.024,"15min":125.024},"last":0.25},{"interface":"front","average":{"1min":7.234,"5min":2.9140000000000001,"15min":1.266},"min":{"1min":0.24299999999999999,"5min":0.192,"15min":0.192},"max":{"1min":124.901,"5min":124.901,"15min":124.901},"last":0.20899999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:29:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":2.1469999999999998,"15min":1.071},"min":{"1min":0.247,"5min":0.218,"15min":0.218},"max":{"1min":55.823999999999998,"5min":55.823999999999998,"15min":55.823999999999998},"last":0.29499999999999998},{"interface":"front","average":{"1min":3.8820000000000001,"5min":2.3340000000000001,"15min":1.1779999999999999},"min":{"1min":0.32200000000000001,"5min":0.245,"15min":0.245},"max":{"1min":55.790999999999997,"5min":55.790999999999997,"15min":55.790999999999997},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:28:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":2.2949999999999999,"15min":1.121},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":55.68,"5min":55.68,"15min":55.68},"last":0.504},{"interface":"front","average":{"1min":2.7160000000000002,"5min":2.069,"15min":1.032},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":55.244,"5min":55.244,"15min":55.244},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":2.5880000000000001,"15min":1.3340000000000001},"min":{"1min":0.36499999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.6240000000000001,"5min":121.486,"15min":121.486},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.495,"5min":3.0739999999999998,"15min":1.5289999999999999},"min":{"1min":0.32600000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.70999999999999996,"5min":121.44499999999999,"15min":121.44499999999999},"last":0.45100000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.8879999999999999,"15min":1.1619999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79200000000000004,"5min":15.162000000000001,"15min":15.162000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.708,"15min":1.121},"min":{"1min":0.35199999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.80500000000000005,"5min":13.766,"15min":13.766},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.373,"15min":0.99299999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.84799999999999998,"5min":13.811999999999999,"15min":13.811999999999999},"last":1.9430000000000001},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.2889999999999999,"15min":0.97699999999999998},"min":{"1min":0.41799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.7130000000000001,"5min":7.556,"15min":7.556},"last":1.9730000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":2.9009999999999998,"15min":1.4850000000000001},"min":{"1min":0.39300000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81200000000000006,"5min":127.199,"15min":127.199},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.68300000000000005,"5min":2.6440000000000001,"15min":1.4179999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.6719999999999999,"5min":118.551,"15min":118.551},"last":0.78600000000000003}]}]},{"osd":2,"up_from":0,"seq":446676598887,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435359,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3045620,"kb_used_data":2883556,"kb_used_omap":2,"kb_used_meta":162045,"kb_avail":90699532,"statfs":{"total":95995035648,"available":92876320768,"internally_reserved":0,"allocated":2952761344,"data_stored":2951464223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":165934439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":2.149,"15min":1.421},"min":{"1min":0.222,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.3929999999999998,"5min":62.206000000000003,"15min":130.946},"last":0.248},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.1280000000000001,"15min":1.4279999999999999},"min":{"1min":0.26800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.1110000000000007,"5min":60.305999999999997,"15min":130.96899999999999},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":1.6160000000000001,"15min":0.84599999999999997},"min":{"1min":0.251,"5min":0.23200000000000001,"15min":0.224},"max":{"1min":3.5169999999999999,"5min":13.039999999999999,"15min":13.039999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.6990000000000001,"15min":0.86299999999999999},"min":{"1min":0.25900000000000001,"5min":0.221,"15min":0.221},"max":{"1min":9.0860000000000003,"5min":14.057,"15min":14.057},"last":0.28899999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":2.7000000000000002,"15min":1.7210000000000001},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.8420000000000001,"5min":61.704999999999998,"15min":145.75899999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.70699999999999996,"5min":2.395,"15min":1.633},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.2429999999999999,"5min":59.094000000000001,"15min":150.358},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.0569999999999999,"15min":0.77100000000000002},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.9830000000000001,"5min":10.301,"15min":10.301},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.204,"5min":1.4870000000000001,"15min":0.92100000000000004},"min":{"1min":0.443,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.9870000000000001,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.607,"15min":0.98699999999999999},"min":{"1min":0.443,"5min":0.314,"15min":0.308},"max":{"1min":9.0310000000000006,"5min":10.048999999999999,"15min":10.048999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.8049999999999999,"15min":1.077},"min":{"1min":0.46400000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.3479999999999999,"5min":12.785,"15min":12.785},"last":0.61599999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.2769999999999999,"15min":0.88200000000000001},"min":{"1min":0.42499999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.4209999999999998,"5min":10.348000000000001,"15min":10.348000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.722,"15min":1.0609999999999999},"min":{"1min":0.44700000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.0589999999999993,"5min":14.69,"15min":14.69},"last":0.55400000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.7090000000000001,"15min":1.034},"min":{"1min":0.40799999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.3239999999999998,"5min":12.742000000000001,"15min":12.742000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.262,"15min":0.88300000000000001},"min":{"1min":0.49299999999999999,"5min":0.377,"15min":0.372},"max":{"1min":7.242,"5min":10.43,"15min":10.43},"last":0.50800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019122,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1294604,"kb_used_data":1155588,"kb_used_omap":0,"kb_used_meta":139008,"kb_avail":92450548,"statfs":{"total":95995035648,"available":94669361152,"internally_reserved":0,"allocated":1183322112,"data_stored":1182465661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":142344192},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":6.2649999999999997,"15min":2.4990000000000001},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":20.542000000000002,"5min":128.79300000000001,"15min":128.79300000000001},"last":2.6589999999999998},{"interface":"front","average":{"1min":1.593,"5min":5.8819999999999997,"15min":2.3769999999999998},"min":{"1min":0.30099999999999999,"5min":0.248,"15min":0.248},"max":{"1min":20.469999999999999,"5min":128.38800000000001,"15min":128.38800000000001},"last":2.3679999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":2.7410000000000001,"15min":1.407},"min":{"1min":0.36399999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.95199999999999996,"5min":93.244,"15min":93.244},"last":2.2879999999999998},{"interface":"front","average":{"1min":0.627,"5min":2.915,"15min":1.474},"min":{"1min":0.36599999999999999,"5min":0.26800000000000002,"15min":0.23999999999999999},"max":{"1min":2.3279999999999998,"5min":93.465999999999994,"15min":93.465999999999994},"last":2.5459999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":6.2380000000000004,"5min":5.274,"15min":2.379},"min":{"1min":0.37,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":77.620000000000005,"5min":140.601,"15min":140.601},"last":2.573},{"interface":"front","average":{"1min":6.2610000000000001,"5min":5.1459999999999999,"15min":2.327},"min":{"1min":0.35999999999999999,"5min":0.35199999999999998,"15min":0.34200000000000003},"max":{"1min":77.638999999999996,"5min":140.54900000000001,"15min":140.54900000000001},"last":2.6850000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":4.4509999999999996,"5min":2.9990000000000001,"15min":1.4179999999999999},"min":{"1min":0.46000000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":75.593999999999994,"5min":86.620000000000005,"15min":86.620000000000005},"last":2.4449999999999998},{"interface":"front","average":{"1min":4.3890000000000002,"5min":2.8849999999999998,"15min":1.401},"min":{"1min":0.45100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":75.510999999999996,"5min":86.786000000000001,"15min":86.786000000000001},"last":2.3380000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:28:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.5649999999999999,"15min":0.84699999999999998},"min":{"1min":0.35799999999999998,"5min":0.252,"15min":0.252},"max":{"1min":2.109,"5min":86.691000000000003,"15min":86.691000000000003},"last":2.3940000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.8360000000000001,"15min":0.96599999999999997},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.80200000000000005,"5min":86.822000000000003,"15min":86.822000000000003},"last":2.5169999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.84699999999999998,"15min":0.63100000000000001},"min":{"1min":0.32400000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.1579999999999999,"5min":5.6310000000000002,"15min":5.6310000000000002},"last":2.6280000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.92800000000000005,"15min":0.69499999999999995},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.0760000000000001,"5min":6.327,"15min":6.327},"last":2.4860000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.992,"15min":1.075},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.391,"5min":86.728999999999999,"15min":86.728999999999999},"last":2.601},{"interface":"front","average":{"1min":0.626,"5min":1.1259999999999999,"15min":0.79600000000000004},"min":{"1min":0.35499999999999998,"5min":0.32600000000000001,"15min":0.314},"max":{"1min":2.3700000000000001,"5min":8.5869999999999997,"15min":8.5869999999999997},"last":2.8610000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757486,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1686392,"kb_used_data":1542448,"kb_used_omap":0,"kb_used_meta":143936,"kb_avail":92058760,"statfs":{"total":95995035648,"available":94268170240,"internally_reserved":0,"allocated":1579466752,"data_stored":1578625595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":147390464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":4.4160000000000004,"15min":1.8460000000000001},"min":{"1min":0.252,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.7879999999999998,"5min":148.79900000000001,"15min":148.79900000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.61399999999999999,"5min":4.9560000000000004,"15min":2.0590000000000002},"min":{"1min":0.26200000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.9100000000000001,"5min":148.779,"15min":148.779},"last":0.311}]},{"osd":1,"last update":"Sat Sep 30 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":3.9940000000000002,"15min":2.1150000000000002},"min":{"1min":0.29199999999999998,"5min":0.251,"15min":0.251},"max":{"1min":2.0339999999999998,"5min":166.708,"15min":166.708},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":3.8450000000000002,"15min":2.081},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5590000000000002,"5min":166.607,"15min":166.607},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":5.8339999999999996,"15min":2.5649999999999999},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.931,"5min":165.768,"15min":165.768},"last":0.372},{"interface":"front","average":{"1min":1.123,"5min":7.1920000000000002,"15min":3.0880000000000001},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.3440000000000003,"5min":166.24000000000001,"15min":166.24000000000001},"last":1.6950000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:28:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.9159999999999999,"15min":1.1639999999999999},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.311},"max":{"1min":2.0059999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.613,"15min":1.026},"min":{"1min":0.45700000000000002,"5min":0.313,"15min":0.313},"max":{"1min":1.6339999999999999,"5min":28.039999999999999,"15min":28.039999999999999},"last":0.39200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.88600000000000001,"15min":0.61299999999999999},"min":{"1min":0.30199999999999999,"5min":0.222,"15min":0.222},"max":{"1min":1.5469999999999999,"5min":8.5359999999999996,"15min":8.5359999999999996},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.054,"15min":0.69999999999999996},"min":{"1min":0.313,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.653,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":1.7310000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:28:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.1779999999999999,"15min":0.76200000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.9199999999999999,"5min":14.23,"15min":14.23},"last":1.1899999999999999},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.4059999999999999,"15min":0.92200000000000004},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":3.7200000000000002,"5min":18.960000000000001,"15min":18.960000000000001},"last":1.202}]},{"osd":7,"last update":"Sat Sep 30 09:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.246,"15min":0.85099999999999998},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.9459999999999997,"5min":19.175999999999998,"15min":19.175999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.95499999999999996,"15min":0.748},"min":{"1min":0.376,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.5099999999999998,"5min":8.8810000000000002,"15min":8.8810000000000002},"last":0.49199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":536346624,"data_stored":536346624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1548812288,"data_stored":1548812288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2306539520,"data_stored":2306539520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2939944960,"data_stored":2939944960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1192165376,"data_stored":1192165376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1591541760,"data_stored":1591541760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1516371968,"data_stored":1516371968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1704067072,"data_stored":1704067072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:29:24.158 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:29:24.159 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pg_num 36 2023-09-30T09:29:24.159 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:24.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:24.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:24.261 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:24.328 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:24.437 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:24.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:24.508 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:24.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:24 smithi019 ceph-mon[224043]: pgmap v548: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 191 op/s; 30084/196179 objects degraded (15.335%); 2555/196179 objects misplaced (1.302%); 11 MiB/s, 171 objects/s recovering 2023-09-30T09:29:24.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:29:24.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:29:24.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:24.517 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:24 smithi019 ceph-mon[216815]: pgmap v548: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 191 op/s; 30084/196179 objects degraded (15.335%); 2555/196179 objects misplaced (1.302%); 11 MiB/s, 171 objects/s recovering 2023-09-30T09:29:24.517 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:29:24.517 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:29:24.517 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:24.564 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:24.626 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:24 smithi167 ceph-mon[192880]: pgmap v548: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 191 op/s; 30084/196179 objects degraded (15.335%); 2555/196179 objects misplaced (1.302%); 11 MiB/s, 171 objects/s recovering 2023-09-30T09:29:24.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:29:24.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:29:24.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:24.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 64 16 70161 70145 68.4717 56.5625 0.0123689 0.0145979 2023-09-30T09:29:25.480 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:25.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:25.507 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:25.593 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:25.614 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:25.657 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:25.747 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:25.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:25.855 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:25.907 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:25.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 65 16 71194 71178 68.4115 64.5625 0.00655773 0.0146116 2023-09-30T09:29:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:26 smithi019 ceph-mon[216815]: pgmap v549: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 40 MiB/s wr, 642 op/s; 28083/207003 objects degraded (13.566%); 4226/207003 objects misplaced (2.042%); 14 MiB/s, 225 objects/s recovering 2023-09-30T09:29:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:26 smithi019 ceph-mon[224043]: pgmap v549: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 40 MiB/s wr, 642 op/s; 28083/207003 objects degraded (13.566%); 4226/207003 objects misplaced (2.042%); 14 MiB/s, 225 objects/s recovering 2023-09-30T09:29:26.636 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:26 smithi167 ceph-mon[192880]: pgmap v549: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 40 MiB/s wr, 642 op/s; 28083/207003 objects degraded (13.566%); 4226/207003 objects misplaced (2.042%); 14 MiB/s, 225 objects/s recovering 2023-09-30T09:29:26.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 66 16 72041 72025 68.177 52.9375 0.00944395 0.0146616 2023-09-30T09:29:27.022 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:27.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:27.129 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:27.158 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:27.281 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:27.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:27.323 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2871922076' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:27.331 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:29:27.385 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:27.394 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:29:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:29:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:29:27.394 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:29:27.442 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2871922076' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2871922076' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:27.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 67 16 73047 73031 68.0979 62.875 0.00348133 0.014677 2023-09-30T09:29:28.513 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:29:28.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: pgmap v550: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 36 MiB/s wr, 571 op/s; 28083/208779 objects degraded (13.451%); 4226/208779 objects misplaced (2.024%); 3.4 MiB/s, 54 objects/s recovering 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2871922076' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: osdmap e236: 8 total, 8 up, 7 in 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28083/208779 objects degraded (13.451%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:28.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: pgmap v550: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 36 MiB/s wr, 571 op/s; 28083/208779 objects degraded (13.451%); 4226/208779 objects misplaced (2.024%); 3.4 MiB/s, 54 objects/s recovering 2023-09-30T09:29:28.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:29:28.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2871922076' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:28.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: osdmap e236: 8 total, 8 up, 7 in 2023-09-30T09:29:28.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28083/208779 objects degraded (13.451%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:28.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: pgmap v550: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 36 MiB/s wr, 571 op/s; 28083/208779 objects degraded (13.451%); 4226/208779 objects misplaced (2.024%); 3.4 MiB/s, 54 objects/s recovering 2023-09-30T09:29:28.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:29:28.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2871922076' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:28.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: osdmap e236: 8 total, 8 up, 7 in 2023-09-30T09:29:28.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: failed to encode map e236 with expected crc 2023-09-30T09:29:28.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28083/208779 objects degraded (13.451%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:28.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 68 16 74032 74016 68.0017 61.5625 0.0120764 0.0147 2023-09-30T09:29:29.008 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:29.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:29.114 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:29.174 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:29.185 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:29.285 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:29.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:29.399 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:29.455 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:29.555 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:29.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:29.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:29.662 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:29.720 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:29.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 69 16 74923 74907 67.8232 55.6875 0.0184116 0.0147394 2023-09-30T09:29:30.066 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:30.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:30.178 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:30.246 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:30.327 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:30 smithi019 ceph-mon[216815]: pgmap v552: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s; 28083/220302 objects degraded (12.748%); 3919/220302 objects misplaced (1.779%); 6.3 MiB/s, 100 objects/s recovering 2023-09-30T09:29:30.327 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/768876674' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:30.342 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:29:30.375 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:30.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:30.382 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:29:30.404 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:29:30.428 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:29:30.450 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:29:30.468 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:29:30.479 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:30.487 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:29:30.512 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:30.560 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:30.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:30.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:30.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:30 smithi019 ceph-mon[224043]: pgmap v552: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s; 28083/220302 objects degraded (12.748%); 3919/220302 objects misplaced (1.779%); 6.3 MiB/s, 100 objects/s recovering 2023-09-30T09:29:30.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/768876674' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:30.647 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:29:30.655 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:29:30.655 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:29:30.655 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:29:30.655 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:29:30.656 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:29:30.656 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:29:30.656 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:29:30.656 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:29:30.665 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:30.735 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:30.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:30 smithi167 ceph-mon[192880]: pgmap v552: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s; 28083/220302 objects degraded (12.748%); 3919/220302 objects misplaced (1.779%); 6.3 MiB/s, 100 objects/s recovering 2023-09-30T09:29:30.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/768876674' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:30.923 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:30.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 70 16 75677 75661 67.5276 47.125 0.0239091 0.0148027 2023-09-30T09:29:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:30.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:31.032 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:31.102 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:31.203 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:31.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:31.312 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:31.358 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:31.366 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 6 pg_num to 36 2023-09-30T09:29:31.425 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:31.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:31.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:31.529 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:31 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4170279295' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:29:31.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:31 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4170279295' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:29:31.606 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:31 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4170279295' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:29:31.613 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:31.700 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:29:31.700 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:29:31.747 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:31.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:31.857 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:31.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 71 16 76627 76611 67.4127 59.375 0.0203642 0.0148288 2023-09-30T09:29:31.933 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:32.072 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:32.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:32.114 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:29:32.175 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:32.228 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:32.302 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:32.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:32.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:32.407 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:32.434 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:32.475 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:32.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:32.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:32.578 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:32.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[216815]: pgmap v553: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 28083/221133 objects degraded (12.700%); 3919/221133 objects misplaced (1.772%); 5.6 MiB/s, 89 objects/s recovering 2023-09-30T09:29:32.661 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4170279295' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]': finished 2023-09-30T09:29:32.661 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[216815]: osdmap e237: 8 total, 8 up, 7 in 2023-09-30T09:29:32.661 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[216815]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.661 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[216815]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.662 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[216815]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.662 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[224043]: pgmap v553: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 28083/221133 objects degraded (12.700%); 3919/221133 objects misplaced (1.772%); 5.6 MiB/s, 89 objects/s recovering 2023-09-30T09:29:32.662 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4170279295' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]': finished 2023-09-30T09:29:32.662 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[224043]: osdmap e237: 8 total, 8 up, 7 in 2023-09-30T09:29:32.662 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[224043]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.662 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[224043]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.662 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:32 smithi019 ceph-mon[224043]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.715 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:32 smithi167 ceph-mon[192880]: pgmap v553: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 28083/221133 objects degraded (12.700%); 3919/221133 objects misplaced (1.772%); 5.6 MiB/s, 89 objects/s recovering 2023-09-30T09:29:32.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4170279295' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]': finished 2023-09-30T09:29:32.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:32 smithi167 ceph-mon[192880]: osdmap e237: 8 total, 8 up, 7 in 2023-09-30T09:29:32.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:32 smithi167 ceph-mon[192880]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:32 smithi167 ceph-mon[192880]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.716 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:32 smithi167 ceph-mon[192880]: failed to encode map e237 with expected crc 2023-09-30T09:29:32.727 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:32.856 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:29:32.875 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:32.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:32.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 72 16 77411 77395 67.157 49 0.0903068 0.0148836 2023-09-30T09:29:32.984 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:33.014 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:33.042 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:33.133 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:33.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:33.246 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:33.299 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:33.388 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:33.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:33.499 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:33.533 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:33.585 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:33.809 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:29:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:29:33] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-09-30T09:29:33.822 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:33.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:33.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 73 16 78349 78333 67.0401 58.625 0.017385 0.0149113 2023-09-30T09:29:33.931 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:33.983 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:34.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[216815]: pgmap v555: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 28083/222963 objects degraded (12.595%); 3919/222963 objects misplaced (1.758%); 3.6 MiB/s, 57 objects/s recovering 2023-09-30T09:29:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-30T09:29:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28083/221133 objects degraded (12.700%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:34.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:34.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:34.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:34.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[224043]: pgmap v555: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 28083/222963 objects degraded (12.595%); 3919/222963 objects misplaced (1.758%); 3.6 MiB/s, 57 objects/s recovering 2023-09-30T09:29:34.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-30T09:29:34.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28083/221133 objects degraded (12.700%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:34.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:34.136 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:29:34.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:34.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:34.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:33 smithi167 ceph-mon[192880]: pgmap v555: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 28083/222963 objects degraded (12.595%); 3919/222963 objects misplaced (1.758%); 3.6 MiB/s, 57 objects/s recovering 2023-09-30T09:29:34.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-30T09:29:34.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28083/221133 objects degraded (12.700%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:34.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:34.181 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:34.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:34.287 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:34.315 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:34.410 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:34.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:34.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:34.514 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:34.541 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:34.585 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:34.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:34.670 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:34.691 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:34.729 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:34.796 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:34.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:34.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:34.901 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:34.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 74 16 78425 78409 66.1986 4.75 0.050558 0.0149362 2023-09-30T09:29:34.942 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:35.158 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:34 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-30T09:29:35.158 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:34 smithi019 ceph-mon[224043]: osdmap e238: 8 total, 8 up, 7 in 2023-09-30T09:29:35.159 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:34 smithi019 ceph-mon[224043]: failed to encode map e238 with expected crc 2023-09-30T09:29:35.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:34 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-30T09:29:35.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:34 smithi019 ceph-mon[216815]: osdmap e238: 8 total, 8 up, 7 in 2023-09-30T09:29:35.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:34 smithi019 ceph-mon[216815]: failed to encode map e238 with expected crc 2023-09-30T09:29:35.216 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:35.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:35.237 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:34 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-30T09:29:35.238 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:34 smithi167 ceph-mon[192880]: osdmap e238: 8 total, 8 up, 7 in 2023-09-30T09:29:35.238 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:34 smithi167 ceph-mon[192880]: failed to encode map e238 with expected crc 2023-09-30T09:29:35.320 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:35.352 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:35.508 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:35.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:35.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:35.613 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:35.645 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:35.693 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:35.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:35.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:35.761 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:29:35.761 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:29:35.797 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:35.828 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:35.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 75 16 78498 78482 65.377 4.5625 0.00259619 0.0150861 2023-09-30T09:29:35.976 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:35.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:35.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:36.008 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:29:36.079 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:36.107 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:36.151 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:36.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: failed to encode map e238 with expected crc 2023-09-30T09:29:36.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: osdmap e239: 8 total, 8 up, 7 in 2023-09-30T09:29:36.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: pgmap v558: 119 pgs: 2 peering, 2 remapped+peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21872/215580 objects degraded (10.146%); 3606/215580 objects misplaced (1.673%) 2023-09-30T09:29:36.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1231158016' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:29:36.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:29:36.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: failed to encode map e238 with expected crc 2023-09-30T09:29:36.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: osdmap e239: 8 total, 8 up, 7 in 2023-09-30T09:29:36.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: pgmap v558: 119 pgs: 2 peering, 2 remapped+peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21872/215580 objects degraded (10.146%); 3606/215580 objects misplaced (1.673%) 2023-09-30T09:29:36.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1231158016' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:29:36.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:35 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:29:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:36.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:36.256 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: failed to encode map e238 with expected crc 2023-09-30T09:29:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: osdmap e239: 8 total, 8 up, 7 in 2023-09-30T09:29:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: failed to encode map e239 with expected crc 2023-09-30T09:29:36.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: pgmap v558: 119 pgs: 2 peering, 2 remapped+peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21872/215580 objects degraded (10.146%); 3606/215580 objects misplaced (1.673%) 2023-09-30T09:29:36.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1231158016' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:29:36.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:35 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:29:36.293 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:36.382 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:36.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:36.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:36.486 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:36.535 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:36.675 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:29:36.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 76 16 79065 79049 64.9831 35.4375 0.00487224 0.0152591 2023-09-30T09:29:37.154 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:37.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:37.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:37.185 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:37.244 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: failed to encode map e239 with expected crc 2023-09-30T09:29:37.244 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: failed to encode map e239 with expected crc 2023-09-30T09:29:37.244 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:37.244 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:29:37.244 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.244 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.244 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: osdmap e240: 8 total, 8 up, 7 in 2023-09-30T09:29:37.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[224043]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: failed to encode map e239 with expected crc 2023-09-30T09:29:37.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: failed to encode map e239 with expected crc 2023-09-30T09:29:37.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:37.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:29:37.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: osdmap e240: 8 total, 8 up, 7 in 2023-09-30T09:29:37.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:36 smithi019 ceph-mon[216815]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.258 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:37.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: failed to encode map e239 with expected crc 2023-09-30T09:29:37.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: failed to encode map e239 with expected crc 2023-09-30T09:29:37.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:37.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:29:37.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: osdmap e240: 8 total, 8 up, 7 in 2023-09-30T09:29:37.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:36 smithi167 ceph-mon[192880]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.309 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:37.430 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:37.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:37.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:37.538 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:37.592 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:37.691 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:37.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:37.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:37.803 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:37.860 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:37.919 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:29:37.919 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:29:37.919 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-09-30T09:29:37.920 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:29:37.931 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 77 16 80248 80232 65.0978 73.9375 0.00797399 0.0153519 2023-09-30T09:29:37.971 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:29:37.994 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:37 smithi167 ceph-mon[192880]: failed to encode map e240 with expected crc 2023-09-30T09:29:37.994 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:37 smithi167 ceph-mon[192880]: failed to encode map e241 with expected crc 2023-09-30T09:29:37.994 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:37 smithi167 ceph-mon[192880]: osdmap e241: 8 total, 8 up, 7 in 2023-09-30T09:29:37.994 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:37 smithi167 ceph-mon[192880]: failed to encode map e241 with expected crc 2023-09-30T09:29:37.995 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:37.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:37.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:38.008 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:29:38.047 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:29:38.088 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:29:38.098 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:38.123 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:29:38.139 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:38.156 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:29:38.186 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:29:38.200 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:38.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:38.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:38.216 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:29:38.220 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:29:38.220 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:29:38.220 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:29:38.220 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:29:38.220 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:29:38.220 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:29:38.220 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:29:38.220 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:29:38.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[216815]: failed to encode map e240 with expected crc 2023-09-30T09:29:38.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[216815]: failed to encode map e241 with expected crc 2023-09-30T09:29:38.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[216815]: osdmap e241: 8 total, 8 up, 7 in 2023-09-30T09:29:38.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[216815]: failed to encode map e241 with expected crc 2023-09-30T09:29:38.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[216815]: failed to encode map e241 with expected crc 2023-09-30T09:29:38.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[216815]: pgmap v561: 119 pgs: 2 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 9 unknown, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21449/216564 objects degraded (9.904%); 3212/216564 objects misplaced (1.483%); 936 KiB/s, 14 objects/s recovering 2023-09-30T09:29:38.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/604982072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:38.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[224043]: failed to encode map e240 with expected crc 2023-09-30T09:29:38.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[224043]: failed to encode map e241 with expected crc 2023-09-30T09:29:38.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[224043]: osdmap e241: 8 total, 8 up, 7 in 2023-09-30T09:29:38.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[224043]: failed to encode map e241 with expected crc 2023-09-30T09:29:38.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[224043]: failed to encode map e241 with expected crc 2023-09-30T09:29:38.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[224043]: pgmap v561: 119 pgs: 2 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 9 unknown, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21449/216564 objects degraded (9.904%); 3212/216564 objects misplaced (1.483%); 936 KiB/s, 14 objects/s recovering 2023-09-30T09:29:38.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/604982072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:38.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:37 smithi167 ceph-mon[192880]: failed to encode map e241 with expected crc 2023-09-30T09:29:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:37 smithi167 ceph-mon[192880]: pgmap v561: 119 pgs: 2 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 9 unknown, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21449/216564 objects degraded (9.904%); 3212/216564 objects misplaced (1.483%); 936 KiB/s, 14 objects/s recovering 2023-09-30T09:29:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/604982072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:38.307 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:38.343 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:38.444 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:38.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:38.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:38.548 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:38.600 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:38.931 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 78 16 80913 80897 64.796 41.5625 0.0036019 0.0153905 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: failed to encode map e241 with expected crc 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 21449/216564 objects degraded (9.904%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: osdmap e242: 8 total, 8 up, 7 in 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:38 smithi167 ceph-mon[192880]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: failed to encode map e241 with expected crc 2023-09-30T09:29:39.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 21449/216564 objects degraded (9.904%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: osdmap e242: 8 total, 8 up, 7 in 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[216815]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: failed to encode map e241 with expected crc 2023-09-30T09:29:39.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 21449/216564 objects degraded (9.904%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:39.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: osdmap e242: 8 total, 8 up, 7 in 2023-09-30T09:29:39.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:38 smithi019 ceph-mon[224043]: failed to encode map e242 with expected crc 2023-09-30T09:29:39.362 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:29:39.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:39.429 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:39.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:39.522 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:29:39.522 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:29:39.522 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.522 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:81423 6.1a 6:5a7deb39:::benchmark_data_smithi019_355417_object81422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-30T09:29:39.522 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:39.513530+0000", 2023-09-30T09:29:39.522 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0025993470000000001, 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026367500000000002, 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 81423 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:29:39.523 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.513530+0000", 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.513530+0000", 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.513528+0000", 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:29:39.524 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.513700+0000", 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017163000000000001 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.513714+0000", 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4096e-05 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:29:39.525 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.513720+0000", 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9730000000000002e-06 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.513773+0000", 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3335999999999999e-05 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.513901+0000", 2023-09-30T09:29:39.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012765699999999999 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,6", 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.514060+0000", 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015867299999999999 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:39.515735+0000", 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016751349999999999 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:29:39.527 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:29:39.528 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:29:39.528 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:29:39.528 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:29:39.528 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 1 2023-09-30T09:29:39.528 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:29:39.530 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:39.567 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:39.577 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:39.806 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:39.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:39.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:39.911 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:39.931 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 79 16 81903 81887 64.759 61.875 0.0079905 0.0154357 2023-09-30T09:29:39.969 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:40.123 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:40.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:40.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:40.226 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[216815]: pgmap v563: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 1 creating+peering, 4 unknown, 2 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 354 op/s; 18644/219999 objects degraded (8.475%); 5718/219999 objects misplaced (2.599%); 13 MiB/s, 217 objects/s recovering 2023-09-30T09:29:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[216815]: osdmap e243: 8 total, 8 up, 7 in 2023-09-30T09:29:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[216815]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[216815]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[216815]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[224043]: pgmap v563: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 1 creating+peering, 4 unknown, 2 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 354 op/s; 18644/219999 objects degraded (8.475%); 5718/219999 objects misplaced (2.599%); 13 MiB/s, 217 objects/s recovering 2023-09-30T09:29:40.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[224043]: osdmap e243: 8 total, 8 up, 7 in 2023-09-30T09:29:40.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[224043]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[224043]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:40 smithi019 ceph-mon[224043]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.995 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:29:40.931434+0000 min lat: 0.00113601 max lat: 9.94718 avg lat: 0.0154546 2023-09-30T09:29:40.995 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:29:40.996 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 80 16 82839 82823 64.6807 58.5 0.0129888 0.0154546 2023-09-30T09:29:40.996 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:40 smithi167 ceph-mon[192880]: pgmap v563: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 1 creating+peering, 4 unknown, 2 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 354 op/s; 18644/219999 objects degraded (8.475%); 5718/219999 objects misplaced (2.599%); 13 MiB/s, 217 objects/s recovering 2023-09-30T09:29:40.996 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:40 smithi167 ceph-mon[192880]: osdmap e243: 8 total, 8 up, 7 in 2023-09-30T09:29:40.996 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:40 smithi167 ceph-mon[192880]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.996 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:40 smithi167 ceph-mon[192880]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.997 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:40 smithi167 ceph-mon[192880]: failed to encode map e243 with expected crc 2023-09-30T09:29:40.997 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:29:41.055 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:41.147 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:41.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:41.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:41.201 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:41.254 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:41.299 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:41.412 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:41.433 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:29:41.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:41.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:41.534 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:41.627 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:41.631 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:41.931 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 81 16 83990 83974 64.7702 71.9375 0.00946598 0.0154322 2023-09-30T09:29:42.267 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:42 smithi167 ceph-mon[192880]: pgmap v565: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 creating+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 24855/242421 objects degraded (10.253%); 5796/242421 objects misplaced (2.391%); 23 MiB/s, 370 objects/s recovering 2023-09-30T09:29:42.393 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:42.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:42.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:42.497 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:42.523 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:29:42.526 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:29:42.542 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:42.550 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:42 smithi019 ceph-mon[216815]: pgmap v565: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 creating+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 24855/242421 objects degraded (10.253%); 5796/242421 objects misplaced (2.391%); 23 MiB/s, 370 objects/s recovering 2023-09-30T09:29:42.551 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:42 smithi019 ceph-mon[224043]: pgmap v565: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 creating+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 24855/242421 objects degraded (10.253%); 5796/242421 objects misplaced (2.391%); 23 MiB/s, 370 objects/s recovering 2023-09-30T09:29:42.604 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:42.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:42.711 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:42.762 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:42.931 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 82 16 84806 84790 64.6023 51 0.144622 0.0154528 2023-09-30T09:29:42.974 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:42.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:42.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:43.078 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:43.135 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:43.286 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:43.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:43.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:43.399 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:43.465 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:43.553 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:43.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:43.560 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:43 smithi167 ceph-mon[192880]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:43.561 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 24855/242421 objects degraded (10.253%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:43 smithi019 ceph-mon[216815]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 24855/242421 objects degraded (10.253%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:43 smithi019 ceph-mon[224043]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:43.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 24855/242421 objects degraded (10.253%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:43.661 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:43.706 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:43.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 83 16 85812 85796 64.578 62.875 0.0806155 0.0154764 2023-09-30T09:29:43.937 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:43.939 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:29:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:29:43] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-09-30T09:29:43.959 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:43.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:43.977 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":565,"stamp":"2023-09-30T09:29:41.144075+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5298567378,"num_objects":80807,"num_object_clones":0,"num_object_copies":242421,"num_objects_missing_on_primary":1393,"num_objects_missing":1393,"num_objects_degraded":24855,"num_objects_misplaced":5796,"num_objects_unfound":0,"num_objects_dirty":80807,"num_whiteouts":0,"num_read":6031,"num_read_kb":12313,"num_write":65553,"num_write_kb":4147691,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8275,"num_bytes_recovered":542217432,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149609,"ondisk_log_size":149609,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15078912,"kb_used_data":13888180,"kb_used_omap":14,"kb_used_meta":1190641,"kb_avail":641137152,"statfs":{"total":671965249536,"available":656524443648,"internally_reserved":0,"allocated":14221496320,"data_stored":14213866472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14566,"internal_metadata":1219217178},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,9,2,1,4,4,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":586350592,"num_objects":8947,"num_object_clones":0,"num_object_copies":26841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2983,"num_objects_misplaced":2190,"num_objects_unfound":0,"num_objects_dirty":8947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8240,"num_write_kb":527360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1910,"num_bytes_recovered":124485632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.156125"},"pg_stats":[{"pgid":"6.23","version":"242'3545","reported_seq":7706,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.098093+0000","last_change":"2023-09-30T09:29:35.062393+0000","last_active":"2023-09-30T09:29:39.098093+0000","last_peered":"2023-09-30T09:29:39.098093+0000","last_clean":"2023-09-30T09:29:39.098093+0000","last_became_active":"2023-09-30T09:29:35.061339+0000","last_became_peered":"2023-09-30T09:29:35.061339+0000","last_unstale":"2023-09-30T09:29:39.098093+0000","last_undegraded":"2023-09-30T09:29:39.098093+0000","last_fullsized":"2023-09-30T09:29:39.098093+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":3545,"log_dups_size":0,"ondisk_log_size":3545,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82116608,"num_objects":1253,"num_object_clones":0,"num_object_copies":3759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":546,"num_write_kb":34944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65699840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.22","version":"242'3501","reported_seq":6430,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:38.280163+0000","last_change":"2023-09-30T09:29:37.013899+0000","last_active":"2023-09-30T09:29:38.280163+0000","last_peered":"2023-09-30T09:29:38.280163+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:37.011820+0000","last_became_peered":"2023-09-30T09:29:37.011820+0000","last_unstale":"2023-09-30T09:29:38.280163+0000","last_undegraded":"2023-09-30T09:29:36.997644+0000","last_fullsized":"2023-09-30T09:29:36.996855+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":3501,"log_dups_size":0,"ondisk_log_size":3501,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80543744,"num_objects":1229,"num_object_clones":0,"num_object_copies":3687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":304,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":504,"num_write_kb":32224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":304},{"shards":"3,6","objects":925}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.21","version":"240'3696","reported_seq":3787,"reported_epoch":240,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:36.632538+0000","last_change":"2023-09-30T09:29:35.078377+0000","last_active":"2023-09-30T09:29:36.632538+0000","last_peered":"2023-09-30T09:29:36.632538+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:35.075606+0000","last_became_peered":"2023-09-30T09:29:35.075606+0000","last_unstale":"2023-09-30T09:29:36.632538+0000","last_undegraded":"2023-09-30T09:29:35.051902+0000","last_fullsized":"2023-09-30T09:29:36.632538+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":3696,"log_dups_size":0,"ondisk_log_size":3696,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83787776,"num_objects":1279,"num_object_clones":0,"num_object_copies":3837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1043,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":648,"num_write_kb":41440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1043},{"shards":"1,6,7","objects":236}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.20","version":"243'3638","reported_seq":3739,"reported_epoch":243,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:39.382149+0000","last_change":"2023-09-30T09:29:35.050182+0000","last_active":"2023-09-30T09:29:39.382149+0000","last_peered":"2023-09-30T09:29:39.382149+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:35.036894+0000","last_became_peered":"2023-09-30T09:29:35.036894+0000","last_unstale":"2023-09-30T09:29:39.382149+0000","last_undegraded":"2023-09-30T09:29:35.031725+0000","last_fullsized":"2023-09-30T09:29:39.382149+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":3638,"log_dups_size":0,"ondisk_log_size":3638,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85852160,"num_objects":1310,"num_object_clones":0,"num_object_copies":3930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1054,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":604,"num_write_kb":38624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":256},{"shards":"0,2,3","objects":1054}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1f","version":"242'4971","reported_seq":5039,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:38.299451+0000","last_change":"2023-09-30T09:29:35.091385+0000","last_active":"2023-09-30T09:29:38.299451+0000","last_peered":"2023-09-30T09:29:38.299451+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:35.087685+0000","last_became_peered":"2023-09-30T09:29:35.087685+0000","last_unstale":"2023-09-30T09:29:38.299451+0000","last_undegraded":"2023-09-30T09:29:35.082957+0000","last_fullsized":"2023-09-30T09:29:38.299451+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4971,"log_dups_size":0,"ondisk_log_size":4971,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165347328,"num_objects":2523,"num_object_clones":0,"num_object_copies":7569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2093,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2523,"num_write_kb":161472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":2093},{"shards":"3,4,6","objects":430}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":229,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002437+0000","last_change":"2023-09-30T09:29:09.598494+0000","last_active":"2023-09-30T09:29:37.002437+0000","last_peered":"2023-09-30T09:29:37.002437+0000","last_clean":"2023-09-30T09:29:37.002437+0000","last_became_active":"2023-09-30T09:29:09.598158+0000","last_became_peered":"2023-09-30T09:29:09.598158+0000","last_unstale":"2023-09-30T09:29:37.002437+0000","last_undegraded":"2023-09-30T09:29:37.002437+0000","last_fullsized":"2023-09-30T09:29:37.002437+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:03.225682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":207,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996432+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:29:36.996432+0000","last_peered":"2023-09-30T09:29:36.996432+0000","last_clean":"2023-09-30T09:29:36.996432+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:29:36.996432+0000","last_undegraded":"2023-09-30T09:29:36.996432+0000","last_fullsized":"2023-09-30T09:29:36.996432+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1c","version":"242'4909","reported_seq":4970,"reported_epoch":242,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:29:38.246744+0000","last_change":"2023-09-30T09:29:35.102559+0000","last_active":"2023-09-30T09:29:38.246744+0000","last_peered":"2023-09-30T09:29:38.246744+0000","last_clean":"2023-09-30T09:29:33.948392+0000","last_became_active":"2023-09-30T09:29:35.089045+0000","last_became_peered":"2023-09-30T09:29:35.089045+0000","last_unstale":"2023-09-30T09:29:38.246744+0000","last_undegraded":"2023-09-30T09:29:38.246744+0000","last_fullsized":"2023-09-30T09:29:38.246744+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4909,"log_dups_size":0,"ondisk_log_size":4909,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163250176,"num_objects":2491,"num_object_clones":0,"num_object_copies":7473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2491,"num_write_kb":159424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":2491}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":202,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.006293+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:29:35.006293+0000","last_peered":"2023-09-30T09:29:35.006293+0000","last_clean":"2023-09-30T09:29:35.006293+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:29:35.006293+0000","last_undegraded":"2023-09-30T09:29:35.006293+0000","last_fullsized":"2023-09-30T09:29:35.006293+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":136,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.246014+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:29:38.246014+0000","last_peered":"2023-09-30T09:29:38.246014+0000","last_clean":"2023-09-30T09:29:38.246014+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:29:38.246014+0000","last_undegraded":"2023-09-30T09:29:38.246014+0000","last_fullsized":"2023-09-30T09:29:38.246014+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1d","version":"242'5069","reported_seq":5130,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:39.129365+0000","last_change":"2023-09-30T09:29:35.078954+0000","last_active":"2023-09-30T09:29:39.129365+0000","last_peered":"2023-09-30T09:29:39.129365+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:35.073573+0000","last_became_peered":"2023-09-30T09:29:35.073573+0000","last_unstale":"2023-09-30T09:29:39.129365+0000","last_undegraded":"2023-09-30T09:29:35.043564+0000","last_fullsized":"2023-09-30T09:29:39.129365+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5069,"log_dups_size":0,"ondisk_log_size":5069,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168689664,"num_objects":2574,"num_object_clones":0,"num_object_copies":7722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2034,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2574,"num_write_kb":164736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":2034},{"shards":"1,5,6","objects":540}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":257,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996331+0000","last_change":"2023-09-30T09:29:09.606288+0000","last_active":"2023-09-30T09:29:36.996331+0000","last_peered":"2023-09-30T09:29:36.996331+0000","last_clean":"2023-09-30T09:29:36.996331+0000","last_became_active":"2023-09-30T09:29:09.605776+0000","last_became_peered":"2023-09-30T09:29:09.605776+0000","last_unstale":"2023-09-30T09:29:36.996331+0000","last_undegraded":"2023-09-30T09:29:36.996331+0000","last_fullsized":"2023-09-30T09:29:36.996331+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:07.935982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":201,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.011659+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:29:37.011659+0000","last_peered":"2023-09-30T09:29:37.011659+0000","last_clean":"2023-09-30T09:29:37.011659+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:29:37.011659+0000","last_undegraded":"2023-09-30T09:29:37.011659+0000","last_fullsized":"2023-09-30T09:29:37.011659+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"242'3584","reported_seq":7732,"reported_epoch":242,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:29:39.078809+0000","last_change":"2023-09-30T09:29:10.766595+0000","last_active":"2023-09-30T09:29:39.078809+0000","last_peered":"2023-09-30T09:29:39.078809+0000","last_clean":"2023-09-30T09:29:08.564047+0000","last_became_active":"2023-09-30T09:29:10.763967+0000","last_became_peered":"2023-09-30T09:29:10.763967+0000","last_unstale":"2023-09-30T09:29:39.078809+0000","last_undegraded":"2023-09-30T09:29:39.078809+0000","last_fullsized":"2023-09-30T09:29:39.078809+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2497","last_scrub_stamp":"2023-09-30T09:28:46.026181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:46.026181+0000","objects_scrubbed":1334,"log_size":3584,"log_dups_size":0,"ondisk_log_size":3584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:02.322482+0000","scrub_duration":0.15695645899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2421,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1087,"num_write_kb":69568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1347,"num_bytes_recovered":88276992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,2],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,4,7","objects":2421}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":135,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.246112+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:29:38.246112+0000","last_peered":"2023-09-30T09:29:38.246112+0000","last_clean":"2023-09-30T09:29:38.246112+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:29:38.246112+0000","last_undegraded":"2023-09-30T09:29:38.246112+0000","last_fullsized":"2023-09-30T09:29:38.246112+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":207,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996515+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:29:36.996515+0000","last_peered":"2023-09-30T09:29:36.996515+0000","last_clean":"2023-09-30T09:29:36.996515+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:29:36.996515+0000","last_undegraded":"2023-09-30T09:29:36.996515+0000","last_fullsized":"2023-09-30T09:29:36.996515+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"242'3600","reported_seq":3667,"reported_epoch":242,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:29:38.293304+0000","last_change":"2023-09-30T09:29:38.038561+0000","last_active":"2023-09-30T09:29:38.293304+0000","last_peered":"2023-09-30T09:29:38.293304+0000","last_clean":"2023-09-30T09:28:50.280641+0000","last_became_active":"2023-09-30T09:29:12.506944+0000","last_became_peered":"2023-09-30T09:29:12.506944+0000","last_unstale":"2023-09-30T09:29:38.293304+0000","last_undegraded":"2023-09-30T09:29:38.293304+0000","last_fullsized":"2023-09-30T09:29:10.583109+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3600,"log_dups_size":0,"ondisk_log_size":3600,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163577856,"num_objects":2496,"num_object_clones":0,"num_object_copies":7488,"num_objects_missing_on_primary":1393,"num_objects_missing":1393,"num_objects_degraded":0,"num_objects_misplaced":3375,"num_objects_unfound":0,"num_objects_dirty":2496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2496,"num_write_kb":159744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,3,6","objects":589},{"shards":"2,6,7","objects":1393},{"shards":"3,6","objects":514}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":93,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.008369+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:29:37.008369+0000","last_peered":"2023-09-30T09:29:37.008369+0000","last_clean":"2023-09-30T09:29:37.008369+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:29:37.008369+0000","last_undegraded":"2023-09-30T09:29:37.008369+0000","last_fullsized":"2023-09-30T09:29:37.008369+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":158,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996507+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:29:36.996507+0000","last_peered":"2023-09-30T09:29:36.996507+0000","last_clean":"2023-09-30T09:29:36.996507+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:29:36.996507+0000","last_undegraded":"2023-09-30T09:29:36.996507+0000","last_fullsized":"2023-09-30T09:29:36.996507+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"243'3624","reported_seq":7672,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.447770+0000","last_change":"2023-09-30T09:28:56.382198+0000","last_active":"2023-09-30T09:29:39.447770+0000","last_peered":"2023-09-30T09:29:39.447770+0000","last_clean":"2023-09-30T09:29:39.447770+0000","last_became_active":"2023-09-30T09:28:56.380752+0000","last_became_peered":"2023-09-30T09:28:56.380752+0000","last_unstale":"2023-09-30T09:29:39.447770+0000","last_undegraded":"2023-09-30T09:29:39.447770+0000","last_fullsized":"2023-09-30T09:29:39.447770+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3624,"log_dups_size":0,"ondisk_log_size":3624,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164200448,"num_objects":2506,"num_object_clones":0,"num_object_copies":7518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2506,"num_write_kb":160352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1324,"num_bytes_recovered":86769664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":205,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002062+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:29:37.002062+0000","last_peered":"2023-09-30T09:29:37.002062+0000","last_clean":"2023-09-30T09:29:37.002062+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:29:37.002062+0000","last_undegraded":"2023-09-30T09:29:37.002062+0000","last_fullsized":"2023-09-30T09:29:37.002062+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":187,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:34.987572+0000","last_change":"2023-09-30T09:29:09.620809+0000","last_active":"2023-09-30T09:29:34.987572+0000","last_peered":"2023-09-30T09:29:34.987572+0000","last_clean":"2023-09-30T09:29:34.987572+0000","last_became_active":"2023-09-30T09:29:09.620497+0000","last_became_peered":"2023-09-30T09:29:09.620497+0000","last_unstale":"2023-09-30T09:29:34.987572+0000","last_undegraded":"2023-09-30T09:29:34.987572+0000","last_fullsized":"2023-09-30T09:29:34.987572+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:48.932180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"241'3710","reported_seq":9295,"reported_epoch":242,"state":"creating+peering","last_fresh":"2023-09-30T09:29:38.252635+0000","last_change":"2023-09-30T09:29:38.248462+0000","last_active":"2023-09-30T09:29:38.248462+0000","last_peered":"2023-09-30T09:28:45.597110+0000","last_clean":"2023-09-30T09:28:45.597110+0000","last_became_active":"2023-09-30T09:29:10.829695+0000","last_became_peered":"2023-09-30T09:29:10.829695+0000","last_unstale":"2023-09-30T09:29:38.252635+0000","last_undegraded":"2023-09-30T09:29:38.252635+0000","last_fullsized":"2023-09-30T09:29:38.252635+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2608","last_scrub_stamp":"2023-09-30T09:28:45.597110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.597110+0000","objects_scrubbed":1407,"log_size":3710,"log_dups_size":0,"ondisk_log_size":3710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":0.26278548800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164823040,"num_objects":2515,"num_object_clones":0,"num_object_copies":7545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1108,"num_write_kb":70912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2034,"num_bytes_recovered":133300224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":206,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002442+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:29:37.002442+0000","last_peered":"2023-09-30T09:29:37.002442+0000","last_clean":"2023-09-30T09:29:37.002442+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:29:37.002442+0000","last_undegraded":"2023-09-30T09:29:37.002442+0000","last_fullsized":"2023-09-30T09:29:37.002442+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":220,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002549+0000","last_change":"2023-09-30T09:29:09.612466+0000","last_active":"2023-09-30T09:29:37.002549+0000","last_peered":"2023-09-30T09:29:37.002549+0000","last_clean":"2023-09-30T09:29:37.002549+0000","last_became_active":"2023-09-30T09:29:09.612173+0000","last_became_peered":"2023-09-30T09:29:09.612173+0000","last_unstale":"2023-09-30T09:29:37.002549+0000","last_undegraded":"2023-09-30T09:29:37.002549+0000","last_fullsized":"2023-09-30T09:29:37.002549+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:15.457876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"241'3704","reported_seq":3766,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.220318+0000","last_change":"2023-09-30T09:28:40.282669+0000","last_active":"2023-09-30T09:29:38.220318+0000","last_peered":"2023-09-30T09:29:38.220318+0000","last_clean":"2023-09-30T09:29:38.220318+0000","last_became_active":"2023-09-30T09:28:40.281544+0000","last_became_peered":"2023-09-30T09:28:40.281544+0000","last_unstale":"2023-09-30T09:29:38.220318+0000","last_undegraded":"2023-09-30T09:29:38.220318+0000","last_fullsized":"2023-09-30T09:29:38.220318+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3704,"log_dups_size":0,"ondisk_log_size":3704,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168198144,"num_objects":2567,"num_object_clones":0,"num_object_copies":7701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2567,"num_write_kb":164256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":220,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.001579+0000","last_change":"2023-09-30T09:29:09.626205+0000","last_active":"2023-09-30T09:29:37.001579+0000","last_peered":"2023-09-30T09:29:37.001579+0000","last_clean":"2023-09-30T09:29:37.001579+0000","last_became_active":"2023-09-30T09:29:09.618293+0000","last_became_peered":"2023-09-30T09:29:09.618293+0000","last_unstale":"2023-09-30T09:29:37.001579+0000","last_undegraded":"2023-09-30T09:29:37.001579+0000","last_fullsized":"2023-09-30T09:29:37.001579+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:04.337293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":229,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.997181+0000","last_change":"2023-09-30T09:29:09.606072+0000","last_active":"2023-09-30T09:29:36.997181+0000","last_peered":"2023-09-30T09:29:36.997181+0000","last_clean":"2023-09-30T09:29:36.997181+0000","last_became_active":"2023-09-30T09:29:09.605716+0000","last_became_peered":"2023-09-30T09:29:09.605716+0000","last_unstale":"2023-09-30T09:29:36.997181+0000","last_undegraded":"2023-09-30T09:29:36.997181+0000","last_fullsized":"2023-09-30T09:29:36.997181+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:50.972640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"242'3634","reported_seq":3755,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.028574+0000","last_change":"2023-09-30T09:28:45.602319+0000","last_active":"2023-09-30T09:29:39.028574+0000","last_peered":"2023-09-30T09:29:39.028574+0000","last_clean":"2023-09-30T09:29:39.028574+0000","last_became_active":"2023-09-30T09:28:40.272792+0000","last_became_peered":"2023-09-30T09:28:40.272792+0000","last_unstale":"2023-09-30T09:29:39.028574+0000","last_undegraded":"2023-09-30T09:29:39.028574+0000","last_fullsized":"2023-09-30T09:29:39.028574+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2523","last_scrub_stamp":"2023-09-30T09:28:45.602248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.602248+0000","objects_scrubbed":1419,"log_size":3634,"log_dups_size":0,"ondisk_log_size":3634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:31:54.788871+0000","scrub_duration":0.27069680099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166264832,"num_objects":2537,"num_object_clones":0,"num_object_copies":7611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1118,"num_write_kb":71552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":135,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.243168+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:29:38.243168+0000","last_peered":"2023-09-30T09:29:38.243168+0000","last_clean":"2023-09-30T09:29:38.243168+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:29:38.243168+0000","last_undegraded":"2023-09-30T09:29:38.243168+0000","last_fullsized":"2023-09-30T09:29:38.243168+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":163,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002185+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:29:37.002185+0000","last_peered":"2023-09-30T09:29:37.002185+0000","last_clean":"2023-09-30T09:29:37.002185+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:29:37.002185+0000","last_undegraded":"2023-09-30T09:29:37.002185+0000","last_fullsized":"2023-09-30T09:29:37.002185+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"240'3658","reported_seq":3741,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.535361+0000","last_change":"2023-09-30T09:28:44.596036+0000","last_active":"2023-09-30T09:29:36.535361+0000","last_peered":"2023-09-30T09:29:36.535361+0000","last_clean":"2023-09-30T09:29:36.535361+0000","last_became_active":"2023-09-30T09:28:40.272740+0000","last_became_peered":"2023-09-30T09:28:40.272740+0000","last_unstale":"2023-09-30T09:29:36.535361+0000","last_undegraded":"2023-09-30T09:29:36.535361+0000","last_fullsized":"2023-09-30T09:29:36.535361+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"214'2497","last_scrub_stamp":"2023-09-30T09:28:44.595809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:44.595809+0000","objects_scrubbed":1358,"log_size":3658,"log_dups_size":0,"ondisk_log_size":3658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:46:41.016886+0000","scrub_duration":0.31195125499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165609472,"num_objects":2527,"num_object_clones":0,"num_object_copies":7581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1169,"num_write_kb":74816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":185,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.003141+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:29:37.003141+0000","last_peered":"2023-09-30T09:29:37.003141+0000","last_clean":"2023-09-30T09:29:37.003141+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:29:37.003141+0000","last_undegraded":"2023-09-30T09:29:37.003141+0000","last_fullsized":"2023-09-30T09:29:37.003141+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":135,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.245950+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:29:38.245950+0000","last_peered":"2023-09-30T09:29:38.245950+0000","last_clean":"2023-09-30T09:29:38.245950+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:29:38.245950+0000","last_undegraded":"2023-09-30T09:29:38.245950+0000","last_fullsized":"2023-09-30T09:29:38.245950+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"242'3700","reported_seq":3753,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.115521+0000","last_change":"2023-09-30T09:28:40.288346+0000","last_active":"2023-09-30T09:29:39.115521+0000","last_peered":"2023-09-30T09:29:39.115521+0000","last_clean":"2023-09-30T09:29:39.115521+0000","last_became_active":"2023-09-30T09:28:40.286899+0000","last_became_peered":"2023-09-30T09:28:40.286899+0000","last_unstale":"2023-09-30T09:29:39.115521+0000","last_undegraded":"2023-09-30T09:29:39.115521+0000","last_fullsized":"2023-09-30T09:29:39.115521+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3700,"log_dups_size":0,"ondisk_log_size":3700,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168329216,"num_objects":2569,"num_object_clones":0,"num_object_copies":7707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2569,"num_write_kb":164384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":334,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.011236+0000","last_change":"2023-09-30T09:29:09.634829+0000","last_active":"2023-09-30T09:29:37.011236+0000","last_peered":"2023-09-30T09:29:37.011236+0000","last_clean":"2023-09-30T09:29:37.011236+0000","last_became_active":"2023-09-30T09:29:09.612183+0000","last_became_peered":"2023-09-30T09:29:09.612183+0000","last_unstale":"2023-09-30T09:29:37.011236+0000","last_undegraded":"2023-09-30T09:29:37.011236+0000","last_fullsized":"2023-09-30T09:29:37.011236+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:28.144436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":227,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.001422+0000","last_change":"2023-09-30T09:29:09.605918+0000","last_active":"2023-09-30T09:29:37.001422+0000","last_peered":"2023-09-30T09:29:37.001422+0000","last_clean":"2023-09-30T09:29:37.001422+0000","last_became_active":"2023-09-30T09:29:09.605521+0000","last_became_peered":"2023-09-30T09:29:09.605521+0000","last_unstale":"2023-09-30T09:29:37.001422+0000","last_undegraded":"2023-09-30T09:29:37.001422+0000","last_fullsized":"2023-09-30T09:29:37.001422+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:43.287337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"242'4960","reported_seq":5015,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.032269+0000","last_change":"2023-09-30T09:29:35.005933+0000","last_active":"2023-09-30T09:29:39.032269+0000","last_peered":"2023-09-30T09:29:39.032269+0000","last_clean":"2023-09-30T09:29:39.032269+0000","last_became_active":"2023-09-30T09:29:35.001538+0000","last_became_peered":"2023-09-30T09:29:35.001538+0000","last_unstale":"2023-09-30T09:29:39.032269+0000","last_undegraded":"2023-09-30T09:29:39.032269+0000","last_fullsized":"2023-09-30T09:29:39.032269+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4960,"log_dups_size":0,"ondisk_log_size":4960,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164691968,"num_objects":2513,"num_object_clones":0,"num_object_copies":7539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2513,"num_write_kb":160832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":245,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002560+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:29:37.002560+0000","last_peered":"2023-09-30T09:29:37.002560+0000","last_clean":"2023-09-30T09:29:37.002560+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:29:37.002560+0000","last_undegraded":"2023-09-30T09:29:37.002560+0000","last_fullsized":"2023-09-30T09:29:37.002560+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.003980+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:29:37.003980+0000","last_peered":"2023-09-30T09:29:37.003980+0000","last_clean":"2023-09-30T09:29:37.003980+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:29:37.003980+0000","last_undegraded":"2023-09-30T09:29:37.003980+0000","last_fullsized":"2023-09-30T09:29:37.003980+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"242'5009","reported_seq":5071,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.277972+0000","last_change":"2023-09-30T09:29:35.047558+0000","last_active":"2023-09-30T09:29:38.277972+0000","last_peered":"2023-09-30T09:29:38.277972+0000","last_clean":"2023-09-30T09:29:38.277972+0000","last_became_active":"2023-09-30T09:29:35.044991+0000","last_became_peered":"2023-09-30T09:29:35.044991+0000","last_unstale":"2023-09-30T09:29:38.277972+0000","last_undegraded":"2023-09-30T09:29:38.277972+0000","last_fullsized":"2023-09-30T09:29:38.277972+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5009,"log_dups_size":0,"ondisk_log_size":5009,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166559744,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":162656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":246,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.001889+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:29:37.001889+0000","last_peered":"2023-09-30T09:29:37.001889+0000","last_clean":"2023-09-30T09:29:37.001889+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:29:37.001889+0000","last_undegraded":"2023-09-30T09:29:37.001889+0000","last_fullsized":"2023-09-30T09:29:37.001889+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":217,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.000639+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:29:37.000639+0000","last_peered":"2023-09-30T09:29:37.000639+0000","last_clean":"2023-09-30T09:29:37.000639+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:29:37.000639+0000","last_undegraded":"2023-09-30T09:29:37.000639+0000","last_fullsized":"2023-09-30T09:29:37.000639+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"242'3648","reported_seq":3712,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:39.138191+0000","last_change":"2023-09-30T09:29:10.874488+0000","last_active":"2023-09-30T09:29:39.138191+0000","last_peered":"2023-09-30T09:29:39.138191+0000","last_clean":"2023-09-30T09:29:08.490081+0000","last_became_active":"2023-09-30T09:29:10.870401+0000","last_became_peered":"2023-09-30T09:29:10.870401+0000","last_unstale":"2023-09-30T09:29:39.138191+0000","last_undegraded":"2023-09-30T09:29:10.657673+0000","last_fullsized":"2023-09-30T09:29:10.593223+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3648,"log_dups_size":0,"ondisk_log_size":3648,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166100992,"num_objects":2534,"num_object_clones":0,"num_object_copies":7602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1975,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2534,"num_write_kb":162208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1975},{"shards":"3,7","objects":559}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":228,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.001888+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:29:37.001888+0000","last_peered":"2023-09-30T09:29:37.001888+0000","last_clean":"2023-09-30T09:29:37.001888+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:29:37.001888+0000","last_undegraded":"2023-09-30T09:29:37.001888+0000","last_fullsized":"2023-09-30T09:29:37.001888+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":211,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.010696+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:29:37.010696+0000","last_peered":"2023-09-30T09:29:37.010696+0000","last_clean":"2023-09-30T09:29:37.010696+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:29:37.010696+0000","last_undegraded":"2023-09-30T09:29:37.010696+0000","last_fullsized":"2023-09-30T09:29:37.010696+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"242'3593","reported_seq":3721,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:39.074199+0000","last_change":"2023-09-30T09:29:10.893105+0000","last_active":"2023-09-30T09:29:39.074199+0000","last_peered":"2023-09-30T09:29:39.074199+0000","last_clean":"2023-09-30T09:29:08.514351+0000","last_became_active":"2023-09-30T09:29:10.883964+0000","last_became_peered":"2023-09-30T09:29:10.883964+0000","last_unstale":"2023-09-30T09:29:39.074199+0000","last_undegraded":"2023-09-30T09:29:10.674378+0000","last_fullsized":"2023-09-30T09:29:10.596450+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2471","last_scrub_stamp":"2023-09-30T09:28:47.207706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:47.207706+0000","objects_scrubbed":1369,"log_size":3593,"log_dups_size":0,"ondisk_log_size":3593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:51.408261+0000","scrub_duration":0.14908584899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163250176,"num_objects":2491,"num_object_clones":0,"num_object_copies":7473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1122,"num_write_kb":71808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1997},{"shards":"3,7","objects":494}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":201,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.008641+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:29:37.008641+0000","last_peered":"2023-09-30T09:29:37.008641+0000","last_clean":"2023-09-30T09:29:37.008641+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:29:37.008641+0000","last_undegraded":"2023-09-30T09:29:37.008641+0000","last_fullsized":"2023-09-30T09:29:37.008641+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":132,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.245639+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:29:38.245639+0000","last_peered":"2023-09-30T09:29:38.245639+0000","last_clean":"2023-09-30T09:29:38.245639+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:29:38.245639+0000","last_undegraded":"2023-09-30T09:29:38.245639+0000","last_fullsized":"2023-09-30T09:29:38.245639+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"242'4880","reported_seq":4947,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:38.777066+0000","last_change":"2023-09-30T09:29:36.032408+0000","last_active":"2023-09-30T09:29:38.777066+0000","last_peered":"2023-09-30T09:29:38.777066+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:36.028267+0000","last_became_peered":"2023-09-30T09:29:36.028267+0000","last_unstale":"2023-09-30T09:29:38.777066+0000","last_undegraded":"2023-09-30T09:29:36.007788+0000","last_fullsized":"2023-09-30T09:29:38.777066+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4880,"log_dups_size":0,"ondisk_log_size":4880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162594816,"num_objects":2481,"num_object_clones":0,"num_object_copies":7443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2481,"num_write_kb":158784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1921},{"shards":"1,4,7","objects":560}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":721,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.003069+0000","last_change":"2023-09-30T09:29:12.637524+0000","last_active":"2023-09-30T09:29:37.003069+0000","last_peered":"2023-09-30T09:29:37.003069+0000","last_clean":"2023-09-30T09:29:37.003069+0000","last_became_active":"2023-09-30T09:29:12.637171+0000","last_became_peered":"2023-09-30T09:29:12.637171+0000","last_unstale":"2023-09-30T09:29:37.003069+0000","last_undegraded":"2023-09-30T09:29:37.003069+0000","last_fullsized":"2023-09-30T09:29:37.003069+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:14:09.735059+0000","scrub_duration":0.0079898150000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":241,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.997082+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:29:36.997082+0000","last_peered":"2023-09-30T09:29:36.997082+0000","last_clean":"2023-09-30T09:29:36.997082+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:29:36.997082+0000","last_undegraded":"2023-09-30T09:29:36.997082+0000","last_fullsized":"2023-09-30T09:29:36.997082+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":200,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002432+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:29:37.002432+0000","last_peered":"2023-09-30T09:29:37.002432+0000","last_clean":"2023-09-30T09:29:37.002432+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:29:37.002432+0000","last_undegraded":"2023-09-30T09:29:37.002432+0000","last_fullsized":"2023-09-30T09:29:37.002432+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"242'4970","reported_seq":5038,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:38.253351+0000","last_change":"2023-09-30T09:29:36.082560+0000","last_active":"2023-09-30T09:29:38.253351+0000","last_peered":"2023-09-30T09:29:38.253351+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:36.077849+0000","last_became_peered":"2023-09-30T09:29:36.077849+0000","last_unstale":"2023-09-30T09:29:38.253351+0000","last_undegraded":"2023-09-30T09:29:35.995233+0000","last_fullsized":"2023-09-30T09:29:38.253351+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4970,"log_dups_size":0,"ondisk_log_size":4970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165281792,"num_objects":2522,"num_object_clones":0,"num_object_copies":7566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2522,"num_write_kb":161408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":1940},{"shards":"3,4,6","objects":582}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":618,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996106+0000","last_change":"2023-09-30T09:29:12.631999+0000","last_active":"2023-09-30T09:29:36.996106+0000","last_peered":"2023-09-30T09:29:36.996106+0000","last_clean":"2023-09-30T09:29:36.996106+0000","last_became_active":"2023-09-30T09:29:12.631620+0000","last_became_peered":"2023-09-30T09:29:12.631620+0000","last_unstale":"2023-09-30T09:29:36.996106+0000","last_undegraded":"2023-09-30T09:29:36.996106+0000","last_fullsized":"2023-09-30T09:29:36.996106+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.999760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.999760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":531037,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":202,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.010559+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:29:37.010559+0000","last_peered":"2023-09-30T09:29:37.010559+0000","last_clean":"2023-09-30T09:29:37.010559+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:29:37.010559+0000","last_undegraded":"2023-09-30T09:29:37.010559+0000","last_fullsized":"2023-09-30T09:29:37.010559+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":124,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.006852+0000","last_change":"2023-09-30T09:29:09.612070+0000","last_active":"2023-09-30T09:29:35.006852+0000","last_peered":"2023-09-30T09:29:35.006852+0000","last_clean":"2023-09-30T09:29:35.006852+0000","last_became_active":"2023-09-30T09:29:09.611725+0000","last_became_peered":"2023-09-30T09:29:09.611725+0000","last_unstale":"2023-09-30T09:29:35.006852+0000","last_undegraded":"2023-09-30T09:29:35.006852+0000","last_fullsized":"2023-09-30T09:29:35.006852+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:55.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"241'4908","reported_seq":4969,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.164396+0000","last_change":"2023-09-30T09:29:35.074390+0000","last_active":"2023-09-30T09:29:38.164396+0000","last_peered":"2023-09-30T09:29:38.164396+0000","last_clean":"2023-09-30T09:29:38.164396+0000","last_became_active":"2023-09-30T09:29:35.072113+0000","last_became_peered":"2023-09-30T09:29:35.072113+0000","last_unstale":"2023-09-30T09:29:38.164396+0000","last_undegraded":"2023-09-30T09:29:38.164396+0000","last_fullsized":"2023-09-30T09:29:38.164396+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4908,"log_dups_size":0,"ondisk_log_size":4908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163184640,"num_objects":2490,"num_object_clones":0,"num_object_copies":7470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2490,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":211,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.000468+0000","last_change":"2023-09-30T09:29:09.607621+0000","last_active":"2023-09-30T09:29:37.000468+0000","last_peered":"2023-09-30T09:29:37.000468+0000","last_clean":"2023-09-30T09:29:37.000468+0000","last_became_active":"2023-09-30T09:29:09.607245+0000","last_became_peered":"2023-09-30T09:29:09.607245+0000","last_unstale":"2023-09-30T09:29:37.000468+0000","last_undegraded":"2023-09-30T09:29:37.000468+0000","last_fullsized":"2023-09-30T09:29:37.000468+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:42:42.429421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":200,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.004510+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:29:37.004510+0000","last_peered":"2023-09-30T09:29:37.004510+0000","last_clean":"2023-09-30T09:29:37.004510+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:29:37.004510+0000","last_undegraded":"2023-09-30T09:29:37.004510+0000","last_fullsized":"2023-09-30T09:29:37.004510+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"242'5069","reported_seq":5133,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:39.036602+0000","last_change":"2023-09-30T09:29:36.240193+0000","last_active":"2023-09-30T09:29:39.036602+0000","last_peered":"2023-09-30T09:29:39.036602+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:36.233690+0000","last_became_peered":"2023-09-30T09:29:36.233690+0000","last_unstale":"2023-09-30T09:29:39.036602+0000","last_undegraded":"2023-09-30T09:29:36.012206+0000","last_fullsized":"2023-09-30T09:29:39.036602+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5069,"log_dups_size":0,"ondisk_log_size":5069,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168689664,"num_objects":2574,"num_object_clones":0,"num_object_copies":7722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2078,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2574,"num_write_kb":164736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":2078},{"shards":"1,5,6","objects":496}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":449,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.010477+0000","last_change":"2023-09-30T09:29:12.635044+0000","last_active":"2023-09-30T09:29:37.010477+0000","last_peered":"2023-09-30T09:29:37.010477+0000","last_clean":"2023-09-30T09:29:37.010477+0000","last_became_active":"2023-09-30T09:29:12.634703+0000","last_became_peered":"2023-09-30T09:29:12.634703+0000","last_unstale":"2023-09-30T09:29:37.010477+0000","last_undegraded":"2023-09-30T09:29:37.010477+0000","last_fullsized":"2023-09-30T09:29:37.010477+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.980385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.980385+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":208,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996181+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:29:36.996181+0000","last_peered":"2023-09-30T09:29:36.996181+0000","last_clean":"2023-09-30T09:29:36.996181+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:29:36.996181+0000","last_undegraded":"2023-09-30T09:29:36.996181+0000","last_fullsized":"2023-09-30T09:29:36.996181+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":180,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.005896+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:29:35.005896+0000","last_peered":"2023-09-30T09:29:35.005896+0000","last_clean":"2023-09-30T09:29:35.005896+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:29:35.005896+0000","last_undegraded":"2023-09-30T09:29:35.005896+0000","last_fullsized":"2023-09-30T09:29:35.005896+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"243'3613","reported_seq":3714,"reported_epoch":243,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:39.254062+0000","last_change":"2023-09-30T09:29:36.014896+0000","last_active":"2023-09-30T09:29:39.254062+0000","last_peered":"2023-09-30T09:29:39.254062+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:36.012719+0000","last_became_peered":"2023-09-30T09:29:36.012719+0000","last_unstale":"2023-09-30T09:29:39.254062+0000","last_undegraded":"2023-09-30T09:29:35.992619+0000","last_fullsized":"2023-09-30T09:29:39.254062+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":3613,"log_dups_size":0,"ondisk_log_size":3613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84213760,"num_objects":1285,"num_object_clones":0,"num_object_copies":3855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":578,"num_write_kb":37024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":240},{"shards":"0,2,3","objects":1045}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":486,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.011586+0000","last_change":"2023-09-30T09:28:30.495699+0000","last_active":"2023-09-30T09:29:37.011586+0000","last_peered":"2023-09-30T09:29:37.011586+0000","last_clean":"2023-09-30T09:29:37.011586+0000","last_became_active":"2023-09-30T09:28:09.228266+0000","last_became_peered":"2023-09-30T09:28:09.228266+0000","last_unstale":"2023-09-30T09:29:37.011586+0000","last_undegraded":"2023-09-30T09:29:37.011586+0000","last_fullsized":"2023-09-30T09:29:37.011586+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:10:19.705285+0000","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":584,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.005720+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:29:35.005720+0000","last_peered":"2023-09-30T09:29:35.005720+0000","last_clean":"2023-09-30T09:29:35.005720+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:29:35.005720+0000","last_undegraded":"2023-09-30T09:29:35.005720+0000","last_fullsized":"2023-09-30T09:29:35.005720+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":228,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.011207+0000","last_change":"2023-09-30T09:29:09.611292+0000","last_active":"2023-09-30T09:29:37.011207+0000","last_peered":"2023-09-30T09:29:37.011207+0000","last_clean":"2023-09-30T09:29:37.011207+0000","last_became_active":"2023-09-30T09:29:09.611015+0000","last_became_peered":"2023-09-30T09:29:09.611015+0000","last_unstale":"2023-09-30T09:29:37.011207+0000","last_undegraded":"2023-09-30T09:29:37.011207+0000","last_fullsized":"2023-09-30T09:29:37.011207+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:48.124378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":199,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.006511+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:29:35.006511+0000","last_peered":"2023-09-30T09:29:35.006511+0000","last_clean":"2023-09-30T09:29:35.006511+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:29:35.006511+0000","last_undegraded":"2023-09-30T09:29:35.006511+0000","last_fullsized":"2023-09-30T09:29:35.006511+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"241'3696","reported_seq":3780,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.230429+0000","last_change":"2023-09-30T09:28:42.250412+0000","last_active":"2023-09-30T09:29:38.230429+0000","last_peered":"2023-09-30T09:29:38.230429+0000","last_clean":"2023-09-30T09:29:38.230429+0000","last_became_active":"2023-09-30T09:28:40.266645+0000","last_became_peered":"2023-09-30T09:28:40.266645+0000","last_unstale":"2023-09-30T09:29:38.230429+0000","last_undegraded":"2023-09-30T09:29:38.230429+0000","last_fullsized":"2023-09-30T09:29:38.230429+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2368","last_scrub_stamp":"2023-09-30T09:28:42.250204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.250204+0000","objects_scrubbed":1224,"log_size":3696,"log_dups_size":0,"ondisk_log_size":3696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:20:30.287549+0000","scrub_duration":0.32304901400000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167641088,"num_objects":2558,"num_object_clones":0,"num_object_copies":7674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1334,"num_write_kb":85376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":664,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.007982+0000","last_change":"2023-09-30T09:29:12.640269+0000","last_active":"2023-09-30T09:29:37.007982+0000","last_peered":"2023-09-30T09:29:37.007982+0000","last_clean":"2023-09-30T09:29:37.007982+0000","last_became_active":"2023-09-30T09:29:12.639925+0000","last_became_peered":"2023-09-30T09:29:12.639925+0000","last_unstale":"2023-09-30T09:29:37.007982+0000","last_undegraded":"2023-09-30T09:29:37.007982+0000","last_fullsized":"2023-09-30T09:29:37.007982+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:45:10.404382+0000","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":387,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.006133+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:29:35.006133+0000","last_peered":"2023-09-30T09:29:35.006133+0000","last_clean":"2023-09-30T09:29:35.006133+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:29:35.006133+0000","last_undegraded":"2023-09-30T09:29:35.006133+0000","last_fullsized":"2023-09-30T09:29:35.006133+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":221,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002544+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:29:37.002544+0000","last_peered":"2023-09-30T09:29:37.002544+0000","last_clean":"2023-09-30T09:29:37.002544+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:29:37.002544+0000","last_undegraded":"2023-09-30T09:29:37.002544+0000","last_fullsized":"2023-09-30T09:29:37.002544+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.997412+0000","last_change":"2023-09-30T09:29:09.611507+0000","last_active":"2023-09-30T09:29:36.997412+0000","last_peered":"2023-09-30T09:29:36.997412+0000","last_clean":"2023-09-30T09:29:36.997412+0000","last_became_active":"2023-09-30T09:29:09.611120+0000","last_became_peered":"2023-09-30T09:29:09.611120+0000","last_unstale":"2023-09-30T09:29:36.997412+0000","last_undegraded":"2023-09-30T09:29:36.997412+0000","last_fullsized":"2023-09-30T09:29:36.997412+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:31.622647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.5","version":"242'3712","reported_seq":3773,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.126969+0000","last_change":"2023-09-30T09:28:46.022537+0000","last_active":"2023-09-30T09:29:39.126969+0000","last_peered":"2023-09-30T09:29:39.126969+0000","last_clean":"2023-09-30T09:29:39.126969+0000","last_became_active":"2023-09-30T09:28:40.276165+0000","last_became_peered":"2023-09-30T09:28:40.276165+0000","last_unstale":"2023-09-30T09:29:39.126969+0000","last_undegraded":"2023-09-30T09:29:39.126969+0000","last_fullsized":"2023-09-30T09:29:39.126969+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3712,"log_dups_size":0,"ondisk_log_size":3712,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169115648,"num_objects":2580,"num_object_clones":0,"num_object_copies":7740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2580,"num_write_kb":165152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":721,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.001208+0000","last_change":"2023-09-30T09:29:12.637491+0000","last_active":"2023-09-30T09:29:37.001208+0000","last_peered":"2023-09-30T09:29:37.001208+0000","last_clean":"2023-09-30T09:29:37.001208+0000","last_became_active":"2023-09-30T09:29:12.637133+0000","last_became_peered":"2023-09-30T09:29:12.637133+0000","last_unstale":"2023-09-30T09:29:37.001208+0000","last_undegraded":"2023-09-30T09:29:37.001208+0000","last_fullsized":"2023-09-30T09:29:37.001208+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:54:24.000552+0000","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":415,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996666+0000","last_change":"2023-09-30T09:29:36.005128+0000","last_active":"2023-09-30T09:29:36.996666+0000","last_peered":"2023-09-30T09:29:36.996666+0000","last_clean":"2023-09-30T09:29:36.996666+0000","last_became_active":"2023-09-30T09:29:36.004563+0000","last_became_peered":"2023-09-30T09:29:36.004563+0000","last_unstale":"2023-09-30T09:29:36.996666+0000","last_undegraded":"2023-09-30T09:29:36.996666+0000","last_fullsized":"2023-09-30T09:29:36.996666+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:59.839484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":187,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.007359+0000","last_change":"2023-09-30T09:29:09.607111+0000","last_active":"2023-09-30T09:29:35.007359+0000","last_peered":"2023-09-30T09:29:35.007359+0000","last_clean":"2023-09-30T09:29:35.007359+0000","last_became_active":"2023-09-30T09:29:09.606807+0000","last_became_peered":"2023-09-30T09:29:09.606807+0000","last_unstale":"2023-09-30T09:29:35.007359+0000","last_undegraded":"2023-09-30T09:29:35.007359+0000","last_fullsized":"2023-09-30T09:29:35.007359+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:49.380239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":235,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.005811+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:29:35.005811+0000","last_peered":"2023-09-30T09:29:35.005811+0000","last_clean":"2023-09-30T09:29:35.005811+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:29:35.005811+0000","last_undegraded":"2023-09-30T09:29:35.005811+0000","last_fullsized":"2023-09-30T09:29:35.005811+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"242'3679","reported_seq":3793,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.065188+0000","last_change":"2023-09-30T09:28:42.687918+0000","last_active":"2023-09-30T09:29:39.065188+0000","last_peered":"2023-09-30T09:29:39.065188+0000","last_clean":"2023-09-30T09:29:39.065188+0000","last_became_active":"2023-09-30T09:28:40.262566+0000","last_became_peered":"2023-09-30T09:28:40.262566+0000","last_unstale":"2023-09-30T09:29:39.065188+0000","last_undegraded":"2023-09-30T09:29:39.065188+0000","last_fullsized":"2023-09-30T09:29:39.065188+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2390","last_scrub_stamp":"2023-09-30T09:28:42.687821+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.687821+0000","objects_scrubbed":1235,"log_size":3679,"log_dups_size":0,"ondisk_log_size":3679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:07:56.313767+0000","scrub_duration":0.36697584500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166133760,"num_objects":2535,"num_object_clones":0,"num_object_copies":7605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1300,"num_write_kb":83200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":659,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:37.008227+0000","last_change":"2023-09-30T09:29:12.096468+0000","last_active":"2023-09-30T09:29:37.008227+0000","last_peered":"2023-09-30T09:29:37.008227+0000","last_clean":"2023-09-30T09:29:03.507390+0000","last_became_active":"2023-09-30T09:29:12.096051+0000","last_became_peered":"2023-09-30T09:29:12.096051+0000","last_unstale":"2023-09-30T09:29:37.008227+0000","last_undegraded":"2023-09-30T09:29:10.593445+0000","last_fullsized":"2023-09-30T09:29:10.592749+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:49.279327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":682,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.003816+0000","last_change":"2023-09-30T09:27:54.119268+0000","last_active":"2023-09-30T09:29:37.003816+0000","last_peered":"2023-09-30T09:29:37.003816+0000","last_clean":"2023-09-30T09:29:37.003816+0000","last_became_active":"2023-09-30T09:27:49.106373+0000","last_became_peered":"2023-09-30T09:27:49.106373+0000","last_unstale":"2023-09-30T09:29:37.003816+0000","last_undegraded":"2023-09-30T09:29:37.003816+0000","last_fullsized":"2023-09-30T09:29:37.003816+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:28.856049+0000","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.997728+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:29:36.997728+0000","last_peered":"2023-09-30T09:29:36.997728+0000","last_clean":"2023-09-30T09:29:36.997728+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:29:36.997728+0000","last_undegraded":"2023-09-30T09:29:36.997728+0000","last_fullsized":"2023-09-30T09:29:36.997728+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":184,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.010605+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:29:37.010605+0000","last_peered":"2023-09-30T09:29:37.010605+0000","last_clean":"2023-09-30T09:29:37.010605+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:29:37.010605+0000","last_undegraded":"2023-09-30T09:29:37.010605+0000","last_fullsized":"2023-09-30T09:29:37.010605+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"240'3675","reported_seq":3756,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.666643+0000","last_change":"2023-09-30T09:28:43.582313+0000","last_active":"2023-09-30T09:29:36.666643+0000","last_peered":"2023-09-30T09:29:36.666643+0000","last_clean":"2023-09-30T09:29:36.666643+0000","last_became_active":"2023-09-30T09:28:40.264204+0000","last_became_peered":"2023-09-30T09:28:40.264204+0000","last_unstale":"2023-09-30T09:29:36.666643+0000","last_undegraded":"2023-09-30T09:29:36.666643+0000","last_fullsized":"2023-09-30T09:29:36.666643+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2440","last_scrub_stamp":"2023-09-30T09:28:43.582001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:43.582001+0000","objects_scrubbed":1291,"log_size":3675,"log_dups_size":0,"ondisk_log_size":3675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:38.964795+0000","scrub_duration":0.30691791200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166199296,"num_objects":2536,"num_object_clones":0,"num_object_copies":7608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1245,"num_write_kb":79680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":644,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.006974+0000","last_change":"2023-09-30T09:28:24.276678+0000","last_active":"2023-09-30T09:29:35.006974+0000","last_peered":"2023-09-30T09:29:35.006974+0000","last_clean":"2023-09-30T09:29:35.006974+0000","last_became_active":"2023-09-30T09:28:09.226029+0000","last_became_peered":"2023-09-30T09:28:09.226029+0000","last_unstale":"2023-09-30T09:29:35.006974+0000","last_undegraded":"2023-09-30T09:29:35.006974+0000","last_fullsized":"2023-09-30T09:29:35.006974+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:03:49.533925+0000","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":802,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.003735+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:29:37.003735+0000","last_peered":"2023-09-30T09:29:37.003735+0000","last_clean":"2023-09-30T09:29:37.003735+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:29:37.003735+0000","last_undegraded":"2023-09-30T09:29:37.003735+0000","last_fullsized":"2023-09-30T09:29:37.003735+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":160,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.244520+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:29:38.244520+0000","last_peered":"2023-09-30T09:29:38.244520+0000","last_clean":"2023-09-30T09:29:38.244520+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:29:38.244520+0000","last_undegraded":"2023-09-30T09:29:38.244520+0000","last_fullsized":"2023-09-30T09:29:38.244520+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":145,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.005774+0000","last_change":"2023-09-30T09:29:09.611376+0000","last_active":"2023-09-30T09:29:35.005774+0000","last_peered":"2023-09-30T09:29:35.005774+0000","last_clean":"2023-09-30T09:29:35.005774+0000","last_became_active":"2023-09-30T09:29:09.611090+0000","last_became_peered":"2023-09-30T09:29:09.611090+0000","last_unstale":"2023-09-30T09:29:35.005774+0000","last_undegraded":"2023-09-30T09:29:35.005774+0000","last_fullsized":"2023-09-30T09:29:35.005774+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:07:15.039397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"240'3690","reported_seq":3781,"reported_epoch":240,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:36.409001+0000","last_change":"2023-09-30T09:29:36.030348+0000","last_active":"2023-09-30T09:29:36.409001+0000","last_peered":"2023-09-30T09:29:36.409001+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:36.027562+0000","last_became_peered":"2023-09-30T09:29:36.027562+0000","last_unstale":"2023-09-30T09:29:36.409001+0000","last_undegraded":"2023-09-30T09:29:35.999894+0000","last_fullsized":"2023-09-30T09:29:36.409001+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":3690,"log_dups_size":0,"ondisk_log_size":3690,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83394560,"num_objects":1272,"num_object_clones":0,"num_object_copies":3816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":641,"num_write_kb":41056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1043},{"shards":"1,6","objects":1},{"shards":"1,6,7","objects":228}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":486,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.011040+0000","last_change":"2023-09-30T09:28:28.453752+0000","last_active":"2023-09-30T09:29:37.011040+0000","last_peered":"2023-09-30T09:29:37.011040+0000","last_clean":"2023-09-30T09:29:37.011040+0000","last_became_active":"2023-09-30T09:28:09.228162+0000","last_became_peered":"2023-09-30T09:28:09.228162+0000","last_unstale":"2023-09-30T09:29:37.011040+0000","last_undegraded":"2023-09-30T09:29:37.011040+0000","last_fullsized":"2023-09-30T09:29:37.011040+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:43:23.683089+0000","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":932,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.001976+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:29:37.001976+0000","last_peered":"2023-09-30T09:29:37.001976+0000","last_clean":"2023-09-30T09:29:37.001976+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:29:37.001976+0000","last_undegraded":"2023-09-30T09:29:37.001976+0000","last_fullsized":"2023-09-30T09:29:37.001976+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":203,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.005903+0000","last_change":"2023-09-30T09:29:09.611808+0000","last_active":"2023-09-30T09:29:35.005903+0000","last_peered":"2023-09-30T09:29:35.005903+0000","last_clean":"2023-09-30T09:29:35.005903+0000","last_became_active":"2023-09-30T09:29:09.611484+0000","last_became_peered":"2023-09-30T09:29:09.611484+0000","last_unstale":"2023-09-30T09:29:35.005903+0000","last_undegraded":"2023-09-30T09:29:35.005903+0000","last_fullsized":"2023-09-30T09:29:35.005903+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:19.562461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":132,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:38.245713+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:29:38.245713+0000","last_peered":"2023-09-30T09:29:38.245713+0000","last_clean":"2023-09-30T09:29:38.245713+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:29:38.245713+0000","last_undegraded":"2023-09-30T09:29:38.245713+0000","last_fullsized":"2023-09-30T09:29:38.245713+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"241'3493","reported_seq":6422,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:38.052888+0000","last_change":"2023-09-30T09:29:35.026309+0000","last_active":"2023-09-30T09:29:38.052888+0000","last_peered":"2023-09-30T09:29:38.052888+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:35.024657+0000","last_became_peered":"2023-09-30T09:29:35.024657+0000","last_unstale":"2023-09-30T09:29:38.052888+0000","last_undegraded":"2023-09-30T09:29:35.010418+0000","last_fullsized":"2023-09-30T09:29:35.009540+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":3493,"log_dups_size":0,"ondisk_log_size":3493,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80019456,"num_objects":1221,"num_object_clones":0,"num_object_copies":3663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":495,"num_write_kb":31712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":276},{"shards":"3,6","objects":945}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":664,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.003676+0000","last_change":"2023-09-30T09:29:12.634588+0000","last_active":"2023-09-30T09:29:37.003676+0000","last_peered":"2023-09-30T09:29:37.003676+0000","last_clean":"2023-09-30T09:29:37.003676+0000","last_became_active":"2023-09-30T09:29:12.634225+0000","last_became_peered":"2023-09-30T09:29:12.634225+0000","last_unstale":"2023-09-30T09:29:37.003676+0000","last_undegraded":"2023-09-30T09:29:37.003676+0000","last_fullsized":"2023-09-30T09:29:37.003676+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:22.392203+0000","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":227,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.001335+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:29:37.001335+0000","last_peered":"2023-09-30T09:29:37.001335+0000","last_clean":"2023-09-30T09:29:37.001335+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:29:37.001335+0000","last_undegraded":"2023-09-30T09:29:37.001335+0000","last_fullsized":"2023-09-30T09:29:37.001335+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1189,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.005862+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:29:35.005862+0000","last_peered":"2023-09-30T09:29:35.005862+0000","last_clean":"2023-09-30T09:29:35.005862+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:29:35.005862+0000","last_undegraded":"2023-09-30T09:29:35.005862+0000","last_fullsized":"2023-09-30T09:29:35.005862+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":227,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996268+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:29:36.996268+0000","last_peered":"2023-09-30T09:29:36.996268+0000","last_clean":"2023-09-30T09:29:36.996268+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:29:36.996268+0000","last_undegraded":"2023-09-30T09:29:36.996268+0000","last_fullsized":"2023-09-30T09:29:36.996268+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"242'3551","reported_seq":7714,"reported_epoch":242,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:29:39.055633+0000","last_change":"2023-09-30T09:29:36.003902+0000","last_active":"2023-09-30T09:29:39.055633+0000","last_peered":"2023-09-30T09:29:39.055633+0000","last_clean":"2023-09-30T09:29:33.909482+0000","last_became_active":"2023-09-30T09:29:35.026292+0000","last_became_peered":"2023-09-30T09:29:35.026292+0000","last_unstale":"2023-09-30T09:29:39.055633+0000","last_undegraded":"2023-09-30T09:29:39.055633+0000","last_fullsized":"2023-09-30T09:29:39.055633+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":3551,"log_dups_size":0,"ondisk_log_size":3551,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82509824,"num_objects":1259,"num_object_clones":0,"num_object_copies":3777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":552,"num_write_kb":35328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65765376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":["7","1","2","6"],"object_location_counts":[{"shards":"1,6,7","objects":1259}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":641,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.997489+0000","last_change":"2023-09-30T09:28:26.894367+0000","last_active":"2023-09-30T09:29:36.997489+0000","last_peered":"2023-09-30T09:29:36.997489+0000","last_clean":"2023-09-30T09:29:36.997489+0000","last_became_active":"2023-09-30T09:28:09.227189+0000","last_became_peered":"2023-09-30T09:28:09.227189+0000","last_unstale":"2023-09-30T09:29:36.997489+0000","last_undegraded":"2023-09-30T09:29:36.997489+0000","last_fullsized":"2023-09-30T09:29:36.997489+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:17.282328+0000","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":519,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.011503+0000","last_change":"2023-09-30T09:29:09.643760+0000","last_active":"2023-09-30T09:29:37.011503+0000","last_peered":"2023-09-30T09:29:37.011503+0000","last_clean":"2023-09-30T09:29:37.011503+0000","last_became_active":"2023-09-30T09:29:09.620803+0000","last_became_peered":"2023-09-30T09:29:09.620803+0000","last_unstale":"2023-09-30T09:29:37.011503+0000","last_undegraded":"2023-09-30T09:29:37.011503+0000","last_fullsized":"2023-09-30T09:29:37.011503+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:54:49.883760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":117,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.006028+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:29:35.006028+0000","last_peered":"2023-09-30T09:29:35.006028+0000","last_clean":"2023-09-30T09:29:35.006028+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:29:35.006028+0000","last_undegraded":"2023-09-30T09:29:35.006028+0000","last_fullsized":"2023-09-30T09:29:35.006028+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":183,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.000559+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:29:37.000559+0000","last_peered":"2023-09-30T09:29:37.000559+0000","last_clean":"2023-09-30T09:29:37.000559+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:29:37.000559+0000","last_undegraded":"2023-09-30T09:29:37.000559+0000","last_fullsized":"2023-09-30T09:29:37.000559+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1e","version":"242'4900","reported_seq":4964,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:38.759222+0000","last_change":"2023-09-30T09:29:35.080846+0000","last_active":"2023-09-30T09:29:38.759222+0000","last_peered":"2023-09-30T09:29:38.759222+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:35.061073+0000","last_became_peered":"2023-09-30T09:29:35.061073+0000","last_unstale":"2023-09-30T09:29:38.759222+0000","last_undegraded":"2023-09-30T09:29:35.049064+0000","last_fullsized":"2023-09-30T09:29:38.759222+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4900,"log_dups_size":0,"ondisk_log_size":4900,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163905536,"num_objects":2501,"num_object_clones":0,"num_object_copies":7503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1977,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2501,"num_write_kb":160064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1977},{"shards":"1,4,7","objects":524}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":100,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.002635+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:29:37.002635+0000","last_peered":"2023-09-30T09:29:37.002635+0000","last_clean":"2023-09-30T09:29:37.002635+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:29:37.002635+0000","last_undegraded":"2023-09-30T09:29:37.002635+0000","last_fullsized":"2023-09-30T09:29:37.002635+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":151,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-30T09:29:35.005813+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:29:35.005813+0000","last_peered":"2023-09-30T09:29:35.005813+0000","last_clean":"2023-09-30T09:29:35.005813+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:29:35.005813+0000","last_undegraded":"2023-09-30T09:29:35.005813+0000","last_fullsized":"2023-09-30T09:29:35.005813+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"242'3620","reported_seq":3686,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:39.182523+0000","last_change":"2023-09-30T09:29:10.893253+0000","last_active":"2023-09-30T09:29:39.182523+0000","last_peered":"2023-09-30T09:29:39.182523+0000","last_clean":"2023-09-30T09:29:08.494599+0000","last_became_active":"2023-09-30T09:29:10.886737+0000","last_became_peered":"2023-09-30T09:29:10.886737+0000","last_unstale":"2023-09-30T09:29:39.182523+0000","last_undegraded":"2023-09-30T09:29:10.766629+0000","last_fullsized":"2023-09-30T09:29:10.598246+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3620,"log_dups_size":0,"ondisk_log_size":3620,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165085184,"num_objects":2519,"num_object_clones":0,"num_object_copies":7557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2001,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2519,"num_write_kb":161216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":2001},{"shards":"3,7","objects":518}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":203,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.008507+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:29:37.008507+0000","last_peered":"2023-09-30T09:29:37.008507+0000","last_clean":"2023-09-30T09:29:37.008507+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:29:37.008507+0000","last_undegraded":"2023-09-30T09:29:37.008507+0000","last_fullsized":"2023-09-30T09:29:37.008507+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":184,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996790+0000","last_change":"2023-09-30T09:29:09.609752+0000","last_active":"2023-09-30T09:29:36.996790+0000","last_peered":"2023-09-30T09:29:36.996790+0000","last_clean":"2023-09-30T09:29:36.996790+0000","last_became_active":"2023-09-30T09:29:09.609242+0000","last_became_peered":"2023-09-30T09:29:09.609242+0000","last_unstale":"2023-09-30T09:29:36.996790+0000","last_undegraded":"2023-09-30T09:29:36.996790+0000","last_fullsized":"2023-09-30T09:29:36.996790+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:31.366509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"242'3621","reported_seq":3685,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:39.073610+0000","last_change":"2023-09-30T09:29:10.864651+0000","last_active":"2023-09-30T09:29:39.073610+0000","last_peered":"2023-09-30T09:29:39.073610+0000","last_clean":"2023-09-30T09:29:08.532983+0000","last_became_active":"2023-09-30T09:29:10.860470+0000","last_became_peered":"2023-09-30T09:29:10.860470+0000","last_unstale":"2023-09-30T09:29:39.073610+0000","last_undegraded":"2023-09-30T09:29:10.638243+0000","last_fullsized":"2023-09-30T09:29:10.593564+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3621,"log_dups_size":0,"ondisk_log_size":3621,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164331520,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2508,"num_write_kb":160480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":2057},{"shards":"3,7","objects":451}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":186,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.011333+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:29:37.011333+0000","last_peered":"2023-09-30T09:29:37.011333+0000","last_clean":"2023-09-30T09:29:37.011333+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:29:37.011333+0000","last_undegraded":"2023-09-30T09:29:37.011333+0000","last_fullsized":"2023-09-30T09:29:37.011333+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":215,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.997812+0000","last_change":"2023-09-30T09:29:09.621218+0000","last_active":"2023-09-30T09:29:36.997812+0000","last_peered":"2023-09-30T09:29:36.997812+0000","last_clean":"2023-09-30T09:29:36.997812+0000","last_became_active":"2023-09-30T09:29:09.620972+0000","last_became_peered":"2023-09-30T09:29:09.620972+0000","last_unstale":"2023-09-30T09:29:36.997812+0000","last_undegraded":"2023-09-30T09:29:36.997812+0000","last_fullsized":"2023-09-30T09:29:36.997812+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:25.640288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1b","version":"242'4999","reported_seq":5061,"reported_epoch":242,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:29:38.278079+0000","last_change":"2023-09-30T09:29:35.093036+0000","last_active":"2023-09-30T09:29:38.278079+0000","last_peered":"2023-09-30T09:29:38.278079+0000","last_clean":"2023-09-30T09:29:33.992127+0000","last_became_active":"2023-09-30T09:29:35.070155+0000","last_became_peered":"2023-09-30T09:29:35.070155+0000","last_unstale":"2023-09-30T09:29:38.278079+0000","last_undegraded":"2023-09-30T09:29:38.278079+0000","last_fullsized":"2023-09-30T09:29:38.278079+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4999,"log_dups_size":0,"ondisk_log_size":4999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165904384,"num_objects":2532,"num_object_clones":0,"num_object_copies":7596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2532,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":2532}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":222,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996607+0000","last_change":"2023-09-30T09:29:09.615385+0000","last_active":"2023-09-30T09:29:36.996607+0000","last_peered":"2023-09-30T09:29:36.996607+0000","last_clean":"2023-09-30T09:29:36.996607+0000","last_became_active":"2023-09-30T09:29:09.614479+0000","last_became_peered":"2023-09-30T09:29:09.614479+0000","last_unstale":"2023-09-30T09:29:36.996607+0000","last_undegraded":"2023-09-30T09:29:36.996607+0000","last_fullsized":"2023-09-30T09:29:36.996607+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:14.671222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":252,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:36.996428+0000","last_change":"2023-09-30T09:29:09.600457+0000","last_active":"2023-09-30T09:29:36.996428+0000","last_peered":"2023-09-30T09:29:36.996428+0000","last_clean":"2023-09-30T09:29:36.996428+0000","last_became_active":"2023-09-30T09:29:09.600080+0000","last_became_peered":"2023-09-30T09:29:09.600080+0000","last_unstale":"2023-09-30T09:29:36.996428+0000","last_undegraded":"2023-09-30T09:29:36.996428+0000","last_fullsized":"2023-09-30T09:29:36.996428+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:10:02.250349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1a","version":"242'4978","reported_seq":5032,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.042565+0000","last_change":"2023-09-30T09:29:35.021535+0000","last_active":"2023-09-30T09:29:39.042565+0000","last_peered":"2023-09-30T09:29:39.042565+0000","last_clean":"2023-09-30T09:29:39.042565+0000","last_became_active":"2023-09-30T09:29:35.014808+0000","last_became_peered":"2023-09-30T09:29:35.014808+0000","last_unstale":"2023-09-30T09:29:39.042565+0000","last_undegraded":"2023-09-30T09:29:39.042565+0000","last_fullsized":"2023-09-30T09:29:39.042565+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4978,"log_dups_size":0,"ondisk_log_size":4978,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165871616,"num_objects":2531,"num_object_clones":0,"num_object_copies":7593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2531,"num_write_kb":161984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":125,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.008733+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:29:37.008733+0000","last_peered":"2023-09-30T09:29:37.008733+0000","last_clean":"2023-09-30T09:29:37.008733+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:29:37.008733+0000","last_undegraded":"2023-09-30T09:29:37.008733+0000","last_fullsized":"2023-09-30T09:29:37.008733+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":200,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-30T09:29:37.000295+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:29:37.000295+0000","last_peered":"2023-09-30T09:29:37.000295+0000","last_clean":"2023-09-30T09:29:37.000295+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:29:37.000295+0000","last_undegraded":"2023-09-30T09:29:37.000295+0000","last_fullsized":"2023-09-30T09:29:37.000295+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":36,"stat_sum":{"num_bytes":5293932544,"num_objects":80780,"num_object_clones":0,"num_object_copies":242340,"num_objects_missing_on_primary":1393,"num_objects_missing":1393,"num_objects_degraded":24839,"num_objects_misplaced":5796,"num_objects_unfound":0,"num_objects_dirty":80780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64603,"num_write_kb":4134528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8135,"num_bytes_recovered":533135360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16516055040,"data_stored":16516055040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146807,"ondisk_log_size":146807,"up":108,"acting":101,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":1418,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":435019,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":43,"num_write_kb":438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234216,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2377276,"kb_used_data":2247284,"kb_used_omap":0,"kb_used_meta":129984,"kb_avail":91367876,"statfs":{"total":95995035648,"available":93560705024,"internally_reserved":0,"allocated":2301218816,"data_stored":2300323157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":133103616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:27 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.5980000000000001,"15min":1.0149999999999999},"min":{"1min":0.30099999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.839,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.4770000000000001,"15min":0.93700000000000006},"min":{"1min":0.29499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.8949999999999996,"5min":18.097999999999999,"15min":18.097999999999999},"last":0.70599999999999996}]},{"osd":1,"last update":"Sat Sep 30 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":10.125,"5min":4.8070000000000004,"15min":2.214},"min":{"1min":0.38200000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":70.566999999999993,"5min":82.025999999999996,"15min":82.025999999999996},"last":2.21},{"interface":"front","average":{"1min":10.115,"5min":4.7279999999999998,"15min":2.2000000000000002},"min":{"1min":0.36599999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":70.626000000000005,"5min":82.012,"15min":82.012},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:29:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7200000000000002,"5min":2.0009999999999999,"15min":1.145},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":39.445,"5min":39.445,"15min":39.445},"last":2.2789999999999999},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.9870000000000001,"15min":1.2190000000000001},"min":{"1min":0.39700000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":40.988999999999997,"5min":40.988999999999997,"15min":40.988999999999997},"last":0.79200000000000004}]},{"osd":3,"last update":"Sat Sep 30 09:28:42 2023","interfaces":[{"interface":"back","average":{"1min":6.5199999999999996,"5min":2.6920000000000002,"15min":1.3540000000000001},"min":{"1min":0.38800000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":57.000999999999998,"5min":57.000999999999998,"15min":57.000999999999998},"last":2.0009999999999999},{"interface":"front","average":{"1min":6.8689999999999998,"5min":2.4289999999999998,"15min":1.2569999999999999},"min":{"1min":0.377,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":56.515000000000001,"5min":56.515000000000001,"15min":56.515000000000001},"last":4.1870000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.95699999999999996,"15min":0.64900000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.78900000000000003,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":2.0750000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.81000000000000005,"15min":0.63700000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.92400000000000004,"5min":6.282,"15min":6.282},"last":0.76600000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.97699999999999998,"15min":0.70999999999999996},"min":{"1min":0.28100000000000003,"5min":0.248,"15min":0.248},"max":{"1min":2.4470000000000001,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":2.2410000000000001},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.121,"15min":0.77600000000000002},"min":{"1min":0.30199999999999999,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":2.4670000000000001,"5min":11.022,"15min":11.022},"last":2.0289999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.052,"15min":0.75700000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33000000000000002},"max":{"1min":1.121,"5min":12.701000000000001,"15min":12.701000000000001},"last":2.0550000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.879,"15min":0.70299999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.88900000000000001,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":6.2270000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495853,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2235128,"kb_used_data":2110256,"kb_used_omap":0,"kb_used_meta":124864,"kb_avail":91510024,"statfs":{"total":95995035648,"available":93706264576,"internally_reserved":0,"allocated":2160902144,"data_stored":2160010998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":127860736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":2.2829999999999999,"15min":1.242},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.222},"max":{"1min":2.4119999999999999,"5min":45.927999999999997,"15min":45.927999999999997},"last":0.627},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.581,"15min":1.3520000000000001},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.23499999999999999},"max":{"1min":1.6899999999999999,"5min":45.316000000000003,"15min":45.316000000000003},"last":2.0659999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.8399999999999999,"15min":1.444},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":37.988,"5min":69.150000000000006,"15min":69.150000000000006},"last":0.60199999999999998},{"interface":"front","average":{"1min":3.0259999999999998,"5min":3.2240000000000002,"15min":1.5649999999999999},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":37.970999999999997,"5min":69.132000000000005,"15min":69.132000000000005},"last":2.0139999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":2.4780000000000002,"15min":1.294},"min":{"1min":0.34999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.4399999999999999,"5min":37.978000000000002,"15min":37.978000000000002},"last":2.8969999999999998},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.9630000000000001,"15min":1.167},"min":{"1min":0.39400000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.9049999999999998,"5min":37.759999999999998,"15min":37.759999999999998},"last":0.65300000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":1.716,"15min":1.054},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.165,"5min":20.818999999999999,"15min":20.818999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.9430000000000001,"15min":1.169},"min":{"1min":0.36499999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.6439999999999999,"5min":21.015000000000001,"15min":21.015000000000001},"last":2.097}]},{"osd":4,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.921,"15min":1.216},"min":{"1min":0.26700000000000002,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":1.72,"5min":49.286999999999999,"15min":73.656000000000006},"last":3.2959999999999998},{"interface":"front","average":{"1min":0.505,"5min":1.5169999999999999,"15min":1.1120000000000001},"min":{"1min":0.32400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.1100000000000001,"5min":49.252000000000002,"15min":73.882999999999996},"last":3.1859999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.6439999999999999,"15min":0.93400000000000005},"min":{"1min":0.32400000000000001,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":2.5750000000000002,"5min":47.408999999999999,"15min":47.408999999999999},"last":2.9780000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.528,"15min":0.93999999999999995},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.311},"max":{"1min":2.746,"5min":47.344000000000001,"15min":47.344000000000001},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.897,"15min":1.0529999999999999},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.466,"5min":53.927999999999997,"15min":53.927999999999997},"last":2.1240000000000001},{"interface":"front","average":{"1min":0.628,"5min":1.8340000000000001,"15min":1.1000000000000001},"min":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":1.454,"5min":51.886000000000003,"15min":51.886000000000003},"last":0.69699999999999995}]}]},{"osd":1,"up_from":99,"seq":425201762417,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2136072,"kb_used_data":1951736,"kb_used_omap":5,"kb_used_meta":184314,"kb_avail":91609080,"statfs":{"total":95995035648,"available":93807697920,"internally_reserved":0,"allocated":1998577664,"data_stored":1997257666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":188737718},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":5.7160000000000002,"15min":2.1829999999999998},"min":{"1min":0.217,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.9160000000000004,"5min":150.93899999999999,"15min":150.93899999999999},"last":6.4400000000000004},{"interface":"front","average":{"1min":1.496,"5min":5.7619999999999996,"15min":2.2370000000000001},"min":{"1min":0.224,"5min":0.216,"15min":0.216},"max":{"1min":9.6699999999999999,"5min":156.74199999999999,"15min":156.74199999999999},"last":7.9470000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:29:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":7.2290000000000001,"15min":2.7360000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.3159999999999998,"5min":144.48699999999999,"15min":144.48699999999999},"last":1.389},{"interface":"front","average":{"1min":1.1499999999999999,"5min":7.0300000000000002,"15min":2.6909999999999998},"min":{"1min":0.28799999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.7560000000000002,"5min":144.62,"15min":144.62},"last":1.9710000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:28:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.5760000000000001,"15min":0.82899999999999996},"min":{"1min":0.27500000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.8300000000000001,"5min":18.757999999999999,"15min":18.757999999999999},"last":0.62},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.2250000000000001,"15min":0.74199999999999999},"min":{"1min":0.33600000000000002,"5min":0.214,"15min":0.214},"max":{"1min":7.8769999999999998,"5min":11.173,"15min":11.173},"last":1.1339999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.242,"15min":0.86299999999999999},"min":{"1min":0.41499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.94799999999999995,"5min":10.173,"15min":10.173},"last":1.171},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.29,"15min":0.88300000000000001},"min":{"1min":0.42399999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79300000000000004,"5min":8.9299999999999997,"15min":8.9299999999999997},"last":8.0700000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.1080000000000001,"15min":0.81100000000000005},"min":{"1min":0.38500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5349999999999999,"5min":10.26,"15min":10.26},"last":7.9989999999999997},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.3420000000000001,"15min":0.89300000000000002},"min":{"1min":0.44700000000000001,"5min":0.35499999999999998,"15min":0.33300000000000002},"max":{"1min":0.76400000000000001,"5min":9.3729999999999993,"15min":9.3729999999999993},"last":10.128}]},{"osd":6,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.409,"15min":0.89600000000000002},"min":{"1min":0.40999999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73899999999999999,"5min":14.356999999999999,"15min":14.356999999999999},"last":10.058},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.508,"15min":0.97999999999999998},"min":{"1min":0.377,"5min":0.313,"15min":0.313},"max":{"1min":2.855,"5min":15.864000000000001,"15min":15.864000000000001},"last":6.399}]},{"osd":7,"last update":"Sat Sep 30 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.7869999999999999,"15min":1.048},"min":{"1min":0.42499999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.7929999999999999,"5min":14.403,"15min":14.403},"last":10.103},{"interface":"front","average":{"1min":0.95299999999999996,"5min":2.9860000000000002,"15min":1.4810000000000001},"min":{"1min":0.50900000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.8799999999999999,"5min":122.56699999999999,"15min":122.56699999999999},"last":1.0860000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925950,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":892148,"kb_used_data":651048,"kb_used_omap":5,"kb_used_meta":241082,"kb_avail":92853004,"statfs":{"total":95995035648,"available":95081476096,"internally_reserved":0,"allocated":666673152,"data_stored":665273909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":246868221},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:28:41 2023","interfaces":[{"interface":"back","average":{"1min":7.0549999999999997,"5min":2.915,"15min":1.2390000000000001},"min":{"1min":0.191,"5min":0.188,"15min":0.188},"max":{"1min":125.024,"5min":125.024,"15min":125.024},"last":0.77900000000000003},{"interface":"front","average":{"1min":7.234,"5min":2.9140000000000001,"15min":1.266},"min":{"1min":0.24299999999999999,"5min":0.192,"15min":0.192},"max":{"1min":124.901,"5min":124.901,"15min":124.901},"last":2.024}]},{"osd":2,"last update":"Sat Sep 30 09:29:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":2.1469999999999998,"15min":1.071},"min":{"1min":0.247,"5min":0.218,"15min":0.218},"max":{"1min":55.823999999999998,"5min":55.823999999999998,"15min":55.823999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":3.8820000000000001,"5min":2.3340000000000001,"15min":1.1779999999999999},"min":{"1min":0.32200000000000001,"5min":0.245,"15min":0.245},"max":{"1min":55.790999999999997,"5min":55.790999999999997,"15min":55.790999999999997},"last":0.57299999999999995}]},{"osd":3,"last update":"Sat Sep 30 09:28:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":2.2949999999999999,"15min":1.121},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":55.68,"5min":55.68,"15min":55.68},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.7160000000000002,"5min":2.069,"15min":1.032},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":55.244,"5min":55.244,"15min":55.244},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":2.4039999999999999,"15min":1.349},"min":{"1min":0.39500000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.863,"5min":121.486,"15min":121.486},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.91200000000000003,"5min":2.8450000000000002,"15min":1.538},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.31},"max":{"1min":2.2050000000000001,"5min":121.44499999999999,"15min":121.44499999999999},"last":0.76100000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.5,"15min":1.173},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.738,"5min":15.162000000000001,"15min":15.162000000000001},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.4059999999999999,"5min":1.45,"15min":1.159},"min":{"1min":0.38600000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.0270000000000001,"5min":13.766,"15min":13.766},"last":1.0920000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":1.3340000000000001,"15min":1.03},"min":{"1min":0.374,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.9960000000000004,"5min":13.811999999999999,"15min":13.811999999999999},"last":0.82199999999999995},{"interface":"front","average":{"1min":1.25,"5min":1.242,"15min":1.002},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":2.7799999999999998,"5min":6.407,"15min":7.556},"last":0.71899999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.8279999999999998,"15min":1.53},"min":{"1min":0.41999999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.9640000000000004,"5min":127.199,"15min":127.199},"last":0.438},{"interface":"front","average":{"1min":1.45,"5min":2.6139999999999999,"15min":1.456},"min":{"1min":0.44600000000000001,"5min":0.371,"15min":0.371},"max":{"1min":4.3040000000000003,"5min":118.551,"15min":118.551},"last":0.79800000000000004}]}]},{"osd":2,"up_from":0,"seq":446676598891,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435363,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3746752,"kb_used_data":3540272,"kb_used_omap":2,"kb_used_meta":206461,"kb_avail":89998400,"statfs":{"total":95995035648,"available":92158361600,"internally_reserved":0,"allocated":3625238528,"data_stored":3623877612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":211416423},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,7,0,0,4,4,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":2.149,"15min":1.421},"min":{"1min":0.222,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.3929999999999998,"5min":62.206000000000003,"15min":130.946},"last":1.425},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.1280000000000001,"15min":1.4279999999999999},"min":{"1min":0.26800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.1110000000000007,"5min":60.305999999999997,"15min":130.96899999999999},"last":0.29099999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":1.6160000000000001,"15min":0.84599999999999997},"min":{"1min":0.251,"5min":0.23200000000000001,"15min":0.224},"max":{"1min":3.5169999999999999,"5min":13.039999999999999,"15min":13.039999999999999},"last":1.3560000000000001},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.6990000000000001,"15min":0.86299999999999999},"min":{"1min":0.25900000000000001,"5min":0.221,"15min":0.221},"max":{"1min":9.0860000000000003,"5min":14.057,"15min":14.057},"last":1.1919999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":2.7000000000000002,"15min":1.7210000000000001},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.8420000000000001,"5min":61.704999999999998,"15min":145.75899999999999},"last":1.5449999999999999},{"interface":"front","average":{"1min":0.70699999999999996,"5min":2.395,"15min":1.633},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.2429999999999999,"5min":59.094000000000001,"15min":150.358},"last":0.28000000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.0569999999999999,"15min":0.77100000000000002},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.9830000000000001,"5min":10.301,"15min":10.301},"last":1.925},{"interface":"front","average":{"1min":1.204,"5min":1.4870000000000001,"15min":0.92100000000000004},"min":{"1min":0.443,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.9870000000000001,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":1.8919999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.607,"15min":0.98699999999999999},"min":{"1min":0.443,"5min":0.314,"15min":0.308},"max":{"1min":9.0310000000000006,"5min":10.048999999999999,"15min":10.048999999999999},"last":1.7669999999999999},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.8049999999999999,"15min":1.077},"min":{"1min":0.46400000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.3479999999999999,"5min":12.785,"15min":12.785},"last":1.7509999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.2769999999999999,"15min":0.88200000000000001},"min":{"1min":0.42499999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.4209999999999998,"5min":10.348000000000001,"15min":10.348000000000001},"last":1.5800000000000001},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.722,"15min":1.0609999999999999},"min":{"1min":0.44700000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.0589999999999993,"5min":14.69,"15min":14.69},"last":1.601}]},{"osd":7,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.7090000000000001,"15min":1.034},"min":{"1min":0.40799999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.3239999999999998,"5min":12.742000000000001,"15min":12.742000000000001},"last":1.8080000000000001},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.262,"15min":0.88300000000000001},"min":{"1min":0.49299999999999999,"5min":0.377,"15min":0.372},"max":{"1min":7.242,"5min":10.43,"15min":10.43},"last":1.79}]}]},{"osd":4,"up_from":30,"seq":128849019126,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1591480,"kb_used_data":1447472,"kb_used_omap":0,"kb_used_meta":144000,"kb_avail":92153672,"statfs":{"total":95995035648,"available":94365360128,"internally_reserved":0,"allocated":1482211328,"data_stored":1481320263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":147456000},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":4.6929999999999996,"15min":2.4929999999999999},"min":{"1min":0.25700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.6589999999999998,"5min":119.986,"15min":128.79300000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":4.5350000000000001,"15min":2.371},"min":{"1min":0.253,"5min":0.253,"15min":0.248},"max":{"1min":2.3679999999999999,"5min":119.863,"15min":128.38800000000001},"last":2.9060000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.9630000000000001,"15min":1.54},"min":{"1min":0.28799999999999998,"5min":0.245,"15min":0.245},"max":{"1min":27.440999999999999,"5min":93.244,"15min":93.244},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.117,"15min":1.597},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.23999999999999999},"max":{"1min":27.489000000000001,"5min":93.465999999999994,"15min":93.465999999999994},"last":3.0840000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":6.2380000000000004,"5min":5.274,"15min":2.379},"min":{"1min":0.37,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":77.620000000000005,"5min":140.601,"15min":140.601},"last":3.137},{"interface":"front","average":{"1min":6.2610000000000001,"5min":5.1459999999999999,"15min":2.327},"min":{"1min":0.35999999999999999,"5min":0.35199999999999998,"15min":0.34200000000000003},"max":{"1min":77.638999999999996,"5min":140.54900000000001,"15min":140.54900000000001},"last":0.33500000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":15.814,"5min":5.1239999999999997,"15min":2.4319999999999999},"min":{"1min":0.34200000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":281.42200000000003,"5min":281.42200000000003,"15min":281.42200000000003},"last":0.45500000000000002},{"interface":"front","average":{"1min":16.393000000000001,"5min":5.1680000000000001,"15min":2.452},"min":{"1min":0.35999999999999999,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":281.44999999999999,"5min":281.44999999999999,"15min":281.44999999999999},"last":3.0099999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.5569999999999999,"15min":0.84799999999999998},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.3940000000000001,"5min":86.691000000000003,"15min":86.691000000000003},"last":3.0449999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.7969999999999999,"15min":0.96599999999999997},"min":{"1min":0.28299999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5169999999999999,"5min":86.822000000000003,"15min":86.822000000000003},"last":3.1560000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.84699999999999998,"15min":0.63100000000000001},"min":{"1min":0.32400000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.1579999999999999,"5min":5.6310000000000002,"15min":5.6310000000000002},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.92800000000000005,"15min":0.69499999999999995},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.0760000000000001,"5min":6.327,"15min":6.327},"last":2.9700000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.992,"15min":1.075},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.391,"5min":86.728999999999999,"15min":86.728999999999999},"last":3.109},{"interface":"front","average":{"1min":0.626,"5min":1.1259999999999999,"15min":0.79600000000000004},"min":{"1min":0.35499999999999998,"5min":0.32600000000000001,"15min":0.314},"max":{"1min":2.3700000000000001,"5min":8.5869999999999997,"15min":8.5869999999999997},"last":0.434}]}]},{"osd":5,"up_from":38,"seq":163208757490,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2100056,"kb_used_data":1940112,"kb_used_omap":0,"kb_used_meta":159936,"kb_avail":91645096,"statfs":{"total":95995035648,"available":93844578304,"internally_reserved":0,"allocated":1986674688,"data_stored":1985802867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":163774464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":4.2439999999999998,"15min":1.8640000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.665,"5min":148.79900000000001,"15min":148.79900000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.3340000000000001,"5min":4.8499999999999996,"15min":2.1070000000000002},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.4169999999999998,"5min":148.779,"15min":148.779},"last":0.52300000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":3.9940000000000002,"15min":2.1150000000000002},"min":{"1min":0.29199999999999998,"5min":0.251,"15min":0.251},"max":{"1min":2.0339999999999998,"5min":166.708,"15min":166.708},"last":0.505},{"interface":"front","average":{"1min":0.68799999999999994,"5min":3.8450000000000002,"15min":2.081},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5590000000000002,"5min":166.607,"15min":166.607},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":5.8339999999999996,"15min":2.5649999999999999},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.931,"5min":165.768,"15min":165.768},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.123,"5min":7.1920000000000002,"15min":3.0880000000000001},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.3440000000000003,"5min":166.24000000000001,"15min":166.24000000000001},"last":0.66000000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:28:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.9159999999999999,"15min":1.1639999999999999},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.311},"max":{"1min":2.0059999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.613,"15min":1.026},"min":{"1min":0.45700000000000002,"5min":0.313,"15min":0.313},"max":{"1min":1.6339999999999999,"5min":28.039999999999999,"15min":28.039999999999999},"last":0.61499999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:29:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.89500000000000002,"15min":0.64200000000000002},"min":{"1min":0.32800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":4.3979999999999997,"5min":8.5359999999999996,"15min":8.5359999999999996},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.95299999999999996,"15min":0.71799999999999997},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.5099999999999998,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":0.29099999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:28:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.1779999999999999,"15min":0.76200000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.9199999999999999,"5min":14.23,"15min":14.23},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.4059999999999999,"15min":0.92200000000000004},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":3.7200000000000002,"5min":18.960000000000001,"15min":18.960000000000001},"last":0.32000000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.246,"15min":0.85099999999999998},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.9459999999999997,"5min":19.175999999999998,"15min":19.175999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.95499999999999996,"15min":0.748},"min":{"1min":0.376,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.5099999999999998,"5min":8.8810000000000002,"15min":8.8810000000000002},"last":0.371}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":663552000,"data_stored":663552000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2006974464,"data_stored":2006974464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2260664320,"data_stored":2260664320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3629121536,"data_stored":3629121536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1486553088,"data_stored":1486553088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1994915840,"data_stored":1994915840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2160787456,"data_stored":2160787456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2313486336,"data_stored":2313486336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:29:44.068 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:44.147 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:44.153 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:29:44.153 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:29:44.153 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:29:44.153 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:29:44.154 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:29:44.407 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:44.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:44 smithi167 ceph-mon[192880]: pgmap v566: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 creating+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 24855/243858 objects degraded (10.192%); 5796/243858 objects misplaced (2.377%); 13 MiB/s, 212 objects/s recovering 2023-09-30T09:29:44.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:44 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:44.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:44.513 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:44.563 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:44 smithi019 ceph-mon[216815]: pgmap v566: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 creating+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 24855/243858 objects degraded (10.192%); 5796/243858 objects misplaced (2.377%); 13 MiB/s, 212 objects/s recovering 2023-09-30T09:29:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:44 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:44.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:44 smithi019 ceph-mon[224043]: pgmap v566: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 creating+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 24855/243858 objects degraded (10.192%); 5796/243858 objects misplaced (2.377%); 13 MiB/s, 212 objects/s recovering 2023-09-30T09:29:44.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:44 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:44.618 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:44.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:44.721 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:44.776 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:44.818 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:44.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:44.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:44.921 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:44.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 84 16 86756 86740 64.5115 59 0.00223968 0.0154803 2023-09-30T09:29:44.946 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:44.995 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:44.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:44.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:45.098 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:45.141 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:45.328 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:29:45.349 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:45.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:45.366 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:29:45.451 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:45 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1742091966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:45.451 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:45 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:45.464 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:45.523 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:45 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1742091966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:45.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:45 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:45.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:45 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1742091966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:45.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:45 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:29:45.602 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:45.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:45.712 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:45.740 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:45.838 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:45.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:45.906 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:45.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 85 16 87692 87676 64.4407 58.5 0.0070006 0.0155127 2023-09-30T09:29:45.945 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:46.016 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:46.142 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:46.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:46.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:46.247 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:46.299 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:46.434 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:46.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:46.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: pgmap v567: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s; 24855/256917 objects degraded (9.674%); 5863/256917 objects misplaced (2.282%); 10 MiB/s, 167 objects/s recovering 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: osdmap e244: 8 total, 8 up, 7 in 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:46 smithi167 ceph-mon[192880]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.541 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:46.542 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:29:46.572 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:29:46.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: pgmap v567: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s; 24855/256917 objects degraded (9.674%); 5863/256917 objects misplaced (2.282%); 10 MiB/s, 167 objects/s recovering 2023-09-30T09:29:46.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:46.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:46.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: osdmap e244: 8 total, 8 up, 7 in 2023-09-30T09:29:46.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[216815]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: pgmap v567: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s; 24855/256917 objects degraded (9.674%); 5863/256917 objects misplaced (2.282%); 10 MiB/s, 167 objects/s recovering 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: osdmap e244: 8 total, 8 up, 7 in 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:46 smithi019 ceph-mon[224043]: failed to encode map e244 with expected crc 2023-09-30T09:29:46.585 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:46.610 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:29:46.659 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:29:46.675 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:46.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:46.701 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:29:46.735 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:29:46.764 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:29:46.788 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:46.799 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:29:46.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:29:46.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:29:46.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:29:46.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:29:46.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:29:46.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:29:46.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:29:46.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:29:46.849 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:46.926 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:46.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:46.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:46.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 86 16 88811 88795 64.5046 69.9375 0.00449142 0.0154928 2023-09-30T09:29:47.031 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:47.061 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:47.109 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:47.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:47.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:47.213 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:47.239 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:47.281 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:47.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:47.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:47.384 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:47.428 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/279688904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:47.428 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/279688904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:47.433 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/279688904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:47.716 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:47.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:47.824 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:47.867 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:47.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 87 16 89565 89549 64.3048 47.125 0.0028328 0.0155422 2023-09-30T09:29:47.940 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:47.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:47.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:48.047 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:48.113 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:48.238 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:29:48.238 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:29:48.245 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:48.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:48.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:48.337 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:29:48.351 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:48.478 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:48.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:48 smithi167 ceph-mon[192880]: pgmap v569: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 24855/258171 objects degraded (9.627%); 5863/258171 objects misplaced (2.271%); 8.1 MiB/s, 130 objects/s recovering 2023-09-30T09:29:48.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:48.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:48.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 24855/258171 objects degraded (9.627%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:48.574 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:48.583 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:48 smithi019 ceph-mon[216815]: pgmap v569: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 24855/258171 objects degraded (9.627%); 5863/258171 objects misplaced (2.271%); 8.1 MiB/s, 130 objects/s recovering 2023-09-30T09:29:48.583 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:48.583 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:48.583 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 24855/258171 objects degraded (9.627%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:48.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:48 smithi019 ceph-mon[224043]: pgmap v569: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 24855/258171 objects degraded (9.627%); 5863/258171 objects misplaced (2.271%); 8.1 MiB/s, 130 objects/s recovering 2023-09-30T09:29:48.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:48.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:29:48.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 24855/258171 objects degraded (9.627%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:48.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:48.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:48.686 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:48.726 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:48.844 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:48.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:48.865 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:48.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 88 16 90501 90485 64.2388 58.5 0.12013 0.0155624 2023-09-30T09:29:48.951 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:48.988 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:29:48.988 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:29:48.988 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-09-30T09:29:48.988 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:29:49.009 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:49.290 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:49.298 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:29:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:49.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:49.404 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:49.474 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:49.610 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:49.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:49.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:49.714 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:49.725 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:29:49.766 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:49.936 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:49.937 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 89 16 91671 91655 64.3386 73.125 0.00599709 0.015536 2023-09-30T09:29:50.462 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:50.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:50.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:50.502 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:50 smithi167 ceph-mon[192880]: pgmap v570: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 24855/264225 objects degraded (9.407%); 5822/264225 objects misplaced (2.203%); 369 KiB/s, 5 objects/s recovering 2023-09-30T09:29:50.567 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:50.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:50 smithi019 ceph-mon[224043]: pgmap v570: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 24855/264225 objects degraded (9.407%); 5822/264225 objects misplaced (2.203%); 369 KiB/s, 5 objects/s recovering 2023-09-30T09:29:50.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:50 smithi019 ceph-mon[216815]: pgmap v570: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 24855/264225 objects degraded (9.407%); 5822/264225 objects misplaced (2.203%); 369 KiB/s, 5 objects/s recovering 2023-09-30T09:29:50.640 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:50.701 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:50.757 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:50.789 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:50.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:50.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:50.893 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:50.923 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:50.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 90 16 92301 92285 64.0612 39.375 0.101768 0.0155814 2023-09-30T09:29:50.967 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:50.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:51.070 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:51.105 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:51.317 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:51.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:51.421 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:29:51.449 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:51.507 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:29:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:51.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:29:51.610 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:51.644 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:51.847 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:51.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:51.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:51.864 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:29:51.864 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:29:51.937 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total time run: 90.0866 2023-09-30T09:29:51.937 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total writes made: 92301 2023-09-30T09:29:51.937 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Write size: 65536 2023-09-30T09:29:51.937 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object size: 65536 2023-09-30T09:29:51.937 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Bandwidth (MB/sec): 64.0363 2023-09-30T09:29:51.937 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Bandwidth: 43.9639 2023-09-30T09:29:51.937 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max bandwidth (MB/sec): 191.375 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min bandwidth (MB/sec): 0 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average IOPS: 1024 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev IOPS: 703.436 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max IOPS: 3062 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min IOPS: 0 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average Latency(s): 0.0156115 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Latency(s): 0.122168 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max latency(s): 9.94718 2023-09-30T09:29:51.938 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min latency(s): 0.00113601 2023-09-30T09:29:51.945 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Cleaning up (deleting benchmark objects) 2023-09-30T09:29:51.952 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:52.033 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:52.138 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:52.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:52.244 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:52.272 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:52.345 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:29:52.377 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:52.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:52.490 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:29:52.556 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:52 smithi019 ceph-mon[224043]: pgmap v571: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 24855/271272 objects degraded (9.162%); 5965/271272 objects misplaced (2.199%); 365 KiB/s, 5 objects/s recovering 2023-09-30T09:29:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2114861094' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:29:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:52 smithi019 ceph-mon[216815]: pgmap v571: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 24855/271272 objects degraded (9.162%); 5965/271272 objects misplaced (2.199%); 365 KiB/s, 5 objects/s recovering 2023-09-30T09:29:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2114861094' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:29:52.726 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:29:52.731 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:52 smithi167 ceph-mon[192880]: pgmap v571: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 24855/271272 objects degraded (9.162%); 5965/271272 objects misplaced (2.199%); 365 KiB/s, 5 objects/s recovering 2023-09-30T09:29:52.731 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2114861094' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:29:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:52.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:29:52.816 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:29:52.834 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:52.887 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:53.379 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2114861094' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:29:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[216815]: osdmap e245: 8 total, 8 up, 7 in 2023-09-30T09:29:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[216815]: failed to encode map e245 with expected crc 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[216815]: failed to encode map e245 with expected crc 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 24855/271272 objects degraded (9.162%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2114861094' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[224043]: osdmap e245: 8 total, 8 up, 7 in 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[224043]: failed to encode map e245 with expected crc 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[224043]: failed to encode map e245 with expected crc 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 24855/271272 objects degraded (9.162%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:53.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:53.627 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:53 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:29:53.628 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2114861094' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:29:53.628 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:53 smithi167 ceph-mon[192880]: osdmap e245: 8 total, 8 up, 7 in 2023-09-30T09:29:53.628 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:53 smithi167 ceph-mon[192880]: failed to encode map e245 with expected crc 2023-09-30T09:29:53.628 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:53 smithi167 ceph-mon[192880]: failed to encode map e245 with expected crc 2023-09-30T09:29:53.628 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 24855/271272 objects degraded (9.162%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:53.628 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:29:53.760 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:53.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:53.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:53.867 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:53.933 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:54.059 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:29:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:29:53] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-09-30T09:29:54.076 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:54.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:54.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:54.183 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:54.235 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:54.327 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:54.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:54.434 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:29:54.438 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:29:54.477 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:29:54.502 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:54.511 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:29:54.554 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:29:54.600 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:29:54.614 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:29:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:54.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:29:54.632 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:54 smithi019 ceph-mon[216815]: failed to encode map e245 with expected crc 2023-09-30T09:29:54.632 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:54 smithi019 ceph-mon[216815]: failed to encode map e245 with expected crc 2023-09-30T09:29:54.632 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:54 smithi019 ceph-mon[216815]: pgmap v573: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 24855/272331 objects degraded (9.127%); 5965/272331 objects misplaced (2.190%); 160 KiB/s, 2 objects/s recovering 2023-09-30T09:29:54.632 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:54 smithi019 ceph-mon[224043]: failed to encode map e245 with expected crc 2023-09-30T09:29:54.632 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:54 smithi019 ceph-mon[224043]: failed to encode map e245 with expected crc 2023-09-30T09:29:54.632 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:54 smithi019 ceph-mon[224043]: pgmap v573: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 24855/272331 objects degraded (9.127%); 5965/272331 objects misplaced (2.190%); 160 KiB/s, 2 objects/s recovering 2023-09-30T09:29:54.726 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:54.749 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:29:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:54 smithi167 ceph-mon[192880]: failed to encode map e245 with expected crc 2023-09-30T09:29:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:54 smithi167 ceph-mon[192880]: failed to encode map e245 with expected crc 2023-09-30T09:29:54.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:54 smithi167 ceph-mon[192880]: pgmap v573: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 24855/272331 objects degraded (9.127%); 5965/272331 objects misplaced (2.190%); 160 KiB/s, 2 objects/s recovering 2023-09-30T09:29:54.777 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:54.802 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:29:54.840 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:29:54.862 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:29:54.862 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:29:54.862 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:29:54.862 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:29:54.862 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:29:54.862 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:29:54.862 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:29:54.863 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:29:54.898 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:54.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:55.007 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:55.047 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:29:55.052 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:29:55.063 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:55.176 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:55.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:55.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:55.285 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:29:55.356 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3591093431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:55.361 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:55.456 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:29:55.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:55.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:29:55.563 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:55.602 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3591093431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:55.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3591093431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:29:55.609 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:55.833 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:29:55.862 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:29:55.863 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:29:55.863 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:29:55.863 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:29:55.863 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:29:56.169 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:56.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:56.279 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:56.331 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:56.391 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:56.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:56.495 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:29:56.524 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:56.567 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:29:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:56.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:29:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:56 smithi019 ceph-mon[216815]: from='client.35250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:56 smithi019 ceph-mon[216815]: pgmap v574: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 104 B/s rd, 43 MiB/s wr, 853 op/s; 24423/272283 objects degraded (8.970%); 5811/272283 objects misplaced (2.134%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:29:56.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:56 smithi019 ceph-mon[224043]: from='client.35250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:56.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:56 smithi019 ceph-mon[224043]: pgmap v574: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 104 B/s rd, 43 MiB/s wr, 853 op/s; 24423/272283 objects degraded (8.970%); 5811/272283 objects misplaced (2.134%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:29:56.670 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:56.702 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:56.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:56 smithi167 ceph-mon[192880]: from='client.35250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:29:56.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:56 smithi167 ceph-mon[192880]: pgmap v574: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 104 B/s rd, 43 MiB/s wr, 853 op/s; 24423/272283 objects degraded (8.970%); 5811/272283 objects misplaced (2.134%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:29:56.968 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:56.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:56.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:56.987 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":573,"stamp":"2023-09-30T09:29:53.148410+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5951961298,"num_objects":90777,"num_object_clones":0,"num_object_copies":272331,"num_objects_missing_on_primary":1335,"num_objects_missing":1335,"num_objects_degraded":24855,"num_objects_misplaced":5965,"num_objects_unfound":0,"num_objects_dirty":90777,"num_whiteouts":0,"num_read":6031,"num_read_kb":12313,"num_write":75523,"num_write_kb":4785771,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8332,"num_bytes_recovered":545952984,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":159579,"ondisk_log_size":159579,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17051004,"kb_used_data":15811200,"kb_used_omap":14,"kb_used_meta":1239729,"kb_avail":639165060,"statfs":{"total":671965249536,"available":654505021440,"internally_reserved":0,"allocated":16190668800,"data_stored":16182926179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14566,"internal_metadata":1269483290},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,2,1,4,1,4,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":336723968,"num_objects":5138,"num_object_clones":0,"num_object_copies":15414,"num_objects_missing_on_primary":-21,"num_objects_missing":-21,"num_objects_degraded":0,"num_objects_misplaced":102,"num_objects_unfound":0,"num_objects_dirty":5138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5138,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002979"},"pg_stats":[{"pgid":"6.23","version":"244'3701","reported_seq":7864,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:49.102736+0000","last_change":"2023-09-30T09:29:35.062393+0000","last_active":"2023-09-30T09:29:49.102736+0000","last_peered":"2023-09-30T09:29:49.102736+0000","last_clean":"2023-09-30T09:29:49.102736+0000","last_became_active":"2023-09-30T09:29:35.061339+0000","last_became_peered":"2023-09-30T09:29:35.061339+0000","last_unstale":"2023-09-30T09:29:49.102736+0000","last_undegraded":"2023-09-30T09:29:49.102736+0000","last_fullsized":"2023-09-30T09:29:49.102736+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":3701,"log_dups_size":0,"ondisk_log_size":3701,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92340224,"num_objects":1409,"num_object_clones":0,"num_object_copies":4227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":702,"num_write_kb":44928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65699840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.22","version":"244'3619","reported_seq":6550,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:48.052835+0000","last_change":"2023-09-30T09:29:37.013899+0000","last_active":"2023-09-30T09:29:48.052835+0000","last_peered":"2023-09-30T09:29:48.052835+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:37.011820+0000","last_became_peered":"2023-09-30T09:29:37.011820+0000","last_unstale":"2023-09-30T09:29:48.052835+0000","last_undegraded":"2023-09-30T09:29:36.997644+0000","last_fullsized":"2023-09-30T09:29:36.996855+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":3619,"log_dups_size":0,"ondisk_log_size":3619,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88276992,"num_objects":1347,"num_object_clones":0,"num_object_copies":4041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":304,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":622,"num_write_kb":39776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":304},{"shards":"3,6","objects":1043}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.21","version":"244'3890","reported_seq":3985,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:50.718278+0000","last_change":"2023-09-30T09:29:35.078377+0000","last_active":"2023-09-30T09:29:50.718278+0000","last_peered":"2023-09-30T09:29:50.718278+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:35.075606+0000","last_became_peered":"2023-09-30T09:29:35.075606+0000","last_unstale":"2023-09-30T09:29:50.718278+0000","last_undegraded":"2023-09-30T09:29:35.051902+0000","last_fullsized":"2023-09-30T09:29:50.718278+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":3890,"log_dups_size":0,"ondisk_log_size":3890,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96501760,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1043,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":842,"num_write_kb":53856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1043},{"shards":"1,6,7","objects":430}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.20","version":"244'3805","reported_seq":3907,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:49.442772+0000","last_change":"2023-09-30T09:29:35.050182+0000","last_active":"2023-09-30T09:29:49.442772+0000","last_peered":"2023-09-30T09:29:49.442772+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:35.036894+0000","last_became_peered":"2023-09-30T09:29:35.036894+0000","last_unstale":"2023-09-30T09:29:49.442772+0000","last_undegraded":"2023-09-30T09:29:35.031725+0000","last_fullsized":"2023-09-30T09:29:49.442772+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":3805,"log_dups_size":0,"ondisk_log_size":3805,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96796672,"num_objects":1477,"num_object_clones":0,"num_object_copies":4431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1054,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":771,"num_write_kb":49312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":423},{"shards":"0,2,3","objects":1054}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1f","version":"244'5281","reported_seq":5351,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:48.325643+0000","last_change":"2023-09-30T09:29:35.091385+0000","last_active":"2023-09-30T09:29:48.325643+0000","last_peered":"2023-09-30T09:29:48.325643+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:35.087685+0000","last_became_peered":"2023-09-30T09:29:35.087685+0000","last_unstale":"2023-09-30T09:29:48.325643+0000","last_undegraded":"2023-09-30T09:29:35.082957+0000","last_fullsized":"2023-09-30T09:29:48.325643+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5281,"log_dups_size":0,"ondisk_log_size":5281,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185663488,"num_objects":2833,"num_object_clones":0,"num_object_copies":8499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2093,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2833,"num_write_kb":181312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":2093},{"shards":"3,4,6","objects":740}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":231,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.267930+0000","last_change":"2023-09-30T09:29:09.598494+0000","last_active":"2023-09-30T09:29:39.267930+0000","last_peered":"2023-09-30T09:29:39.267930+0000","last_clean":"2023-09-30T09:29:39.267930+0000","last_became_active":"2023-09-30T09:29:09.598158+0000","last_became_peered":"2023-09-30T09:29:09.598158+0000","last_unstale":"2023-09-30T09:29:39.267930+0000","last_undegraded":"2023-09-30T09:29:39.267930+0000","last_fullsized":"2023-09-30T09:29:39.267930+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:03.225682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":209,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.256282+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:29:39.256282+0000","last_peered":"2023-09-30T09:29:39.256282+0000","last_clean":"2023-09-30T09:29:39.256282+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:29:39.256282+0000","last_undegraded":"2023-09-30T09:29:39.256282+0000","last_fullsized":"2023-09-30T09:29:39.256282+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1c","version":"244'5166","reported_seq":5230,"reported_epoch":244,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:29:48.279746+0000","last_change":"2023-09-30T09:29:35.102559+0000","last_active":"2023-09-30T09:29:48.279746+0000","last_peered":"2023-09-30T09:29:48.279746+0000","last_clean":"2023-09-30T09:29:33.948392+0000","last_became_active":"2023-09-30T09:29:35.089045+0000","last_became_peered":"2023-09-30T09:29:35.089045+0000","last_unstale":"2023-09-30T09:29:48.279746+0000","last_undegraded":"2023-09-30T09:29:48.279746+0000","last_fullsized":"2023-09-30T09:29:48.279746+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5166,"log_dups_size":0,"ondisk_log_size":5166,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180092928,"num_objects":2748,"num_object_clones":0,"num_object_copies":8244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2748,"num_write_kb":175872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":2748}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":206,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269058+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:29:39.269058+0000","last_peered":"2023-09-30T09:29:39.269058+0000","last_clean":"2023-09-30T09:29:39.269058+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:29:39.269058+0000","last_undegraded":"2023-09-30T09:29:39.269058+0000","last_fullsized":"2023-09-30T09:29:39.269058+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":137,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.253154+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:29:39.253154+0000","last_peered":"2023-09-30T09:29:39.253154+0000","last_clean":"2023-09-30T09:29:39.253154+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:29:39.253154+0000","last_undegraded":"2023-09-30T09:29:39.253154+0000","last_fullsized":"2023-09-30T09:29:39.253154+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1d","version":"244'5340","reported_seq":5403,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:49.036786+0000","last_change":"2023-09-30T09:29:35.078954+0000","last_active":"2023-09-30T09:29:49.036786+0000","last_peered":"2023-09-30T09:29:49.036786+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:35.073573+0000","last_became_peered":"2023-09-30T09:29:35.073573+0000","last_unstale":"2023-09-30T09:29:49.036786+0000","last_undegraded":"2023-09-30T09:29:35.043564+0000","last_fullsized":"2023-09-30T09:29:49.036786+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186449920,"num_objects":2845,"num_object_clones":0,"num_object_copies":8535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2034,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2845,"num_write_kb":182080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":2034},{"shards":"1,5,6","objects":811}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":259,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255515+0000","last_change":"2023-09-30T09:29:09.606288+0000","last_active":"2023-09-30T09:29:39.255515+0000","last_peered":"2023-09-30T09:29:39.255515+0000","last_clean":"2023-09-30T09:29:39.255515+0000","last_became_active":"2023-09-30T09:29:09.605776+0000","last_became_peered":"2023-09-30T09:29:09.605776+0000","last_unstale":"2023-09-30T09:29:39.255515+0000","last_undegraded":"2023-09-30T09:29:39.255515+0000","last_fullsized":"2023-09-30T09:29:39.255515+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:07.935982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.266747+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:29:39.266747+0000","last_peered":"2023-09-30T09:29:39.266747+0000","last_clean":"2023-09-30T09:29:39.266747+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:29:39.266747+0000","last_undegraded":"2023-09-30T09:29:39.266747+0000","last_fullsized":"2023-09-30T09:29:39.266747+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"244'3868","reported_seq":8018,"reported_epoch":244,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:29:49.176338+0000","last_change":"2023-09-30T09:29:10.766595+0000","last_active":"2023-09-30T09:29:49.176338+0000","last_peered":"2023-09-30T09:29:49.176338+0000","last_clean":"2023-09-30T09:29:08.564047+0000","last_became_active":"2023-09-30T09:29:10.763967+0000","last_became_peered":"2023-09-30T09:29:10.763967+0000","last_unstale":"2023-09-30T09:29:49.176338+0000","last_undegraded":"2023-09-30T09:29:49.176338+0000","last_fullsized":"2023-09-30T09:29:49.176338+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2497","last_scrub_stamp":"2023-09-30T09:28:46.026181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:46.026181+0000","objects_scrubbed":1334,"log_size":3868,"log_dups_size":0,"ondisk_log_size":3868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:02.322482+0000","scrub_duration":0.15695645899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2705,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1371,"num_write_kb":87744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1347,"num_bytes_recovered":88276992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,2],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,4,7","objects":2705}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":136,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.253260+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:29:39.253260+0000","last_peered":"2023-09-30T09:29:39.253260+0000","last_clean":"2023-09-30T09:29:39.253260+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:29:39.253260+0000","last_undegraded":"2023-09-30T09:29:39.253260+0000","last_fullsized":"2023-09-30T09:29:39.253260+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":209,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255592+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:29:39.255592+0000","last_peered":"2023-09-30T09:29:39.255592+0000","last_clean":"2023-09-30T09:29:39.255592+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:29:39.255592+0000","last_undegraded":"2023-09-30T09:29:39.255592+0000","last_fullsized":"2023-09-30T09:29:39.255592+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"244'3928","reported_seq":4169,"reported_epoch":244,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:29:48.328273+0000","last_change":"2023-09-30T09:29:38.038561+0000","last_active":"2023-09-30T09:29:48.328273+0000","last_peered":"2023-09-30T09:29:48.328273+0000","last_clean":"2023-09-30T09:28:50.280641+0000","last_became_active":"2023-09-30T09:29:12.506944+0000","last_became_peered":"2023-09-30T09:29:12.506944+0000","last_unstale":"2023-09-30T09:29:48.328273+0000","last_undegraded":"2023-09-30T09:29:48.328273+0000","last_fullsized":"2023-09-30T09:29:10.583109+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3928,"log_dups_size":0,"ondisk_log_size":3928,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185073664,"num_objects":2824,"num_object_clones":0,"num_object_copies":8472,"num_objects_missing_on_primary":1335,"num_objects_missing":1335,"num_objects_degraded":0,"num_objects_misplaced":3260,"num_objects_unfound":0,"num_objects_dirty":2824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2824,"num_write_kb":180736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":3735552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,3,6","objects":589},{"shards":"2,3,6,7","objects":1},{"shards":"2,6,7","objects":1335},{"shards":"3,6","objects":899}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":95,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.261025+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:29:39.261025+0000","last_peered":"2023-09-30T09:29:39.261025+0000","last_clean":"2023-09-30T09:29:39.261025+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:29:39.261025+0000","last_undegraded":"2023-09-30T09:29:39.261025+0000","last_fullsized":"2023-09-30T09:29:39.261025+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":160,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255824+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:29:39.255824+0000","last_peered":"2023-09-30T09:29:39.255824+0000","last_clean":"2023-09-30T09:29:39.255824+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:29:39.255824+0000","last_undegraded":"2023-09-30T09:29:39.255824+0000","last_fullsized":"2023-09-30T09:29:39.255824+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"244'3925","reported_seq":7974,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:49.295672+0000","last_change":"2023-09-30T09:28:56.382198+0000","last_active":"2023-09-30T09:29:49.295672+0000","last_peered":"2023-09-30T09:29:49.295672+0000","last_clean":"2023-09-30T09:29:49.295672+0000","last_became_active":"2023-09-30T09:28:56.380752+0000","last_became_peered":"2023-09-30T09:28:56.380752+0000","last_unstale":"2023-09-30T09:29:49.295672+0000","last_undegraded":"2023-09-30T09:29:49.295672+0000","last_fullsized":"2023-09-30T09:29:49.295672+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3925,"log_dups_size":0,"ondisk_log_size":3925,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183926784,"num_objects":2807,"num_object_clones":0,"num_object_copies":8421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2807,"num_write_kb":179616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1324,"num_bytes_recovered":86769664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":207,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.262257+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:29:39.262257+0000","last_peered":"2023-09-30T09:29:39.262257+0000","last_clean":"2023-09-30T09:29:39.262257+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:29:39.262257+0000","last_undegraded":"2023-09-30T09:29:39.262257+0000","last_fullsized":"2023-09-30T09:29:39.262257+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":192,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:41.794770+0000","last_change":"2023-09-30T09:29:09.620809+0000","last_active":"2023-09-30T09:29:41.794770+0000","last_peered":"2023-09-30T09:29:41.794770+0000","last_clean":"2023-09-30T09:29:41.794770+0000","last_became_active":"2023-09-30T09:29:09.620497+0000","last_became_peered":"2023-09-30T09:29:09.620497+0000","last_unstale":"2023-09-30T09:29:41.794770+0000","last_undegraded":"2023-09-30T09:29:41.794770+0000","last_fullsized":"2023-09-30T09:29:41.794770+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:48.932180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"244'4016","reported_seq":9612,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:48.286116+0000","last_change":"2023-09-30T09:29:39.270828+0000","last_active":"2023-09-30T09:29:48.286116+0000","last_peered":"2023-09-30T09:29:48.286116+0000","last_clean":"2023-09-30T09:29:48.286116+0000","last_became_active":"2023-09-30T09:29:39.266753+0000","last_became_peered":"2023-09-30T09:29:39.266753+0000","last_unstale":"2023-09-30T09:29:48.286116+0000","last_undegraded":"2023-09-30T09:29:48.286116+0000","last_fullsized":"2023-09-30T09:29:48.286116+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2608","last_scrub_stamp":"2023-09-30T09:28:45.597110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.597110+0000","objects_scrubbed":1407,"log_size":4016,"log_dups_size":0,"ondisk_log_size":4016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:18:41.672155+0000","scrub_duration":0.26278548800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184877056,"num_objects":2821,"num_object_clones":0,"num_object_copies":8463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1414,"num_write_kb":90496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2034,"num_bytes_recovered":133300224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":208,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.268106+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:29:39.268106+0000","last_peered":"2023-09-30T09:29:39.268106+0000","last_clean":"2023-09-30T09:29:39.268106+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:29:39.268106+0000","last_undegraded":"2023-09-30T09:29:39.268106+0000","last_fullsized":"2023-09-30T09:29:39.268106+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.267802+0000","last_change":"2023-09-30T09:29:09.612466+0000","last_active":"2023-09-30T09:29:39.267802+0000","last_peered":"2023-09-30T09:29:39.267802+0000","last_clean":"2023-09-30T09:29:39.267802+0000","last_became_active":"2023-09-30T09:29:09.612173+0000","last_became_peered":"2023-09-30T09:29:09.612173+0000","last_unstale":"2023-09-30T09:29:39.267802+0000","last_undegraded":"2023-09-30T09:29:39.267802+0000","last_fullsized":"2023-09-30T09:29:39.267802+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:15.457876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"244'4010","reported_seq":4074,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:48.339213+0000","last_change":"2023-09-30T09:28:40.282669+0000","last_active":"2023-09-30T09:29:48.339213+0000","last_peered":"2023-09-30T09:29:48.339213+0000","last_clean":"2023-09-30T09:29:48.339213+0000","last_became_active":"2023-09-30T09:28:40.281544+0000","last_became_peered":"2023-09-30T09:28:40.281544+0000","last_unstale":"2023-09-30T09:29:48.339213+0000","last_undegraded":"2023-09-30T09:29:48.339213+0000","last_fullsized":"2023-09-30T09:29:48.339213+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4010,"log_dups_size":0,"ondisk_log_size":4010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188252160,"num_objects":2873,"num_object_clones":0,"num_object_copies":8619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2873,"num_write_kb":183840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.265757+0000","last_change":"2023-09-30T09:29:09.626205+0000","last_active":"2023-09-30T09:29:39.265757+0000","last_peered":"2023-09-30T09:29:39.265757+0000","last_clean":"2023-09-30T09:29:39.265757+0000","last_became_active":"2023-09-30T09:29:09.618293+0000","last_became_peered":"2023-09-30T09:29:09.618293+0000","last_unstale":"2023-09-30T09:29:39.265757+0000","last_undegraded":"2023-09-30T09:29:39.265757+0000","last_fullsized":"2023-09-30T09:29:39.265757+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:04.337293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":231,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.256163+0000","last_change":"2023-09-30T09:29:09.606072+0000","last_active":"2023-09-30T09:29:39.256163+0000","last_peered":"2023-09-30T09:29:39.256163+0000","last_clean":"2023-09-30T09:29:39.256163+0000","last_became_active":"2023-09-30T09:29:09.605716+0000","last_became_peered":"2023-09-30T09:29:09.605716+0000","last_unstale":"2023-09-30T09:29:39.256163+0000","last_undegraded":"2023-09-30T09:29:39.256163+0000","last_fullsized":"2023-09-30T09:29:39.256163+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:50.972640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"244'3961","reported_seq":4084,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:49.099008+0000","last_change":"2023-09-30T09:28:45.602319+0000","last_active":"2023-09-30T09:29:49.099008+0000","last_peered":"2023-09-30T09:29:49.099008+0000","last_clean":"2023-09-30T09:29:49.099008+0000","last_became_active":"2023-09-30T09:28:40.272792+0000","last_became_peered":"2023-09-30T09:28:40.272792+0000","last_unstale":"2023-09-30T09:29:49.099008+0000","last_undegraded":"2023-09-30T09:29:49.099008+0000","last_fullsized":"2023-09-30T09:29:49.099008+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2523","last_scrub_stamp":"2023-09-30T09:28:45.602248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.602248+0000","objects_scrubbed":1419,"log_size":3961,"log_dups_size":0,"ondisk_log_size":3961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:31:54.788871+0000","scrub_duration":0.27069680099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187695104,"num_objects":2864,"num_object_clones":0,"num_object_copies":8592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1445,"num_write_kb":92480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":136,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.252904+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:29:39.252904+0000","last_peered":"2023-09-30T09:29:39.252904+0000","last_clean":"2023-09-30T09:29:39.252904+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:29:39.252904+0000","last_undegraded":"2023-09-30T09:29:39.252904+0000","last_fullsized":"2023-09-30T09:29:39.252904+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":165,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.262396+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:29:39.262396+0000","last_peered":"2023-09-30T09:29:39.262396+0000","last_clean":"2023-09-30T09:29:39.262396+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:29:39.262396+0000","last_undegraded":"2023-09-30T09:29:39.262396+0000","last_fullsized":"2023-09-30T09:29:39.262396+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"244'4035","reported_seq":4122,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:50.786457+0000","last_change":"2023-09-30T09:28:44.596036+0000","last_active":"2023-09-30T09:29:50.786457+0000","last_peered":"2023-09-30T09:29:50.786457+0000","last_clean":"2023-09-30T09:29:50.786457+0000","last_became_active":"2023-09-30T09:28:40.272740+0000","last_became_peered":"2023-09-30T09:28:40.272740+0000","last_unstale":"2023-09-30T09:29:50.786457+0000","last_undegraded":"2023-09-30T09:29:50.786457+0000","last_fullsized":"2023-09-30T09:29:50.786457+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"214'2497","last_scrub_stamp":"2023-09-30T09:28:44.595809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:44.595809+0000","objects_scrubbed":1358,"log_size":4035,"log_dups_size":0,"ondisk_log_size":4035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:46:41.016886+0000","scrub_duration":0.31195125499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190316544,"num_objects":2904,"num_object_clones":0,"num_object_copies":8712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1546,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":187,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.263463+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:29:39.263463+0000","last_peered":"2023-09-30T09:29:39.263463+0000","last_clean":"2023-09-30T09:29:39.263463+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:29:39.263463+0000","last_undegraded":"2023-09-30T09:29:39.263463+0000","last_fullsized":"2023-09-30T09:29:39.263463+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":136,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.253076+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:29:39.253076+0000","last_peered":"2023-09-30T09:29:39.253076+0000","last_clean":"2023-09-30T09:29:39.253076+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:29:39.253076+0000","last_undegraded":"2023-09-30T09:29:39.253076+0000","last_fullsized":"2023-09-30T09:29:39.253076+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"244'3980","reported_seq":4034,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:49.171613+0000","last_change":"2023-09-30T09:28:40.288346+0000","last_active":"2023-09-30T09:29:49.171613+0000","last_peered":"2023-09-30T09:29:49.171613+0000","last_clean":"2023-09-30T09:29:49.171613+0000","last_became_active":"2023-09-30T09:28:40.286899+0000","last_became_peered":"2023-09-30T09:28:40.286899+0000","last_unstale":"2023-09-30T09:29:49.171613+0000","last_undegraded":"2023-09-30T09:29:49.171613+0000","last_fullsized":"2023-09-30T09:29:49.171613+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3980,"log_dups_size":0,"ondisk_log_size":3980,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186679296,"num_objects":2849,"num_object_clones":0,"num_object_copies":8547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2849,"num_write_kb":182304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":336,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.258838+0000","last_change":"2023-09-30T09:29:09.634829+0000","last_active":"2023-09-30T09:29:39.258838+0000","last_peered":"2023-09-30T09:29:39.258838+0000","last_clean":"2023-09-30T09:29:39.258838+0000","last_became_active":"2023-09-30T09:29:09.612183+0000","last_became_peered":"2023-09-30T09:29:09.612183+0000","last_unstale":"2023-09-30T09:29:39.258838+0000","last_undegraded":"2023-09-30T09:29:39.258838+0000","last_fullsized":"2023-09-30T09:29:39.258838+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:28.144436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":229,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.258299+0000","last_change":"2023-09-30T09:29:09.605918+0000","last_active":"2023-09-30T09:29:39.258299+0000","last_peered":"2023-09-30T09:29:39.258299+0000","last_clean":"2023-09-30T09:29:39.258299+0000","last_became_active":"2023-09-30T09:29:09.605521+0000","last_became_peered":"2023-09-30T09:29:09.605521+0000","last_unstale":"2023-09-30T09:29:39.258299+0000","last_undegraded":"2023-09-30T09:29:39.258299+0000","last_fullsized":"2023-09-30T09:29:39.258299+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:43.287337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"244'5278","reported_seq":5335,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:49.052931+0000","last_change":"2023-09-30T09:29:35.005933+0000","last_active":"2023-09-30T09:29:49.052931+0000","last_peered":"2023-09-30T09:29:49.052931+0000","last_clean":"2023-09-30T09:29:49.052931+0000","last_became_active":"2023-09-30T09:29:35.001538+0000","last_became_peered":"2023-09-30T09:29:35.001538+0000","last_unstale":"2023-09-30T09:29:49.052931+0000","last_undegraded":"2023-09-30T09:29:49.052931+0000","last_fullsized":"2023-09-30T09:29:49.052931+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5278,"log_dups_size":0,"ondisk_log_size":5278,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185532416,"num_objects":2831,"num_object_clones":0,"num_object_copies":8493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2831,"num_write_kb":181184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":247,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.260840+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:29:39.260840+0000","last_peered":"2023-09-30T09:29:39.260840+0000","last_clean":"2023-09-30T09:29:39.260840+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:29:39.260840+0000","last_undegraded":"2023-09-30T09:29:39.260840+0000","last_fullsized":"2023-09-30T09:29:39.260840+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":107,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.264923+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:29:39.264923+0000","last_peered":"2023-09-30T09:29:39.264923+0000","last_clean":"2023-09-30T09:29:39.264923+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:29:39.264923+0000","last_undegraded":"2023-09-30T09:29:39.264923+0000","last_fullsized":"2023-09-30T09:29:39.264923+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"244'5297","reported_seq":5361,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:48.347621+0000","last_change":"2023-09-30T09:29:35.047558+0000","last_active":"2023-09-30T09:29:48.347621+0000","last_peered":"2023-09-30T09:29:48.347621+0000","last_clean":"2023-09-30T09:29:48.347621+0000","last_became_active":"2023-09-30T09:29:35.044991+0000","last_became_peered":"2023-09-30T09:29:35.044991+0000","last_unstale":"2023-09-30T09:29:48.347621+0000","last_undegraded":"2023-09-30T09:29:48.347621+0000","last_fullsized":"2023-09-30T09:29:48.347621+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5297,"log_dups_size":0,"ondisk_log_size":5297,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185434112,"num_objects":2829,"num_object_clones":0,"num_object_copies":8487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2829,"num_write_kb":181088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":248,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.261344+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:29:39.261344+0000","last_peered":"2023-09-30T09:29:39.261344+0000","last_clean":"2023-09-30T09:29:39.261344+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:29:39.261344+0000","last_undegraded":"2023-09-30T09:29:39.261344+0000","last_fullsized":"2023-09-30T09:29:39.261344+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":219,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269262+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:29:39.269262+0000","last_peered":"2023-09-30T09:29:39.269262+0000","last_clean":"2023-09-30T09:29:39.269262+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:29:39.269262+0000","last_undegraded":"2023-09-30T09:29:39.269262+0000","last_fullsized":"2023-09-30T09:29:39.269262+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"244'3936","reported_seq":4003,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:49.166418+0000","last_change":"2023-09-30T09:29:10.874488+0000","last_active":"2023-09-30T09:29:49.166418+0000","last_peered":"2023-09-30T09:29:49.166418+0000","last_clean":"2023-09-30T09:29:08.490081+0000","last_became_active":"2023-09-30T09:29:10.870401+0000","last_became_peered":"2023-09-30T09:29:10.870401+0000","last_unstale":"2023-09-30T09:29:49.166418+0000","last_undegraded":"2023-09-30T09:29:10.657673+0000","last_fullsized":"2023-09-30T09:29:10.593223+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3936,"log_dups_size":0,"ondisk_log_size":3936,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184975360,"num_objects":2822,"num_object_clones":0,"num_object_copies":8466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1975,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2822,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1975},{"shards":"3,7","objects":847}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":230,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.262094+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:29:39.262094+0000","last_peered":"2023-09-30T09:29:39.262094+0000","last_clean":"2023-09-30T09:29:39.262094+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:29:39.262094+0000","last_undegraded":"2023-09-30T09:29:39.262094+0000","last_fullsized":"2023-09-30T09:29:39.262094+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":213,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.259230+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:29:39.259230+0000","last_peered":"2023-09-30T09:29:39.259230+0000","last_clean":"2023-09-30T09:29:39.259230+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:29:39.259230+0000","last_undegraded":"2023-09-30T09:29:39.259230+0000","last_fullsized":"2023-09-30T09:29:39.259230+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"244'3909","reported_seq":4039,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:49.035002+0000","last_change":"2023-09-30T09:29:10.893105+0000","last_active":"2023-09-30T09:29:49.035002+0000","last_peered":"2023-09-30T09:29:49.035002+0000","last_clean":"2023-09-30T09:29:08.514351+0000","last_became_active":"2023-09-30T09:29:10.883964+0000","last_became_peered":"2023-09-30T09:29:10.883964+0000","last_unstale":"2023-09-30T09:29:49.035002+0000","last_undegraded":"2023-09-30T09:29:10.674378+0000","last_fullsized":"2023-09-30T09:29:10.596450+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2471","last_scrub_stamp":"2023-09-30T09:28:47.207706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:47.207706+0000","objects_scrubbed":1369,"log_size":3909,"log_dups_size":0,"ondisk_log_size":3909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:51.408261+0000","scrub_duration":0.14908584899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183959552,"num_objects":2807,"num_object_clones":0,"num_object_copies":8421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1438,"num_write_kb":92032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1997},{"shards":"3,7","objects":810}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":203,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.262000+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:29:39.262000+0000","last_peered":"2023-09-30T09:29:39.262000+0000","last_clean":"2023-09-30T09:29:39.262000+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:29:39.262000+0000","last_undegraded":"2023-09-30T09:29:39.262000+0000","last_fullsized":"2023-09-30T09:29:39.262000+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":133,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.253140+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:29:39.253140+0000","last_peered":"2023-09-30T09:29:39.253140+0000","last_clean":"2023-09-30T09:29:39.253140+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:29:39.253140+0000","last_undegraded":"2023-09-30T09:29:39.253140+0000","last_fullsized":"2023-09-30T09:29:39.253140+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"244'5190","reported_seq":5259,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:48.943302+0000","last_change":"2023-09-30T09:29:36.032408+0000","last_active":"2023-09-30T09:29:48.943302+0000","last_peered":"2023-09-30T09:29:48.943302+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:36.028267+0000","last_became_peered":"2023-09-30T09:29:36.028267+0000","last_unstale":"2023-09-30T09:29:48.943302+0000","last_undegraded":"2023-09-30T09:29:36.007788+0000","last_fullsized":"2023-09-30T09:29:48.943302+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5190,"log_dups_size":0,"ondisk_log_size":5190,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182910976,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2791,"num_write_kb":178624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1921},{"shards":"1,4,7","objects":870}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":723,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.263373+0000","last_change":"2023-09-30T09:29:12.637524+0000","last_active":"2023-09-30T09:29:39.263373+0000","last_peered":"2023-09-30T09:29:39.263373+0000","last_clean":"2023-09-30T09:29:39.263373+0000","last_became_active":"2023-09-30T09:29:12.637171+0000","last_became_peered":"2023-09-30T09:29:12.637171+0000","last_unstale":"2023-09-30T09:29:39.263373+0000","last_undegraded":"2023-09-30T09:29:39.263373+0000","last_fullsized":"2023-09-30T09:29:39.263373+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:14:09.735059+0000","scrub_duration":0.0079898150000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":243,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.266596+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:29:39.266596+0000","last_peered":"2023-09-30T09:29:39.266596+0000","last_clean":"2023-09-30T09:29:39.266596+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:29:39.266596+0000","last_undegraded":"2023-09-30T09:29:39.266596+0000","last_fullsized":"2023-09-30T09:29:39.266596+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":202,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.268023+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:29:39.268023+0000","last_peered":"2023-09-30T09:29:39.268023+0000","last_clean":"2023-09-30T09:29:39.268023+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:29:39.268023+0000","last_undegraded":"2023-09-30T09:29:39.268023+0000","last_fullsized":"2023-09-30T09:29:39.268023+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"244'5261","reported_seq":5331,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:48.298692+0000","last_change":"2023-09-30T09:29:36.082560+0000","last_active":"2023-09-30T09:29:48.298692+0000","last_peered":"2023-09-30T09:29:48.298692+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:36.077849+0000","last_became_peered":"2023-09-30T09:29:36.077849+0000","last_unstale":"2023-09-30T09:29:48.298692+0000","last_undegraded":"2023-09-30T09:29:35.995233+0000","last_fullsized":"2023-09-30T09:29:48.298692+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5261,"log_dups_size":0,"ondisk_log_size":5261,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184352768,"num_objects":2813,"num_object_clones":0,"num_object_copies":8439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2813,"num_write_kb":180032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":1940},{"shards":"3,4,6","objects":873}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":620,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255103+0000","last_change":"2023-09-30T09:29:12.631999+0000","last_active":"2023-09-30T09:29:39.255103+0000","last_peered":"2023-09-30T09:29:39.255103+0000","last_clean":"2023-09-30T09:29:39.255103+0000","last_became_active":"2023-09-30T09:29:12.631620+0000","last_became_peered":"2023-09-30T09:29:12.631620+0000","last_unstale":"2023-09-30T09:29:39.255103+0000","last_undegraded":"2023-09-30T09:29:39.255103+0000","last_fullsized":"2023-09-30T09:29:39.255103+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.999760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.999760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":531037,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":204,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.259146+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:29:39.259146+0000","last_peered":"2023-09-30T09:29:39.259146+0000","last_clean":"2023-09-30T09:29:39.259146+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:29:39.259146+0000","last_undegraded":"2023-09-30T09:29:39.259146+0000","last_fullsized":"2023-09-30T09:29:39.259146+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":128,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269767+0000","last_change":"2023-09-30T09:29:09.612070+0000","last_active":"2023-09-30T09:29:39.269767+0000","last_peered":"2023-09-30T09:29:39.269767+0000","last_clean":"2023-09-30T09:29:39.269767+0000","last_became_active":"2023-09-30T09:29:09.611725+0000","last_became_peered":"2023-09-30T09:29:09.611725+0000","last_unstale":"2023-09-30T09:29:39.269767+0000","last_undegraded":"2023-09-30T09:29:39.269767+0000","last_fullsized":"2023-09-30T09:29:39.269767+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:55.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"244'5207","reported_seq":5271,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:48.325951+0000","last_change":"2023-09-30T09:29:35.074390+0000","last_active":"2023-09-30T09:29:48.325951+0000","last_peered":"2023-09-30T09:29:48.325951+0000","last_clean":"2023-09-30T09:29:48.325951+0000","last_became_active":"2023-09-30T09:29:35.072113+0000","last_became_peered":"2023-09-30T09:29:35.072113+0000","last_unstale":"2023-09-30T09:29:48.325951+0000","last_undegraded":"2023-09-30T09:29:48.325951+0000","last_fullsized":"2023-09-30T09:29:48.325951+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5207,"log_dups_size":0,"ondisk_log_size":5207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182779904,"num_objects":2789,"num_object_clones":0,"num_object_copies":8367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2789,"num_write_kb":178496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":213,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.267617+0000","last_change":"2023-09-30T09:29:09.607621+0000","last_active":"2023-09-30T09:29:39.267617+0000","last_peered":"2023-09-30T09:29:39.267617+0000","last_clean":"2023-09-30T09:29:39.267617+0000","last_became_active":"2023-09-30T09:29:09.607245+0000","last_became_peered":"2023-09-30T09:29:09.607245+0000","last_unstale":"2023-09-30T09:29:39.267617+0000","last_undegraded":"2023-09-30T09:29:39.267617+0000","last_fullsized":"2023-09-30T09:29:39.267617+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:42:42.429421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":202,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.268519+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:29:39.268519+0000","last_peered":"2023-09-30T09:29:39.268519+0000","last_clean":"2023-09-30T09:29:39.268519+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:29:39.268519+0000","last_undegraded":"2023-09-30T09:29:39.268519+0000","last_fullsized":"2023-09-30T09:29:39.268519+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"244'5372","reported_seq":5438,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:49.109861+0000","last_change":"2023-09-30T09:29:36.240193+0000","last_active":"2023-09-30T09:29:49.109861+0000","last_peered":"2023-09-30T09:29:49.109861+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:36.233690+0000","last_became_peered":"2023-09-30T09:29:36.233690+0000","last_unstale":"2023-09-30T09:29:49.109861+0000","last_undegraded":"2023-09-30T09:29:36.012206+0000","last_fullsized":"2023-09-30T09:29:49.109861+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5372,"log_dups_size":0,"ondisk_log_size":5372,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188547072,"num_objects":2877,"num_object_clones":0,"num_object_copies":8631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2078,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2877,"num_write_kb":184128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":2078},{"shards":"1,5,6","objects":799}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":451,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.259059+0000","last_change":"2023-09-30T09:29:12.635044+0000","last_active":"2023-09-30T09:29:39.259059+0000","last_peered":"2023-09-30T09:29:39.259059+0000","last_clean":"2023-09-30T09:29:39.259059+0000","last_became_active":"2023-09-30T09:29:12.634703+0000","last_became_peered":"2023-09-30T09:29:12.634703+0000","last_unstale":"2023-09-30T09:29:39.259059+0000","last_undegraded":"2023-09-30T09:29:39.259059+0000","last_fullsized":"2023-09-30T09:29:39.259059+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.980385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.980385+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":210,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255287+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:29:39.255287+0000","last_peered":"2023-09-30T09:29:39.255287+0000","last_clean":"2023-09-30T09:29:39.255287+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:29:39.255287+0000","last_undegraded":"2023-09-30T09:29:39.255287+0000","last_fullsized":"2023-09-30T09:29:39.255287+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":184,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269453+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:29:39.269453+0000","last_peered":"2023-09-30T09:29:39.269453+0000","last_clean":"2023-09-30T09:29:39.269453+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:29:39.269453+0000","last_undegraded":"2023-09-30T09:29:39.269453+0000","last_fullsized":"2023-09-30T09:29:39.269453+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"244'3766","reported_seq":3868,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:49.355494+0000","last_change":"2023-09-30T09:29:36.014896+0000","last_active":"2023-09-30T09:29:49.355494+0000","last_peered":"2023-09-30T09:29:49.355494+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:36.012719+0000","last_became_peered":"2023-09-30T09:29:36.012719+0000","last_unstale":"2023-09-30T09:29:49.355494+0000","last_undegraded":"2023-09-30T09:29:35.992619+0000","last_fullsized":"2023-09-30T09:29:49.355494+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":3766,"log_dups_size":0,"ondisk_log_size":3766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94240768,"num_objects":1438,"num_object_clones":0,"num_object_copies":4314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":731,"num_write_kb":46816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":393},{"shards":"0,2,3","objects":1045}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":488,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.266678+0000","last_change":"2023-09-30T09:28:30.495699+0000","last_active":"2023-09-30T09:29:39.266678+0000","last_peered":"2023-09-30T09:29:39.266678+0000","last_clean":"2023-09-30T09:29:39.266678+0000","last_became_active":"2023-09-30T09:28:09.228266+0000","last_became_peered":"2023-09-30T09:28:09.228266+0000","last_unstale":"2023-09-30T09:29:39.266678+0000","last_undegraded":"2023-09-30T09:29:39.266678+0000","last_fullsized":"2023-09-30T09:29:39.266678+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:10:19.705285+0000","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":588,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269243+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:29:39.269243+0000","last_peered":"2023-09-30T09:29:39.269243+0000","last_clean":"2023-09-30T09:29:39.269243+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:29:39.269243+0000","last_undegraded":"2023-09-30T09:29:39.269243+0000","last_fullsized":"2023-09-30T09:29:39.269243+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":230,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.264528+0000","last_change":"2023-09-30T09:29:09.611292+0000","last_active":"2023-09-30T09:29:39.264528+0000","last_peered":"2023-09-30T09:29:39.264528+0000","last_clean":"2023-09-30T09:29:39.264528+0000","last_became_active":"2023-09-30T09:29:09.611015+0000","last_became_peered":"2023-09-30T09:29:09.611015+0000","last_unstale":"2023-09-30T09:29:39.264528+0000","last_undegraded":"2023-09-30T09:29:39.264528+0000","last_fullsized":"2023-09-30T09:29:39.264528+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:48.124378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":203,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.268933+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:29:39.268933+0000","last_peered":"2023-09-30T09:29:39.268933+0000","last_clean":"2023-09-30T09:29:39.268933+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:29:39.268933+0000","last_undegraded":"2023-09-30T09:29:39.268933+0000","last_fullsized":"2023-09-30T09:29:39.268933+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"244'4016","reported_seq":4102,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:48.185814+0000","last_change":"2023-09-30T09:28:42.250412+0000","last_active":"2023-09-30T09:29:48.185814+0000","last_peered":"2023-09-30T09:29:48.185814+0000","last_clean":"2023-09-30T09:29:48.185814+0000","last_became_active":"2023-09-30T09:28:40.266645+0000","last_became_peered":"2023-09-30T09:28:40.266645+0000","last_unstale":"2023-09-30T09:29:48.185814+0000","last_undegraded":"2023-09-30T09:29:48.185814+0000","last_fullsized":"2023-09-30T09:29:48.185814+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2368","last_scrub_stamp":"2023-09-30T09:28:42.250204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.250204+0000","objects_scrubbed":1224,"log_size":4016,"log_dups_size":0,"ondisk_log_size":4016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:20:30.287549+0000","scrub_duration":0.32304901400000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188612608,"num_objects":2878,"num_object_clones":0,"num_object_copies":8634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1654,"num_write_kb":105856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":666,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.261564+0000","last_change":"2023-09-30T09:29:12.640269+0000","last_active":"2023-09-30T09:29:39.261564+0000","last_peered":"2023-09-30T09:29:39.261564+0000","last_clean":"2023-09-30T09:29:39.261564+0000","last_became_active":"2023-09-30T09:29:12.639925+0000","last_became_peered":"2023-09-30T09:29:12.639925+0000","last_unstale":"2023-09-30T09:29:39.261564+0000","last_undegraded":"2023-09-30T09:29:39.261564+0000","last_fullsized":"2023-09-30T09:29:39.261564+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:45:10.404382+0000","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":392,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:41.794808+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:29:41.794808+0000","last_peered":"2023-09-30T09:29:41.794808+0000","last_clean":"2023-09-30T09:29:41.794808+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:29:41.794808+0000","last_undegraded":"2023-09-30T09:29:41.794808+0000","last_fullsized":"2023-09-30T09:29:41.794808+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":223,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.267858+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:29:39.267858+0000","last_peered":"2023-09-30T09:29:39.267858+0000","last_clean":"2023-09-30T09:29:39.267858+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:29:39.267858+0000","last_undegraded":"2023-09-30T09:29:39.267858+0000","last_fullsized":"2023-09-30T09:29:39.267858+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":102,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.256406+0000","last_change":"2023-09-30T09:29:09.611507+0000","last_active":"2023-09-30T09:29:39.256406+0000","last_peered":"2023-09-30T09:29:39.256406+0000","last_clean":"2023-09-30T09:29:39.256406+0000","last_became_active":"2023-09-30T09:29:09.611120+0000","last_became_peered":"2023-09-30T09:29:09.611120+0000","last_unstale":"2023-09-30T09:29:39.256406+0000","last_undegraded":"2023-09-30T09:29:39.256406+0000","last_fullsized":"2023-09-30T09:29:39.256406+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:31.622647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.5","version":"244'4026","reported_seq":4088,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:49.153645+0000","last_change":"2023-09-30T09:28:46.022537+0000","last_active":"2023-09-30T09:29:49.153645+0000","last_peered":"2023-09-30T09:29:49.153645+0000","last_clean":"2023-09-30T09:29:49.153645+0000","last_became_active":"2023-09-30T09:28:40.276165+0000","last_became_peered":"2023-09-30T09:28:40.276165+0000","last_unstale":"2023-09-30T09:29:49.153645+0000","last_undegraded":"2023-09-30T09:29:49.153645+0000","last_fullsized":"2023-09-30T09:29:49.153645+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4026,"log_dups_size":0,"ondisk_log_size":4026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189693952,"num_objects":2894,"num_object_clones":0,"num_object_copies":8682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2894,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":723,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.260660+0000","last_change":"2023-09-30T09:29:12.637491+0000","last_active":"2023-09-30T09:29:39.260660+0000","last_peered":"2023-09-30T09:29:39.260660+0000","last_clean":"2023-09-30T09:29:39.260660+0000","last_became_active":"2023-09-30T09:29:12.637133+0000","last_became_peered":"2023-09-30T09:29:12.637133+0000","last_unstale":"2023-09-30T09:29:39.260660+0000","last_undegraded":"2023-09-30T09:29:39.260660+0000","last_fullsized":"2023-09-30T09:29:39.260660+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:54:24.000552+0000","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":417,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255929+0000","last_change":"2023-09-30T09:29:36.005128+0000","last_active":"2023-09-30T09:29:39.255929+0000","last_peered":"2023-09-30T09:29:39.255929+0000","last_clean":"2023-09-30T09:29:39.255929+0000","last_became_active":"2023-09-30T09:29:36.004563+0000","last_became_peered":"2023-09-30T09:29:36.004563+0000","last_unstale":"2023-09-30T09:29:39.255929+0000","last_undegraded":"2023-09-30T09:29:39.255929+0000","last_fullsized":"2023-09-30T09:29:39.255929+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:59.839484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":191,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.268743+0000","last_change":"2023-09-30T09:29:09.607111+0000","last_active":"2023-09-30T09:29:39.268743+0000","last_peered":"2023-09-30T09:29:39.268743+0000","last_clean":"2023-09-30T09:29:39.268743+0000","last_became_active":"2023-09-30T09:29:09.606807+0000","last_became_peered":"2023-09-30T09:29:09.606807+0000","last_unstale":"2023-09-30T09:29:39.268743+0000","last_undegraded":"2023-09-30T09:29:39.268743+0000","last_fullsized":"2023-09-30T09:29:39.268743+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:49.380239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":239,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269369+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:29:39.269369+0000","last_peered":"2023-09-30T09:29:39.269369+0000","last_clean":"2023-09-30T09:29:39.269369+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:29:39.269369+0000","last_undegraded":"2023-09-30T09:29:39.269369+0000","last_fullsized":"2023-09-30T09:29:39.269369+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"244'3956","reported_seq":4071,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:49.117646+0000","last_change":"2023-09-30T09:28:42.687918+0000","last_active":"2023-09-30T09:29:49.117646+0000","last_peered":"2023-09-30T09:29:49.117646+0000","last_clean":"2023-09-30T09:29:49.117646+0000","last_became_active":"2023-09-30T09:28:40.262566+0000","last_became_peered":"2023-09-30T09:28:40.262566+0000","last_unstale":"2023-09-30T09:29:49.117646+0000","last_undegraded":"2023-09-30T09:29:49.117646+0000","last_fullsized":"2023-09-30T09:29:49.117646+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2390","last_scrub_stamp":"2023-09-30T09:28:42.687821+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.687821+0000","objects_scrubbed":1235,"log_size":3956,"log_dups_size":0,"ondisk_log_size":3956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:07:56.313767+0000","scrub_duration":0.36697584500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184287232,"num_objects":2812,"num_object_clones":0,"num_object_copies":8436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1577,"num_write_kb":100928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":661,"reported_epoch":243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:39.260670+0000","last_change":"2023-09-30T09:29:12.096468+0000","last_active":"2023-09-30T09:29:39.260670+0000","last_peered":"2023-09-30T09:29:39.260670+0000","last_clean":"2023-09-30T09:29:03.507390+0000","last_became_active":"2023-09-30T09:29:12.096051+0000","last_became_peered":"2023-09-30T09:29:12.096051+0000","last_unstale":"2023-09-30T09:29:39.260670+0000","last_undegraded":"2023-09-30T09:29:10.593445+0000","last_fullsized":"2023-09-30T09:29:10.592749+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:49.279327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":684,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.264738+0000","last_change":"2023-09-30T09:27:54.119268+0000","last_active":"2023-09-30T09:29:39.264738+0000","last_peered":"2023-09-30T09:29:39.264738+0000","last_clean":"2023-09-30T09:29:39.264738+0000","last_became_active":"2023-09-30T09:27:49.106373+0000","last_became_peered":"2023-09-30T09:27:49.106373+0000","last_unstale":"2023-09-30T09:29:39.264738+0000","last_undegraded":"2023-09-30T09:29:39.264738+0000","last_fullsized":"2023-09-30T09:29:39.264738+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:28.856049+0000","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":160,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:45.313307+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:29:45.313307+0000","last_peered":"2023-09-30T09:29:45.313307+0000","last_clean":"2023-09-30T09:29:45.313307+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:29:45.313307+0000","last_undegraded":"2023-09-30T09:29:45.313307+0000","last_fullsized":"2023-09-30T09:29:45.313307+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":186,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.260594+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:29:39.260594+0000","last_peered":"2023-09-30T09:29:39.260594+0000","last_clean":"2023-09-30T09:29:39.260594+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:29:39.260594+0000","last_undegraded":"2023-09-30T09:29:39.260594+0000","last_fullsized":"2023-09-30T09:29:39.260594+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"244'4139","reported_seq":4224,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:50.748532+0000","last_change":"2023-09-30T09:28:43.582313+0000","last_active":"2023-09-30T09:29:50.748532+0000","last_peered":"2023-09-30T09:29:50.748532+0000","last_clean":"2023-09-30T09:29:50.748532+0000","last_became_active":"2023-09-30T09:28:40.264204+0000","last_became_peered":"2023-09-30T09:28:40.264204+0000","last_unstale":"2023-09-30T09:29:50.748532+0000","last_undegraded":"2023-09-30T09:29:50.748532+0000","last_fullsized":"2023-09-30T09:29:50.748532+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2440","last_scrub_stamp":"2023-09-30T09:28:43.582001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:43.582001+0000","objects_scrubbed":1291,"log_size":4139,"log_dups_size":0,"ondisk_log_size":4139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:38.964795+0000","scrub_duration":0.30691791200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608000,"num_objects":3000,"num_object_clones":0,"num_object_copies":9000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1709,"num_write_kb":109376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":648,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.270307+0000","last_change":"2023-09-30T09:28:24.276678+0000","last_active":"2023-09-30T09:29:39.270307+0000","last_peered":"2023-09-30T09:29:39.270307+0000","last_clean":"2023-09-30T09:29:39.270307+0000","last_became_active":"2023-09-30T09:28:09.226029+0000","last_became_peered":"2023-09-30T09:28:09.226029+0000","last_unstale":"2023-09-30T09:29:39.270307+0000","last_undegraded":"2023-09-30T09:29:39.270307+0000","last_fullsized":"2023-09-30T09:29:39.270307+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:03:49.533925+0000","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":804,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.264654+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:29:39.264654+0000","last_peered":"2023-09-30T09:29:39.264654+0000","last_clean":"2023-09-30T09:29:39.264654+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:29:39.264654+0000","last_undegraded":"2023-09-30T09:29:39.264654+0000","last_fullsized":"2023-09-30T09:29:39.264654+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.252808+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:29:39.252808+0000","last_peered":"2023-09-30T09:29:39.252808+0000","last_clean":"2023-09-30T09:29:39.252808+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:29:39.252808+0000","last_undegraded":"2023-09-30T09:29:39.252808+0000","last_fullsized":"2023-09-30T09:29:39.252808+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":149,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269416+0000","last_change":"2023-09-30T09:29:09.611376+0000","last_active":"2023-09-30T09:29:39.269416+0000","last_peered":"2023-09-30T09:29:39.269416+0000","last_clean":"2023-09-30T09:29:39.269416+0000","last_became_active":"2023-09-30T09:29:09.611090+0000","last_became_peered":"2023-09-30T09:29:09.611090+0000","last_unstale":"2023-09-30T09:29:39.269416+0000","last_undegraded":"2023-09-30T09:29:39.269416+0000","last_fullsized":"2023-09-30T09:29:39.269416+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:07:15.039397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"244'3905","reported_seq":4000,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:50.631863+0000","last_change":"2023-09-30T09:29:36.030348+0000","last_active":"2023-09-30T09:29:50.631863+0000","last_peered":"2023-09-30T09:29:50.631863+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:36.027562+0000","last_became_peered":"2023-09-30T09:29:36.027562+0000","last_unstale":"2023-09-30T09:29:50.631863+0000","last_undegraded":"2023-09-30T09:29:35.999894+0000","last_fullsized":"2023-09-30T09:29:50.631863+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":3905,"log_dups_size":0,"ondisk_log_size":3905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97484800,"num_objects":1487,"num_object_clones":0,"num_object_copies":4461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":856,"num_write_kb":54816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1043},{"shards":"1,6","objects":1},{"shards":"1,6,7","objects":443}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":488,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.258798+0000","last_change":"2023-09-30T09:28:28.453752+0000","last_active":"2023-09-30T09:29:39.258798+0000","last_peered":"2023-09-30T09:29:39.258798+0000","last_clean":"2023-09-30T09:29:39.258798+0000","last_became_active":"2023-09-30T09:28:09.228162+0000","last_became_peered":"2023-09-30T09:28:09.228162+0000","last_unstale":"2023-09-30T09:29:39.258798+0000","last_undegraded":"2023-09-30T09:29:39.258798+0000","last_fullsized":"2023-09-30T09:29:39.258798+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:43:23.683089+0000","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":934,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.262179+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:29:39.262179+0000","last_peered":"2023-09-30T09:29:39.262179+0000","last_clean":"2023-09-30T09:29:39.262179+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:29:39.262179+0000","last_undegraded":"2023-09-30T09:29:39.262179+0000","last_fullsized":"2023-09-30T09:29:39.262179+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":207,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269398+0000","last_change":"2023-09-30T09:29:09.611808+0000","last_active":"2023-09-30T09:29:39.269398+0000","last_peered":"2023-09-30T09:29:39.269398+0000","last_clean":"2023-09-30T09:29:39.269398+0000","last_became_active":"2023-09-30T09:29:09.611484+0000","last_became_peered":"2023-09-30T09:29:09.611484+0000","last_unstale":"2023-09-30T09:29:39.269398+0000","last_undegraded":"2023-09-30T09:29:39.269398+0000","last_fullsized":"2023-09-30T09:29:39.269398+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:19.562461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":133,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.253235+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:29:39.253235+0000","last_peered":"2023-09-30T09:29:39.253235+0000","last_clean":"2023-09-30T09:29:39.253235+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:29:39.253235+0000","last_undegraded":"2023-09-30T09:29:39.253235+0000","last_fullsized":"2023-09-30T09:29:39.253235+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"244'3632","reported_seq":6563,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:48.281003+0000","last_change":"2023-09-30T09:29:35.026309+0000","last_active":"2023-09-30T09:29:48.281003+0000","last_peered":"2023-09-30T09:29:48.281003+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:35.024657+0000","last_became_peered":"2023-09-30T09:29:35.024657+0000","last_unstale":"2023-09-30T09:29:48.281003+0000","last_undegraded":"2023-09-30T09:29:35.010418+0000","last_fullsized":"2023-09-30T09:29:35.009540+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":3632,"log_dups_size":0,"ondisk_log_size":3632,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89128960,"num_objects":1360,"num_object_clones":0,"num_object_copies":4080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":634,"num_write_kb":40608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":276},{"shards":"3,6","objects":1084}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":666,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.267989+0000","last_change":"2023-09-30T09:29:12.634588+0000","last_active":"2023-09-30T09:29:39.267989+0000","last_peered":"2023-09-30T09:29:39.267989+0000","last_clean":"2023-09-30T09:29:39.267989+0000","last_became_active":"2023-09-30T09:29:12.634225+0000","last_became_peered":"2023-09-30T09:29:12.634225+0000","last_unstale":"2023-09-30T09:29:39.267989+0000","last_undegraded":"2023-09-30T09:29:39.267989+0000","last_fullsized":"2023-09-30T09:29:39.267989+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:22.392203+0000","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":229,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.258255+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:29:39.258255+0000","last_peered":"2023-09-30T09:29:39.258255+0000","last_clean":"2023-09-30T09:29:39.258255+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:29:39.258255+0000","last_undegraded":"2023-09-30T09:29:39.258255+0000","last_fullsized":"2023-09-30T09:29:39.258255+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1193,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269506+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:29:39.269506+0000","last_peered":"2023-09-30T09:29:39.269506+0000","last_clean":"2023-09-30T09:29:39.269506+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:29:39.269506+0000","last_undegraded":"2023-09-30T09:29:39.269506+0000","last_fullsized":"2023-09-30T09:29:39.269506+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":229,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255435+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:29:39.255435+0000","last_peered":"2023-09-30T09:29:39.255435+0000","last_clean":"2023-09-30T09:29:39.255435+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:29:39.255435+0000","last_undegraded":"2023-09-30T09:29:39.255435+0000","last_fullsized":"2023-09-30T09:29:39.255435+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"244'3702","reported_seq":7867,"reported_epoch":244,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:29:48.974115+0000","last_change":"2023-09-30T09:29:36.003902+0000","last_active":"2023-09-30T09:29:48.974115+0000","last_peered":"2023-09-30T09:29:48.974115+0000","last_clean":"2023-09-30T09:29:33.909482+0000","last_became_active":"2023-09-30T09:29:35.026292+0000","last_became_peered":"2023-09-30T09:29:35.026292+0000","last_unstale":"2023-09-30T09:29:48.974115+0000","last_undegraded":"2023-09-30T09:29:48.974115+0000","last_fullsized":"2023-09-30T09:29:48.974115+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":3702,"log_dups_size":0,"ondisk_log_size":3702,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92405760,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":703,"num_write_kb":44992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65765376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":["7","1","2","6"],"object_location_counts":[{"shards":"1,6,7","objects":1410}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":643,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.256481+0000","last_change":"2023-09-30T09:28:26.894367+0000","last_active":"2023-09-30T09:29:39.256481+0000","last_peered":"2023-09-30T09:29:39.256481+0000","last_clean":"2023-09-30T09:29:39.256481+0000","last_became_active":"2023-09-30T09:28:09.227189+0000","last_became_peered":"2023-09-30T09:28:09.227189+0000","last_unstale":"2023-09-30T09:29:39.256481+0000","last_undegraded":"2023-09-30T09:29:39.256481+0000","last_fullsized":"2023-09-30T09:29:39.256481+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:17.282328+0000","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":521,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.266677+0000","last_change":"2023-09-30T09:29:09.643760+0000","last_active":"2023-09-30T09:29:39.266677+0000","last_peered":"2023-09-30T09:29:39.266677+0000","last_clean":"2023-09-30T09:29:39.266677+0000","last_became_active":"2023-09-30T09:29:09.620803+0000","last_became_peered":"2023-09-30T09:29:09.620803+0000","last_unstale":"2023-09-30T09:29:39.266677+0000","last_undegraded":"2023-09-30T09:29:39.266677+0000","last_fullsized":"2023-09-30T09:29:39.266677+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:54:49.883760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":121,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269627+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:29:39.269627+0000","last_peered":"2023-09-30T09:29:39.269627+0000","last_clean":"2023-09-30T09:29:39.269627+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:29:39.269627+0000","last_undegraded":"2023-09-30T09:29:39.269627+0000","last_fullsized":"2023-09-30T09:29:39.269627+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":185,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.267629+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:29:39.267629+0000","last_peered":"2023-09-30T09:29:39.267629+0000","last_clean":"2023-09-30T09:29:39.267629+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:29:39.267629+0000","last_undegraded":"2023-09-30T09:29:39.267629+0000","last_fullsized":"2023-09-30T09:29:39.267629+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1e","version":"244'5194","reported_seq":5260,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:29:48.950512+0000","last_change":"2023-09-30T09:29:35.080846+0000","last_active":"2023-09-30T09:29:48.950512+0000","last_peered":"2023-09-30T09:29:48.950512+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:35.061073+0000","last_became_peered":"2023-09-30T09:29:35.061073+0000","last_unstale":"2023-09-30T09:29:48.950512+0000","last_undegraded":"2023-09-30T09:29:35.049064+0000","last_fullsized":"2023-09-30T09:29:48.950512+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5194,"log_dups_size":0,"ondisk_log_size":5194,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183173120,"num_objects":2795,"num_object_clones":0,"num_object_copies":8385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1977,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2795,"num_write_kb":178880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1977},{"shards":"1,4,7","objects":818}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":102,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.267895+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:29:39.267895+0000","last_peered":"2023-09-30T09:29:39.267895+0000","last_clean":"2023-09-30T09:29:39.267895+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:29:39.267895+0000","last_undegraded":"2023-09-30T09:29:39.267895+0000","last_fullsized":"2023-09-30T09:29:39.267895+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":155,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.269318+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:29:39.269318+0000","last_peered":"2023-09-30T09:29:39.269318+0000","last_clean":"2023-09-30T09:29:39.269318+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:29:39.269318+0000","last_undegraded":"2023-09-30T09:29:39.269318+0000","last_fullsized":"2023-09-30T09:29:39.269318+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"244'3943","reported_seq":4011,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:49.101750+0000","last_change":"2023-09-30T09:29:10.893253+0000","last_active":"2023-09-30T09:29:49.101750+0000","last_peered":"2023-09-30T09:29:49.101750+0000","last_clean":"2023-09-30T09:29:08.494599+0000","last_became_active":"2023-09-30T09:29:10.886737+0000","last_became_peered":"2023-09-30T09:29:10.886737+0000","last_unstale":"2023-09-30T09:29:49.101750+0000","last_undegraded":"2023-09-30T09:29:10.766629+0000","last_fullsized":"2023-09-30T09:29:10.598246+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3943,"log_dups_size":0,"ondisk_log_size":3943,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186253312,"num_objects":2842,"num_object_clones":0,"num_object_copies":8526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2001,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2842,"num_write_kb":181888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":2001},{"shards":"3,7","objects":841}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":205,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.261882+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:29:39.261882+0000","last_peered":"2023-09-30T09:29:39.261882+0000","last_clean":"2023-09-30T09:29:39.261882+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:29:39.261882+0000","last_undegraded":"2023-09-30T09:29:39.261882+0000","last_fullsized":"2023-09-30T09:29:39.261882+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":186,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.266153+0000","last_change":"2023-09-30T09:29:09.609752+0000","last_active":"2023-09-30T09:29:39.266153+0000","last_peered":"2023-09-30T09:29:39.266153+0000","last_clean":"2023-09-30T09:29:39.266153+0000","last_became_active":"2023-09-30T09:29:09.609242+0000","last_became_peered":"2023-09-30T09:29:09.609242+0000","last_unstale":"2023-09-30T09:29:39.266153+0000","last_undegraded":"2023-09-30T09:29:39.266153+0000","last_fullsized":"2023-09-30T09:29:39.266153+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:31.366509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"244'3934","reported_seq":4000,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:29:49.184740+0000","last_change":"2023-09-30T09:29:10.864651+0000","last_active":"2023-09-30T09:29:49.184740+0000","last_peered":"2023-09-30T09:29:49.184740+0000","last_clean":"2023-09-30T09:29:08.532983+0000","last_became_active":"2023-09-30T09:29:10.860470+0000","last_became_peered":"2023-09-30T09:29:10.860470+0000","last_unstale":"2023-09-30T09:29:49.184740+0000","last_undegraded":"2023-09-30T09:29:10.638243+0000","last_fullsized":"2023-09-30T09:29:10.593564+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":3934,"log_dups_size":0,"ondisk_log_size":3934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184844288,"num_objects":2821,"num_object_clones":0,"num_object_copies":8463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2821,"num_write_kb":180512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":2057},{"shards":"3,7","objects":764}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":188,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.264646+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:29:39.264646+0000","last_peered":"2023-09-30T09:29:39.264646+0000","last_clean":"2023-09-30T09:29:39.264646+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:29:39.264646+0000","last_undegraded":"2023-09-30T09:29:39.264646+0000","last_fullsized":"2023-09-30T09:29:39.264646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":217,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255901+0000","last_change":"2023-09-30T09:29:09.621218+0000","last_active":"2023-09-30T09:29:39.255901+0000","last_peered":"2023-09-30T09:29:39.255901+0000","last_clean":"2023-09-30T09:29:39.255901+0000","last_became_active":"2023-09-30T09:29:09.620972+0000","last_became_peered":"2023-09-30T09:29:09.620972+0000","last_unstale":"2023-09-30T09:29:39.255901+0000","last_undegraded":"2023-09-30T09:29:39.255901+0000","last_fullsized":"2023-09-30T09:29:39.255901+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:25.640288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1b","version":"244'5298","reported_seq":5362,"reported_epoch":244,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:29:48.333858+0000","last_change":"2023-09-30T09:29:35.093036+0000","last_active":"2023-09-30T09:29:48.333858+0000","last_peered":"2023-09-30T09:29:48.333858+0000","last_clean":"2023-09-30T09:29:33.992127+0000","last_became_active":"2023-09-30T09:29:35.070155+0000","last_became_peered":"2023-09-30T09:29:35.070155+0000","last_unstale":"2023-09-30T09:29:48.333858+0000","last_undegraded":"2023-09-30T09:29:48.333858+0000","last_fullsized":"2023-09-30T09:29:48.333858+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5298,"log_dups_size":0,"ondisk_log_size":5298,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185499648,"num_objects":2831,"num_object_clones":0,"num_object_copies":8493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2831,"num_write_kb":181152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":2831}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":224,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.256517+0000","last_change":"2023-09-30T09:29:09.615385+0000","last_active":"2023-09-30T09:29:39.256517+0000","last_peered":"2023-09-30T09:29:39.256517+0000","last_clean":"2023-09-30T09:29:39.256517+0000","last_became_active":"2023-09-30T09:29:09.614479+0000","last_became_peered":"2023-09-30T09:29:09.614479+0000","last_unstale":"2023-09-30T09:29:39.256517+0000","last_undegraded":"2023-09-30T09:29:39.256517+0000","last_fullsized":"2023-09-30T09:29:39.256517+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:14.671222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":254,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.255781+0000","last_change":"2023-09-30T09:29:09.600457+0000","last_active":"2023-09-30T09:29:39.255781+0000","last_peered":"2023-09-30T09:29:39.255781+0000","last_clean":"2023-09-30T09:29:39.255781+0000","last_became_active":"2023-09-30T09:29:09.600080+0000","last_became_peered":"2023-09-30T09:29:09.600080+0000","last_unstale":"2023-09-30T09:29:39.255781+0000","last_undegraded":"2023-09-30T09:29:39.255781+0000","last_fullsized":"2023-09-30T09:29:39.255781+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:10:02.250349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1a","version":"244'5291","reported_seq":5347,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-30T09:29:49.136297+0000","last_change":"2023-09-30T09:29:35.021535+0000","last_active":"2023-09-30T09:29:49.136297+0000","last_peered":"2023-09-30T09:29:49.136297+0000","last_clean":"2023-09-30T09:29:49.136297+0000","last_became_active":"2023-09-30T09:29:35.014808+0000","last_became_peered":"2023-09-30T09:29:35.014808+0000","last_unstale":"2023-09-30T09:29:49.136297+0000","last_undegraded":"2023-09-30T09:29:49.136297+0000","last_fullsized":"2023-09-30T09:29:49.136297+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5291,"log_dups_size":0,"ondisk_log_size":5291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186384384,"num_objects":2844,"num_object_clones":0,"num_object_copies":8532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2844,"num_write_kb":182016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":127,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.263819+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:29:39.263819+0000","last_peered":"2023-09-30T09:29:39.263819+0000","last_clean":"2023-09-30T09:29:39.263819+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:29:39.263819+0000","last_undegraded":"2023-09-30T09:29:39.263819+0000","last_fullsized":"2023-09-30T09:29:39.263819+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":202,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-30T09:29:39.267482+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:29:39.267482+0000","last_peered":"2023-09-30T09:29:39.267482+0000","last_clean":"2023-09-30T09:29:39.267482+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:29:39.267482+0000","last_undegraded":"2023-09-30T09:29:39.267482+0000","last_fullsized":"2023-09-30T09:29:39.267482+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":36,"stat_sum":{"num_bytes":5947326464,"num_objects":90750,"num_object_clones":0,"num_object_copies":272250,"num_objects_missing_on_primary":1335,"num_objects_missing":1335,"num_objects_degraded":24839,"num_objects_misplaced":5965,"num_objects_unfound":0,"num_objects_dirty":90750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":74573,"num_write_kb":4772608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8192,"num_bytes_recovered":536870912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18459131904,"data_stored":18459131904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156777,"ondisk_log_size":156777,"up":108,"acting":101,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":1418,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":435019,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":43,"num_write_kb":438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234218,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2714068,"kb_used_data":2576524,"kb_used_omap":0,"kb_used_meta":137536,"kb_avail":91031084,"statfs":{"total":95995035648,"available":93215830016,"internally_reserved":0,"allocated":2638360576,"data_stored":2637457017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":140836864},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:27 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.5980000000000001,"15min":1.0149999999999999},"min":{"1min":0.30099999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.839,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.4770000000000001,"15min":0.93700000000000006},"min":{"1min":0.29499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.8949999999999996,"5min":18.097999999999999,"15min":18.097999999999999},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:29:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":3.8029999999999999,"15min":2.2669999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.944,"5min":70.566999999999993,"15min":82.025999999999996},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.1950000000000001,"5min":3.746,"15min":2.242},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.6580000000000004,"5min":70.626000000000005,"15min":82.012},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:29:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7200000000000002,"5min":2.0009999999999999,"15min":1.145},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":39.445,"5min":39.445,"15min":39.445},"last":0.78100000000000003},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.9870000000000001,"15min":1.2190000000000001},"min":{"1min":0.39700000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":40.988999999999997,"5min":40.988999999999997,"15min":40.988999999999997},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:29:43 2023","interfaces":[{"interface":"back","average":{"1min":5.298,"5min":3.552,"15min":1.6659999999999999},"min":{"1min":0.371,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":76.909000000000006,"5min":76.909000000000006,"15min":76.909000000000006},"last":0.432},{"interface":"front","average":{"1min":5.0410000000000004,"5min":3.262,"15min":1.5509999999999999},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":77.048000000000002,"5min":77.048000000000002,"15min":77.048000000000002},"last":2.1480000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.95699999999999996,"15min":0.64900000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.78900000000000003,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.81000000000000005,"15min":0.63700000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.92400000000000004,"5min":6.282,"15min":6.282},"last":2.206}]},{"osd":5,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.97699999999999998,"15min":0.70999999999999996},"min":{"1min":0.28100000000000003,"5min":0.248,"15min":0.248},"max":{"1min":2.4470000000000001,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.121,"15min":0.77600000000000002},"min":{"1min":0.30199999999999999,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":2.4670000000000001,"5min":11.022,"15min":11.022},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.052,"15min":0.75700000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33000000000000002},"max":{"1min":1.121,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.879,"15min":0.70299999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.88900000000000001,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":2.2349999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495855,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2608720,"kb_used_data":2475464,"kb_used_omap":0,"kb_used_meta":133248,"kb_avail":91136432,"statfs":{"total":95995035648,"available":93323706368,"internally_reserved":0,"allocated":2534875136,"data_stored":2533976090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":136445952},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":2.2829999999999999,"15min":1.242},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.222},"max":{"1min":2.4119999999999999,"5min":45.927999999999997,"15min":45.927999999999997},"last":0.248},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.581,"15min":1.3520000000000001},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.23499999999999999},"max":{"1min":1.6899999999999999,"5min":45.316000000000003,"15min":45.316000000000003},"last":1.363}]},{"osd":1,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.8399999999999999,"15min":1.444},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":37.988,"5min":69.150000000000006,"15min":69.150000000000006},"last":0.28299999999999997},{"interface":"front","average":{"1min":3.0259999999999998,"5min":3.2240000000000002,"15min":1.5649999999999999},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":37.970999999999997,"5min":69.132000000000005,"15min":69.132000000000005},"last":1.341}]},{"osd":2,"last update":"Sat Sep 30 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":2.4780000000000002,"15min":1.294},"min":{"1min":0.34999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.4399999999999999,"5min":37.978000000000002,"15min":37.978000000000002},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.9630000000000001,"15min":1.167},"min":{"1min":0.39400000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.9049999999999998,"5min":37.759999999999998,"15min":37.759999999999998},"last":0.497}]},{"osd":3,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":1.716,"15min":1.054},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.165,"5min":20.818999999999999,"15min":20.818999999999999},"last":3.3180000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.9430000000000001,"15min":1.169},"min":{"1min":0.36499999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.6439999999999999,"5min":21.015000000000001,"15min":21.015000000000001},"last":1.3899999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.921,"15min":1.216},"min":{"1min":0.26700000000000002,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":1.72,"5min":49.286999999999999,"15min":73.656000000000006},"last":1.2789999999999999},{"interface":"front","average":{"1min":0.505,"5min":1.5169999999999999,"15min":1.1120000000000001},"min":{"1min":0.32400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.1100000000000001,"5min":49.252000000000002,"15min":73.882999999999996},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.6439999999999999,"15min":0.93400000000000005},"min":{"1min":0.32400000000000001,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":2.5750000000000002,"5min":47.408999999999999,"15min":47.408999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.528,"15min":0.93999999999999995},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.311},"max":{"1min":2.746,"5min":47.344000000000001,"15min":47.344000000000001},"last":0.435}]},{"osd":7,"last update":"Sat Sep 30 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.897,"15min":1.0529999999999999},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.466,"5min":53.927999999999997,"15min":53.927999999999997},"last":1.3149999999999999},{"interface":"front","average":{"1min":0.628,"5min":1.8340000000000001,"15min":1.1000000000000001},"min":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":1.454,"5min":51.886000000000003,"15min":51.886000000000003},"last":0.48099999999999998}]}]},{"osd":1,"up_from":99,"seq":425201762420,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2531024,"kb_used_data":2323080,"kb_used_omap":5,"kb_used_meta":207930,"kb_avail":91214128,"statfs":{"total":95995035648,"available":93403267072,"internally_reserved":0,"allocated":2378833920,"data_stored":2377475374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":212920502},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":5.1429999999999998,"15min":2.2749999999999999},"min":{"1min":0.23799999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.4400000000000004,"5min":150.93899999999999,"15min":150.93899999999999},"last":0.20599999999999999},{"interface":"front","average":{"1min":1.762,"5min":5.0979999999999999,"15min":2.3279999999999998},"min":{"1min":0.28999999999999998,"5min":0.216,"15min":0.216},"max":{"1min":7.9470000000000001,"5min":156.74199999999999,"15min":156.74199999999999},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:29:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":7.2290000000000001,"15min":2.7360000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.3159999999999998,"5min":144.48699999999999,"15min":144.48699999999999},"last":1.9790000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":7.0300000000000002,"15min":2.6909999999999998},"min":{"1min":0.28799999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.7560000000000002,"5min":144.62,"15min":144.62},"last":2.0489999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:29:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":1.8759999999999999,"15min":0.96399999999999997},"min":{"1min":0.26400000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.1109999999999998,"5min":18.757999999999999,"15min":18.757999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.5509999999999999,"15min":0.879},"min":{"1min":0.35099999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.9809999999999999,"5min":11.173,"15min":11.173},"last":0.49399999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.484,"5min":1.286,"15min":0.91900000000000004},"min":{"1min":0.441,"5min":0.317,"15min":0.28299999999999997},"max":{"1min":8.3900000000000006,"5min":10.173,"15min":10.173},"last":2.8450000000000002},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.4019999999999999,"15min":0.94999999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.0700000000000003,"5min":8.9299999999999997,"15min":8.9299999999999997},"last":2.867}]},{"osd":5,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":1.3720000000000001,"15min":0.91100000000000003},"min":{"1min":0.45400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.678000000000001,"5min":12.678000000000001,"15min":12.678000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.794,"5min":1.4830000000000001,"15min":0.96899999999999997},"min":{"1min":0.432,"5min":0.35499999999999998,"15min":0.33300000000000002},"max":{"1min":10.128,"5min":10.128,"15min":10.128},"last":0.35899999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":1.5369999999999999,"15min":0.96899999999999997},"min":{"1min":0.44700000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":10.058,"5min":14.356999999999999,"15min":14.356999999999999},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.881,"5min":1.6479999999999999,"15min":1.0600000000000001},"min":{"1min":0.39700000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.399,"5min":15.864000000000001,"15min":15.864000000000001},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":1.8420000000000001,"15min":1.145},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.103,"5min":14.403,"15min":14.403},"last":1.083},{"interface":"front","average":{"1min":1.538,"5min":2.9729999999999999,"15min":1.5369999999999999},"min":{"1min":0.42499999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.6589999999999998,"5min":122.56699999999999,"15min":122.56699999999999},"last":1.0660000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925952,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":976144,"kb_used_data":730888,"kb_used_omap":5,"kb_used_meta":245242,"kb_avail":92769008,"statfs":{"total":95995035648,"available":94995464192,"internally_reserved":0,"allocated":748429312,"data_stored":747014589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":251128061},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":2.8759999999999999,"15min":1.3440000000000001},"min":{"1min":0.22,"5min":0.188,"15min":0.188},"max":{"1min":15.513,"5min":125.024,"15min":125.024},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.032,"5min":2.706,"15min":1.3089999999999999},"min":{"1min":0.20899999999999999,"5min":0.192,"15min":0.192},"max":{"1min":2.827,"5min":124.901,"15min":124.901},"last":1.012}]},{"osd":2,"last update":"Sat Sep 30 09:29:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":2.1469999999999998,"15min":1.071},"min":{"1min":0.247,"5min":0.218,"15min":0.218},"max":{"1min":55.823999999999998,"5min":55.823999999999998,"15min":55.823999999999998},"last":1.0509999999999999},{"interface":"front","average":{"1min":3.8820000000000001,"5min":2.3340000000000001,"15min":1.1779999999999999},"min":{"1min":0.32200000000000001,"5min":0.245,"15min":0.245},"max":{"1min":55.790999999999997,"5min":55.790999999999997,"15min":55.790999999999997},"last":1.2470000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:29:42 2023","interfaces":[{"interface":"back","average":{"1min":3.6419999999999999,"5min":2.698,"15min":1.3320000000000001},"min":{"1min":0.28799999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":42.917000000000002,"5min":55.68,"15min":55.68},"last":4.8239999999999998},{"interface":"front","average":{"1min":3.8399999999999999,"5min":2.48,"15min":1.2589999999999999},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":43.023000000000003,"5min":55.244,"15min":55.244},"last":0.32200000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":2.4039999999999999,"15min":1.349},"min":{"1min":0.39500000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.863,"5min":121.486,"15min":121.486},"last":1.1419999999999999},{"interface":"front","average":{"1min":0.91200000000000003,"5min":2.8450000000000002,"15min":1.538},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.31},"max":{"1min":2.2050000000000001,"5min":121.44499999999999,"15min":121.44499999999999},"last":4.8810000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.5,"15min":1.173},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.738,"5min":15.162000000000001,"15min":15.162000000000001},"last":4.9459999999999997},{"interface":"front","average":{"1min":1.4059999999999999,"5min":1.45,"15min":1.159},"min":{"1min":0.38600000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.0270000000000001,"5min":13.766,"15min":13.766},"last":1.9239999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":1.3340000000000001,"15min":1.03},"min":{"1min":0.374,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.9960000000000004,"5min":13.811999999999999,"15min":13.811999999999999},"last":1.8460000000000001},{"interface":"front","average":{"1min":1.25,"5min":1.242,"15min":1.002},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":2.7799999999999998,"5min":6.407,"15min":7.556},"last":1.1850000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.8279999999999998,"15min":1.53},"min":{"1min":0.41999999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.9640000000000004,"5min":127.199,"15min":127.199},"last":1.8899999999999999},{"interface":"front","average":{"1min":1.45,"5min":2.6139999999999999,"15min":1.456},"min":{"1min":0.44600000000000001,"5min":0.371,"15min":0.371},"max":{"1min":4.3040000000000003,"5min":118.551,"15min":118.551},"last":1.222}]}]},{"osd":2,"up_from":0,"seq":446676598893,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435365,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4124588,"kb_used_data":3912544,"kb_used_omap":2,"kb_used_meta":212029,"kb_avail":89620564,"statfs":{"total":95995035648,"available":91771457536,"internally_reserved":0,"allocated":4006445056,"data_stored":4005060945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":217118055},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,2,1,4,1,4,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":2.149,"15min":1.421},"min":{"1min":0.222,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.3929999999999998,"5min":62.206000000000003,"15min":130.946},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.1280000000000001,"15min":1.4279999999999999},"min":{"1min":0.26800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.1110000000000007,"5min":60.305999999999997,"15min":130.96899999999999},"last":0.248}]},{"osd":1,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":1.6160000000000001,"15min":0.84599999999999997},"min":{"1min":0.251,"5min":0.23200000000000001,"15min":0.224},"max":{"1min":3.5169999999999999,"5min":13.039999999999999,"15min":13.039999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.6990000000000001,"15min":0.86299999999999999},"min":{"1min":0.25900000000000001,"5min":0.221,"15min":0.221},"max":{"1min":9.0860000000000003,"5min":14.057,"15min":14.057},"last":0.28599999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":2.7000000000000002,"15min":1.7210000000000001},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.8420000000000001,"5min":61.704999999999998,"15min":145.75899999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.70699999999999996,"5min":2.395,"15min":1.633},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.2429999999999999,"5min":59.094000000000001,"15min":150.358},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.0569999999999999,"15min":0.77100000000000002},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.9830000000000001,"5min":10.301,"15min":10.301},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.204,"5min":1.4870000000000001,"15min":0.92100000000000004},"min":{"1min":0.443,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.9870000000000001,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.607,"15min":0.98699999999999999},"min":{"1min":0.443,"5min":0.314,"15min":0.308},"max":{"1min":9.0310000000000006,"5min":10.048999999999999,"15min":10.048999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.8049999999999999,"15min":1.077},"min":{"1min":0.46400000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.3479999999999999,"5min":12.785,"15min":12.785},"last":0.36599999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.2769999999999999,"15min":0.88200000000000001},"min":{"1min":0.42499999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.4209999999999998,"5min":10.348000000000001,"15min":10.348000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.722,"15min":1.0609999999999999},"min":{"1min":0.44700000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.0589999999999993,"5min":14.69,"15min":14.69},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 30 09:28:47 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.7090000000000001,"15min":1.034},"min":{"1min":0.40799999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.3239999999999998,"5min":12.742000000000001,"15min":12.742000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.262,"15min":0.88300000000000001},"min":{"1min":0.49299999999999999,"5min":0.377,"15min":0.372},"max":{"1min":7.242,"5min":10.43,"15min":10.43},"last":0.46200000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019128,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761552,"kb_used_data":1617928,"kb_used_omap":0,"kb_used_meta":143616,"kb_avail":91983600,"statfs":{"total":95995035648,"available":94191206400,"internally_reserved":0,"allocated":1656758272,"data_stored":1655859307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":147062784},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":4.6929999999999996,"15min":2.4929999999999999},"min":{"1min":0.25700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.6589999999999998,"5min":119.986,"15min":128.79300000000001},"last":3.6749999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":4.5350000000000001,"15min":2.371},"min":{"1min":0.253,"5min":0.253,"15min":0.248},"max":{"1min":2.3679999999999999,"5min":119.863,"15min":128.38800000000001},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.9630000000000001,"15min":1.54},"min":{"1min":0.28799999999999998,"5min":0.245,"15min":0.245},"max":{"1min":27.440999999999999,"5min":93.244,"15min":93.244},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.117,"15min":1.597},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.23999999999999999},"max":{"1min":27.489000000000001,"5min":93.465999999999994,"15min":93.465999999999994},"last":0.432}]},{"osd":2,"last update":"Sat Sep 30 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":6.2380000000000004,"5min":5.274,"15min":2.379},"min":{"1min":0.37,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":77.620000000000005,"5min":140.601,"15min":140.601},"last":4.6639999999999997},{"interface":"front","average":{"1min":6.2610000000000001,"5min":5.1459999999999999,"15min":2.327},"min":{"1min":0.35999999999999999,"5min":0.35199999999999998,"15min":0.34200000000000003},"max":{"1min":77.638999999999996,"5min":140.54900000000001,"15min":140.54900000000001},"last":3.7170000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":15.814,"5min":5.1239999999999997,"15min":2.4319999999999999},"min":{"1min":0.34200000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":281.42200000000003,"5min":281.42200000000003,"15min":281.42200000000003},"last":0.62},{"interface":"front","average":{"1min":16.393000000000001,"5min":5.1680000000000001,"15min":2.452},"min":{"1min":0.35999999999999999,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":281.44999999999999,"5min":281.44999999999999,"15min":281.44999999999999},"last":0.63200000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.5569999999999999,"15min":0.84799999999999998},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.3940000000000001,"5min":86.691000000000003,"15min":86.691000000000003},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.7969999999999999,"15min":0.96599999999999997},"min":{"1min":0.28299999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5169999999999999,"5min":86.822000000000003,"15min":86.822000000000003},"last":0.34499999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.84699999999999998,"15min":0.63100000000000001},"min":{"1min":0.32400000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.1579999999999999,"5min":5.6310000000000002,"15min":5.6310000000000002},"last":1.1080000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.92800000000000005,"15min":0.69499999999999995},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.0760000000000001,"5min":6.327,"15min":6.327},"last":0.47999999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.992,"15min":1.075},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.391,"5min":86.728999999999999,"15min":86.728999999999999},"last":4.7069999999999999},{"interface":"front","average":{"1min":0.626,"5min":1.1259999999999999,"15min":0.79600000000000004},"min":{"1min":0.35499999999999998,"5min":0.32600000000000001,"15min":0.314},"max":{"1min":2.3700000000000001,"5min":8.5869999999999997,"15min":8.5869999999999997},"last":0.64800000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757492,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2334908,"kb_used_data":2174772,"kb_used_omap":0,"kb_used_meta":160128,"kb_avail":91410244,"statfs":{"total":95995035648,"available":93604089856,"internally_reserved":0,"allocated":2226966528,"data_stored":2226082857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":163971072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":4.2439999999999998,"15min":1.8640000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.665,"5min":148.79900000000001,"15min":148.79900000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.3340000000000001,"5min":4.8499999999999996,"15min":2.1070000000000002},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.4169999999999998,"5min":148.779,"15min":148.779},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:29:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9089999999999998,"5min":4.2809999999999997,"15min":2.2709999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":38.487000000000002,"5min":166.708,"15min":166.708},"last":0.28499999999999998},{"interface":"front","average":{"1min":2.9009999999999998,"5min":4.1609999999999996,"15min":2.2309999999999999},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":38.512,"5min":166.607,"15min":166.607},"last":2.1699999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":5.8339999999999996,"15min":2.5649999999999999},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.931,"5min":165.768,"15min":165.768},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.123,"5min":7.1920000000000002,"15min":3.0880000000000001},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.3440000000000003,"5min":166.24000000000001,"15min":166.24000000000001},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:29:44 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.9590000000000001,"15min":1.256},"min":{"1min":0.35099999999999998,"5min":0.311,"15min":0.311},"max":{"1min":20.079000000000001,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.37},{"interface":"front","average":{"1min":1.919,"5min":1.673,"15min":1.111},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":20.021000000000001,"5min":28.039999999999999,"15min":28.039999999999999},"last":2.242}]},{"osd":4,"last update":"Sat Sep 30 09:29:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.89500000000000002,"15min":0.64200000000000002},"min":{"1min":0.32800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":4.3979999999999997,"5min":8.5359999999999996,"15min":8.5359999999999996},"last":2.2679999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.95299999999999996,"15min":0.71799999999999997},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.5099999999999998,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":0.70799999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.27,"15min":0.80200000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.5179999999999998,"5min":14.23,"15min":14.23},"last":2.2869999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.4339999999999999,"15min":0.94999999999999996},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28199999999999997},"max":{"1min":4.1600000000000001,"5min":18.960000000000001,"15min":18.960000000000001},"last":0.73299999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.246,"15min":0.85099999999999998},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.9459999999999997,"5min":19.175999999999998,"15min":19.175999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.95499999999999996,"15min":0.748},"min":{"1min":0.376,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.5099999999999998,"5min":8.8810000000000002,"15min":8.8810000000000002},"last":2.2090000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":745537536,"data_stored":745537536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2373713920,"data_stored":2373713920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2220163072,"data_stored":2220163072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4054515712,"data_stored":4054515712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1665138688,"data_stored":1665138688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2231238656,"data_stored":2231238656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2536898560,"data_stored":2536898560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2631925760,"data_stored":2631925760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:29:56.995 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 36 2023-09-30T09:29:57.036 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:29:57.078 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:57.115 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:57.143 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:57.255 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:57.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:57.362 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:57.407 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:57.500 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:57.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:57.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:57.610 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:57.643 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:29:57.665 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:57.912 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:57.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:58.023 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:58.079 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:58.138 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:58.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:58.208 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:29:58.241 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:29:58.286 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:58.391 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:29:58.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:58.400 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:58 smithi019 ceph-mon[216815]: pgmap v575: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 40 MiB/s wr, 842 op/s; 24274/270936 objects degraded (8.959%); 5811/270936 objects misplaced (2.145%); 4.2 MiB/s, 66 objects/s recovering 2023-09-30T09:29:58.401 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:29:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 24274/270936 objects degraded (8.959%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:58.401 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:58 smithi019 ceph-mon[224043]: pgmap v575: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 40 MiB/s wr, 842 op/s; 24274/270936 objects degraded (8.959%); 5811/270936 objects misplaced (2.145%); 4.2 MiB/s, 66 objects/s recovering 2023-09-30T09:29:58.401 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:29:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 24274/270936 objects degraded (8.959%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:58.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:29:58.502 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:58.551 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:58.734 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:58.757 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:29:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:58.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:29:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:58 smithi167 ceph-mon[192880]: pgmap v575: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 40 MiB/s wr, 842 op/s; 24274/270936 objects degraded (8.959%); 5811/270936 objects misplaced (2.145%); 4.2 MiB/s, 66 objects/s recovering 2023-09-30T09:29:58.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:29:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 24274/270936 objects degraded (8.959%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:29:58.866 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:29:58.900 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:29:58.916 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:29:59.009 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:29:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:29:59.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:29:59.115 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:29:59.193 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:00.024 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:43734 5.2 5:4c196cde:::benchmark_data_smithi019_287194_object43733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:24.122679+0000", 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 395.89810209199999, 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.182332521, 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 43734 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.122679+0000", 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.122679+0000", 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.122678+0000", 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.122742+0000", 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4041000000000006e-05 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.122759+0000", 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7010000000000001e-05 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.122766+0000", 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5220000000000004e-06 2023-09-30T09:30:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.122817+0000", 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1097999999999998e-05 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.122963+0000", 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014608000000000001 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.123108+0000", 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014531200000000001 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.126955+0000", 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003847086 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.127283+0000", 2023-09-30T09:30:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00032817099999999999 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.303794+0000", 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.17651098400000001 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.303868+0000", 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3744000000000004e-05 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.305012+0000", 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001143856 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:43745 5.2 5:45c31f79:::benchmark_data_smithi019_287194_object43744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:24.133406+0000", 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 395.88737512900002, 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.17242716899999999, 2023-09-30T09:30:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 43745 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133406+0000", 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133406+0000", 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133404+0000", 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133443+0000", 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8526999999999999e-05 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133456+0000", 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3383999999999999e-05 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133463+0000", 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6309999999999997e-06 2023-09-30T09:30:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133512+0000", 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9042999999999998e-05 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133652+0000", 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013955700000000001 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.133814+0000", 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016247000000000001 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.135562+0000", 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001747612 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.136474+0000", 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00091198199999999998 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.305073+0000", 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16859976700000001 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.305138+0000", 2023-09-30T09:30:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4849000000000002e-05 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.305834+0000", 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00069543399999999998 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:43750 5.2 5:44c7a3a6:::benchmark_data_smithi019_287194_object43749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-30T09:30:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:24.138123+0000", 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 395.88265874400003, 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16885088000000001, 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 43750 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138123+0000", 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138123+0000", 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138121+0000", 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138174+0000", 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2790000000000001e-05 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138189+0000", 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4722000000000001e-05 2023-09-30T09:30:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138195+0000", 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0009999999999998e-06 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138234+0000", 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9436e-05 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138348+0000", 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000113527 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.138484+0000", 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013617999999999999 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.140002+0000", 2023-09-30T09:30:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001518291 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.140869+0000", 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00086689799999999995 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.306216+0000", 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.165347195 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.306282+0000", 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5965999999999998e-05 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.306974+0000", 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00069135799999999999 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:106961 5.2 5:4e1f0aa2:::benchmark_data_smithi019_287194_object24183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:51.680462+0000", 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 248.34031945300001, 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20695609000000001, 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106961 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680462+0000", 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680462+0000", 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680461+0000", 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680466+0000", 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7899999999999996e-06 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680481+0000", 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4514e-05 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680488+0000", 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.25e-06 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680585+0000", 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6861000000000007e-05 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680713+0000", 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012833199999999999 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680825+0000", 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112069 2023-09-30T09:30:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691038+0000", 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010212809999999999 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.694221+0000", 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031824589999999999 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.887308+0000", 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.19308688199999999 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.887363+0000", 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5711999999999998e-05 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.887418+0000", 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4857e-05 2023-09-30T09:30:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:106964 5.2 5:488c836a:::benchmark_data_smithi019_287194_object24186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-30T09:30:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:51.680503+0000", 2023-09-30T09:30:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 248.340279039, 2023-09-30T09:30:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20706771600000001, 2023-09-30T09:30:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106964 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680503+0000", 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680503+0000", 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680502+0000", 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:30:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680505+0000", 2023-09-30T09:30:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6180000000000002e-06 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680513+0000", 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1450000000000008e-06 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.680517+0000", 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7629999999999998e-06 2023-09-30T09:30:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.684131+0000", 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003614342 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.684250+0000", 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000118714 2023-09-30T09:30:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.684357+0000", 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000107277 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.694175+0000", 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0098180279999999995 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.694261+0000", 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5945999999999993e-05 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.887468+0000", 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.19320670300000001 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.887524+0000", 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6033999999999999e-05 2023-09-30T09:30:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.887570+0000", 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6539000000000003e-05 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:106966 5.2 5:44afbc63:::benchmark_data_smithi019_287194_object24188:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:51.681797+0000", 2023-09-30T09:30:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 248.33898424399999, 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21166030999999999, 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106966 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.681797+0000", 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.681797+0000", 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.681796+0000", 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:30:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.681803+0000", 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2550000000000003e-06 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.681830+0000", 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7637e-05 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.681839+0000", 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0750000000000004e-06 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.687661+0000", 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0058220310000000001 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.687786+0000", 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000124645 2023-09-30T09:30:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.687880+0000", 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3888999999999995e-05 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.694271+0000", 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0063911660000000002 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.694309+0000", 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7598999999999997e-05 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.893352+0000", 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.19904348199999999 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.893409+0000", 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7061000000000001e-05 2023-09-30T09:30:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.893458+0000", 2023-09-30T09:30:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8470000000000002e-05 2023-09-30T09:30:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:106969 5.2 5:4db4be5f:::benchmark_data_smithi019_287194_object24191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:51.683980+0000", 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 248.33680173400001, 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.209613097, 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106969 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.683980+0000", 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.683980+0000", 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.683978+0000", 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.683985+0000", 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3259999999999998e-06 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.683998+0000", 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3046e-05 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.684005+0000", 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2709999999999998e-06 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691063+0000", 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0070585400000000003 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691169+0000", 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010560699999999999 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.691249+0000", 2023-09-30T09:30:00.075 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0142999999999995e-05 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.694343+0000", 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030938530000000001 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.694349+0000", 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0020000000000003e-06 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.893511+0000", 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.19916198600000001 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.893555+0000", 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4394999999999998e-05 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:51.893593+0000", 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7662e-05 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:109420 5.2 5:4ebce9b3:::benchmark_data_smithi019_287194_object26642:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:54.806764+0000", 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 245.214017431, 2023-09-30T09:30:00.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16728001100000001, 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 109420 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.806764+0000", 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.806764+0000", 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.806762+0000", 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:30:00.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.806770+0000", 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6449999999999994e-06 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.806791+0000", 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0628999999999998e-05 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.806799+0000", 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.605e-06 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.806847+0000", 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8140000000000003e-05 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.082 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.806979+0000", 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000131977 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.807097+0000", 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000118164 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.818308+0000", 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011211502999999999 2023-09-30T09:30:00.083 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.822458+0000", 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041493509999999999 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.973967+0000", 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.151509009 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.974012+0000", 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5348e-05 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.974044+0000", 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2104999999999998e-05 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:109434 5.2 5:478c7634:::benchmark_data_smithi019_287194_object26656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:25:54.825988+0000", 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 245.19479388299999, 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16424170599999999, 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 109434 2023-09-30T09:30:00.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.825988+0000", 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.825988+0000", 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.825986+0000", 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.826000+0000", 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3706999999999999e-05 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.826022+0000", 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2045999999999999e-05 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.826030+0000", 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3149999999999992e-06 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.826082+0000", 2023-09-30T09:30:00.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1922999999999997e-05 2023-09-30T09:30:00.090 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.090 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.826219+0000", 2023-09-30T09:30:00.090 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000137026 2023-09-30T09:30:00.090 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.826344+0000", 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012511 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.829801+0000", 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034572959999999999 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.834139+0000", 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043381180000000002 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.990131+0000", 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.155991719 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.990188+0000", 2023-09-30T09:30:00.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6974e-05 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:25:54.990229+0000", 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1341000000000003e-05 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.093 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:135231 5.2 5:459965d9:::benchmark_data_smithi019_287194_object52453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.094830+0000", 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 205.925951166, 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.31595177000000002, 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 135231 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.094 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094830+0000", 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094830+0000", 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094829+0000", 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094839+0000", 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0516000000000001e-05 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094863+0000", 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4389999999999999e-05 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094882+0000", 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8090000000000001e-05 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094981+0000", 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9071000000000001e-05 2023-09-30T09:30:00.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.095276+0000", 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00029558299999999998 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.095388+0000", 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000111636 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.103594+0000", 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0082064189999999995 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.104575+0000", 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00098032400000000008 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.099 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.410689+0000", 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.30611437400000002 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.410740+0000", 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1408e-05 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.100 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.410782+0000", 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1819000000000001e-05 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:135234 5.2 5:4499f6bb:::benchmark_data_smithi019_287194_object52456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.094917+0000", 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 205.92586443799999, 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32106875600000001, 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.101 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 135234 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094917+0000", 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.102 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094917+0000", 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094916+0000", 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094923+0000", 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0299999999999996e-06 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094942+0000", 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9259999999999999e-05 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.094954+0000", 2023-09-30T09:30:00.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1518999999999999e-05 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.099532+0000", 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045785039999999997 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.099663+0000", 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013103799999999999 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.099780+0000", 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011636700000000001 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.103602+0000", 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003822448 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.106599+0000", 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029973370000000001 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.415893+0000", 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.30929357299999999 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.415942+0000", 2023-09-30T09:30:00.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9478999999999998e-05 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.415986+0000", 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3464000000000003e-05 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.108 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:137746 5.2 5:4ebbcc36:::benchmark_data_smithi019_287194_object54968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:38.253236+0000", 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 201.76754535800001, 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21886172500000001, 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 137746 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.109 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253236+0000", 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253236+0000", 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.110 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253234+0000", 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253242+0000", 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3760000000000008e-06 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253263+0000", 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0653999999999999e-05 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253272+0000", 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7150000000000004e-06 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253429+0000", 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015712300000000001 2023-09-30T09:30:00.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253555+0000", 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000126212 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.253668+0000", 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011336300000000001 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.258010+0000", 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004341106 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.258320+0000", 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031085999999999999 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.471992+0000", 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21367123599999999 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.472050+0000", 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8335999999999998e-05 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:38.472098+0000", 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7970000000000003e-05 2023-09-30T09:30:00.115 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:147841 5.2 5:4a29d1e1:::benchmark_data_smithi019_287194_object65063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:56.787511+0000", 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 183.23327056599999, 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.230919979, 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.116 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 147841 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787511+0000", 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.117 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787511+0000", 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787509+0000", 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787517+0000", 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2980000000000003e-06 2023-09-30T09:30:00.118 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787534+0000", 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7053000000000001e-05 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787540+0000", 2023-09-30T09:30:00.119 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5640000000000002e-06 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787595+0000", 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4960000000000002e-05 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.121 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787737+0000", 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014138599999999999 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.787878+0000", 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000141719 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.796087+0000", 2023-09-30T09:30:00.122 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0082088370000000001 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.799960+0000", 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0038732039999999999 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.018323+0000", 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21836282100000001 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.123 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.018389+0000", 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5270000000000004e-05 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.018431+0000", 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2401999999999998e-05 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.124 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:147842 5.2 5:48619fb8:::benchmark_data_smithi019_287194_object65064:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:56.788263+0000", 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 183.23251857899999, 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.23447486100000001, 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.125 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.126 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 147842 2023-09-30T09:30:00.126 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.126 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.126 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.126 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.126 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.788263+0000", 2023-09-30T09:30:00.126 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.788263+0000", 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.788262+0000", 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.127 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.788267+0000", 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2370000000000003e-06 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.788278+0000", 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.094e-05 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.128 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.788282+0000", 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3490000000000002e-06 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.791688+0000", 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034057979999999998 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.791806+0000", 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000118354 2023-09-30T09:30:00.129 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.791935+0000", 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012929 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.799931+0000", 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0079951929999999994 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.802380+0000", 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024495290000000002 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.022648+0000", 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.220267612 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.022697+0000", 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8921999999999997e-05 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.022738+0000", 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1291999999999997e-05 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:147846 5.2 5:41d876a5:::benchmark_data_smithi019_287194_object65068:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:56.792911+0000", 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 183.227870712, 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.229928941, 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 147846 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.133 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.792911+0000", 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.792911+0000", 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.792910+0000", 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.792915+0000", 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5690000000000004e-06 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.792928+0000", 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2568000000000001e-05 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.792935+0000", 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8950000000000001e-06 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.796115+0000", 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00318017 2023-09-30T09:30:00.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.796239+0000", 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012426200000000001 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.796382+0000", 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014240999999999999 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.800046+0000", 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036644680000000002 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:56.802531+0000", 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024850520000000002 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.022778+0000", 2023-09-30T09:30:00.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22024696299999999 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.022808+0000", 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0012e-05 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:57.022840+0000", 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1643000000000002e-05 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.139 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:157648 5.2 5:4bc522f0:::benchmark_data_smithi019_287194_object74870:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:27:14.180174+0000", 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 165.84060753599999, 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.25750764999999998, 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.140 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 157648 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180174+0000", 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180174+0000", 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180173+0000", 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180179+0000", 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9859999999999999e-06 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.142 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180193+0000", 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4012999999999999e-05 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180199+0000", 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0009999999999998e-06 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180243+0000", 2023-09-30T09:30:00.143 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4240000000000003e-05 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180397+0000", 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000153678 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.180539+0000", 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000142041 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.185516+0000", 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049776029999999997 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.185735+0000", 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000218469 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.437586+0000", 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.251851135 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.437639+0000", 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2896000000000003e-05 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.437682+0000", 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2938000000000001e-05 2023-09-30T09:30:00.146 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:157664 5.2 5:4672a5ba:::benchmark_data_smithi019_287194_object74886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:27:14.206903+0000", 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 165.81387871199999, 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.23087447899999999, 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.147 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 157664 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.206903+0000", 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.206903+0000", 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.206901+0000", 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.206908+0000", 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5279999999999998e-06 2023-09-30T09:30:00.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.206924+0000", 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5173e-05 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.206929+0000", 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8730000000000004e-06 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.150 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.206973+0000", 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3628999999999999e-05 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.207066+0000", 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3317000000000001e-05 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.207144+0000", 2023-09-30T09:30:00.151 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7674000000000002e-05 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.210841+0000", 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036973010000000001 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.214136+0000", 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003294843 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.437715+0000", 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.223578473 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.437748+0000", 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2941999999999997e-05 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.437777+0000", 2023-09-30T09:30:00.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9680999999999999e-05 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:157675 5.2 5:43b73193:::benchmark_data_smithi019_287194_object74897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:27:14.231364+0000", 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 165.78941794599999, 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21622683500000001, 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.154 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 157675 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231364+0000", 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231364+0000", 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231362+0000", 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231372+0000", 2023-09-30T09:30:00.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9680000000000001e-06 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231392+0000", 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0928999999999999e-05 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231401+0000", 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3429999999999996e-06 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231468+0000", 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7410999999999995e-05 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231608+0000", 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013971599999999999 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.231763+0000", 2023-09-30T09:30:00.158 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015498700000000001 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.238953+0000", 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0071904569999999999 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.247425+0000", 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0084716549999999998 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.159 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.447528+0000", 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20010309800000001 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.447563+0000", 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4861999999999999e-05 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.447590+0000", 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7501999999999999e-05 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:157683 5.2 5:474c5a6c:::benchmark_data_smithi019_287194_object74905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:27:14.242794+0000", 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 165.777987104, 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20488093099999999, 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.161 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 157683 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.242794+0000", 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.162 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.242794+0000", 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.242793+0000", 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.242802+0000", 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1030000000000008e-06 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.242816+0000", 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3380000000000001e-05 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.242823+0000", 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5660000000000001e-06 2023-09-30T09:30:00.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.242873+0000", 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9329000000000002e-05 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.243024+0000", 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000151008 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.243168+0000", 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014464600000000001 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.247308+0000", 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041391500000000003 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.247581+0000", 2023-09-30T09:30:00.166 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027306599999999998 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.447624+0000", 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20004328299999999 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.447654+0000", 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9658e-05 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.167 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.447675+0000", 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1858e-05 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:157690 5.2 5:48a35690:::benchmark_data_smithi019_287194_object74912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:27:14.250667+0000", 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 165.77011459799999, 2023-09-30T09:30:00.168 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20996076799999999, 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 157690 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250667+0000", 2023-09-30T09:30:00.169 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250667+0000", 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250666+0000", 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250673+0000", 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8469999999999996e-06 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250693+0000", 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9568999999999998e-05 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:30:00.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250700+0000", 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3590000000000002e-06 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250741+0000", 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0268000000000001e-05 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250860+0000", 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000118921 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.250960+0000", 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100647 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.255252+0000", 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042922589999999997 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.255344+0000", 2023-09-30T09:30:00.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.179e-05 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.460528+0000", 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.205183791 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.460585+0000", 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6953999999999997e-05 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:30:00.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:27:14.460628+0000", 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2747999999999998e-05 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:30:00.175 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:30:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:00.176 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:00.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:00.277 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:00.324 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:00.382 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:30:00.401 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:00.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:00.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:00.505 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:00.535 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:00.576 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:00.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pgmap v576: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 834 op/s; 23673/265119 objects degraded (8.929%); 5707/265119 objects misplaced (2.153%); 8.1 MiB/s, 130 objects/s recovering 2023-09-30T09:30:00.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3957436314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:00.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:00.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 23673/265119 objects degraded (8.929%), 17 pgs degraded 2023-09-30T09:30:00.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:30:00.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: [WRN] PG_DEGRADED: Degraded data redundancy: 23673/265119 objects degraded (8.929%), 17 pgs degraded 2023-09-30T09:30:00.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 2.3 is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.0 is active+recovery_wait+degraded, acting [0,3,1] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.1 is active+recovery_wait+degraded, acting [1,6,7] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.8 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.9 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.d is active+recovery_wait+degraded, acting [5,1,6] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.e is active+recovery_wait+degraded, acting [4,1,7] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.f is active+recovery_wait+degraded, acting [3,4,6] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.18 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.19 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.1d is active+recovery_wait+degraded, acting [5,1,6] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.1e is active+recovery_wait+degraded, acting [4,1,7] 2023-09-30T09:30:00.579 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.1f is active+recovery_wait+degraded, acting [3,4,6] 2023-09-30T09:30:00.580 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.20 is active+recovery_wait+degraded, acting [0,3,1] 2023-09-30T09:30:00.580 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.21 is active+recovery_wait+degraded, acting [1,6,7] 2023-09-30T09:30:00.580 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:00 smithi167 ceph-mon[192880]: pg 6.22 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-09-30T09:30:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:00.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:00.681 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:00.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pgmap v576: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 834 op/s; 23673/265119 objects degraded (8.929%); 5707/265119 objects misplaced (2.153%); 8.1 MiB/s, 130 objects/s recovering 2023-09-30T09:30:00.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3957436314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:00.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:00.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 23673/265119 objects degraded (8.929%), 17 pgs degraded 2023-09-30T09:30:00.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:30:00.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: [WRN] PG_DEGRADED: Degraded data redundancy: 23673/265119 objects degraded (8.929%), 17 pgs degraded 2023-09-30T09:30:00.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 2.3 is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.0 is active+recovery_wait+degraded, acting [0,3,1] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.1 is active+recovery_wait+degraded, acting [1,6,7] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.8 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.9 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.d is active+recovery_wait+degraded, acting [5,1,6] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.e is active+recovery_wait+degraded, acting [4,1,7] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.f is active+recovery_wait+degraded, acting [3,4,6] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.18 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.19 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.1d is active+recovery_wait+degraded, acting [5,1,6] 2023-09-30T09:30:00.717 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.1e is active+recovery_wait+degraded, acting [4,1,7] 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.1f is active+recovery_wait+degraded, acting [3,4,6] 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.20 is active+recovery_wait+degraded, acting [0,3,1] 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.21 is active+recovery_wait+degraded, acting [1,6,7] 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[224043]: pg 6.22 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pgmap v576: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 834 op/s; 23673/265119 objects degraded (8.929%); 5707/265119 objects misplaced (2.153%); 8.1 MiB/s, 130 objects/s recovering 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3957436314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 23673/265119 objects degraded (8.929%), 17 pgs degraded 2023-09-30T09:30:00.718 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: [WRN] PG_DEGRADED: Degraded data redundancy: 23673/265119 objects degraded (8.929%), 17 pgs degraded 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 2.3 is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.0 is active+recovery_wait+degraded, acting [0,3,1] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.1 is active+recovery_wait+degraded, acting [1,6,7] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.8 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.9 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.d is active+recovery_wait+degraded, acting [5,1,6] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.e is active+recovery_wait+degraded, acting [4,1,7] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.f is active+recovery_wait+degraded, acting [3,4,6] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.18 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.719 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.19 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-30T09:30:00.720 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.1d is active+recovery_wait+degraded, acting [5,1,6] 2023-09-30T09:30:00.720 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.1e is active+recovery_wait+degraded, acting [4,1,7] 2023-09-30T09:30:00.720 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.1f is active+recovery_wait+degraded, acting [3,4,6] 2023-09-30T09:30:00.720 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.20 is active+recovery_wait+degraded, acting [0,3,1] 2023-09-30T09:30:00.720 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.21 is active+recovery_wait+degraded, acting [1,6,7] 2023-09-30T09:30:00.720 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:00 smithi019 ceph-mon[216815]: pg 6.22 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-09-30T09:30:00.724 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:01.090 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:30:01.385 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:01 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:30:01.385 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:01 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:01.386 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:01 smithi167 ceph-mon[192880]: failed to encode map e246 with expected crc 2023-09-30T09:30:01.386 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:01 smithi167 ceph-mon[192880]: osdmap e246: 8 total, 8 up, 7 in 2023-09-30T09:30:01.386 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:01 smithi167 ceph-mon[192880]: failed to encode map e246 with expected crc 2023-09-30T09:30:01.386 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:01 smithi167 ceph-mon[192880]: failed to encode map e246 with expected crc 2023-09-30T09:30:01.386 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:01 smithi167 ceph-mon[192880]: failed to encode map e246 with expected crc 2023-09-30T09:30:01.457 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:01.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:01.475 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:30:01.561 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:02.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:30:02.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:02.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[216815]: failed to encode map e246 with expected crc 2023-09-30T09:30:02.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[216815]: osdmap e246: 8 total, 8 up, 7 in 2023-09-30T09:30:02.284 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[216815]: failed to encode map e246 with expected crc 2023-09-30T09:30:02.284 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[216815]: failed to encode map e246 with expected crc 2023-09-30T09:30:02.284 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[216815]: failed to encode map e246 with expected crc 2023-09-30T09:30:02.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:30:02.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:02.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[224043]: failed to encode map e246 with expected crc 2023-09-30T09:30:02.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[224043]: osdmap e246: 8 total, 8 up, 7 in 2023-09-30T09:30:02.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[224043]: failed to encode map e246 with expected crc 2023-09-30T09:30:02.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[224043]: failed to encode map e246 with expected crc 2023-09-30T09:30:02.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:01 smithi019 ceph-mon[224043]: failed to encode map e246 with expected crc 2023-09-30T09:30:02.328 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:02.384 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:02.385 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:02 smithi167 ceph-mon[192880]: pgmap v578: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 116 B/s rd, 11 MiB/s wr, 895 op/s; 23256/258405 objects degraded (9.000%); 5578/258405 objects misplaced (2.159%); 12 MiB/s, 194 objects/s recovering 2023-09-30T09:30:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:02.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:02.487 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:02.515 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:02.599 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:02.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:02.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:02.705 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:02.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:02 smithi019 ceph-mon[224043]: pgmap v578: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 116 B/s rd, 11 MiB/s wr, 895 op/s; 23256/258405 objects degraded (9.000%); 5578/258405 objects misplaced (2.159%); 12 MiB/s, 194 objects/s recovering 2023-09-30T09:30:02.762 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:02 smithi019 ceph-mon[216815]: pgmap v578: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 116 B/s rd, 11 MiB/s wr, 895 op/s; 23256/258405 objects degraded (9.000%); 5578/258405 objects misplaced (2.159%); 12 MiB/s, 194 objects/s recovering 2023-09-30T09:30:02.777 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:02.884 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:02.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:02.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:02.991 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:03.032 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:03.118 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:03.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:03.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:03.222 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:03.294 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:03.384 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:03.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:03.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:03.409 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:30:03.434 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:30:03.449 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:30:03.450 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:30:03.450 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:30:03.450 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:30:03.464 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:30:03.490 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:03.498 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:30:03.528 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:03.529 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:30:03.553 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:30:03.577 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:30:03.609 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:30:03.613 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:30:03.613 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:30:03.613 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:30:03.613 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:30:03.613 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:30:03.613 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:30:03.614 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:30:03.614 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:30:03.635 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:03.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:03.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:03.739 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:03.795 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:03.857 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:03.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:03.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:03.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:03.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[216815]: pgmap v579: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 10 MiB/s wr, 824 op/s; 23114/257322 objects degraded (8.983%); 5578/257322 objects misplaced (2.168%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:30:03.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/131076693' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "36"}]: dispatch 2023-09-30T09:30:03.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23256/258405 objects degraded (9.000%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:30:03.963 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4269255350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:03.963 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:30:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:30:03] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-09-30T09:30:03.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:03.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:03.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[224043]: pgmap v579: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 10 MiB/s wr, 824 op/s; 23114/257322 objects degraded (8.983%); 5578/257322 objects misplaced (2.168%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:30:03.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/131076693' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "36"}]: dispatch 2023-09-30T09:30:03.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23256/258405 objects degraded (9.000%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:30:03.964 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4269255350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:03.964 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:03.976 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 6 pgp_num to 36 2023-09-30T09:30:04.018 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:04.096 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:04.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:04.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:04.204 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:04.257 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:03 smithi167 ceph-mon[192880]: pgmap v579: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 10 MiB/s wr, 824 op/s; 23114/257322 objects degraded (8.983%); 5578/257322 objects misplaced (2.168%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:30:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/131076693' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "36"}]: dispatch 2023-09-30T09:30:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23256/258405 objects degraded (9.000%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:30:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4269255350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:04.385 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:30:04.421 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:30:04.926 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:30:05.218 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/131076693' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "36"}]': finished 2023-09-30T09:30:05.218 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:04 smithi167 ceph-mon[192880]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.218 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:04 smithi167 ceph-mon[192880]: osdmap e247: 8 total, 8 up, 7 in 2023-09-30T09:30:05.218 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:04 smithi167 ceph-mon[192880]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.218 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:04 smithi167 ceph-mon[192880]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/131076693' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "36"}]': finished 2023-09-30T09:30:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[216815]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[216815]: osdmap e247: 8 total, 8 up, 7 in 2023-09-30T09:30:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[216815]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[216815]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/131076693' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "36"}]': finished 2023-09-30T09:30:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[224043]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[224043]: osdmap e247: 8 total, 8 up, 7 in 2023-09-30T09:30:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[224043]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:04 smithi019 ceph-mon[224043]: failed to encode map e247 with expected crc 2023-09-30T09:30:05.529 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:05.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:05.635 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:05.675 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:05.739 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:05.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:05.844 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:05.878 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:05.927 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:05.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:05.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:05.935 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:06.030 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:06.062 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:06.142 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:06.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:06.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:06.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:05 smithi167 ceph-mon[192880]: failed to encode map e247 with expected crc 2023-09-30T09:30:06.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:05 smithi167 ceph-mon[192880]: pgmap v581: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 22386/248235 objects degraded (9.018%); 5258/248235 objects misplaced (2.118%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:30:06.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:05 smithi019 ceph-mon[224043]: failed to encode map e247 with expected crc 2023-09-30T09:30:06.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:05 smithi019 ceph-mon[224043]: pgmap v581: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 22386/248235 objects degraded (9.018%); 5258/248235 objects misplaced (2.118%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:30:06.215 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:05 smithi019 ceph-mon[216815]: failed to encode map e247 with expected crc 2023-09-30T09:30:06.215 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:05 smithi019 ceph-mon[216815]: pgmap v581: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 22386/248235 objects degraded (9.018%); 5258/248235 objects misplaced (2.118%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:30:06.274 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:06.318 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:06.588 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:06.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:06.695 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:06.752 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:06.930 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:06.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:07.008 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:30:07.008 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:30:07.032 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:07.069 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:07.332 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:07.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:07.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:07.495 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:07.523 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:30:07 smithi167 conmon[172457]: logger=sqlstore.transactions t=2023-09-30T09:30:07.231686052Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-30T09:30:07.580 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:30:07.682 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:07.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:07.793 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:07.834 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:08.188 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:08.201 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:08.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:08.225 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:30:08.312 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:08.344 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:08.411 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:08.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:08.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:08 smithi019 ceph-mon[224043]: pgmap v582: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 22258/247194 objects degraded (9.004%); 5258/247194 objects misplaced (2.127%); 12 MiB/s, 189 objects/s recovering 2023-09-30T09:30:08.471 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:08 smithi019 ceph-mon[216815]: pgmap v582: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 22258/247194 objects degraded (9.004%); 5258/247194 objects misplaced (2.127%); 12 MiB/s, 189 objects/s recovering 2023-09-30T09:30:08.519 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:08 smithi167 ceph-mon[192880]: pgmap v582: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 22258/247194 objects degraded (9.004%); 5258/247194 objects misplaced (2.127%); 12 MiB/s, 189 objects/s recovering 2023-09-30T09:30:08.563 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:08.757 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:30:08.889 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:08.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:08.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:08.994 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:09.027 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:09.115 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:09.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:09.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:09.221 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22258/247194 objects degraded (9.004%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:30:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/592297496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:09.274 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:09.276 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:30:09.473 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:09.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:09.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:09.578 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:09.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22258/247194 objects degraded (9.004%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:30:09.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/592297496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22258/247194 objects degraded (9.004%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:30:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/592297496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:09.657 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:10.088 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:10.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:10.192 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:10.216 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[216815]: pgmap v583: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 21835/243069 objects degraded (8.983%); 5044/243069 objects misplaced (2.075%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:30:10.216 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:10.216 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/592297496' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:30:10.216 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[216815]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.216 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[216815]: osdmap e248: 8 total, 8 up, 7 in 2023-09-30T09:30:10.216 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[216815]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.216 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[216815]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.216 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[216815]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.225 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:10.495 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[224043]: pgmap v583: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 21835/243069 objects degraded (8.983%); 5044/243069 objects misplaced (2.075%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:30:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/592297496' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:30:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[224043]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[224043]: osdmap e248: 8 total, 8 up, 7 in 2023-09-30T09:30:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[224043]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[224043]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.498 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:10 smithi019 ceph-mon[224043]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:10.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:10 smithi167 ceph-mon[192880]: pgmap v583: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 21835/243069 objects degraded (8.983%); 5044/243069 objects misplaced (2.075%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:30:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:10 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/592297496' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:30:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:10 smithi167 ceph-mon[192880]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:10 smithi167 ceph-mon[192880]: osdmap e248: 8 total, 8 up, 7 in 2023-09-30T09:30:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:10 smithi167 ceph-mon[192880]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:10 smithi167 ceph-mon[192880]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:10 smithi167 ceph-mon[192880]: failed to encode map e248 with expected crc 2023-09-30T09:30:10.600 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:10.654 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:10.834 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:10.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:10.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:10.846 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:30:10.846 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:30:10.846 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-30T09:30:10.847 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:30:10.926 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:30:10.926 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:30:10.940 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:11.005 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:11.092 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:30:11.099 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:11.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:11.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:11.123 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:30:11.151 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:30:11.188 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:30:11.211 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:11.219 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:30:11.229 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2991757337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:11.254 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:30:11.257 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:11.283 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:30:11.325 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:30:11.331 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:30:11.331 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:30:11.331 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:30:11.332 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:30:11.332 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:30:11.332 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:30:11.332 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:30:11.332 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:30:11.382 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:11.383 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2991757337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:11.383 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2991757337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:11.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:11.404 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:11.485 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:11.526 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:11.853 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:11.855 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:30:11.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:11.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:11.958 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:11.986 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:12.096 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:12.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:12.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:12.210 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:12.230 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:12 smithi019 ceph-mon[224043]: pgmap v585: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 21491/237783 objects degraded (9.038%); 4953/237783 objects misplaced (2.083%); 14 MiB/s, 228 objects/s recovering 2023-09-30T09:30:12.261 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:12.344 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:12.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:12.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:12.451 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:12.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:12 smithi019 ceph-mon[216815]: pgmap v585: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 21491/237783 objects degraded (9.038%); 4953/237783 objects misplaced (2.083%); 14 MiB/s, 228 objects/s recovering 2023-09-30T09:30:12.487 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:12 smithi167 ceph-mon[192880]: pgmap v585: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 21491/237783 objects degraded (9.038%); 4953/237783 objects misplaced (2.083%); 14 MiB/s, 228 objects/s recovering 2023-09-30T09:30:12.491 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:12.566 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:12.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:12.591 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:30:12.679 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:12.712 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:12.790 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:12.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:12.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:12.897 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:12.944 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:13.011 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:13.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:13.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:13.115 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:13.155 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:13.750 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:13.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:13.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:13.842 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:13.854 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:13.888 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:14.015 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:14.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:14.023 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:30:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:30:13] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-09-30T09:30:14.115 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:14.153 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:14.219 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:14.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:14.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:14.329 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:14.360 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:14.411 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:14.415 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:14 smithi167 ceph-mon[192880]: pgmap v586: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 21379/236829 objects degraded (9.027%); 4953/236829 objects misplaced (2.091%); 11 MiB/s, 176 objects/s recovering 2023-09-30T09:30:14.415 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 21379/236829 objects degraded (9.027%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:14.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:14.507 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:30:14.517 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:14.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:14 smithi019 ceph-mon[216815]: pgmap v586: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 21379/236829 objects degraded (9.027%); 4953/236829 objects misplaced (2.091%); 11 MiB/s, 176 objects/s recovering 2023-09-30T09:30:14.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 21379/236829 objects degraded (9.027%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:14.544 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:14 smithi019 ceph-mon[224043]: pgmap v586: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 21379/236829 objects degraded (9.027%); 4953/236829 objects misplaced (2.091%); 11 MiB/s, 176 objects/s recovering 2023-09-30T09:30:14.544 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 21379/236829 objects degraded (9.027%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:14.550 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:14.598 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:14.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:14.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:14.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:14.736 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:15.448 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:15.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:15.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:15.554 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:15.634 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:16.113 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:30:16.117 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:30:16.337 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:16.376 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:16.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:16.483 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:16.509 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:16 smithi019 ceph-mon[216815]: pgmap v587: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 20623/227193 objects degraded (9.077%); 4634/227193 objects misplaced (2.040%); 12 MiB/s, 197 objects/s recovering 2023-09-30T09:30:16.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:16 smithi019 ceph-mon[224043]: pgmap v587: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 20623/227193 objects degraded (9.077%); 4634/227193 objects misplaced (2.040%); 12 MiB/s, 197 objects/s recovering 2023-09-30T09:30:16.511 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:16 smithi167 ceph-mon[192880]: pgmap v587: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 20623/227193 objects degraded (9.077%); 4634/227193 objects misplaced (2.040%); 12 MiB/s, 197 objects/s recovering 2023-09-30T09:30:17.501 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:17 smithi019 ceph-mon[224043]: from='client.55051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:30:17.502 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:17 smithi019 ceph-mon[216815]: from='client.55051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:30:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:17 smithi167 ceph-mon[192880]: from='client.55051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:30:17.922 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":587,"stamp":"2023-09-30T09:30:15.158485+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4965841130,"num_objects":75731,"num_object_clones":0,"num_object_copies":227193,"num_objects_missing_on_primary":892,"num_objects_missing":892,"num_objects_degraded":20623,"num_objects_misplaced":4634,"num_objects_unfound":0,"num_objects_dirty":75731,"num_whiteouts":0,"num_read":6032,"num_read_kb":12314,"num_write":93813,"num_write_kb":4889516,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13000,"num_bytes_recovered":851875032,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177869,"ondisk_log_size":177869,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14851328,"kb_used_data":13515272,"kb_used_omap":14,"kb_used_meta":1335921,"kb_avail":641364736,"statfs":{"total":671965249536,"available":656757489664,"internally_reserved":0,"allocated":13839638528,"data_stored":13831732377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14566,"internal_metadata":1367983898},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,6,3,3,0,1,0,0,9,5,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-459669504,"num_objects":-7014,"num_object_clones":0,"num_object_copies":-21042,"num_objects_missing_on_primary":-215,"num_objects_missing":-215,"num_objects_degraded":-1763,"num_objects_misplaced":-624,"num_objects_unfound":0,"num_objects_dirty":-7014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7014,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1978,"num_bytes_recovered":129630208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005934"},"pg_stats":[{"pgid":"6.23","version":"248'4002","reported_seq":8169,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:13.980182+0000","last_change":"2023-09-30T09:29:35.062393+0000","last_active":"2023-09-30T09:30:13.980182+0000","last_peered":"2023-09-30T09:30:13.980182+0000","last_clean":"2023-09-30T09:30:13.980182+0000","last_became_active":"2023-09-30T09:29:35.061339+0000","last_became_peered":"2023-09-30T09:29:35.061339+0000","last_unstale":"2023-09-30T09:30:13.980182+0000","last_undegraded":"2023-09-30T09:30:13.980182+0000","last_fullsized":"2023-09-30T09:30:13.980182+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":4002,"log_dups_size":0,"ondisk_log_size":4002,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75497472,"num_objects":1152,"num_object_clones":0,"num_object_copies":3456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1003,"num_write_kb":46336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65699840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.22","version":"248'3933","reported_seq":6869,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:13.343322+0000","last_change":"2023-09-30T09:29:37.013899+0000","last_active":"2023-09-30T09:30:13.343322+0000","last_peered":"2023-09-30T09:30:13.343322+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:37.011820+0000","last_became_peered":"2023-09-30T09:29:37.011820+0000","last_unstale":"2023-09-30T09:30:13.343322+0000","last_undegraded":"2023-09-30T09:29:36.997644+0000","last_fullsized":"2023-09-30T09:29:36.996855+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":3933,"log_dups_size":0,"ondisk_log_size":3933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73203712,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":304,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":936,"num_write_kb":42464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":304},{"shards":"3,6","objects":813}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.21","version":"248'4151","reported_seq":4773,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:11.709199+0000","last_change":"2023-09-30T09:29:35.078377+0000","last_active":"2023-09-30T09:30:11.709199+0000","last_peered":"2023-09-30T09:30:11.709199+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:35.075606+0000","last_became_peered":"2023-09-30T09:29:35.075606+0000","last_unstale":"2023-09-30T09:30:11.709199+0000","last_undegraded":"2023-09-30T09:29:35.051902+0000","last_fullsized":"2023-09-30T09:30:11.709199+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":4151,"log_dups_size":0,"ondisk_log_size":4151,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79396864,"num_objects":1212,"num_object_clones":0,"num_object_copies":3636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":782,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1103,"num_write_kb":53856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":261,"num_bytes_recovered":17104896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":782},{"shards":"1,6,7","objects":430}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.20","version":"248'4102","reported_seq":4769,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:14.520716+0000","last_change":"2023-09-30T09:29:35.050182+0000","last_active":"2023-09-30T09:30:14.520716+0000","last_peered":"2023-09-30T09:30:14.520716+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:35.036894+0000","last_became_peered":"2023-09-30T09:29:35.036894+0000","last_unstale":"2023-09-30T09:30:14.520716+0000","last_undegraded":"2023-09-30T09:29:35.031725+0000","last_fullsized":"2023-09-30T09:30:14.520716+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":4102,"log_dups_size":0,"ondisk_log_size":4102,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79560704,"num_objects":1214,"num_object_clones":0,"num_object_copies":3642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":774,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1068,"num_write_kb":50400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":281,"num_bytes_recovered":18382848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":440},{"shards":"0,2,3","objects":774}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1f","version":"248'5910","reported_seq":7101,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:13.530891+0000","last_change":"2023-09-30T09:29:35.091385+0000","last_active":"2023-09-30T09:30:13.530891+0000","last_peered":"2023-09-30T09:30:13.530891+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:35.087685+0000","last_became_peered":"2023-09-30T09:29:35.087685+0000","last_unstale":"2023-09-30T09:30:13.530891+0000","last_undegraded":"2023-09-30T09:29:35.082957+0000","last_fullsized":"2023-09-30T09:30:13.530891+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153747456,"num_objects":2346,"num_object_clones":0,"num_object_copies":7038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1535,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3462,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":559,"num_bytes_recovered":36634624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":1535},{"shards":"3,4,6","objects":811}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":235,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.979805+0000","last_change":"2023-09-30T09:29:09.598494+0000","last_active":"2023-09-30T09:30:03.979805+0000","last_peered":"2023-09-30T09:30:03.979805+0000","last_clean":"2023-09-30T09:30:03.979805+0000","last_became_active":"2023-09-30T09:29:09.598158+0000","last_became_peered":"2023-09-30T09:29:09.598158+0000","last_unstale":"2023-09-30T09:30:03.979805+0000","last_undegraded":"2023-09-30T09:30:03.979805+0000","last_fullsized":"2023-09-30T09:30:03.979805+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:03.225682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":213,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.959484+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:30:03.959484+0000","last_peered":"2023-09-30T09:30:03.959484+0000","last_clean":"2023-09-30T09:30:03.959484+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:30:03.959484+0000","last_undegraded":"2023-09-30T09:30:03.959484+0000","last_fullsized":"2023-09-30T09:30:03.959484+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1c","version":"248'5743","reported_seq":5812,"reported_epoch":248,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:30:13.547503+0000","last_change":"2023-09-30T09:29:35.102559+0000","last_active":"2023-09-30T09:30:13.547503+0000","last_peered":"2023-09-30T09:30:13.547503+0000","last_clean":"2023-09-30T09:29:33.948392+0000","last_became_active":"2023-09-30T09:29:35.089045+0000","last_became_peered":"2023-09-30T09:29:35.089045+0000","last_unstale":"2023-09-30T09:30:13.547503+0000","last_undegraded":"2023-09-30T09:30:13.547503+0000","last_fullsized":"2023-09-30T09:30:13.547503+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152240128,"num_objects":2323,"num_object_clones":0,"num_object_copies":6969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3325,"num_write_kb":180736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":2323}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":211,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.969683+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:30:03.969683+0000","last_peered":"2023-09-30T09:30:03.969683+0000","last_clean":"2023-09-30T09:30:03.969683+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:30:03.969683+0000","last_undegraded":"2023-09-30T09:30:03.969683+0000","last_fullsized":"2023-09-30T09:30:03.969683+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":141,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.961406+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:30:03.961406+0000","last_peered":"2023-09-30T09:30:03.961406+0000","last_clean":"2023-09-30T09:30:03.961406+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:30:03.961406+0000","last_undegraded":"2023-09-30T09:30:03.961406+0000","last_fullsized":"2023-09-30T09:30:03.961406+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1d","version":"248'5911","reported_seq":7033,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:14.118779+0000","last_change":"2023-09-30T09:29:35.078954+0000","last_active":"2023-09-30T09:30:14.118779+0000","last_peered":"2023-09-30T09:30:14.118779+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:35.073573+0000","last_became_peered":"2023-09-30T09:29:35.073573+0000","last_unstale":"2023-09-30T09:30:14.118779+0000","last_undegraded":"2023-09-30T09:29:35.043564+0000","last_fullsized":"2023-09-30T09:30:14.118779+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154730520,"num_objects":2362,"num_object_clones":0,"num_object_copies":7086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1507,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2362,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3416,"num_write_kb":184833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":527,"num_bytes_recovered":34537472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1507},{"shards":"1,5,6","objects":855}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":263,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.964197+0000","last_change":"2023-09-30T09:29:09.606288+0000","last_active":"2023-09-30T09:30:03.964197+0000","last_peered":"2023-09-30T09:30:03.964197+0000","last_clean":"2023-09-30T09:30:03.964197+0000","last_became_active":"2023-09-30T09:29:09.605776+0000","last_became_peered":"2023-09-30T09:29:09.605776+0000","last_unstale":"2023-09-30T09:30:03.964197+0000","last_undegraded":"2023-09-30T09:30:03.964197+0000","last_fullsized":"2023-09-30T09:30:03.964197+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:07.935982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":207,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968537+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:30:03.968537+0000","last_peered":"2023-09-30T09:30:03.968537+0000","last_clean":"2023-09-30T09:30:03.968537+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:30:03.968537+0000","last_undegraded":"2023-09-30T09:30:03.968537+0000","last_fullsized":"2023-09-30T09:30:03.968537+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"248'4408","reported_seq":8562,"reported_epoch":248,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:30:14.111509+0000","last_change":"2023-09-30T09:29:10.766595+0000","last_active":"2023-09-30T09:30:14.111509+0000","last_peered":"2023-09-30T09:30:14.111509+0000","last_clean":"2023-09-30T09:29:08.564047+0000","last_became_active":"2023-09-30T09:29:10.763967+0000","last_became_peered":"2023-09-30T09:29:10.763967+0000","last_unstale":"2023-09-30T09:30:14.111509+0000","last_undegraded":"2023-09-30T09:30:14.111509+0000","last_fullsized":"2023-09-30T09:30:14.111509+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2497","last_scrub_stamp":"2023-09-30T09:28:46.026181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:46.026181+0000","objects_scrubbed":1334,"log_size":4408,"log_dups_size":0,"ondisk_log_size":4408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:02.322482+0000","scrub_duration":0.15695645899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147652608,"num_objects":2253,"num_object_clones":0,"num_object_copies":6759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2253,"num_objects_unfound":0,"num_objects_dirty":2253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1911,"num_write_kb":90560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1347,"num_bytes_recovered":88276992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,2],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,4,7","objects":2253}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":140,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.961503+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:30:03.961503+0000","last_peered":"2023-09-30T09:30:03.961503+0000","last_clean":"2023-09-30T09:30:03.961503+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:30:03.961503+0000","last_undegraded":"2023-09-30T09:30:03.961503+0000","last_fullsized":"2023-09-30T09:30:03.961503+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":213,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.964276+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:30:03.964276+0000","last_peered":"2023-09-30T09:30:03.964276+0000","last_clean":"2023-09-30T09:30:03.964276+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:30:03.964276+0000","last_undegraded":"2023-09-30T09:30:03.964276+0000","last_fullsized":"2023-09-30T09:30:03.964276+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"248'4512","reported_seq":6073,"reported_epoch":248,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:30:13.544806+0000","last_change":"2023-09-30T09:29:38.038561+0000","last_active":"2023-09-30T09:30:13.544806+0000","last_peered":"2023-09-30T09:30:13.544806+0000","last_clean":"2023-09-30T09:28:50.280641+0000","last_became_active":"2023-09-30T09:29:12.506944+0000","last_became_peered":"2023-09-30T09:29:12.506944+0000","last_unstale":"2023-09-30T09:30:13.544806+0000","last_undegraded":"2023-09-30T09:30:13.544806+0000","last_fullsized":"2023-09-30T09:29:10.583109+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4512,"log_dups_size":0,"ondisk_log_size":4512,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158859264,"num_objects":2424,"num_object_clones":0,"num_object_copies":7272,"num_objects_missing_on_primary":892,"num_objects_missing":892,"num_objects_degraded":0,"num_objects_misplaced":2381,"num_objects_unfound":0,"num_objects_dirty":2424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3408,"num_write_kb":186624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":493,"num_bytes_recovered":32309248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,3,6","objects":589},{"shards":"2,3,6,7","objects":8},{"shards":"2,6,7","objects":892},{"shards":"3,6","objects":935}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.983968+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:30:03.983968+0000","last_peered":"2023-09-30T09:30:03.983968+0000","last_clean":"2023-09-30T09:30:03.983968+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:30:03.983968+0000","last_undegraded":"2023-09-30T09:30:03.983968+0000","last_fullsized":"2023-09-30T09:30:03.983968+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":164,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.959396+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:30:03.959396+0000","last_peered":"2023-09-30T09:30:03.959396+0000","last_clean":"2023-09-30T09:30:03.959396+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:30:03.959396+0000","last_undegraded":"2023-09-30T09:30:03.959396+0000","last_fullsized":"2023-09-30T09:30:03.959396+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"248'4452","reported_seq":8506,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:14.416731+0000","last_change":"2023-09-30T09:28:56.382198+0000","last_active":"2023-09-30T09:30:14.416731+0000","last_peered":"2023-09-30T09:30:14.416731+0000","last_clean":"2023-09-30T09:30:14.416731+0000","last_became_active":"2023-09-30T09:28:56.380752+0000","last_became_peered":"2023-09-30T09:28:56.380752+0000","last_unstale":"2023-09-30T09:30:14.416731+0000","last_undegraded":"2023-09-30T09:30:14.416731+0000","last_fullsized":"2023-09-30T09:30:14.416731+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4452,"log_dups_size":0,"ondisk_log_size":4452,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153059328,"num_objects":2336,"num_object_clones":0,"num_object_copies":7008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3334,"num_write_kb":181408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1324,"num_bytes_recovered":86769664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":211,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980738+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:30:03.980738+0000","last_peered":"2023-09-30T09:30:03.980738+0000","last_clean":"2023-09-30T09:30:03.980738+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:30:03.980738+0000","last_undegraded":"2023-09-30T09:30:03.980738+0000","last_fullsized":"2023-09-30T09:30:03.980738+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":196,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.967342+0000","last_change":"2023-09-30T09:29:09.620809+0000","last_active":"2023-09-30T09:30:03.967342+0000","last_peered":"2023-09-30T09:30:03.967342+0000","last_clean":"2023-09-30T09:30:03.967342+0000","last_became_active":"2023-09-30T09:29:09.620497+0000","last_became_peered":"2023-09-30T09:29:09.620497+0000","last_unstale":"2023-09-30T09:30:03.967342+0000","last_undegraded":"2023-09-30T09:30:03.967342+0000","last_fullsized":"2023-09-30T09:30:03.967342+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:48.932180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"248'4609","reported_seq":10210,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:13.473156+0000","last_change":"2023-09-30T09:29:39.270828+0000","last_active":"2023-09-30T09:30:13.473156+0000","last_peered":"2023-09-30T09:30:13.473156+0000","last_clean":"2023-09-30T09:30:13.473156+0000","last_became_active":"2023-09-30T09:29:39.266753+0000","last_became_peered":"2023-09-30T09:29:39.266753+0000","last_unstale":"2023-09-30T09:30:13.473156+0000","last_undegraded":"2023-09-30T09:30:13.473156+0000","last_fullsized":"2023-09-30T09:30:13.473156+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2608","last_scrub_stamp":"2023-09-30T09:28:45.597110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.597110+0000","objects_scrubbed":1407,"log_size":4609,"log_dups_size":0,"ondisk_log_size":4609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:06.120738+0000","scrub_duration":0.26278548800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154009600,"num_objects":2350,"num_object_clones":0,"num_object_copies":7050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2007,"num_write_kb":94400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2034,"num_bytes_recovered":133300224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980695+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:30:03.980695+0000","last_peered":"2023-09-30T09:30:03.980695+0000","last_clean":"2023-09-30T09:30:03.980695+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:30:03.980695+0000","last_undegraded":"2023-09-30T09:30:03.980695+0000","last_fullsized":"2023-09-30T09:30:03.980695+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":226,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980128+0000","last_change":"2023-09-30T09:29:09.612466+0000","last_active":"2023-09-30T09:30:03.980128+0000","last_peered":"2023-09-30T09:30:03.980128+0000","last_clean":"2023-09-30T09:30:03.980128+0000","last_became_active":"2023-09-30T09:29:09.612173+0000","last_became_peered":"2023-09-30T09:29:09.612173+0000","last_unstale":"2023-09-30T09:30:03.980128+0000","last_undegraded":"2023-09-30T09:30:03.980128+0000","last_fullsized":"2023-09-30T09:30:03.980128+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:15.457876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"248'4627","reported_seq":4696,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:13.476975+0000","last_change":"2023-09-30T09:28:40.282669+0000","last_active":"2023-09-30T09:30:13.476975+0000","last_peered":"2023-09-30T09:30:13.476975+0000","last_clean":"2023-09-30T09:30:13.476975+0000","last_became_active":"2023-09-30T09:28:40.281544+0000","last_became_peered":"2023-09-30T09:28:40.281544+0000","last_unstale":"2023-09-30T09:30:13.476975+0000","last_undegraded":"2023-09-30T09:30:13.476975+0000","last_fullsized":"2023-09-30T09:30:13.476975+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4627,"log_dups_size":0,"ondisk_log_size":4627,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157515776,"num_objects":2404,"num_object_clones":0,"num_object_copies":7212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3490,"num_write_kb":188576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":226,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.978549+0000","last_change":"2023-09-30T09:29:09.626205+0000","last_active":"2023-09-30T09:30:03.978549+0000","last_peered":"2023-09-30T09:30:03.978549+0000","last_clean":"2023-09-30T09:30:03.978549+0000","last_became_active":"2023-09-30T09:29:09.618293+0000","last_became_peered":"2023-09-30T09:29:09.618293+0000","last_unstale":"2023-09-30T09:30:03.978549+0000","last_undegraded":"2023-09-30T09:30:03.978549+0000","last_fullsized":"2023-09-30T09:30:03.978549+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:04.337293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":235,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.964540+0000","last_change":"2023-09-30T09:29:09.606072+0000","last_active":"2023-09-30T09:30:03.964540+0000","last_peered":"2023-09-30T09:30:03.964540+0000","last_clean":"2023-09-30T09:30:03.964540+0000","last_became_active":"2023-09-30T09:29:09.605716+0000","last_became_peered":"2023-09-30T09:29:09.605716+0000","last_unstale":"2023-09-30T09:30:03.964540+0000","last_undegraded":"2023-09-30T09:30:03.964540+0000","last_fullsized":"2023-09-30T09:30:03.964540+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:50.972640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"248'4571","reported_seq":4698,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:14.101605+0000","last_change":"2023-09-30T09:28:45.602319+0000","last_active":"2023-09-30T09:30:14.101605+0000","last_peered":"2023-09-30T09:30:14.101605+0000","last_clean":"2023-09-30T09:30:14.101605+0000","last_became_active":"2023-09-30T09:28:40.272792+0000","last_became_peered":"2023-09-30T09:28:40.272792+0000","last_unstale":"2023-09-30T09:30:14.101605+0000","last_undegraded":"2023-09-30T09:30:14.101605+0000","last_fullsized":"2023-09-30T09:30:14.101605+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2523","last_scrub_stamp":"2023-09-30T09:28:45.602248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.602248+0000","objects_scrubbed":1419,"log_size":4571,"log_dups_size":0,"ondisk_log_size":4571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:31:54.788871+0000","scrub_duration":0.27069680099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154927104,"num_objects":2364,"num_object_clones":0,"num_object_copies":7092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2055,"num_write_kb":96000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":141,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:09.270275+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:30:09.270275+0000","last_peered":"2023-09-30T09:30:09.270275+0000","last_clean":"2023-09-30T09:30:09.270275+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:30:09.270275+0000","last_undegraded":"2023-09-30T09:30:09.270275+0000","last_fullsized":"2023-09-30T09:30:09.270275+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":169,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980851+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:30:03.980851+0000","last_peered":"2023-09-30T09:30:03.980851+0000","last_clean":"2023-09-30T09:30:03.980851+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:30:03.980851+0000","last_undegraded":"2023-09-30T09:30:03.980851+0000","last_fullsized":"2023-09-30T09:30:03.980851+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"248'4505","reported_seq":4597,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:11.304581+0000","last_change":"2023-09-30T09:28:44.596036+0000","last_active":"2023-09-30T09:30:11.304581+0000","last_peered":"2023-09-30T09:30:11.304581+0000","last_clean":"2023-09-30T09:30:11.304581+0000","last_became_active":"2023-09-30T09:28:40.272740+0000","last_became_peered":"2023-09-30T09:28:40.272740+0000","last_unstale":"2023-09-30T09:30:11.304581+0000","last_undegraded":"2023-09-30T09:30:11.304581+0000","last_fullsized":"2023-09-30T09:30:11.304581+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"214'2497","last_scrub_stamp":"2023-09-30T09:28:44.595809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:44.595809+0000","objects_scrubbed":1358,"log_size":4505,"log_dups_size":0,"ondisk_log_size":4505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:18:56.692373+0000","scrub_duration":0.31195125499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2016,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":191,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.979027+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:30:03.979027+0000","last_peered":"2023-09-30T09:30:03.979027+0000","last_clean":"2023-09-30T09:30:03.979027+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:30:03.979027+0000","last_undegraded":"2023-09-30T09:30:03.979027+0000","last_fullsized":"2023-09-30T09:30:03.979027+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":140,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.961336+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:30:03.961336+0000","last_peered":"2023-09-30T09:30:03.961336+0000","last_clean":"2023-09-30T09:30:03.961336+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:30:03.961336+0000","last_undegraded":"2023-09-30T09:30:03.961336+0000","last_fullsized":"2023-09-30T09:30:03.961336+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"248'4563","reported_seq":4622,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:14.131195+0000","last_change":"2023-09-30T09:28:40.288346+0000","last_active":"2023-09-30T09:30:14.131195+0000","last_peered":"2023-09-30T09:30:14.131195+0000","last_clean":"2023-09-30T09:30:14.131195+0000","last_became_active":"2023-09-30T09:28:40.286899+0000","last_became_peered":"2023-09-30T09:28:40.286899+0000","last_unstale":"2023-09-30T09:30:14.131195+0000","last_undegraded":"2023-09-30T09:30:14.131195+0000","last_fullsized":"2023-09-30T09:30:14.131195+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4563,"log_dups_size":0,"ondisk_log_size":4563,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154632192,"num_objects":2360,"num_object_clones":0,"num_object_copies":7080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3432,"num_write_kb":185312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":340,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.967802+0000","last_change":"2023-09-30T09:29:09.634829+0000","last_active":"2023-09-30T09:30:03.967802+0000","last_peered":"2023-09-30T09:30:03.967802+0000","last_clean":"2023-09-30T09:30:03.967802+0000","last_became_active":"2023-09-30T09:29:09.612183+0000","last_became_peered":"2023-09-30T09:29:09.612183+0000","last_unstale":"2023-09-30T09:30:03.967802+0000","last_undegraded":"2023-09-30T09:30:03.967802+0000","last_fullsized":"2023-09-30T09:30:03.967802+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:28.144436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":233,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.964575+0000","last_change":"2023-09-30T09:29:09.605918+0000","last_active":"2023-09-30T09:30:03.964575+0000","last_peered":"2023-09-30T09:30:03.964575+0000","last_clean":"2023-09-30T09:30:03.964575+0000","last_became_active":"2023-09-30T09:29:09.605521+0000","last_became_peered":"2023-09-30T09:29:09.605521+0000","last_unstale":"2023-09-30T09:30:03.964575+0000","last_undegraded":"2023-09-30T09:30:03.964575+0000","last_fullsized":"2023-09-30T09:30:03.964575+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:43.287337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"248'5834","reported_seq":5895,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:14.134956+0000","last_change":"2023-09-30T09:29:35.005933+0000","last_active":"2023-09-30T09:30:14.134956+0000","last_peered":"2023-09-30T09:30:14.134956+0000","last_clean":"2023-09-30T09:30:14.134956+0000","last_became_active":"2023-09-30T09:29:35.001538+0000","last_became_peered":"2023-09-30T09:29:35.001538+0000","last_unstale":"2023-09-30T09:30:14.134956+0000","last_undegraded":"2023-09-30T09:30:14.134956+0000","last_fullsized":"2023-09-30T09:30:14.134956+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5834,"log_dups_size":0,"ondisk_log_size":5834,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153550848,"num_objects":2343,"num_object_clones":0,"num_object_copies":7029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3387,"num_write_kb":183360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":251,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.978549+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:30:03.978549+0000","last_peered":"2023-09-30T09:30:03.978549+0000","last_clean":"2023-09-30T09:30:03.978549+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:30:03.978549+0000","last_undegraded":"2023-09-30T09:30:03.978549+0000","last_fullsized":"2023-09-30T09:30:03.978549+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":111,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.978242+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:30:03.978242+0000","last_peered":"2023-09-30T09:30:03.978242+0000","last_clean":"2023-09-30T09:30:03.978242+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:30:03.978242+0000","last_undegraded":"2023-09-30T09:30:03.978242+0000","last_fullsized":"2023-09-30T09:30:03.978242+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"248'5879","reported_seq":5948,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:13.530375+0000","last_change":"2023-09-30T09:29:35.047558+0000","last_active":"2023-09-30T09:30:13.530375+0000","last_peered":"2023-09-30T09:30:13.530375+0000","last_clean":"2023-09-30T09:30:13.530375+0000","last_became_active":"2023-09-30T09:29:35.044991+0000","last_became_peered":"2023-09-30T09:29:35.044991+0000","last_unstale":"2023-09-30T09:30:13.530375+0000","last_undegraded":"2023-09-30T09:30:13.530375+0000","last_fullsized":"2023-09-30T09:30:13.530375+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5879,"log_dups_size":0,"ondisk_log_size":5879,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158171136,"num_objects":2413,"num_object_clones":0,"num_object_copies":7239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3411,"num_write_kb":186400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":252,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.978011+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:30:03.978011+0000","last_peered":"2023-09-30T09:30:03.978011+0000","last_clean":"2023-09-30T09:30:03.978011+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:30:03.978011+0000","last_undegraded":"2023-09-30T09:30:03.978011+0000","last_fullsized":"2023-09-30T09:30:03.978011+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":223,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.979518+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:30:03.979518+0000","last_peered":"2023-09-30T09:30:03.979518+0000","last_clean":"2023-09-30T09:30:03.979518+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:30:03.979518+0000","last_undegraded":"2023-09-30T09:30:03.979518+0000","last_fullsized":"2023-09-30T09:30:03.979518+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"248'4422","reported_seq":4493,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:14.172362+0000","last_change":"2023-09-30T09:29:10.874488+0000","last_active":"2023-09-30T09:30:14.172362+0000","last_peered":"2023-09-30T09:30:14.172362+0000","last_clean":"2023-09-30T09:29:08.490081+0000","last_became_active":"2023-09-30T09:29:10.870401+0000","last_became_peered":"2023-09-30T09:29:10.870401+0000","last_unstale":"2023-09-30T09:30:14.172362+0000","last_undegraded":"2023-09-30T09:29:10.657673+0000","last_fullsized":"2023-09-30T09:29:10.593223+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4422,"log_dups_size":0,"ondisk_log_size":4422,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157581312,"num_objects":2404,"num_object_clones":0,"num_object_copies":7212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1975,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3308,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1523},{"shards":"3,7","objects":429}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":234,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980587+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:30:03.980587+0000","last_peered":"2023-09-30T09:30:03.980587+0000","last_clean":"2023-09-30T09:30:03.980587+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:30:03.980587+0000","last_undegraded":"2023-09-30T09:30:03.980587+0000","last_fullsized":"2023-09-30T09:30:03.980587+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968466+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:30:03.968466+0000","last_peered":"2023-09-30T09:30:03.968466+0000","last_clean":"2023-09-30T09:30:03.968466+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:30:03.968466+0000","last_undegraded":"2023-09-30T09:30:03.968466+0000","last_fullsized":"2023-09-30T09:30:03.968466+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"248'4474","reported_seq":4608,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:14.142959+0000","last_change":"2023-09-30T09:29:10.893105+0000","last_active":"2023-09-30T09:30:14.142959+0000","last_peered":"2023-09-30T09:30:14.142959+0000","last_clean":"2023-09-30T09:29:08.514351+0000","last_became_active":"2023-09-30T09:29:10.883964+0000","last_became_peered":"2023-09-30T09:29:10.883964+0000","last_unstale":"2023-09-30T09:30:14.142959+0000","last_undegraded":"2023-09-30T09:29:10.674378+0000","last_fullsized":"2023-09-30T09:29:10.596450+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2471","last_scrub_stamp":"2023-09-30T09:28:47.207706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:47.207706+0000","objects_scrubbed":1369,"log_size":4474,"log_dups_size":0,"ondisk_log_size":4474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:51.408261+0000","scrub_duration":0.14908584899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153485312,"num_objects":2342,"num_object_clones":0,"num_object_copies":7026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2003,"num_write_kb":95232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1482},{"shards":"3,7","objects":345}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":207,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.984517+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:30:03.984517+0000","last_peered":"2023-09-30T09:30:03.984517+0000","last_clean":"2023-09-30T09:30:03.984517+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:30:03.984517+0000","last_undegraded":"2023-09-30T09:30:03.984517+0000","last_fullsized":"2023-09-30T09:30:03.984517+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":138,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.961058+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:30:03.961058+0000","last_peered":"2023-09-30T09:30:03.961058+0000","last_clean":"2023-09-30T09:30:03.961058+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:30:03.961058+0000","last_undegraded":"2023-09-30T09:30:03.961058+0000","last_fullsized":"2023-09-30T09:30:03.961058+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"248'5730","reported_seq":6757,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:13.960800+0000","last_change":"2023-09-30T09:29:36.032408+0000","last_active":"2023-09-30T09:30:13.960800+0000","last_peered":"2023-09-30T09:30:13.960800+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:36.028267+0000","last_became_peered":"2023-09-30T09:29:36.028267+0000","last_unstale":"2023-09-30T09:30:13.960800+0000","last_undegraded":"2023-09-30T09:29:36.007788+0000","last_fullsized":"2023-09-30T09:30:13.960800+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155779072,"num_objects":2377,"num_object_clones":0,"num_object_copies":7131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1444,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3331,"num_write_kb":182656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":477,"num_bytes_recovered":31293440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1444},{"shards":"1,4,7","objects":933}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":727,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.978944+0000","last_change":"2023-09-30T09:29:12.637524+0000","last_active":"2023-09-30T09:30:03.978944+0000","last_peered":"2023-09-30T09:30:03.978944+0000","last_clean":"2023-09-30T09:30:03.978944+0000","last_became_active":"2023-09-30T09:29:12.637171+0000","last_became_peered":"2023-09-30T09:29:12.637171+0000","last_unstale":"2023-09-30T09:30:03.978944+0000","last_undegraded":"2023-09-30T09:30:03.978944+0000","last_fullsized":"2023-09-30T09:30:03.978944+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:14:09.735059+0000","scrub_duration":0.0079898150000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":247,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:04.000082+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:30:04.000082+0000","last_peered":"2023-09-30T09:30:04.000082+0000","last_clean":"2023-09-30T09:30:04.000082+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:30:04.000082+0000","last_undegraded":"2023-09-30T09:30:04.000082+0000","last_fullsized":"2023-09-30T09:30:04.000082+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":206,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980603+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:30:03.980603+0000","last_peered":"2023-09-30T09:30:03.980603+0000","last_clean":"2023-09-30T09:30:03.980603+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:30:03.980603+0000","last_undegraded":"2023-09-30T09:30:03.980603+0000","last_fullsized":"2023-09-30T09:30:03.980603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"248'5812","reported_seq":6853,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:13.492017+0000","last_change":"2023-09-30T09:29:36.082560+0000","last_active":"2023-09-30T09:30:13.492017+0000","last_peered":"2023-09-30T09:30:13.492017+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:36.077849+0000","last_became_peered":"2023-09-30T09:29:36.077849+0000","last_unstale":"2023-09-30T09:30:13.492017+0000","last_undegraded":"2023-09-30T09:29:35.995233+0000","last_fullsized":"2023-09-30T09:30:13.492017+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5812,"log_dups_size":0,"ondisk_log_size":5812,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157155328,"num_objects":2398,"num_object_clones":0,"num_object_copies":7194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1457,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3364,"num_write_kb":184384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":484,"num_bytes_recovered":31719424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":1457},{"shards":"3,4,6","objects":941}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":624,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.963838+0000","last_change":"2023-09-30T09:29:12.631999+0000","last_active":"2023-09-30T09:30:03.963838+0000","last_peered":"2023-09-30T09:30:03.963838+0000","last_clean":"2023-09-30T09:30:03.963838+0000","last_became_active":"2023-09-30T09:29:12.631620+0000","last_became_peered":"2023-09-30T09:29:12.631620+0000","last_unstale":"2023-09-30T09:30:03.963838+0000","last_undegraded":"2023-09-30T09:30:03.963838+0000","last_fullsized":"2023-09-30T09:30:03.963838+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.999760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.999760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":531037,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968367+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:30:03.968367+0000","last_peered":"2023-09-30T09:30:03.968367+0000","last_clean":"2023-09-30T09:30:03.968367+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:30:03.968367+0000","last_undegraded":"2023-09-30T09:30:03.968367+0000","last_fullsized":"2023-09-30T09:30:03.968367+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":132,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968467+0000","last_change":"2023-09-30T09:29:09.612070+0000","last_active":"2023-09-30T09:30:03.968467+0000","last_peered":"2023-09-30T09:30:03.968467+0000","last_clean":"2023-09-30T09:30:03.968467+0000","last_became_active":"2023-09-30T09:29:09.611725+0000","last_became_peered":"2023-09-30T09:29:09.611725+0000","last_unstale":"2023-09-30T09:30:03.968467+0000","last_undegraded":"2023-09-30T09:30:03.968467+0000","last_fullsized":"2023-09-30T09:30:03.968467+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:55.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"248'5806","reported_seq":5875,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:13.498278+0000","last_change":"2023-09-30T09:29:35.074390+0000","last_active":"2023-09-30T09:30:13.498278+0000","last_peered":"2023-09-30T09:30:13.498278+0000","last_clean":"2023-09-30T09:30:13.498278+0000","last_became_active":"2023-09-30T09:29:35.072113+0000","last_became_peered":"2023-09-30T09:29:35.072113+0000","last_unstale":"2023-09-30T09:30:13.498278+0000","last_undegraded":"2023-09-30T09:30:13.498278+0000","last_fullsized":"2023-09-30T09:30:13.498278+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152305664,"num_objects":2324,"num_object_clones":0,"num_object_copies":6972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3388,"num_write_kb":182784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.979391+0000","last_change":"2023-09-30T09:29:09.607621+0000","last_active":"2023-09-30T09:30:03.979391+0000","last_peered":"2023-09-30T09:30:03.979391+0000","last_clean":"2023-09-30T09:30:03.979391+0000","last_became_active":"2023-09-30T09:29:09.607245+0000","last_became_peered":"2023-09-30T09:29:09.607245+0000","last_unstale":"2023-09-30T09:30:03.979391+0000","last_undegraded":"2023-09-30T09:30:03.979391+0000","last_fullsized":"2023-09-30T09:30:03.979391+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:42:42.429421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":206,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.981339+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:30:03.981339+0000","last_peered":"2023-09-30T09:30:03.981339+0000","last_clean":"2023-09-30T09:30:03.981339+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:30:03.981339+0000","last_undegraded":"2023-09-30T09:30:03.981339+0000","last_fullsized":"2023-09-30T09:30:03.981339+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"248'5942","reported_seq":7052,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:14.157797+0000","last_change":"2023-09-30T09:29:36.240193+0000","last_active":"2023-09-30T09:30:14.157797+0000","last_peered":"2023-09-30T09:30:14.157797+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:36.233690+0000","last_became_peered":"2023-09-30T09:29:36.233690+0000","last_unstale":"2023-09-30T09:30:14.157797+0000","last_undegraded":"2023-09-30T09:29:36.012206+0000","last_fullsized":"2023-09-30T09:30:14.157797+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157876224,"num_objects":2409,"num_object_clones":0,"num_object_copies":7227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1558,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3447,"num_write_kb":187392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":520,"num_bytes_recovered":34078720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1558},{"shards":"1,5,6","objects":851}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":455,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968278+0000","last_change":"2023-09-30T09:29:12.635044+0000","last_active":"2023-09-30T09:30:03.968278+0000","last_peered":"2023-09-30T09:30:03.968278+0000","last_clean":"2023-09-30T09:30:03.968278+0000","last_became_active":"2023-09-30T09:29:12.634703+0000","last_became_peered":"2023-09-30T09:29:12.634703+0000","last_unstale":"2023-09-30T09:30:03.968278+0000","last_undegraded":"2023-09-30T09:30:03.968278+0000","last_fullsized":"2023-09-30T09:30:03.968278+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.980385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.980385+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":215,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:08.529518+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:30:08.529518+0000","last_peered":"2023-09-30T09:30:08.529518+0000","last_clean":"2023-09-30T09:30:08.529518+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:30:08.529518+0000","last_undegraded":"2023-09-30T09:30:08.529518+0000","last_fullsized":"2023-09-30T09:30:08.529518+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":188,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.969423+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:30:03.969423+0000","last_peered":"2023-09-30T09:30:03.969423+0000","last_clean":"2023-09-30T09:30:03.969423+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:30:03.969423+0000","last_undegraded":"2023-09-30T09:30:03.969423+0000","last_fullsized":"2023-09-30T09:30:03.969423+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"248'4103","reported_seq":4825,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:14.252352+0000","last_change":"2023-09-30T09:29:36.014896+0000","last_active":"2023-09-30T09:30:14.252352+0000","last_peered":"2023-09-30T09:30:14.252352+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:36.012719+0000","last_became_peered":"2023-09-30T09:29:36.012719+0000","last_unstale":"2023-09-30T09:30:14.252352+0000","last_undegraded":"2023-09-30T09:29:35.992619+0000","last_fullsized":"2023-09-30T09:30:14.252352+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":4103,"log_dups_size":0,"ondisk_log_size":4103,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76087296,"num_objects":1161,"num_object_clones":0,"num_object_copies":3483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":738,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1068,"num_write_kb":48736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":307,"num_bytes_recovered":20152320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":423},{"shards":"0,2,3","objects":738}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":492,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968597+0000","last_change":"2023-09-30T09:28:30.495699+0000","last_active":"2023-09-30T09:30:03.968597+0000","last_peered":"2023-09-30T09:30:03.968597+0000","last_clean":"2023-09-30T09:30:03.968597+0000","last_became_active":"2023-09-30T09:28:09.228266+0000","last_became_peered":"2023-09-30T09:28:09.228266+0000","last_unstale":"2023-09-30T09:30:03.968597+0000","last_undegraded":"2023-09-30T09:30:03.968597+0000","last_fullsized":"2023-09-30T09:30:03.968597+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:10:19.705285+0000","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":592,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968969+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:30:03.968969+0000","last_peered":"2023-09-30T09:30:03.968969+0000","last_clean":"2023-09-30T09:30:03.968969+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:30:03.968969+0000","last_undegraded":"2023-09-30T09:30:03.968969+0000","last_fullsized":"2023-09-30T09:30:03.968969+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":234,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.985794+0000","last_change":"2023-09-30T09:29:09.611292+0000","last_active":"2023-09-30T09:30:03.985794+0000","last_peered":"2023-09-30T09:30:03.985794+0000","last_clean":"2023-09-30T09:30:03.985794+0000","last_became_active":"2023-09-30T09:29:09.611015+0000","last_became_peered":"2023-09-30T09:29:09.611015+0000","last_unstale":"2023-09-30T09:30:03.985794+0000","last_undegraded":"2023-09-30T09:30:03.985794+0000","last_fullsized":"2023-09-30T09:30:03.985794+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:48.124378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":207,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.967426+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:30:03.967426+0000","last_peered":"2023-09-30T09:30:03.967426+0000","last_clean":"2023-09-30T09:30:03.967426+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:30:03.967426+0000","last_undegraded":"2023-09-30T09:30:03.967426+0000","last_fullsized":"2023-09-30T09:30:03.967426+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"248'4640","reported_seq":4731,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:13.558753+0000","last_change":"2023-09-30T09:28:42.250412+0000","last_active":"2023-09-30T09:30:13.558753+0000","last_peered":"2023-09-30T09:30:13.558753+0000","last_clean":"2023-09-30T09:30:13.558753+0000","last_became_active":"2023-09-30T09:28:40.266645+0000","last_became_peered":"2023-09-30T09:28:40.266645+0000","last_unstale":"2023-09-30T09:30:13.558753+0000","last_undegraded":"2023-09-30T09:30:13.558753+0000","last_fullsized":"2023-09-30T09:30:13.558753+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2368","last_scrub_stamp":"2023-09-30T09:28:42.250204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.250204+0000","objects_scrubbed":1224,"log_size":4640,"log_dups_size":0,"ondisk_log_size":4640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:50:26.843055+0000","scrub_duration":0.32304901400000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155582464,"num_objects":2374,"num_object_clones":0,"num_object_copies":7122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2278,"num_write_kb":109696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":670,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.983233+0000","last_change":"2023-09-30T09:29:12.640269+0000","last_active":"2023-09-30T09:30:03.983233+0000","last_peered":"2023-09-30T09:30:03.983233+0000","last_clean":"2023-09-30T09:30:03.983233+0000","last_became_active":"2023-09-30T09:29:12.639925+0000","last_became_peered":"2023-09-30T09:29:12.639925+0000","last_unstale":"2023-09-30T09:30:03.983233+0000","last_undegraded":"2023-09-30T09:30:03.983233+0000","last_fullsized":"2023-09-30T09:30:03.983233+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:45:10.404382+0000","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":396,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.967574+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:30:03.967574+0000","last_peered":"2023-09-30T09:30:03.967574+0000","last_clean":"2023-09-30T09:30:03.967574+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:30:03.967574+0000","last_undegraded":"2023-09-30T09:30:03.967574+0000","last_fullsized":"2023-09-30T09:30:03.967574+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":227,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980218+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:30:03.980218+0000","last_peered":"2023-09-30T09:30:03.980218+0000","last_clean":"2023-09-30T09:30:03.980218+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:30:03.980218+0000","last_undegraded":"2023-09-30T09:30:03.980218+0000","last_fullsized":"2023-09-30T09:30:03.980218+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":106,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.964663+0000","last_change":"2023-09-30T09:29:09.611507+0000","last_active":"2023-09-30T09:30:03.964663+0000","last_peered":"2023-09-30T09:30:03.964663+0000","last_clean":"2023-09-30T09:30:03.964663+0000","last_became_active":"2023-09-30T09:29:09.611120+0000","last_became_peered":"2023-09-30T09:29:09.611120+0000","last_unstale":"2023-09-30T09:30:03.964663+0000","last_undegraded":"2023-09-30T09:30:03.964663+0000","last_fullsized":"2023-09-30T09:30:03.964663+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:31.622647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.5","version":"248'4619","reported_seq":4686,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:14.174609+0000","last_change":"2023-09-30T09:28:46.022537+0000","last_active":"2023-09-30T09:30:14.174609+0000","last_peered":"2023-09-30T09:30:14.174609+0000","last_clean":"2023-09-30T09:30:14.174609+0000","last_became_active":"2023-09-30T09:28:40.276165+0000","last_became_peered":"2023-09-30T09:28:40.276165+0000","last_unstale":"2023-09-30T09:30:14.174609+0000","last_undegraded":"2023-09-30T09:30:14.174609+0000","last_fullsized":"2023-09-30T09:30:14.174609+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4619,"log_dups_size":0,"ondisk_log_size":4619,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157384704,"num_objects":2401,"num_object_clones":0,"num_object_copies":7203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3487,"num_write_kb":188448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":727,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.977609+0000","last_change":"2023-09-30T09:29:12.637491+0000","last_active":"2023-09-30T09:30:03.977609+0000","last_peered":"2023-09-30T09:30:03.977609+0000","last_clean":"2023-09-30T09:30:03.977609+0000","last_became_active":"2023-09-30T09:29:12.637133+0000","last_became_peered":"2023-09-30T09:29:12.637133+0000","last_unstale":"2023-09-30T09:30:03.977609+0000","last_undegraded":"2023-09-30T09:30:03.977609+0000","last_fullsized":"2023-09-30T09:30:03.977609+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:54:24.000552+0000","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":421,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.959530+0000","last_change":"2023-09-30T09:29:36.005128+0000","last_active":"2023-09-30T09:30:03.959530+0000","last_peered":"2023-09-30T09:30:03.959530+0000","last_clean":"2023-09-30T09:30:03.959530+0000","last_became_active":"2023-09-30T09:29:36.004563+0000","last_became_peered":"2023-09-30T09:29:36.004563+0000","last_unstale":"2023-09-30T09:30:03.959530+0000","last_undegraded":"2023-09-30T09:30:03.959530+0000","last_fullsized":"2023-09-30T09:30:03.959530+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:59.839484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":195,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.967780+0000","last_change":"2023-09-30T09:29:09.607111+0000","last_active":"2023-09-30T09:30:03.967780+0000","last_peered":"2023-09-30T09:30:03.967780+0000","last_clean":"2023-09-30T09:30:03.967780+0000","last_became_active":"2023-09-30T09:29:09.606807+0000","last_became_peered":"2023-09-30T09:29:09.606807+0000","last_unstale":"2023-09-30T09:30:03.967780+0000","last_undegraded":"2023-09-30T09:30:03.967780+0000","last_fullsized":"2023-09-30T09:30:03.967780+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:49.380239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":243,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.969348+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:30:03.969348+0000","last_peered":"2023-09-30T09:30:03.969348+0000","last_clean":"2023-09-30T09:30:03.969348+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:30:03.969348+0000","last_undegraded":"2023-09-30T09:30:03.969348+0000","last_fullsized":"2023-09-30T09:30:03.969348+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"248'4487","reported_seq":4607,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:14.068199+0000","last_change":"2023-09-30T09:28:42.687918+0000","last_active":"2023-09-30T09:30:14.068199+0000","last_peered":"2023-09-30T09:30:14.068199+0000","last_clean":"2023-09-30T09:30:14.068199+0000","last_became_active":"2023-09-30T09:28:40.262566+0000","last_became_peered":"2023-09-30T09:28:40.262566+0000","last_unstale":"2023-09-30T09:30:14.068199+0000","last_undegraded":"2023-09-30T09:30:14.068199+0000","last_fullsized":"2023-09-30T09:30:14.068199+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2390","last_scrub_stamp":"2023-09-30T09:28:42.687821+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.687821+0000","objects_scrubbed":1235,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:07:56.313767+0000","scrub_duration":0.36697584500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154992640,"num_objects":2365,"num_object_clones":0,"num_object_copies":7095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2108,"num_write_kb":103616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":665,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:03.983526+0000","last_change":"2023-09-30T09:29:12.096468+0000","last_active":"2023-09-30T09:30:03.983526+0000","last_peered":"2023-09-30T09:30:03.983526+0000","last_clean":"2023-09-30T09:29:03.507390+0000","last_became_active":"2023-09-30T09:29:12.096051+0000","last_became_peered":"2023-09-30T09:29:12.096051+0000","last_unstale":"2023-09-30T09:30:03.983526+0000","last_undegraded":"2023-09-30T09:29:10.593445+0000","last_fullsized":"2023-09-30T09:29:10.592749+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:49.279327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":688,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.976942+0000","last_change":"2023-09-30T09:27:54.119268+0000","last_active":"2023-09-30T09:30:03.976942+0000","last_peered":"2023-09-30T09:30:03.976942+0000","last_clean":"2023-09-30T09:30:03.976942+0000","last_became_active":"2023-09-30T09:27:49.106373+0000","last_became_peered":"2023-09-30T09:27:49.106373+0000","last_unstale":"2023-09-30T09:30:03.976942+0000","last_undegraded":"2023-09-30T09:30:03.976942+0000","last_fullsized":"2023-09-30T09:30:03.976942+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:28.856049+0000","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":164,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.959934+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:30:03.959934+0000","last_peered":"2023-09-30T09:30:03.959934+0000","last_clean":"2023-09-30T09:30:03.959934+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:30:03.959934+0000","last_undegraded":"2023-09-30T09:30:03.959934+0000","last_fullsized":"2023-09-30T09:30:03.959934+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":190,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968663+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:30:03.968663+0000","last_peered":"2023-09-30T09:30:03.968663+0000","last_clean":"2023-09-30T09:30:03.968663+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:30:03.968663+0000","last_undegraded":"2023-09-30T09:30:03.968663+0000","last_fullsized":"2023-09-30T09:30:03.968663+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"248'4613","reported_seq":4703,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:11.689754+0000","last_change":"2023-09-30T09:28:43.582313+0000","last_active":"2023-09-30T09:30:11.689754+0000","last_peered":"2023-09-30T09:30:11.689754+0000","last_clean":"2023-09-30T09:30:11.689754+0000","last_became_active":"2023-09-30T09:28:40.264204+0000","last_became_peered":"2023-09-30T09:28:40.264204+0000","last_unstale":"2023-09-30T09:30:11.689754+0000","last_undegraded":"2023-09-30T09:30:11.689754+0000","last_fullsized":"2023-09-30T09:30:11.689754+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2440","last_scrub_stamp":"2023-09-30T09:28:43.582001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:43.582001+0000","objects_scrubbed":1291,"log_size":4613,"log_dups_size":0,"ondisk_log_size":4613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:50.257550+0000","scrub_duration":0.30691791200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165543936,"num_objects":2526,"num_object_clones":0,"num_object_copies":7578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2183,"num_write_kb":109376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":653,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:09.286259+0000","last_change":"2023-09-30T09:28:24.276678+0000","last_active":"2023-09-30T09:30:09.286259+0000","last_peered":"2023-09-30T09:30:09.286259+0000","last_clean":"2023-09-30T09:30:09.286259+0000","last_became_active":"2023-09-30T09:28:09.226029+0000","last_became_peered":"2023-09-30T09:28:09.226029+0000","last_unstale":"2023-09-30T09:30:09.286259+0000","last_undegraded":"2023-09-30T09:30:09.286259+0000","last_fullsized":"2023-09-30T09:30:09.286259+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:03:49.533925+0000","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":808,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.976845+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:30:03.976845+0000","last_peered":"2023-09-30T09:30:03.976845+0000","last_clean":"2023-09-30T09:30:03.976845+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:30:03.976845+0000","last_undegraded":"2023-09-30T09:30:03.976845+0000","last_fullsized":"2023-09-30T09:30:03.976845+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":165,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.960879+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:30:03.960879+0000","last_peered":"2023-09-30T09:30:03.960879+0000","last_clean":"2023-09-30T09:30:03.960879+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:30:03.960879+0000","last_undegraded":"2023-09-30T09:30:03.960879+0000","last_fullsized":"2023-09-30T09:30:03.960879+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":153,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968810+0000","last_change":"2023-09-30T09:29:09.611376+0000","last_active":"2023-09-30T09:30:03.968810+0000","last_peered":"2023-09-30T09:30:03.968810+0000","last_clean":"2023-09-30T09:30:03.968810+0000","last_became_active":"2023-09-30T09:29:09.611090+0000","last_became_peered":"2023-09-30T09:29:09.611090+0000","last_unstale":"2023-09-30T09:30:03.968810+0000","last_undegraded":"2023-09-30T09:30:03.968810+0000","last_fullsized":"2023-09-30T09:30:03.968810+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:07:15.039397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"248'4175","reported_seq":4815,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:11.667170+0000","last_change":"2023-09-30T09:29:36.030348+0000","last_active":"2023-09-30T09:30:11.667170+0000","last_peered":"2023-09-30T09:30:11.667170+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:36.027562+0000","last_became_peered":"2023-09-30T09:29:36.027562+0000","last_unstale":"2023-09-30T09:30:11.667170+0000","last_undegraded":"2023-09-30T09:29:35.999894+0000","last_fullsized":"2023-09-30T09:30:11.667170+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":4175,"log_dups_size":0,"ondisk_log_size":4175,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79790080,"num_objects":1217,"num_object_clones":0,"num_object_copies":3651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":774,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1126,"num_write_kb":54816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":270,"num_bytes_recovered":17694720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":773},{"shards":"1,6","objects":1},{"shards":"1,6,7","objects":443}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":492,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968291+0000","last_change":"2023-09-30T09:28:28.453752+0000","last_active":"2023-09-30T09:30:03.968291+0000","last_peered":"2023-09-30T09:30:03.968291+0000","last_clean":"2023-09-30T09:30:03.968291+0000","last_became_active":"2023-09-30T09:28:09.228162+0000","last_became_peered":"2023-09-30T09:28:09.228162+0000","last_unstale":"2023-09-30T09:30:03.968291+0000","last_undegraded":"2023-09-30T09:30:03.968291+0000","last_fullsized":"2023-09-30T09:30:03.968291+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:43:23.683089+0000","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":938,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980662+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:30:03.980662+0000","last_peered":"2023-09-30T09:30:03.980662+0000","last_clean":"2023-09-30T09:30:03.980662+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:30:03.980662+0000","last_undegraded":"2023-09-30T09:30:03.980662+0000","last_fullsized":"2023-09-30T09:30:03.980662+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":211,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.969121+0000","last_change":"2023-09-30T09:29:09.611808+0000","last_active":"2023-09-30T09:30:03.969121+0000","last_peered":"2023-09-30T09:30:03.969121+0000","last_clean":"2023-09-30T09:30:03.969121+0000","last_became_active":"2023-09-30T09:29:09.611484+0000","last_became_peered":"2023-09-30T09:29:09.611484+0000","last_unstale":"2023-09-30T09:30:03.969121+0000","last_undegraded":"2023-09-30T09:30:03.969121+0000","last_fullsized":"2023-09-30T09:30:03.969121+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:19.562461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":137,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.961121+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:30:03.961121+0000","last_peered":"2023-09-30T09:30:03.961121+0000","last_clean":"2023-09-30T09:30:03.961121+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:30:03.961121+0000","last_undegraded":"2023-09-30T09:30:03.961121+0000","last_fullsized":"2023-09-30T09:30:03.961121+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"248'3924","reported_seq":6860,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:13.459600+0000","last_change":"2023-09-30T09:29:35.026309+0000","last_active":"2023-09-30T09:30:13.459600+0000","last_peered":"2023-09-30T09:30:13.459600+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:35.024657+0000","last_became_peered":"2023-09-30T09:29:35.024657+0000","last_unstale":"2023-09-30T09:30:13.459600+0000","last_undegraded":"2023-09-30T09:29:35.010418+0000","last_fullsized":"2023-09-30T09:29:35.009540+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":3924,"log_dups_size":0,"ondisk_log_size":3924,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75104256,"num_objects":1146,"num_object_clones":0,"num_object_copies":3438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":926,"num_write_kb":43104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":276},{"shards":"3,6","objects":870}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":670,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980809+0000","last_change":"2023-09-30T09:29:12.634588+0000","last_active":"2023-09-30T09:30:03.980809+0000","last_peered":"2023-09-30T09:30:03.980809+0000","last_clean":"2023-09-30T09:30:03.980809+0000","last_became_active":"2023-09-30T09:29:12.634225+0000","last_became_peered":"2023-09-30T09:29:12.634225+0000","last_unstale":"2023-09-30T09:30:03.980809+0000","last_undegraded":"2023-09-30T09:30:03.980809+0000","last_fullsized":"2023-09-30T09:30:03.980809+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:22.392203+0000","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":234,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:08.539805+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:30:08.539805+0000","last_peered":"2023-09-30T09:30:08.539805+0000","last_clean":"2023-09-30T09:30:08.539805+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:30:08.539805+0000","last_undegraded":"2023-09-30T09:30:08.539805+0000","last_fullsized":"2023-09-30T09:30:08.539805+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1197,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968894+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:30:03.968894+0000","last_peered":"2023-09-30T09:30:03.968894+0000","last_clean":"2023-09-30T09:30:03.968894+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:30:03.968894+0000","last_undegraded":"2023-09-30T09:30:03.968894+0000","last_fullsized":"2023-09-30T09:30:03.968894+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":233,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.964116+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:30:03.964116+0000","last_peered":"2023-09-30T09:30:03.964116+0000","last_clean":"2023-09-30T09:30:03.964116+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:30:03.964116+0000","last_undegraded":"2023-09-30T09:30:03.964116+0000","last_fullsized":"2023-09-30T09:30:03.964116+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"248'3988","reported_seq":8157,"reported_epoch":248,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:30:13.965096+0000","last_change":"2023-09-30T09:29:36.003902+0000","last_active":"2023-09-30T09:30:13.965096+0000","last_peered":"2023-09-30T09:30:13.965096+0000","last_clean":"2023-09-30T09:29:33.909482+0000","last_became_active":"2023-09-30T09:29:35.026292+0000","last_became_peered":"2023-09-30T09:29:35.026292+0000","last_unstale":"2023-09-30T09:30:13.965096+0000","last_undegraded":"2023-09-30T09:30:13.965096+0000","last_fullsized":"2023-09-30T09:30:13.965096+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":3988,"log_dups_size":0,"ondisk_log_size":3988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76808192,"num_objects":1172,"num_object_clones":0,"num_object_copies":3516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":989,"num_write_kb":46528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65765376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":["7","1","2","6"],"object_location_counts":[{"shards":"1,6,7","objects":1172}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":647,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.964706+0000","last_change":"2023-09-30T09:28:26.894367+0000","last_active":"2023-09-30T09:30:03.964706+0000","last_peered":"2023-09-30T09:30:03.964706+0000","last_clean":"2023-09-30T09:30:03.964706+0000","last_became_active":"2023-09-30T09:28:09.227189+0000","last_became_peered":"2023-09-30T09:28:09.227189+0000","last_unstale":"2023-09-30T09:30:03.964706+0000","last_undegraded":"2023-09-30T09:30:03.964706+0000","last_fullsized":"2023-09-30T09:30:03.964706+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:17.282328+0000","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":525,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.968433+0000","last_change":"2023-09-30T09:29:09.643760+0000","last_active":"2023-09-30T09:30:03.968433+0000","last_peered":"2023-09-30T09:30:03.968433+0000","last_clean":"2023-09-30T09:30:03.968433+0000","last_became_active":"2023-09-30T09:29:09.620803+0000","last_became_peered":"2023-09-30T09:29:09.620803+0000","last_unstale":"2023-09-30T09:30:03.968433+0000","last_undegraded":"2023-09-30T09:30:03.968433+0000","last_fullsized":"2023-09-30T09:30:03.968433+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:54:49.883760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.969049+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:30:03.969049+0000","last_peered":"2023-09-30T09:30:03.969049+0000","last_clean":"2023-09-30T09:30:03.969049+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:30:03.969049+0000","last_undegraded":"2023-09-30T09:30:03.969049+0000","last_fullsized":"2023-09-30T09:30:03.969049+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":189,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.979457+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:30:03.979457+0000","last_peered":"2023-09-30T09:30:03.979457+0000","last_clean":"2023-09-30T09:30:03.979457+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:30:03.979457+0000","last_undegraded":"2023-09-30T09:30:03.979457+0000","last_fullsized":"2023-09-30T09:30:03.979457+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1e","version":"248'5796","reported_seq":6964,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:13.834491+0000","last_change":"2023-09-30T09:29:35.080846+0000","last_active":"2023-09-30T09:30:13.834491+0000","last_peered":"2023-09-30T09:30:13.834491+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:35.061073+0000","last_became_peered":"2023-09-30T09:29:35.061073+0000","last_unstale":"2023-09-30T09:30:13.834491+0000","last_undegraded":"2023-09-30T09:29:35.049064+0000","last_fullsized":"2023-09-30T09:30:13.834491+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5796,"log_dups_size":0,"ondisk_log_size":5796,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150667264,"num_objects":2299,"num_object_clones":0,"num_object_copies":6897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1428,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3397,"num_write_kb":182272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":550,"num_bytes_recovered":36012032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1428},{"shards":"1,4,7","objects":871}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":106,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.980306+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:30:03.980306+0000","last_peered":"2023-09-30T09:30:03.980306+0000","last_clean":"2023-09-30T09:30:03.980306+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:30:03.980306+0000","last_undegraded":"2023-09-30T09:30:03.980306+0000","last_fullsized":"2023-09-30T09:30:03.980306+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":159,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.969041+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:30:03.969041+0000","last_peered":"2023-09-30T09:30:03.969041+0000","last_clean":"2023-09-30T09:30:03.969041+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:30:03.969041+0000","last_undegraded":"2023-09-30T09:30:03.969041+0000","last_fullsized":"2023-09-30T09:30:03.969041+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"248'4533","reported_seq":4605,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:14.108467+0000","last_change":"2023-09-30T09:29:10.893253+0000","last_active":"2023-09-30T09:30:14.108467+0000","last_peered":"2023-09-30T09:30:14.108467+0000","last_clean":"2023-09-30T09:29:08.494599+0000","last_became_active":"2023-09-30T09:29:10.886737+0000","last_became_peered":"2023-09-30T09:29:10.886737+0000","last_unstale":"2023-09-30T09:30:14.108467+0000","last_undegraded":"2023-09-30T09:29:10.766629+0000","last_fullsized":"2023-09-30T09:29:10.598246+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4533,"log_dups_size":0,"ondisk_log_size":4533,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154009600,"num_objects":2350,"num_object_clones":0,"num_object_copies":7050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2001,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3432,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1460},{"shards":"3,7","objects":349}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":209,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.984343+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:30:03.984343+0000","last_peered":"2023-09-30T09:30:03.984343+0000","last_clean":"2023-09-30T09:30:03.984343+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:30:03.984343+0000","last_undegraded":"2023-09-30T09:30:03.984343+0000","last_fullsized":"2023-09-30T09:30:03.984343+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":190,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.999823+0000","last_change":"2023-09-30T09:29:09.609752+0000","last_active":"2023-09-30T09:30:03.999823+0000","last_peered":"2023-09-30T09:30:03.999823+0000","last_clean":"2023-09-30T09:30:03.999823+0000","last_became_active":"2023-09-30T09:29:09.609242+0000","last_became_peered":"2023-09-30T09:29:09.609242+0000","last_unstale":"2023-09-30T09:30:03.999823+0000","last_undegraded":"2023-09-30T09:30:03.999823+0000","last_fullsized":"2023-09-30T09:30:03.999823+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:31.366509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"248'4504","reported_seq":4574,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:14.094158+0000","last_change":"2023-09-30T09:29:10.864651+0000","last_active":"2023-09-30T09:30:14.094158+0000","last_peered":"2023-09-30T09:30:14.094158+0000","last_clean":"2023-09-30T09:29:08.532983+0000","last_became_active":"2023-09-30T09:29:10.860470+0000","last_became_peered":"2023-09-30T09:29:10.860470+0000","last_unstale":"2023-09-30T09:30:14.094158+0000","last_undegraded":"2023-09-30T09:29:10.638243+0000","last_fullsized":"2023-09-30T09:29:10.593564+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4504,"log_dups_size":0,"ondisk_log_size":4504,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152731648,"num_objects":2331,"num_object_clones":0,"num_object_copies":6993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3391,"num_write_kb":183072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1527},{"shards":"3,7","objects":274}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":192,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.985870+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:30:03.985870+0000","last_peered":"2023-09-30T09:30:03.985870+0000","last_clean":"2023-09-30T09:30:03.985870+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:30:03.985870+0000","last_undegraded":"2023-09-30T09:30:03.985870+0000","last_fullsized":"2023-09-30T09:30:03.985870+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":221,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.960027+0000","last_change":"2023-09-30T09:29:09.621218+0000","last_active":"2023-09-30T09:30:03.960027+0000","last_peered":"2023-09-30T09:30:03.960027+0000","last_clean":"2023-09-30T09:30:03.960027+0000","last_became_active":"2023-09-30T09:29:09.620972+0000","last_became_peered":"2023-09-30T09:29:09.620972+0000","last_unstale":"2023-09-30T09:30:03.960027+0000","last_undegraded":"2023-09-30T09:30:03.960027+0000","last_fullsized":"2023-09-30T09:30:03.960027+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:25.640288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1b","version":"248'5851","reported_seq":5920,"reported_epoch":248,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:30:13.529884+0000","last_change":"2023-09-30T09:29:35.093036+0000","last_active":"2023-09-30T09:30:13.529884+0000","last_peered":"2023-09-30T09:30:13.529884+0000","last_clean":"2023-09-30T09:29:33.992127+0000","last_became_active":"2023-09-30T09:29:35.070155+0000","last_became_peered":"2023-09-30T09:29:35.070155+0000","last_unstale":"2023-09-30T09:30:13.529884+0000","last_undegraded":"2023-09-30T09:30:13.529884+0000","last_fullsized":"2023-09-30T09:30:13.529884+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5851,"log_dups_size":0,"ondisk_log_size":5851,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156991488,"num_objects":2396,"num_object_clones":0,"num_object_copies":7188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3384,"num_write_kb":184928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":2396}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":228,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.960574+0000","last_change":"2023-09-30T09:29:09.615385+0000","last_active":"2023-09-30T09:30:03.960574+0000","last_peered":"2023-09-30T09:30:03.960574+0000","last_clean":"2023-09-30T09:30:03.960574+0000","last_became_active":"2023-09-30T09:29:09.614479+0000","last_became_peered":"2023-09-30T09:29:09.614479+0000","last_unstale":"2023-09-30T09:30:03.960574+0000","last_undegraded":"2023-09-30T09:30:03.960574+0000","last_fullsized":"2023-09-30T09:30:03.960574+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:14.671222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":258,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.959326+0000","last_change":"2023-09-30T09:29:09.600457+0000","last_active":"2023-09-30T09:30:03.959326+0000","last_peered":"2023-09-30T09:30:03.959326+0000","last_clean":"2023-09-30T09:30:03.959326+0000","last_became_active":"2023-09-30T09:29:09.600080+0000","last_became_peered":"2023-09-30T09:29:09.600080+0000","last_unstale":"2023-09-30T09:30:03.959326+0000","last_undegraded":"2023-09-30T09:30:03.959326+0000","last_fullsized":"2023-09-30T09:30:03.959326+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:10:02.250349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1a","version":"248'5936","reported_seq":5996,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-30T09:30:14.089656+0000","last_change":"2023-09-30T09:29:35.021535+0000","last_active":"2023-09-30T09:30:14.089656+0000","last_peered":"2023-09-30T09:30:14.089656+0000","last_clean":"2023-09-30T09:30:14.089656+0000","last_became_active":"2023-09-30T09:29:35.014808+0000","last_became_peered":"2023-09-30T09:29:35.014808+0000","last_unstale":"2023-09-30T09:30:14.089656+0000","last_undegraded":"2023-09-30T09:30:14.089656+0000","last_fullsized":"2023-09-30T09:30:14.089656+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5936,"log_dups_size":0,"ondisk_log_size":5936,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151060480,"num_objects":2305,"num_object_clones":0,"num_object_copies":6915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3489,"num_write_kb":185408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":131,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.985668+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:30:03.985668+0000","last_peered":"2023-09-30T09:30:03.985668+0000","last_clean":"2023-09-30T09:30:03.985668+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:30:03.985668+0000","last_undegraded":"2023-09-30T09:30:03.985668+0000","last_fullsized":"2023-09-30T09:30:03.985668+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":206,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-30T09:30:03.979213+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:30:03.979213+0000","last_peered":"2023-09-30T09:30:03.979213+0000","last_clean":"2023-09-30T09:30:03.979213+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:30:03.979213+0000","last_undegraded":"2023-09-30T09:30:03.979213+0000","last_fullsized":"2023-09-30T09:30:03.979213+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":36,"stat_sum":{"num_bytes":4961206296,"num_objects":75704,"num_object_clones":0,"num_object_copies":227112,"num_objects_missing_on_primary":892,"num_objects_missing":892,"num_objects_degraded":20607,"num_objects_misplaced":4634,"num_objects_unfound":0,"num_objects_dirty":75704,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":92863,"num_write_kb":4876353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12860,"num_bytes_recovered":842792960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15860903936,"data_stored":15860891720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":175067,"ondisk_log_size":175067,"up":108,"acting":101,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":1418,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":435019,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":43,"num_write_kb":438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234223,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2346564,"kb_used_data":2187580,"kb_used_omap":0,"kb_used_meta":158976,"kb_avail":91398588,"statfs":{"total":95995035648,"available":93592154112,"internally_reserved":0,"allocated":2240081920,"data_stored":2239162561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":162791424},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:27 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.5980000000000001,"15min":1.0149999999999999},"min":{"1min":0.30099999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.839,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.4770000000000001,"15min":0.93700000000000006},"min":{"1min":0.29499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.8949999999999996,"5min":18.097999999999999,"15min":18.097999999999999},"last":0.40300000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:29:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":3.8029999999999999,"15min":2.2669999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.944,"5min":70.566999999999993,"15min":82.025999999999996},"last":63.616999999999997},{"interface":"front","average":{"1min":1.1950000000000001,"5min":3.746,"15min":2.242},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.6580000000000004,"5min":70.626000000000005,"15min":82.012},"last":63.597000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:29:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7200000000000002,"5min":2.0009999999999999,"15min":1.145},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":39.445,"5min":39.445,"15min":39.445},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.9870000000000001,"15min":1.2190000000000001},"min":{"1min":0.39700000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":40.988999999999997,"5min":40.988999999999997,"15min":40.988999999999997},"last":0.47299999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:29:43 2023","interfaces":[{"interface":"back","average":{"1min":5.298,"5min":3.552,"15min":1.6659999999999999},"min":{"1min":0.371,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":76.909000000000006,"5min":76.909000000000006,"15min":76.909000000000006},"last":0.38700000000000001},{"interface":"front","average":{"1min":5.0410000000000004,"5min":3.262,"15min":1.5509999999999999},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":77.048000000000002,"5min":77.048000000000002,"15min":77.048000000000002},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":1.155,"15min":0.71899999999999997},"min":{"1min":0.318,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.75,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.436},{"interface":"front","average":{"1min":1.335,"5min":0.97599999999999998,"15min":0.69699999999999995},"min":{"1min":0.314,"5min":0.251,"15min":0.251},"max":{"1min":5.6879999999999997,"5min":6.282,"15min":6.282},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.2070000000000001,"15min":0.79100000000000004},"min":{"1min":0.38100000000000001,"5min":0.248,"15min":0.248},"max":{"1min":7.8490000000000002,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.579,"5min":1.325,"15min":0.84699999999999998},"min":{"1min":0.443,"5min":0.30199999999999999,"15min":0.28599999999999998},"max":{"1min":7.6909999999999998,"5min":11.022,"15min":11.022},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.234,"15min":0.82899999999999996},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33000000000000002},"max":{"1min":7.7210000000000001,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.503},{"interface":"front","average":{"1min":1.492,"5min":1.0660000000000001,"15min":0.76900000000000002},"min":{"1min":0.34699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.4329999999999998,"5min":6.4329999999999998,"15min":6.4329999999999998},"last":0.56899999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495860,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2322180,"kb_used_data":2172284,"kb_used_omap":0,"kb_used_meta":149888,"kb_avail":91422972,"statfs":{"total":95995035648,"available":93617123328,"internally_reserved":0,"allocated":2224418816,"data_stored":2223499898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":153485312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":2.2829999999999999,"15min":1.242},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.222},"max":{"1min":2.4119999999999999,"5min":45.927999999999997,"15min":45.927999999999997},"last":0.25},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.581,"15min":1.3520000000000001},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.23499999999999999},"max":{"1min":1.6899999999999999,"5min":45.316000000000003,"15min":45.316000000000003},"last":0.45200000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.8399999999999999,"15min":1.444},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":37.988,"5min":69.150000000000006,"15min":69.150000000000006},"last":0.35199999999999998},{"interface":"front","average":{"1min":3.0259999999999998,"5min":3.2240000000000002,"15min":1.5649999999999999},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":37.970999999999997,"5min":69.132000000000005,"15min":69.132000000000005},"last":0.42199999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":2.496,"15min":1.3400000000000001},"min":{"1min":0.313,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":3.871,"5min":37.978000000000002,"15min":37.978000000000002},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.9359999999999999,"15min":1.167},"min":{"1min":0.377,"5min":0.31,"15min":0.31},"max":{"1min":1.4390000000000001,"5min":37.759999999999998,"15min":37.759999999999998},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":1.716,"15min":1.054},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.165,"5min":20.818999999999999,"15min":20.818999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.9430000000000001,"15min":1.169},"min":{"1min":0.36499999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.6439999999999999,"5min":21.015000000000001,"15min":21.015000000000001},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.921,"15min":1.216},"min":{"1min":0.26700000000000002,"5min":0.25700000000000001,"15min":0.22700000000000001},"max":{"1min":1.72,"5min":49.286999999999999,"15min":73.656000000000006},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.505,"5min":1.5169999999999999,"15min":1.1120000000000001},"min":{"1min":0.32400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.1100000000000001,"5min":49.252000000000002,"15min":73.882999999999996},"last":0.443}]},{"osd":5,"last update":"Sat Sep 30 09:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.6439999999999999,"15min":0.93400000000000005},"min":{"1min":0.32400000000000001,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":2.5750000000000002,"5min":47.408999999999999,"15min":47.408999999999999},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.528,"15min":0.93999999999999995},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.311},"max":{"1min":2.746,"5min":47.344000000000001,"15min":47.344000000000001},"last":1.016}]},{"osd":7,"last update":"Sat Sep 30 09:30:10 2023","interfaces":[{"interface":"back","average":{"1min":11.069000000000001,"5min":4.0069999999999997,"15min":1.752},"min":{"1min":0.372,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":91.120999999999995,"5min":91.120999999999995,"15min":91.120999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":10.925000000000001,"5min":3.851,"15min":1.7889999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":91.733999999999995,"5min":91.733999999999995,"15min":91.733999999999995},"last":0.505}]}]},{"osd":1,"up_from":99,"seq":425201762424,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2211920,"kb_used_data":1995656,"kb_used_omap":5,"kb_used_meta":216250,"kb_avail":91533232,"statfs":{"total":95995035648,"available":93730029568,"internally_reserved":0,"allocated":2043551744,"data_stored":2042162284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":221440182},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":5.1429999999999998,"15min":2.2749999999999999},"min":{"1min":0.23799999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.4400000000000004,"5min":150.93899999999999,"15min":150.93899999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.762,"5min":5.0979999999999999,"15min":2.3279999999999998},"min":{"1min":0.28999999999999998,"5min":0.216,"15min":0.216},"max":{"1min":7.9470000000000001,"5min":156.74199999999999,"15min":156.74199999999999},"last":1.2450000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:30:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":7.3390000000000004,"15min":2.7959999999999998},"min":{"1min":0.36799999999999999,"5min":0.247,"15min":0.247},"max":{"1min":6.2569999999999997,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.374},{"interface":"front","average":{"1min":1.974,"5min":7.2599999999999998,"15min":2.7959999999999998},"min":{"1min":0.32700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.0030000000000001,"5min":144.62,"15min":144.62},"last":0.433}]},{"osd":3,"last update":"Sat Sep 30 09:29:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":1.8759999999999999,"15min":0.96399999999999997},"min":{"1min":0.26400000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.1109999999999998,"5min":18.757999999999999,"15min":18.757999999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.5509999999999999,"15min":0.879},"min":{"1min":0.35099999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.9809999999999999,"5min":11.173,"15min":11.173},"last":0.73299999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.484,"5min":1.286,"15min":0.91900000000000004},"min":{"1min":0.441,"5min":0.317,"15min":0.28299999999999997},"max":{"1min":8.3900000000000006,"5min":10.173,"15min":10.173},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.4019999999999999,"15min":0.94999999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.0700000000000003,"5min":8.9299999999999997,"15min":8.9299999999999997},"last":2.1789999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":1.3720000000000001,"15min":0.91100000000000003},"min":{"1min":0.45400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.678000000000001,"5min":12.678000000000001,"15min":12.678000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.794,"5min":1.4830000000000001,"15min":0.96899999999999997},"min":{"1min":0.432,"5min":0.35499999999999998,"15min":0.33300000000000002},"max":{"1min":10.128,"5min":10.128,"15min":10.128},"last":0.59799999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":1.5369999999999999,"15min":0.96899999999999997},"min":{"1min":0.44700000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":10.058,"5min":14.356999999999999,"15min":14.356999999999999},"last":0.625},{"interface":"front","average":{"1min":1.881,"5min":1.6479999999999999,"15min":1.0600000000000001},"min":{"1min":0.39700000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.399,"5min":15.864000000000001,"15min":15.864000000000001},"last":4.5339999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:29:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":1.8420000000000001,"15min":1.145},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.103,"5min":14.403,"15min":14.403},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.538,"5min":2.9729999999999999,"15min":1.5369999999999999},"min":{"1min":0.42499999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.6589999999999998,"5min":122.56699999999999,"15min":122.56699999999999},"last":0.503}]}]},{"osd":0,"up_from":94,"seq":403726925957,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":842896,"kb_used_data":605960,"kb_used_omap":5,"kb_used_meta":236922,"kb_avail":92902256,"statfs":{"total":95995035648,"available":95131910144,"internally_reserved":0,"allocated":620503040,"data_stored":619057681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":242608381},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":2.8759999999999999,"15min":1.3440000000000001},"min":{"1min":0.22,"5min":0.188,"15min":0.188},"max":{"1min":15.513,"5min":125.024,"15min":125.024},"last":54.314999999999998},{"interface":"front","average":{"1min":1.032,"5min":2.706,"15min":1.3089999999999999},"min":{"1min":0.20899999999999999,"5min":0.192,"15min":0.192},"max":{"1min":2.827,"5min":124.901,"15min":124.901},"last":53.536999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.968,"15min":1.1120000000000001},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":4.3460000000000001,"5min":55.823999999999998,"15min":55.823999999999998},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.74199999999999999,"5min":2.169,"15min":1.198},"min":{"1min":0.26200000000000001,"5min":0.245,"15min":0.245},"max":{"1min":2.855,"5min":55.790999999999997,"15min":55.790999999999997},"last":0.33500000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:29:42 2023","interfaces":[{"interface":"back","average":{"1min":3.6419999999999999,"5min":2.698,"15min":1.3320000000000001},"min":{"1min":0.28799999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":42.917000000000002,"5min":55.68,"15min":55.68},"last":0.36199999999999999},{"interface":"front","average":{"1min":3.8399999999999999,"5min":2.48,"15min":1.2589999999999999},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":43.023000000000003,"5min":55.244,"15min":55.244},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":2.4039999999999999,"15min":1.349},"min":{"1min":0.39500000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.863,"5min":121.486,"15min":121.486},"last":0.626},{"interface":"front","average":{"1min":0.91200000000000003,"5min":2.8450000000000002,"15min":1.538},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.31},"max":{"1min":2.2050000000000001,"5min":121.44499999999999,"15min":121.44499999999999},"last":0.54400000000000004}]},{"osd":5,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.5,"15min":1.173},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.738,"5min":15.162000000000001,"15min":15.162000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.4059999999999999,"5min":1.45,"15min":1.159},"min":{"1min":0.38600000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.0270000000000001,"5min":13.766,"15min":13.766},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":1.3340000000000001,"15min":1.03},"min":{"1min":0.374,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.9960000000000004,"5min":13.811999999999999,"15min":13.811999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.25,"5min":1.242,"15min":1.002},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":2.7799999999999998,"5min":6.407,"15min":7.556},"last":0.60299999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.8279999999999998,"15min":1.53},"min":{"1min":0.41999999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.9640000000000004,"5min":127.199,"15min":127.199},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.45,"5min":2.6139999999999999,"15min":1.456},"min":{"1min":0.44600000000000001,"5min":0.371,"15min":0.371},"max":{"1min":4.3040000000000003,"5min":118.551,"15min":118.551},"last":0.64600000000000002}]}]},{"osd":2,"up_from":0,"seq":446676598898,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435370,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3629544,"kb_used_data":3371040,"kb_used_omap":2,"kb_used_meta":258493,"kb_avail":90115608,"statfs":{"total":95995035648,"available":92278382592,"internally_reserved":0,"allocated":3451944960,"data_stored":3450530213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":264697191},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,3,0,1,0,0,3,5,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.492,"15min":1.4690000000000001},"min":{"1min":0.20899999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.133,"5min":10.856,"15min":130.946},"last":1.651},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.4299999999999999,"15min":1.4470000000000001},"min":{"1min":0.224,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.532,"5min":12.361000000000001,"15min":130.96899999999999},"last":0.503}]},{"osd":1,"last update":"Sat Sep 30 09:29:49 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":1.734,"15min":0.92800000000000005},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.224},"max":{"1min":11.196,"5min":13.039999999999999,"15min":13.039999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.81699999999999995,"5min":1.641,"15min":0.89000000000000001},"min":{"1min":0.28299999999999997,"5min":0.221,"15min":0.221},"max":{"1min":4.5190000000000001,"5min":14.057,"15min":14.057},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:29:49 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.4420000000000002,"15min":1.869},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26800000000000002},"max":{"1min":29.986000000000001,"5min":29.986000000000001,"15min":145.75899999999999},"last":1.7709999999999999},{"interface":"front","average":{"1min":2.8780000000000001,"5min":2.1760000000000002,"15min":1.796},"min":{"1min":0.28000000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":30.242999999999999,"5min":30.242999999999999,"15min":150.358},"last":1.7270000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:29:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":1.256,"15min":0.86799999999999999},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":15.960000000000001,"5min":15.960000000000001,"15min":15.960000000000001},"last":1.7569999999999999},{"interface":"front","average":{"1min":1.498,"5min":1.5880000000000001,"15min":0.97999999999999998},"min":{"1min":0.29799999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":15.638,"5min":15.638,"15min":15.638},"last":0.57299999999999995}]},{"osd":5,"last update":"Sat Sep 30 09:29:49 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.619,"15min":1.038},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.308},"max":{"1min":7.3609999999999998,"5min":10.048999999999999,"15min":10.048999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.8200000000000001,"15min":1.1200000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.33500000000000002},"max":{"1min":4.6040000000000001,"5min":12.785,"15min":12.785},"last":0.67500000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:29:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.339,"15min":0.93000000000000005},"min":{"1min":0.376,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.2359999999999998,"5min":10.348000000000001,"15min":10.348000000000001},"last":3.3849999999999998},{"interface":"front","average":{"1min":1.272,"5min":1.768,"15min":1.1000000000000001},"min":{"1min":0.46600000000000003,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.3120000000000003,"5min":14.69,"15min":14.69},"last":0.53900000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:29:49 2023","interfaces":[{"interface":"back","average":{"1min":12.000999999999999,"5min":3.9169999999999998,"15min":1.7889999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":226.03999999999999,"5min":226.03999999999999,"15min":226.03999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":12.202999999999999,"5min":3.5070000000000001,"15min":1.651},"min":{"1min":0.42199999999999999,"5min":0.377,"15min":0.372},"max":{"1min":226.06100000000001,"5min":226.06100000000001,"15min":226.06100000000001},"last":1.794}]}]},{"osd":4,"up_from":30,"seq":128849019133,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1519360,"kb_used_data":1365752,"kb_used_omap":0,"kb_used_meta":153536,"kb_avail":92225792,"statfs":{"total":95995035648,"available":94439211008,"internally_reserved":0,"allocated":1398530048,"data_stored":1397615283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":157220864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":4.6929999999999996,"15min":2.4929999999999999},"min":{"1min":0.25700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.6589999999999998,"5min":119.986,"15min":128.79300000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":4.5350000000000001,"15min":2.371},"min":{"1min":0.253,"5min":0.253,"15min":0.248},"max":{"1min":2.3679999999999999,"5min":119.863,"15min":128.38800000000001},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.9630000000000001,"15min":1.54},"min":{"1min":0.28799999999999998,"5min":0.245,"15min":0.245},"max":{"1min":27.440999999999999,"5min":93.244,"15min":93.244},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.117,"15min":1.597},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.23999999999999999},"max":{"1min":27.489000000000001,"5min":93.465999999999994,"15min":93.465999999999994},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":6.2380000000000004,"5min":5.274,"15min":2.379},"min":{"1min":0.37,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":77.620000000000005,"5min":140.601,"15min":140.601},"last":0.51100000000000001},{"interface":"front","average":{"1min":6.2610000000000001,"5min":5.1459999999999999,"15min":2.327},"min":{"1min":0.35999999999999999,"5min":0.35199999999999998,"15min":0.34200000000000003},"max":{"1min":77.638999999999996,"5min":140.54900000000001,"15min":140.54900000000001},"last":0.501}]},{"osd":3,"last update":"Sat Sep 30 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":15.814,"5min":5.1239999999999997,"15min":2.4319999999999999},"min":{"1min":0.34200000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":281.42200000000003,"5min":281.42200000000003,"15min":281.42200000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":16.393000000000001,"5min":5.1680000000000001,"15min":2.452},"min":{"1min":0.35999999999999999,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":281.44999999999999,"5min":281.44999999999999,"15min":281.44999999999999},"last":0.45000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.5569999999999999,"15min":0.84799999999999998},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.3940000000000001,"5min":86.691000000000003,"15min":86.691000000000003},"last":134.30600000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.7969999999999999,"15min":0.96599999999999997},"min":{"1min":0.28299999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5169999999999999,"5min":86.822000000000003,"15min":86.822000000000003},"last":134.35300000000001}]},{"osd":6,"last update":"Sat Sep 30 09:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.876,"15min":0.65600000000000003},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.27200000000000002},"max":{"1min":3.0350000000000001,"5min":5.6310000000000002,"15min":5.6310000000000002},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.0700000000000001,"15min":0.749},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":8.8049999999999997,"5min":8.8049999999999997,"15min":8.8049999999999997},"last":0.38400000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.992,"15min":1.075},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.391,"5min":86.728999999999999,"15min":86.728999999999999},"last":136.27099999999999},{"interface":"front","average":{"1min":0.626,"5min":1.1259999999999999,"15min":0.79600000000000004},"min":{"1min":0.35499999999999998,"5min":0.32600000000000001,"15min":0.314},"max":{"1min":2.3700000000000001,"5min":8.5869999999999997,"15min":8.5869999999999997},"last":136.60400000000001}]}]},{"osd":5,"up_from":38,"seq":163208757497,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1978864,"kb_used_data":1817000,"kb_used_omap":0,"kb_used_meta":161856,"kb_avail":91766288,"statfs":{"total":95995035648,"available":93968678912,"internally_reserved":0,"allocated":1860608000,"data_stored":1859704457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":165740544},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:29:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":4.2439999999999998,"15min":1.8640000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.665,"5min":148.79900000000001,"15min":148.79900000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.3340000000000001,"5min":4.8499999999999996,"15min":2.1070000000000002},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.4169999999999998,"5min":148.779,"15min":148.779},"last":0.30199999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:29:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9089999999999998,"5min":4.2809999999999997,"15min":2.2709999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":38.487000000000002,"5min":166.708,"15min":166.708},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.9009999999999998,"5min":4.1609999999999996,"15min":2.2309999999999999},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":38.512,"5min":166.607,"15min":166.607},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":4.931,"15min":2.601},"min":{"1min":0.372,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.4210000000000003,"5min":165.768,"15min":165.768},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":6.202,"15min":3.105},"min":{"1min":0.33700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5779999999999998,"5min":166.24000000000001,"15min":166.24000000000001},"last":1.0800000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:29:44 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.9590000000000001,"15min":1.256},"min":{"1min":0.35099999999999998,"5min":0.311,"15min":0.311},"max":{"1min":20.079000000000001,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.443},{"interface":"front","average":{"1min":1.919,"5min":1.673,"15min":1.111},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":20.021000000000001,"5min":28.039999999999999,"15min":28.039999999999999},"last":0.501}]},{"osd":4,"last update":"Sat Sep 30 09:29:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.89500000000000002,"15min":0.64200000000000002},"min":{"1min":0.32800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":4.3979999999999997,"5min":8.5359999999999996,"15min":8.5359999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.95299999999999996,"15min":0.71799999999999997},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.5099999999999998,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":0.55000000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.27,"15min":0.80200000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.5179999999999998,"5min":14.23,"15min":14.23},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.4339999999999999,"15min":0.94999999999999996},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28199999999999997},"max":{"1min":4.1600000000000001,"5min":18.960000000000001,"15min":18.960000000000001},"last":0.38400000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.3660000000000001,"15min":0.90400000000000003},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.665,"5min":19.175999999999998,"15min":19.175999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.141,"15min":0.81399999999999995},"min":{"1min":0.371,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.6449999999999996,"5min":8.8810000000000002,"15min":8.8810000000000002},"last":1.5980000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":609746944,"data_stored":609746944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2027753472,"data_stored":2027749400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2099183616,"data_stored":2099183616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3443392512,"data_stored":3443392512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1386348544,"data_stored":1386348544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1850216448,"data_stored":1850212376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2214137856,"data_stored":2214133784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2230124544,"data_stored":2230124544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:30:17.929 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:30:17.930 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-09-30T09:30:18.004 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:18.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:18.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:18.112 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:18.136 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:30:18.136 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:30:18.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.136 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35130.0:112192 6.a e248/238 6:56bdd41c:::benchmark_data_smithi019_355417_object19888:head v 248'5939, mlcod=248'5937)", 2023-09-30T09:30:18.136 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:18.132632+0000", 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0014957869999999999, 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015219470000000001, 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:18.132632+0000", 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:18.132632+0000", 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:18.132633+0000", 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.004e-06 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:18.132641+0000", 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6899999999999992e-06 2023-09-30T09:30:18.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:18.132660+0000", 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8788e-05 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:18.132670+0000", 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0101999999999999e-05 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:18.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:18.132749+0000", 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9499999999999994e-05 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:18.132775+0000", 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6295000000000002e-05 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:18.140 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2023-09-30T09:30:18.141 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 1 2023-09-30T09:30:18.141 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:30:18.205 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:18.446 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:18 smithi019 ceph-mon[216815]: pgmap v588: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 20479/225990 objects degraded (9.062%); 4634/225990 objects misplaced (2.051%); 12 MiB/s, 199 objects/s recovering 2023-09-30T09:30:18.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:18.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:18.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:18 smithi019 ceph-mon[224043]: pgmap v588: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 20479/225990 objects degraded (9.062%); 4634/225990 objects misplaced (2.051%); 12 MiB/s, 199 objects/s recovering 2023-09-30T09:30:18.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:18.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:18.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:18 smithi167 ceph-mon[192880]: pgmap v588: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 20479/225990 objects degraded (9.062%); 4634/225990 objects misplaced (2.051%); 12 MiB/s, 199 objects/s recovering 2023-09-30T09:30:18.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:18.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20479/225990 objects degraded (9.062%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20479/225990 objects degraded (9.062%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20479/225990 objects degraded (9.062%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:19.714 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:19.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:19.821 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:19.829 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:30:19.854 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:19.918 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:19.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:20.029 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:20.102 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:20.241 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:20.242 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:20.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:20.315 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:30:20.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:20 smithi019 ceph-mon[216815]: pgmap v589: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 20005/221421 objects degraded (9.035%); 4409/221421 objects misplaced (1.991%); 13 MiB/s, 205 objects/s recovering 2023-09-30T09:30:20.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1841473998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:20 smithi019 ceph-mon[224043]: pgmap v589: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 20005/221421 objects degraded (9.035%); 4409/221421 objects misplaced (1.991%); 13 MiB/s, 205 objects/s recovering 2023-09-30T09:30:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1841473998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:20.352 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:20.385 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:30:20.426 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:30:20.427 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:20.469 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:30:20.514 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:30:20.547 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:30:20.581 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:20.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:20.585 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:30:20.627 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:30:20.659 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:30:20.659 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:30:20.659 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:30:20.659 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:30:20.659 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:30:20.659 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:30:20.659 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:30:20.660 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:30:20.660 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:30:20.686 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:20.726 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:20 smithi167 ceph-mon[192880]: pgmap v589: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 20005/221421 objects degraded (9.035%); 4409/221421 objects misplaced (1.991%); 13 MiB/s, 205 objects/s recovering 2023-09-30T09:30:20.726 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1841473998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:20.729 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:20.818 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:20.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:20.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:20.931 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:20.995 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:21.042 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:21.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:21.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:21.148 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:21.191 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:21.573 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:21.574 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1841473998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:21.574 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[224043]: osdmap e249: 8 total, 8 up, 7 in 2023-09-30T09:30:21.574 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[224043]: failed to encode map e249 with expected crc 2023-09-30T09:30:21.574 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[224043]: failed to encode map e249 with expected crc 2023-09-30T09:30:21.574 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3503800928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:21.574 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:21.574 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1841473998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:21.574 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[216815]: osdmap e249: 8 total, 8 up, 7 in 2023-09-30T09:30:21.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[216815]: failed to encode map e249 with expected crc 2023-09-30T09:30:21.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[216815]: failed to encode map e249 with expected crc 2023-09-30T09:30:21.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3503800928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:21 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1841473998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:21 smithi167 ceph-mon[192880]: osdmap e249: 8 total, 8 up, 7 in 2023-09-30T09:30:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:21 smithi167 ceph-mon[192880]: failed to encode map e249 with expected crc 2023-09-30T09:30:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:21 smithi167 ceph-mon[192880]: failed to encode map e249 with expected crc 2023-09-30T09:30:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3503800928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:21.906 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:21.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:21.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:22.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:22.049 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:22.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:22 smithi019 ceph-mon[216815]: pgmap v591: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 19668/216114 objects degraded (9.101%); 4304/216114 objects misplaced (1.992%); 13 MiB/s, 204 objects/s recovering 2023-09-30T09:30:22.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:22 smithi019 ceph-mon[224043]: pgmap v591: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 19668/216114 objects degraded (9.101%); 4304/216114 objects misplaced (1.992%); 13 MiB/s, 204 objects/s recovering 2023-09-30T09:30:22.620 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:22.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:22.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:22.726 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:22.766 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:22 smithi167 ceph-mon[192880]: pgmap v591: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 19668/216114 objects degraded (9.101%); 4304/216114 objects misplaced (1.992%); 13 MiB/s, 204 objects/s recovering 2023-09-30T09:30:22.769 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:22.833 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:23.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:23.364 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:23.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:23.464 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:23.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 19668/216114 objects degraded (9.101%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:23.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 19668/216114 objects degraded (9.101%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:23.514 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:23.677 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:23.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:23.764 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:30:23.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 19668/216114 objects degraded (9.101%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:23.783 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:23.872 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:30:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:30:23] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-09-30T09:30:24.157 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:24.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:24.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:24.266 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:24.312 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:24.410 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:24.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:24.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:24.516 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:24.545 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:24 smithi019 ceph-mon[216815]: pgmap v592: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 19573/215136 objects degraded (9.098%); 4304/215136 objects misplaced (2.001%); 13 MiB/s, 202 objects/s recovering 2023-09-30T09:30:24.545 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:30:24.545 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:24 smithi019 ceph-mon[224043]: pgmap v592: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 19573/215136 objects degraded (9.098%); 4304/215136 objects misplaced (2.001%); 13 MiB/s, 202 objects/s recovering 2023-09-30T09:30:24.545 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:30:24.552 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:24 smithi167 ceph-mon[192880]: pgmap v592: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 19573/215136 objects degraded (9.098%); 4304/215136 objects misplaced (2.001%); 13 MiB/s, 202 objects/s recovering 2023-09-30T09:30:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:30:24.936 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:24.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:24.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:25.041 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:25.078 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:25.151 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:25.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:25.155 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:25.255 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:25.294 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:25.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:25 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3961704523' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-30T09:30:25.314 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:25 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3961704523' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-30T09:30:25.329 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 1 pg_num to 21 2023-09-30T09:30:25.364 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:30:25.411 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:25.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:25.417 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:25 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3961704523' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-30T09:30:25.518 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:25.574 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:25.665 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:25.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:25.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:25.772 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:25.828 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:25.912 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:25.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:26.025 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:26.069 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:26.139 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:26.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:26.246 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:26.294 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:26.309 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:30:26.309 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:30:26.369 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:26.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:26.475 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:26.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:26 smithi167 ceph-mon[192880]: pgmap v593: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 18793/205326 objects degraded (9.153%); 3969/205326 objects misplaced (1.933%); 13 MiB/s, 205 objects/s recovering 2023-09-30T09:30:26.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3961704523' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-30T09:30:26.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:26 smithi167 ceph-mon[192880]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:26 smithi167 ceph-mon[192880]: osdmap e250: 8 total, 8 up, 7 in 2023-09-30T09:30:26.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:26 smithi167 ceph-mon[192880]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.554 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:26 smithi167 ceph-mon[192880]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.565 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[216815]: pgmap v593: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 18793/205326 objects degraded (9.153%); 3969/205326 objects misplaced (1.933%); 13 MiB/s, 205 objects/s recovering 2023-09-30T09:30:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3961704523' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-30T09:30:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[216815]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[216815]: osdmap e250: 8 total, 8 up, 7 in 2023-09-30T09:30:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[216815]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[216815]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[224043]: pgmap v593: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 18793/205326 objects degraded (9.153%); 3969/205326 objects misplaced (1.933%); 13 MiB/s, 205 objects/s recovering 2023-09-30T09:30:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3961704523' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-30T09:30:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[224043]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[224043]: osdmap e250: 8 total, 8 up, 7 in 2023-09-30T09:30:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[224043]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:26 smithi019 ceph-mon[224043]: failed to encode map e250 with expected crc 2023-09-30T09:30:26.694 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:26.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:26.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:26.800 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:26.848 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:27.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-30T09:30:27.484 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:27.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:27.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-30T09:30:27.588 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:27.654 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-30T09:30:27.660 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:27.846 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:27.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:27.954 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:28.018 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:28.319 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:30:28.338 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:28.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:28.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:28.445 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: pgmap v595: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 18659/204141 objects degraded (9.140%); 3969/204141 objects misplaced (1.944%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:30:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-30T09:30:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: osdmap e251: 8 total, 8 up, 7 in 2023-09-30T09:30:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18659/204141 objects degraded (9.140%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: pgmap v595: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 18659/204141 objects degraded (9.140%); 3969/204141 objects misplaced (1.944%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-30T09:30:28.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: osdmap e251: 8 total, 8 up, 7 in 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:28.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18659/204141 objects degraded (9.140%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:28.585 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:28.728 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: pgmap v595: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 18659/204141 objects degraded (9.140%); 3969/204141 objects misplaced (1.944%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:30:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-30T09:30:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: osdmap e251: 8 total, 8 up, 7 in 2023-09-30T09:30:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: failed to encode map e251 with expected crc 2023-09-30T09:30:28.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:28.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:28.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18659/204141 objects degraded (9.140%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:28.837 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:28.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:28.942 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:28.977 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:29.044 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:29.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:29.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:29.151 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:29.212 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:29.323 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:29.353 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:29.362 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: osdmap e252: 8 total, 8 up, 7 in 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1701244033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:30:29.363 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:30:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:29.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:29.399 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:30:29.465 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:29.506 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:29.564 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:29.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:29.672 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:29.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: osdmap e252: 8 total, 8 up, 7 in 2023-09-30T09:30:29.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1701244033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:30:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:29 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:30:29.716 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: osdmap e252: 8 total, 8 up, 7 in 2023-09-30T09:30:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: failed to encode map e252 with expected crc 2023-09-30T09:30:29.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1701244033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:30:29.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:29 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:30:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:30 smithi167 ceph-mon[192880]: pgmap v598: 129 pgs: 1 active+recovering+undersized+remapped, 5 unknown, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 18178/199503 objects degraded (9.112%); 3720/199503 objects misplaced (1.865%); 17 MiB/s, 271 objects/s recovering 2023-09-30T09:30:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:30 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:30:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:30 smithi167 ceph-mon[192880]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:30 smithi167 ceph-mon[192880]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:30 smithi167 ceph-mon[192880]: osdmap e253: 8 total, 8 up, 7 in 2023-09-30T09:30:30.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:30 smithi167 ceph-mon[192880]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:30 smithi167 ceph-mon[192880]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[216815]: pgmap v598: 129 pgs: 1 active+recovering+undersized+remapped, 5 unknown, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 18178/199503 objects degraded (9.112%); 3720/199503 objects misplaced (1.865%); 17 MiB/s, 271 objects/s recovering 2023-09-30T09:30:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:30:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[216815]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[216815]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[216815]: osdmap e253: 8 total, 8 up, 7 in 2023-09-30T09:30:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[216815]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[216815]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[224043]: pgmap v598: 129 pgs: 1 active+recovering+undersized+remapped, 5 unknown, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 18178/199503 objects degraded (9.112%); 3720/199503 objects misplaced (1.865%); 17 MiB/s, 271 objects/s recovering 2023-09-30T09:30:30.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:30:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[224043]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[224043]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[224043]: osdmap e253: 8 total, 8 up, 7 in 2023-09-30T09:30:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[224043]: failed to encode map e253 with expected crc 2023-09-30T09:30:30.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:30 smithi019 ceph-mon[224043]: failed to encode map e253 with expected crc 2023-09-30T09:30:31.200 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:31.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:31.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:31.305 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:31.388 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:31.898 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:30:31.899 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:30:31.899 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-30T09:30:31.899 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:30:31.899 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:30:31.899 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-30T09:30:31.930 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:31.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:31.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:31.971 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:30:32.016 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:30:32.038 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:32.054 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:30:32.095 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:30:32.122 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:32.147 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:30:32.192 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:30:32.243 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:30:32.276 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:30:32.283 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:30:32.283 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:30:32.283 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:30:32.283 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:30:32.283 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:30:32.284 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:30:32.284 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:30:32.284 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:30:32.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:32 smithi019 ceph-mon[216815]: pgmap v600: 129 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 17828/193917 objects degraded (9.194%); 3591/193917 objects misplaced (1.852%); 12 MiB/s, 186 objects/s recovering 2023-09-30T09:30:32.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/403470712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:32.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:32 smithi019 ceph-mon[224043]: pgmap v600: 129 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 17828/193917 objects degraded (9.194%); 3591/193917 objects misplaced (1.852%); 12 MiB/s, 186 objects/s recovering 2023-09-30T09:30:32.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/403470712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:32 smithi167 ceph-mon[192880]: pgmap v600: 129 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 17828/193917 objects degraded (9.194%); 3591/193917 objects misplaced (1.852%); 12 MiB/s, 186 objects/s recovering 2023-09-30T09:30:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/403470712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:32.968 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:32.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:33.084 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:33.156 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:30:33.156 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:33.255 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:33.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:33.368 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:33.395 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:33.436 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:33.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:33.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 17828/193917 objects degraded (9.194%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:33.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:33.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:33.544 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:33.592 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:33.686 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:33.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:33.792 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:33.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:33.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 17828/193917 objects degraded (9.194%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:33.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:33.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 17828/193917 objects degraded (9.194%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:33.854 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:34.087 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:30:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:30:33] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-09-30T09:30:34.207 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:34.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:34.324 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:34.377 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:34.426 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:34 smithi019 ceph-mon[216815]: pgmap v601: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 17708/192885 objects degraded (9.181%); 3591/192885 objects misplaced (1.862%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:30:34.426 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:34 smithi019 ceph-mon[224043]: pgmap v601: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 17708/192885 objects degraded (9.181%); 3591/192885 objects misplaced (1.862%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:30:34.468 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:34.608 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:34.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:34.645 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80438 5.e 5:7ec295bd:::benchmark_data_smithi019_287194_object80437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 393.08262483499999, 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5009281310000002, 2023-09-30T09:30:34.711 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80438 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.712 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621660+0000", 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.423e-06 2023-09-30T09:30:34.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621684+0000", 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4569000000000001e-05 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621694+0000", 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5210000000000008e-06 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621700+0000", 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6710000000000002e-06 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.622254+0000", 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00055331 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.622263+0000", 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8419999999999994e-06 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603497+0000", 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98123395899999999 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603507+0000", 2023-09-30T09:30:34.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0285999999999999e-05 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.078297+0000", 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4747904549999999 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.084057+0000", 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0057597530000000003 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.717 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122473+0000", 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.038415673999999997 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122586+0000", 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113668 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.718 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80444 5.5 5:a3c25906:::benchmark_data_smithi019_287194_object80443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 393.08257238499999, 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.335433417, 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80444 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.719 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.720 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3099999999999998e-07 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621733+0000", 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1899000000000002e-05 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621740+0000", 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7290000000000003e-06 2023-09-30T09:30:34.721 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621745+0000", 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3489999999999999e-06 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621802+0000", 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7148000000000002e-05 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.722 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621813+0000", 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1136e-05 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.602972+0000", 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98115875399999997 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.602983+0000", 2023-09-30T09:30:34.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1294000000000001e-05 2023-09-30T09:30:34.724 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.725 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.725 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.725 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.941598+0000", 2023-09-30T09:30:34.725 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.338614129 2023-09-30T09:30:34.725 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.725 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.726 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.726 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.941805+0000", 2023-09-30T09:30:34.726 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020759399999999999 2023-09-30T09:30:34.726 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.727 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.727 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:30:34.727 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.942011+0000", 2023-09-30T09:30:34.727 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020628199999999999 2023-09-30T09:30:34.727 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.727 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.727 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949084+0000", 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0070728550000000003 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954359+0000", 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0052750540000000004 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954690+0000", 2023-09-30T09:30:34.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033033200000000002 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954747+0000", 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6943e-05 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957144+0000", 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023974880000000001 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.729 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80452 5.5 5:a7eeee83:::benchmark_data_smithi019_287194_object80451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 393.07178414700002, 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3299363909999999, 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.730 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80452 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:30:34.731 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632500+0000", 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.576e-06 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632525+0000", 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4213999999999999e-05 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.732 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632543+0000", 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8309e-05 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632552+0000", 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5860000000000005e-06 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632615+0000", 2023-09-30T09:30:34.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3384000000000002e-05 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632628+0000", 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2727e-05 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603007+0000", 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97037908100000003 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603015+0000", 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7049999999999999e-06 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947003+0000", 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.343988728 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947114+0000", 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000110746 2023-09-30T09:30:34.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947240+0000", 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000126136 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954457+0000", 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0072169809999999999 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957159+0000", 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00270178 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.960528+0000", 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033696120000000001 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.960588+0000", 2023-09-30T09:30:34.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9361000000000003e-05 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962435+0000", 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001847465 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80456 5.5 5:a29acd63:::benchmark_data_smithi019_287194_object80455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:30:34.738 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 393.06486141200003, 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3250314799999998, 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80456 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.739 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639424+0000", 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0729999999999999e-06 2023-09-30T09:30:34.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.741 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.908 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639449+0000", 2023-09-30T09:30:34.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5324999999999999e-05 2023-09-30T09:30:34.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639467+0000", 2023-09-30T09:30:34.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7923000000000001e-05 2023-09-30T09:30:34.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639477+0000", 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6809999999999993e-06 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639552+0000", 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5191000000000006e-05 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639566+0000", 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3654e-05 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603040+0000", 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96347411699999996 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603047+0000", 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.126e-06 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949179+0000", 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3461320999999999 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949348+0000", 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016921899999999999 2023-09-30T09:30:34.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949468+0000", 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011944599999999999 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954573+0000", 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005105371 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957167+0000", 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025936269999999998 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962500+0000", 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0053332429999999997 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962562+0000", 2023-09-30T09:30:34.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2016999999999999e-05 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.964453+0000", 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018913669999999999 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80465 5.e 5:72d43b9b:::benchmark_data_smithi019_287194_object80464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 393.041180822, 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5017385250000004, 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80465 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.917 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.918 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663104+0000", 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9649999999999998e-06 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663128+0000", 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3878000000000001e-05 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663145+0000", 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.649e-05 2023-09-30T09:30:34.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663153+0000", 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5750000000000001e-06 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663245+0000", 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2249000000000004e-05 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663259+0000", 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3422e-05 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603534+0000", 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94027541000000003 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603543+0000", 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4120000000000001e-06 2023-09-30T09:30:34.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122620+0000", 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5190771290000002 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122769+0000", 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014931999999999999 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:30:34.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122900+0000", 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013077799999999999 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.131893+0000", 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089926810000000006 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135017+0000", 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031246440000000002 2023-09-30T09:30:34.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.162244+0000", 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027226509999999999 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.162314+0000", 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0116000000000004e-05 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.164841+0000", 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002526946 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80467 5.e 5:7e7fcfbf:::benchmark_data_smithi019_287194_object80466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 390.96556053, 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4284252850000003, 2023-09-30T09:30:34.925 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80467 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738724+0000", 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.823e-06 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738752+0000", 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7413e-05 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.740661+0000", 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001908795 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.740676+0000", 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5333999999999999e-05 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125570+0000", 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3848938300000002 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125759+0000", 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018891200000000001 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125968+0000", 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020954400000000001 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.134957+0000", 2023-09-30T09:30:34.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089886380000000002 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135084+0000", 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012721999999999999 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.164930+0000", 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.02984645 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.165000+0000", 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9052000000000002e-05 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167148+0000", 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021482739999999999 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80472 5.e 5:7c0728d2:::benchmark_data_smithi019_287194_object80471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 390.76975517199998, 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2349960290000004, 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80472 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.935 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934529+0000", 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4750000000000001e-06 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934554+0000", 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4953999999999999e-05 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934572+0000", 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7317e-05 2023-09-30T09:30:34.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934580+0000", 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5080000000000005e-06 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.128799+0000", 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1942186069999998 2023-09-30T09:30:34.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.128998+0000", 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000199447 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.129303+0000", 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00030457599999999998 2023-09-30T09:30:34.939 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141779+0000", 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012476716000000001 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141802+0000", 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2903000000000001e-05 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167214+0000", 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.025411309 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167261+0000", 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7747999999999998e-05 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169524+0000", 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002262469 2023-09-30T09:30:34.941 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80493 5.e 5:7e14abc9:::benchmark_data_smithi019_287194_object80492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 390.668244281, 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1357754939999998, 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.942 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80493 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:30:34.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036040+0000", 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5269999999999999e-06 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036063+0000", 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3034999999999999e-05 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036076+0000", 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2979999999999999e-05 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036085+0000", 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6340000000000007e-06 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.945 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.131929+0000", 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.095844402 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.132099+0000", 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016993999999999999 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.132318+0000", 2023-09-30T09:30:34.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021850799999999999 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141810+0000", 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0094919229999999993 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141869+0000", 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9252000000000001e-05 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169591+0000", 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027721632999999999 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169673+0000", 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2330999999999995e-05 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171814+0000", 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002141329 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80495 5.e 5:72ad1293:::benchmark_data_smithi019_287194_object80494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 390.66469524500002, 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1346890580000002, 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.949 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80495 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039590+0000", 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0020000000000001e-06 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039617+0000", 2023-09-30T09:30:34.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6701000000000001e-05 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039628+0000", 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1557e-05 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039635+0000", 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1829999999999997e-06 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135122+0000", 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0954865739999997 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135319+0000", 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019667600000000001 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135541+0000", 2023-09-30T09:30:34.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022274 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141816+0000", 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062751109999999999 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151681+0000", 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0098647409999999998 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171874+0000", 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020192412999999999 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171931+0000", 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7716000000000001e-05 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174277+0000", 2023-09-30T09:30:34.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002345644 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80498 5.e 5:7631c3ad:::benchmark_data_smithi019_287194_object80497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 390.65655243800001, 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1287919449999997, 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.956 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80498 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.957 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047732+0000", 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.832e-06 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047759+0000", 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6628999999999999e-05 2023-09-30T09:30:34.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047779+0000", 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9468000000000001e-05 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047789+0000", 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0864e-05 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138222+0000", 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0904321699999997 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138406+0000", 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018471300000000001 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138613+0000", 2023-09-30T09:30:34.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000206596 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.148131+0000", 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095180230000000005 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151779+0000", 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036479849999999999 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174332+0000", 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022552744 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174400+0000", 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7886999999999996e-05 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176523+0000", 2023-09-30T09:30:34.962 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021230340000000002 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80506 5.e 5:7cef17fd:::benchmark_data_smithi019_287194_object80505:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 390.62036850099997, 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0963020300000004, 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.963 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80506 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.964 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083916+0000", 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9199999999999999e-07 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083934+0000", 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8879e-05 2023-09-30T09:30:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083943+0000", 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0620000000000007e-06 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083950+0000", 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1519999999999997e-06 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.966 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141906+0000", 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0579564440000002 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.142127+0000", 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000220447 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.142337+0000", 2023-09-30T09:30:34.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000210867 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.148142+0000", 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058050389999999997 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.153209+0000", 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050662160000000001 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176580+0000", 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.02337171 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176646+0000", 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5775e-05 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.180217+0000", 2023-09-30T09:30:34.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003570447 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80509 5.e 5:7a2f2184:::benchmark_data_smithi019_287194_object80508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 390.59770645200001, 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0801843260000004, 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.970 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80509 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.971 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106578+0000", 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6199999999999999e-06 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106608+0000", 2023-09-30T09:30:34.972 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9841000000000001e-05 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106623+0000", 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4628e-05 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106632+0000", 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6560000000000004e-06 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.973 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145019+0000", 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0383863680000003 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145216+0000", 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000196942 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145433+0000", 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000217436 2023-09-30T09:30:34.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151985+0000", 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0065521779999999996 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.153262+0000", 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00127673 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.975 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.184424+0000", 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031162419 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.184490+0000", 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5228999999999995e-05 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.186761+0000", 2023-09-30T09:30:34.976 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022712790000000002 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:82845 5.3 5:c557c714:::benchmark_data_smithi019_287194_object67:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 365.51476291300003, 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.525600517, 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.977 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 82845 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.978 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189521+0000", 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9499999999999998e-07 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189523+0000", 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1469999999999999e-06 2023-09-30T09:30:34.979 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189529+0000", 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0719999999999996e-06 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189534+0000", 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4090000000000002e-06 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.980 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.459994+0000", 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.270459909 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.466180+0000", 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061857609999999997 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.331107+0000", 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86492722099999997 2023-09-30T09:30:34.981 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.339884+0000", 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087769049999999998 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.642343+0000", 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.302458745 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.982 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.647436+0000", 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050929809999999999 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.656592+0000", 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091567330000000002 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.983 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.695178+0000", 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.038585174999999999 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.697589+0000", 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024115120000000002 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.698724+0000", 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001134456 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.984 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.701606+0000", 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028820909999999998 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.715121+0000", 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013515005 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.985 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:83209 5.3 5:c8e07ed0:::benchmark_data_smithi019_287194_object431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 364.21955416899999, 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8400215319999997, 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 83209 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.986 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484730+0000", 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3769999999999999e-06 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484736+0000", 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2889999999999997e-06 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484750+0000", 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4929999999999999e-05 2023-09-30T09:30:34.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484758+0000", 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1910000000000003e-06 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.512794+0000", 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028036607000000002 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:30:34.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.515536+0000", 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027418659999999999 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.902937+0000", 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3874005220000001 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.909306+0000", 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063690129999999998 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.219767+0000", 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3104613839999999 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.229321+0000", 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095544229999999994 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:30:34.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.239410+0000", 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010088311000000001 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.268110+0000", 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028700013 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.272145+0000", 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040355829999999997 2023-09-30T09:30:34.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.298154+0000", 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026008963999999999 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.302424+0000", 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042697239999999999 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.993 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.324750+0000", 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022326334999999999 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:85292 5.3 5:c22e0b20:::benchmark_data_smithi019_287194_object2514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 348.59841587599999, 2023-09-30T09:30:34.994 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0057864740000002, 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 85292 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:30:34.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105869+0000", 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6419999999999999e-06 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105875+0000", 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6390000000000003e-06 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105892+0000", 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6092000000000001e-05 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:34.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105902+0000", 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0723000000000001e-05 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.151432+0000", 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.045529575000000003 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.157840+0000", 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0064082469999999997 2023-09-30T09:30:34.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:47.639845+0000", 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4820053099999999 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:47.646225+0000", 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063791960000000002 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:34.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.022311+0000", 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.376086213 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.027327+0000", 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050160719999999999 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.035994+0000", 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086666720000000003 2023-09-30T09:30:35.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.078087+0000", 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.042092989999999997 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.079289+0000", 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001202172 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.081249+0000", 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019603310000000001 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.085070+0000", 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038209870000000001 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.111654+0000", 2023-09-30T09:30:35.002 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026583612999999999 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87684 5.3 5:c18d78e7:::benchmark_data_smithi019_287194_object4906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 329.907879444, 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1651827400000001, 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:35.003 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87684 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796405+0000", 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.406e-06 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796411+0000", 2023-09-30T09:30:35.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9139999999999996e-06 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796422+0000", 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1493e-05 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796429+0000", 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.567e-06 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.070700+0000", 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.274271406 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.077044+0000", 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063433329999999996 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.143755+0000", 2023-09-30T09:30:35.007 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.066710900000000004 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.150381+0000", 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0066263609999999999 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.882648+0000", 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.732266774 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.888309+0000", 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005660867 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.897037+0000", 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087287739999999999 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.939441+0000", 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.042403771999999999 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.941115+0000", 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001673316 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.943881+0000", 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027661420000000001 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.947752+0000", 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038716940000000002 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.961586+0000", 2023-09-30T09:30:35.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013834021 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87688 5.3 5:cc6b6e52:::benchmark_data_smithi019_287194_object4910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 329.90212638499997, 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1848223249999998, 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:35.012 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87688 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802159+0000", 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.322e-06 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:35.014 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802168+0000", 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8470000000000002e-06 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802199+0000", 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1343000000000001e-05 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802212+0000", 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2368000000000001e-05 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.077878+0000", 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27566626799999999 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.087824+0000", 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0099457680000000007 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.196502+0000", 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.10867827200000001 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.211705+0000", 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015202983 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.919095+0000", 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.707390357 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.017 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.924017+0000", 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049221409999999997 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.932712+0000", 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086950120000000002 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:35.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.942222+0000", 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095098939999999996 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.963866+0000", 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.021643335999999999 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.966772+0000", 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029066550000000002 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.019 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.971211+0000", 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0044391040000000001 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.986979+0000", 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015767654999999998 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.020 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87754 5.3 5:cecacda3:::benchmark_data_smithi019_287194_object4976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 329.49207794300003, 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1694036080000001, 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87754 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.021 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:35.022 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212206+0000", 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2869999999999999e-06 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212211+0000", 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6789999999999996e-06 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212224+0000", 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2931999999999999e-05 2023-09-30T09:30:35.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212233+0000", 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0289999999999994e-06 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.299820+0000", 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.087586586999999994 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:30:35.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.303276+0000", 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034559970000000001 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.858386+0000", 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5551105619999999 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.864253+0000", 2023-09-30T09:30:35.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058669530000000003 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.276710+0000", 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.412457187 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.287222+0000", 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010511855000000001 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.300806+0000", 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013584003000000001 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.358020+0000", 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.057214108999999999 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:35.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.359098+0000", 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0010772889999999999 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.361377+0000", 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022794439999999998 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.367047+0000", 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0056699250000000001 2023-09-30T09:30:35.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.381609+0000", 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.01456177 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145688 5.e 5:72fbdf89:::benchmark_data_smithi019_287194_object62910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:30:35.029 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 224.33959374200001, 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.140027033, 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145688 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:35.030 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364691+0000", 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3039999999999999e-06 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364696+0000", 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5860000000000004e-06 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364709+0000", 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2938999999999999e-05 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:35.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364716+0000", 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7009999999999999e-06 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364773+0000", 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7463999999999997e-05 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364784+0000", 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.077e-05 2023-09-30T09:30:35.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305414+0000", 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94062986800000004 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305424+0000", 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6339999999999996e-06 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.474904+0000", 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.169480576 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.475014+0000", 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011030400000000001 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4,5", 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.475093+0000", 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8994999999999998e-05 2023-09-30T09:30:35.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478115+0000", 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030214360000000002 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.481419+0000", 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033041759999999998 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.487360+0000", 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0059407549999999998 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504511+0000", 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017150822 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504581+0000", 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0104000000000005e-05 2023-09-30T09:30:35.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504716+0000", 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000135599 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145692 5.e 5:778ecb27:::benchmark_data_smithi019_287194_object62914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 224.33314820699999, 2023-09-30T09:30:35.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1338203870000001, 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145692 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:30:35.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371136+0000", 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4499999999999996e-07 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371141+0000", 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9710000000000003e-06 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371151+0000", 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0669000000000001e-05 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:30:35.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371157+0000", 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7219999999999996e-06 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371209+0000", 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1801000000000001e-05 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:35.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371218+0000", 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7860000000000002e-06 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305986+0000", 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.934768932 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305996+0000", 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7399999999999999e-06 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478142+0000", 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172145298 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478283+0000", 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000141037 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4,5", 2023-09-30T09:30:35.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478364+0000", 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1600999999999993e-05 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.481503+0000", 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031385779999999999 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.487053+0000", 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055504949999999999 2023-09-30T09:30:35.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.495859+0000", 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088061919999999991 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504770+0000", 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089107639999999998 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504831+0000", 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1209000000000003e-05 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504955+0000", 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012384700000000001 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:35.047 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.048 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:30:35.048 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:30:35.048 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:30:35.048 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:35.049 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:34 smithi167 ceph-mon[192880]: pgmap v601: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 17708/192885 objects degraded (9.181%); 3591/192885 objects misplaced (1.862%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:30:35.078 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:35.167 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:35.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:35.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:35.274 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:35.329 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:36.185 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:30:36.514 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:36 smithi019 ceph-mon[216815]: pgmap v602: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 16756/180801 objects degraded (9.268%); 3212/180801 objects misplaced (1.777%); 17 MiB/s, 275 objects/s recovering 2023-09-30T09:30:36.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:36 smithi019 ceph-mon[224043]: pgmap v602: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 16756/180801 objects degraded (9.268%); 3212/180801 objects misplaced (1.777%); 17 MiB/s, 275 objects/s recovering 2023-09-30T09:30:36.591 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:36.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:36.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:36.700 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:36.756 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:36 smithi167 ceph-mon[192880]: pgmap v602: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 16756/180801 objects degraded (9.268%); 3212/180801 objects misplaced (1.777%); 17 MiB/s, 275 objects/s recovering 2023-09-30T09:30:36.758 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:36.859 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:36.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:36.974 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:37.040 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:37.752 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:30:37.851 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:37.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:37.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:37.957 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:37.994 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:38.587 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:38.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:38.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:38.698 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:38.753 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:38.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:38 smithi167 ceph-mon[192880]: pgmap v603: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 16614/179484 objects degraded (9.257%); 3212/179484 objects misplaced (1.790%); 16 MiB/s, 260 objects/s recovering 2023-09-30T09:30:38.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 16614/179484 objects degraded (9.257%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:38.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:38 smithi019 ceph-mon[216815]: pgmap v603: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 16614/179484 objects degraded (9.257%); 3212/179484 objects misplaced (1.790%); 16 MiB/s, 260 objects/s recovering 2023-09-30T09:30:38.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 16614/179484 objects degraded (9.257%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:38.780 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:38 smithi019 ceph-mon[224043]: pgmap v603: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 16614/179484 objects degraded (9.257%); 3212/179484 objects misplaced (1.790%); 16 MiB/s, 260 objects/s recovering 2023-09-30T09:30:38.780 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 16614/179484 objects degraded (9.257%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:38.797 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:38.894 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:38.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:38.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:39.003 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:39.093 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:39.517 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:30:39.517 INFO:teuthology.orchestra.run.smithi019.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:30:39.517 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:30:39.556 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:39.669 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:39.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:39.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:39.777 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:39.817 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:39.903 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:39.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:40.008 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:40.089 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:40.666 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:30:41.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:40 smithi167 ceph-mon[192880]: pgmap v604: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 16083/173946 objects degraded (9.246%); 2968/173946 objects misplaced (1.706%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:30:41.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:40 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:41.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:40 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:41.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2087323532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:41.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:40 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[224043]: pgmap v604: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 16083/173946 objects degraded (9.246%); 2968/173946 objects misplaced (1.706%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:30:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2087323532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[216815]: pgmap v604: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 16083/173946 objects degraded (9.246%); 2968/173946 objects misplaced (1.706%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:30:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2087323532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:40 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:30:41.326 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:41.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:41.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:41.432 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:41.491 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:41.791 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:30:41.791 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: failed to encode map e254 with expected crc 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: osdmap e254: 8 total, 8 up, 7 in 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: failed to encode map e254 with expected crc 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: failed to encode map e254 with expected crc 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: failed to encode map e254 with expected crc 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:30:41.792 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:41.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: failed to encode map e254 with expected crc 2023-09-30T09:30:41.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: osdmap e254: 8 total, 8 up, 7 in 2023-09-30T09:30:41.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: failed to encode map e254 with expected crc 2023-09-30T09:30:41.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: failed to encode map e254 with expected crc 2023-09-30T09:30:41.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: failed to encode map e254 with expected crc 2023-09-30T09:30:41.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:30:41.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:30:41.793 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:41 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:30:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:30:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: failed to encode map e254 with expected crc 2023-09-30T09:30:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: osdmap e254: 8 total, 8 up, 7 in 2023-09-30T09:30:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: failed to encode map e254 with expected crc 2023-09-30T09:30:42.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: failed to encode map e254 with expected crc 2023-09-30T09:30:42.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: failed to encode map e254 with expected crc 2023-09-30T09:30:42.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:30:42.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:30:42.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:41 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:42.620 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:42.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:42.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:42.730 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:42.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:42 smithi019 ceph-mon[216815]: pgmap v606: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 15630/167058 objects degraded (9.356%); 2835/167058 objects misplaced (1.697%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:30:42.766 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:42 smithi019 ceph-mon[224043]: pgmap v606: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 15630/167058 objects degraded (9.356%); 2835/167058 objects misplaced (1.697%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:30:42.803 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:42.882 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:42.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:42.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:42.987 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:42 smithi167 ceph-mon[192880]: pgmap v606: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 15630/167058 objects degraded (9.356%); 2835/167058 objects misplaced (1.697%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:30:43.030 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:43.112 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:43.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:43.221 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:43.248 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:43.334 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:43.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:43.444 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:43.518 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:43.627 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:43.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:43.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:43.739 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:43.776 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 15630/167058 objects degraded (9.356%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:43.778 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:43.871 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:43.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:43.977 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:44.639 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:30:44.640 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:30:44.640 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:30:44.640 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:30:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:30:43] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-09-30T09:30:44.641 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 15630/167058 objects degraded (9.356%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:44.641 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 15630/167058 objects degraded (9.356%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:44.642 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:30:44.642 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:30:44.642 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:30:44.642 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:30:44.643 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:30:44.644 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:30:44.644 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:30:44.644 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:30:44.696 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:44.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:44 smithi019 ceph-mon[224043]: pgmap v607: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 903 op/s; 15479/165771 objects degraded (9.338%); 2835/165771 objects misplaced (1.710%); 15 MiB/s, 247 objects/s recovering 2023-09-30T09:30:44.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1304368219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:44.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:44 smithi019 ceph-mon[216815]: pgmap v607: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 903 op/s; 15479/165771 objects degraded (9.338%); 2835/165771 objects misplaced (1.710%); 15 MiB/s, 247 objects/s recovering 2023-09-30T09:30:44.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1304368219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:44 smithi167 ceph-mon[192880]: pgmap v607: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 903 op/s; 15479/165771 objects degraded (9.338%); 2835/165771 objects misplaced (1.710%); 15 MiB/s, 247 objects/s recovering 2023-09-30T09:30:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1304368219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:45.386 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:45.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:45.491 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:45.531 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:45.546 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:45.631 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:45.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:45.746 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:45.815 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:45.873 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:45.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:45.980 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:46.035 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:46.302 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:46.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:46.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:46.342 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:30:46.411 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:46.499 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:46.840 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:46.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:46.947 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:47.007 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:47.008 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:46 smithi019 ceph-mon[216815]: pgmap v608: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 904 op/s; 14480/153657 objects degraded (9.424%); 2406/153657 objects misplaced (1.566%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:30:47.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:46 smithi019 ceph-mon[224043]: pgmap v608: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 904 op/s; 14480/153657 objects degraded (9.424%); 2406/153657 objects misplaced (1.566%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:30:47.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:46 smithi167 ceph-mon[192880]: pgmap v608: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 904 op/s; 14480/153657 objects degraded (9.424%); 2406/153657 objects misplaced (1.566%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:30:47.408 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:30:47.682 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:47.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:47.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:47.790 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:47.802 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:30:47.802 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:30:47.802 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-30T09:30:47.802 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:30:47.909 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:48.029 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:48.088 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:48.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:48.201 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:48.272 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:48.368 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:48.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:48.478 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:48.539 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:48.545 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:48.642 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:48.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:48.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:48 smithi019 ceph-mon[224043]: pgmap v609: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 906 op/s; 14319/152283 objects degraded (9.403%); 2406/152283 objects misplaced (1.580%); 16 MiB/s, 256 objects/s recovering 2023-09-30T09:30:48.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:48.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:48.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 14319/152283 objects degraded (9.403%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:48.700 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:48 smithi019 ceph-mon[216815]: pgmap v609: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 906 op/s; 14319/152283 objects degraded (9.403%); 2406/152283 objects misplaced (1.580%); 16 MiB/s, 256 objects/s recovering 2023-09-30T09:30:48.700 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:48.701 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:48.701 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 14319/152283 objects degraded (9.403%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:48.754 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:48.816 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:48 smithi167 ceph-mon[192880]: pgmap v609: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 906 op/s; 14319/152283 objects degraded (9.403%); 2406/152283 objects misplaced (1.580%); 16 MiB/s, 256 objects/s recovering 2023-09-30T09:30:48.816 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:30:48.816 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:30:48.817 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 14319/152283 objects degraded (9.403%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:48.821 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:48.914 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:48.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:49.024 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:49.082 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:49.184 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:49.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:49.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:49.291 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:49.317 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:49.382 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:49.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:49.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:49.496 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:49.554 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:49.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:49 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4068771697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:49.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:49.731 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:30:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:49 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4068771697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:49.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:49 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4068771697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:49.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: pgmap v610: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 13822/147009 objects degraded (9.402%); 2105/147009 objects misplaced (1.432%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: osdmap e255: 8 total, 8 up, 7 in 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: failed to encode map e255 with expected crc 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: failed to encode map e255 with expected crc 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: failed to encode map e255 with expected crc 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: failed to encode map e255 with expected crc 2023-09-30T09:30:50.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: failed to encode map e255 with expected crc 2023-09-30T09:30:50.999 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[224043]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.000 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: pgmap v610: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 13822/147009 objects degraded (9.402%); 2105/147009 objects misplaced (1.432%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:30:51.000 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:51.000 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:30:51.000 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: osdmap e255: 8 total, 8 up, 7 in 2023-09-30T09:30:51.000 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.001 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.001 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.001 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.001 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.001 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:50 smithi019 ceph-mon[216815]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.021 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: pgmap v610: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 13822/147009 objects degraded (9.402%); 2105/147009 objects misplaced (1.432%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:30:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:30:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:30:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: osdmap e255: 8 total, 8 up, 7 in 2023-09-30T09:30:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:50 smithi167 ceph-mon[192880]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:51.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:51.128 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:51.154 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:51.194 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:51.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:51.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:51.299 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:51.361 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:51.481 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:51.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:51.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:51.592 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:51.655 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:51.734 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:51.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:51.743 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:51 smithi167 ceph-mon[192880]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.822 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:51.844 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:51.916 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:51.981 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:51 smithi019 ceph-mon[224043]: failed to encode map e255 with expected crc 2023-09-30T09:30:51.982 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:51 smithi019 ceph-mon[216815]: failed to encode map e255 with expected crc 2023-09-30T09:30:52.025 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:52.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:52.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:52.132 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:30:52.183 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:52.294 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:30:52.296 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:30:52.297 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:30:52.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:52.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:30:52.405 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:52.461 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:52.562 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:52.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:52.679 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:52.731 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:52 smithi167 ceph-mon[192880]: pgmap v612: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 895 op/s; 13350/140172 objects degraded (9.524%); 1968/140172 objects misplaced (1.404%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:30:52.738 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:53.001 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:52 smithi019 ceph-mon[224043]: pgmap v612: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 895 op/s; 13350/140172 objects degraded (9.524%); 1968/140172 objects misplaced (1.404%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:30:53.001 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:52 smithi019 ceph-mon[216815]: pgmap v612: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 895 op/s; 13350/140172 objects degraded (9.524%); 1968/140172 objects misplaced (1.404%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:30:53.253 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:53.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:53.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:53.365 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:53.390 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:53.487 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:53.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:53.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:53.561 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:30:53.562 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:30:53.562 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:30:53.562 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:30:53.562 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:30:53.581 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:30:53.603 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:53.619 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:30:53.651 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:53.660 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:30:53.664 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:30:53.664 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:30:53.664 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:30:53.664 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:30:53.664 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:30:53.664 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:30:53.665 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:30:53.665 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:30:53.748 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 13350/140172 objects degraded (9.524%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:53.748 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2717537779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:53.835 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:53.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:53.863 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:30:53.926 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:30:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:30:53] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-09-30T09:30:53.926 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 13350/140172 objects degraded (9.524%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:53.926 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2717537779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:53.929 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 13350/140172 objects degraded (9.524%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:53.929 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2717537779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:30:53.939 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:54.002 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:54 smithi167 ceph-mon[192880]: pgmap v613: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 13204/138837 objects degraded (9.510%); 1968/138837 objects misplaced (1.417%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:30:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:54 smithi167 ceph-mon[192880]: osdmap e256: 8 total, 8 up, 7 in 2023-09-30T09:30:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:54 smithi167 ceph-mon[192880]: failed to encode map e256 with expected crc 2023-09-30T09:30:55.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:54 smithi019 ceph-mon[216815]: pgmap v613: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 13204/138837 objects degraded (9.510%); 1968/138837 objects misplaced (1.417%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:30:55.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:54 smithi019 ceph-mon[216815]: osdmap e256: 8 total, 8 up, 7 in 2023-09-30T09:30:55.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:54 smithi019 ceph-mon[216815]: failed to encode map e256 with expected crc 2023-09-30T09:30:55.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:54 smithi019 ceph-mon[224043]: pgmap v613: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 13204/138837 objects degraded (9.510%); 1968/138837 objects misplaced (1.417%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:30:55.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:54 smithi019 ceph-mon[224043]: osdmap e256: 8 total, 8 up, 7 in 2023-09-30T09:30:55.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:54 smithi019 ceph-mon[224043]: failed to encode map e256 with expected crc 2023-09-30T09:30:55.306 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:30:55.505 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:55.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:55.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:55.631 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:55.686 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:55.691 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:55.804 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:55.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:55.926 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:30:56.013 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:55 smithi167 ceph-mon[192880]: failed to encode map e256 with expected crc 2023-09-30T09:30:56.013 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:55 smithi167 ceph-mon[192880]: pgmap v615: 129 pgs: 10 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.6 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 12163/125595 objects degraded (9.684%); 1219/125595 objects misplaced (0.971%); 23 MiB/s, 362 objects/s recovering 2023-09-30T09:30:56.013 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:55 smithi167 ceph-mon[192880]: osdmap e257: 8 total, 8 up, 7 in 2023-09-30T09:30:56.013 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:55 smithi167 ceph-mon[192880]: failed to encode map e257 with expected crc 2023-09-30T09:30:56.014 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:55 smithi167 ceph-mon[192880]: failed to encode map e257 with expected crc 2023-09-30T09:30:56.020 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:56.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[224043]: failed to encode map e256 with expected crc 2023-09-30T09:30:56.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[224043]: pgmap v615: 129 pgs: 10 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.6 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 12163/125595 objects degraded (9.684%); 1219/125595 objects misplaced (0.971%); 23 MiB/s, 362 objects/s recovering 2023-09-30T09:30:56.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[224043]: osdmap e257: 8 total, 8 up, 7 in 2023-09-30T09:30:56.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[224043]: failed to encode map e257 with expected crc 2023-09-30T09:30:56.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[224043]: failed to encode map e257 with expected crc 2023-09-30T09:30:56.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[216815]: failed to encode map e256 with expected crc 2023-09-30T09:30:56.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[216815]: pgmap v615: 129 pgs: 10 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.6 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 12163/125595 objects degraded (9.684%); 1219/125595 objects misplaced (0.971%); 23 MiB/s, 362 objects/s recovering 2023-09-30T09:30:56.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[216815]: osdmap e257: 8 total, 8 up, 7 in 2023-09-30T09:30:56.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[216815]: failed to encode map e257 with expected crc 2023-09-30T09:30:56.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:55 smithi019 ceph-mon[216815]: failed to encode map e257 with expected crc 2023-09-30T09:30:56.092 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:56.144 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:30:56.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:56.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:30:56.264 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:30:56.331 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:56.471 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:30:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:56.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:30:56.577 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:30:56.580 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:56.581 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:30:56.639 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:56.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:56 smithi019 ceph-mon[216815]: failed to encode map e257 with expected crc 2023-09-30T09:30:56.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:56 smithi019 ceph-mon[216815]: from='client.45233 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:30:56.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:56 smithi019 ceph-mon[224043]: failed to encode map e257 with expected crc 2023-09-30T09:30:56.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:56 smithi019 ceph-mon[224043]: from='client.45233 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:30:57.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:56 smithi167 ceph-mon[192880]: failed to encode map e257 with expected crc 2023-09-30T09:30:57.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:56 smithi167 ceph-mon[192880]: from='client.45233 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:30:57.454 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:30:57.788 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:57 smithi019 ceph-mon[216815]: pgmap v617: 129 pgs: 9 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 11861/124410 objects degraded (9.534%); 1219/124410 objects misplaced (0.980%); 17 MiB/s, 278 objects/s recovering 2023-09-30T09:30:57.819 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:57.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:57.925 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:30:58.005 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:58.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:57 smithi019 ceph-mon[224043]: pgmap v617: 129 pgs: 9 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 11861/124410 objects degraded (9.534%); 1219/124410 objects misplaced (0.980%); 17 MiB/s, 278 objects/s recovering 2023-09-30T09:30:58.119 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:30:58.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:58.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:30:58.132 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:57 smithi167 ceph-mon[192880]: pgmap v617: 129 pgs: 9 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 11861/124410 objects degraded (9.534%); 1219/124410 objects misplaced (0.980%); 17 MiB/s, 278 objects/s recovering 2023-09-30T09:30:58.233 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:58.260 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:58.357 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:58.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:58.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:58.466 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:30:58.513 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:58.594 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:30:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:58.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:30:58.700 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:30:58.727 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:59.030 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:30:59.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:59.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:30:59.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 11861/124410 objects degraded (9.534%), 16 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:59.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 11861/124410 objects degraded (9.534%), 16 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:59.137 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:30:59.170 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":616,"stamp":"2023-09-30T09:30:55.277289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2746398954,"num_objects":41865,"num_object_clones":0,"num_object_copies":125595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12163,"num_objects_misplaced":1219,"num_objects_unfound":0,"num_objects_dirty":41865,"num_whiteouts":0,"num_read":6032,"num_read_kb":12314,"num_write":127679,"num_write_kb":4889516,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22949,"num_bytes_recovered":1503892696,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":210913,"ondisk_log_size":210913,"up":387,"acting":379,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":390,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9546724,"kb_used_data":8074472,"kb_used_omap":14,"kb_used_meta":1471409,"kb_avail":646669340,"statfs":{"total":671965249536,"available":662189404160,"internally_reserved":0,"allocated":8268259328,"data_stored":8260082373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14566,"internal_metadata":1506723610},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-467795968,"num_objects":-7138,"num_object_clones":0,"num_object_copies":-21414,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-1659,"num_objects_misplaced":-886,"num_objects_unfound":0,"num_objects_dirty":-7138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7138,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2250,"num_bytes_recovered":147456000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.106062"},"pg_stats":[{"pgid":"6.23","version":"255'4506","reported_seq":8680,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:53.988718+0000","last_change":"2023-09-30T09:29:35.062393+0000","last_active":"2023-09-30T09:30:53.988718+0000","last_peered":"2023-09-30T09:30:53.988718+0000","last_clean":"2023-09-30T09:30:53.988718+0000","last_became_active":"2023-09-30T09:29:35.061339+0000","last_became_peered":"2023-09-30T09:29:35.061339+0000","last_unstale":"2023-09-30T09:30:53.988718+0000","last_undegraded":"2023-09-30T09:30:53.988718+0000","last_fullsized":"2023-09-30T09:30:53.988718+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":4506,"log_dups_size":0,"ondisk_log_size":4506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42467328,"num_objects":648,"num_object_clones":0,"num_object_copies":1944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1507,"num_write_kb":46336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65699840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.22","version":"255'4466","reported_seq":7409,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:53.567431+0000","last_change":"2023-09-30T09:29:37.013899+0000","last_active":"2023-09-30T09:30:53.567431+0000","last_peered":"2023-09-30T09:30:53.567431+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:37.011820+0000","last_became_peered":"2023-09-30T09:29:37.011820+0000","last_unstale":"2023-09-30T09:30:53.567431+0000","last_undegraded":"2023-09-30T09:29:36.997644+0000","last_fullsized":"2023-09-30T09:29:36.996855+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":4466,"log_dups_size":0,"ondisk_log_size":4466,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38273024,"num_objects":584,"num_object_clones":0,"num_object_copies":1752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":304,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1469,"num_write_kb":42464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":219},{"shards":"3,6","objects":280}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.21","version":"255'4698","reported_seq":6421,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:52.016963+0000","last_change":"2023-09-30T09:29:35.078377+0000","last_active":"2023-09-30T09:30:52.016963+0000","last_peered":"2023-09-30T09:30:52.016963+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:35.075606+0000","last_became_peered":"2023-09-30T09:29:35.075606+0000","last_unstale":"2023-09-30T09:30:52.016963+0000","last_undegraded":"2023-09-30T09:29:35.051902+0000","last_fullsized":"2023-09-30T09:30:52.016963+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":4698,"log_dups_size":0,"ondisk_log_size":4698,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43548672,"num_objects":665,"num_object_clones":0,"num_object_copies":1995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":235,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1650,"num_write_kb":53856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":808,"num_bytes_recovered":52953088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":235},{"shards":"1,6,7","objects":430}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.20","version":"256'4650","reported_seq":6421,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:54.507223+0000","last_change":"2023-09-30T09:29:35.050182+0000","last_active":"2023-09-30T09:30:54.507223+0000","last_peered":"2023-09-30T09:30:54.507223+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:35.036894+0000","last_became_peered":"2023-09-30T09:29:35.036894+0000","last_unstale":"2023-09-30T09:30:54.507223+0000","last_undegraded":"2023-09-30T09:29:35.031725+0000","last_fullsized":"2023-09-30T09:30:54.507223+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":4650,"log_dups_size":0,"ondisk_log_size":4650,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43646976,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1616,"num_write_kb":50400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":829,"num_bytes_recovered":54296576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":440},{"shards":"0,2,3","objects":226}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1f","version":"255'6997","reported_seq":10369,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:53.702335+0000","last_change":"2023-09-30T09:29:35.091385+0000","last_active":"2023-09-30T09:30:53.702335+0000","last_peered":"2023-09-30T09:30:53.702335+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:35.087685+0000","last_became_peered":"2023-09-30T09:29:35.087685+0000","last_unstale":"2023-09-30T09:30:53.702335+0000","last_undegraded":"2023-09-30T09:29:35.082957+0000","last_fullsized":"2023-09-30T09:30:53.702335+0000","mapping_epoch":238,"log_start":"210'532","ondisk_log_start":"210'532","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6465,"log_dups_size":0,"ondisk_log_size":6465,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82509824,"num_objects":1259,"num_object_clones":0,"num_object_copies":3777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":448,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4549,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1646,"num_bytes_recovered":107872256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":448},{"shards":"3,4,6","objects":811}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":242,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671402+0000","last_change":"2023-09-30T09:29:09.598494+0000","last_active":"2023-09-30T09:30:40.671402+0000","last_peered":"2023-09-30T09:30:40.671402+0000","last_clean":"2023-09-30T09:30:40.671402+0000","last_became_active":"2023-09-30T09:29:09.598158+0000","last_became_peered":"2023-09-30T09:29:09.598158+0000","last_unstale":"2023-09-30T09:30:40.671402+0000","last_undegraded":"2023-09-30T09:30:40.671402+0000","last_fullsized":"2023-09-30T09:30:40.671402+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:03.225682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":220,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.662964+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:30:40.662964+0000","last_peered":"2023-09-30T09:30:40.662964+0000","last_clean":"2023-09-30T09:30:40.662964+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:30:40.662964+0000","last_undegraded":"2023-09-30T09:30:40.662964+0000","last_fullsized":"2023-09-30T09:30:40.662964+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1c","version":"255'6792","reported_seq":6868,"reported_epoch":255,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:30:53.689368+0000","last_change":"2023-09-30T09:29:35.102559+0000","last_active":"2023-09-30T09:30:53.689368+0000","last_peered":"2023-09-30T09:30:53.689368+0000","last_clean":"2023-09-30T09:29:33.948392+0000","last_became_active":"2023-09-30T09:29:35.089045+0000","last_became_peered":"2023-09-30T09:29:35.089045+0000","last_unstale":"2023-09-30T09:30:53.689368+0000","last_undegraded":"2023-09-30T09:30:53.689368+0000","last_fullsized":"2023-09-30T09:30:53.689368+0000","mapping_epoch":238,"log_start":"210'115","ondisk_log_start":"210'115","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6677,"log_dups_size":0,"ondisk_log_size":6677,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83492864,"num_objects":1274,"num_object_clones":0,"num_object_copies":3822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4374,"num_write_kb":180736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1274}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":218,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.767785+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:30:40.767785+0000","last_peered":"2023-09-30T09:30:40.767785+0000","last_clean":"2023-09-30T09:30:40.767785+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:30:40.767785+0000","last_undegraded":"2023-09-30T09:30:40.767785+0000","last_fullsized":"2023-09-30T09:30:40.767785+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":149,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:49.707869+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:30:49.707869+0000","last_peered":"2023-09-30T09:30:49.707869+0000","last_clean":"2023-09-30T09:30:49.707869+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:30:49.707869+0000","last_undegraded":"2023-09-30T09:30:49.707869+0000","last_fullsized":"2023-09-30T09:30:49.707869+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1d","version":"255'6972","reported_seq":10223,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:54.119066+0000","last_change":"2023-09-30T09:29:35.078954+0000","last_active":"2023-09-30T09:30:54.119066+0000","last_peered":"2023-09-30T09:30:54.119066+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:35.073573+0000","last_became_peered":"2023-09-30T09:29:35.073573+0000","last_unstale":"2023-09-30T09:30:54.119066+0000","last_undegraded":"2023-09-30T09:29:35.043564+0000","last_fullsized":"2023-09-30T09:30:54.119066+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6972,"log_dups_size":0,"ondisk_log_size":6972,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85196824,"num_objects":1301,"num_object_clones":0,"num_object_copies":3903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":446,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1301,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4477,"num_write_kb":184833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1588,"num_bytes_recovered":104071168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":446},{"shards":"1,5,6","objects":855}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":270,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.662026+0000","last_change":"2023-09-30T09:29:09.606288+0000","last_active":"2023-09-30T09:30:40.662026+0000","last_peered":"2023-09-30T09:30:40.662026+0000","last_clean":"2023-09-30T09:30:40.662026+0000","last_became_active":"2023-09-30T09:29:09.605776+0000","last_became_peered":"2023-09-30T09:29:09.605776+0000","last_unstale":"2023-09-30T09:30:40.662026+0000","last_undegraded":"2023-09-30T09:30:40.662026+0000","last_fullsized":"2023-09-30T09:30:40.662026+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:07.935982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":214,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671096+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:30:40.671096+0000","last_peered":"2023-09-30T09:30:40.671096+0000","last_clean":"2023-09-30T09:30:40.671096+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:30:40.671096+0000","last_undegraded":"2023-09-30T09:30:40.671096+0000","last_fullsized":"2023-09-30T09:30:40.671096+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"255'5442","reported_seq":9603,"reported_epoch":255,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:30:54.179060+0000","last_change":"2023-09-30T09:29:10.766595+0000","last_active":"2023-09-30T09:30:54.179060+0000","last_peered":"2023-09-30T09:30:54.179060+0000","last_clean":"2023-09-30T09:29:08.564047+0000","last_became_active":"2023-09-30T09:29:10.763967+0000","last_became_peered":"2023-09-30T09:29:10.763967+0000","last_unstale":"2023-09-30T09:30:54.179060+0000","last_undegraded":"2023-09-30T09:30:54.179060+0000","last_fullsized":"2023-09-30T09:30:54.179060+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2497","last_scrub_stamp":"2023-09-30T09:28:46.026181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:46.026181+0000","objects_scrubbed":1334,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:02.322482+0000","scrub_duration":0.15695645899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79888384,"num_objects":1219,"num_object_clones":0,"num_object_copies":3657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1219,"num_objects_unfound":0,"num_objects_dirty":1219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2945,"num_write_kb":90560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1347,"num_bytes_recovered":88276992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,2],"avail_no_missing":["7","2","4","6"],"object_location_counts":[{"shards":"2,4,7","objects":1219}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":149,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:49.707966+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:30:49.707966+0000","last_peered":"2023-09-30T09:30:49.707966+0000","last_clean":"2023-09-30T09:30:49.707966+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:30:49.707966+0000","last_undegraded":"2023-09-30T09:30:49.707966+0000","last_fullsized":"2023-09-30T09:30:49.707966+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":221,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:43.709438+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:30:43.709438+0000","last_peered":"2023-09-30T09:30:43.709438+0000","last_clean":"2023-09-30T09:30:43.709438+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:30:43.709438+0000","last_undegraded":"2023-09-30T09:30:43.709438+0000","last_fullsized":"2023-09-30T09:30:43.709438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":664,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.660947+0000","last_change":"2023-09-30T09:30:28.376570+0000","last_active":"2023-09-30T09:30:40.660947+0000","last_peered":"2023-09-30T09:30:40.660947+0000","last_clean":"2023-09-30T09:30:40.660947+0000","last_became_active":"2023-09-30T09:30:28.376194+0000","last_became_peered":"2023-09-30T09:30:28.376194+0000","last_unstale":"2023-09-30T09:30:40.660947+0000","last_undegraded":"2023-09-30T09:30:40.660947+0000","last_fullsized":"2023-09-30T09:30:40.660947+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"255'5554","reported_seq":10993,"reported_epoch":255,"state":"active+undersized+remapped","last_fresh":"2023-09-30T09:30:53.587538+0000","last_change":"2023-09-30T09:30:53.437288+0000","last_active":"2023-09-30T09:30:53.587538+0000","last_peered":"2023-09-30T09:30:53.587538+0000","last_clean":"2023-09-30T09:28:50.280641+0000","last_became_active":"2023-09-30T09:29:12.506944+0000","last_became_peered":"2023-09-30T09:29:12.506944+0000","last_unstale":"2023-09-30T09:30:53.587538+0000","last_undegraded":"2023-09-30T09:30:53.587538+0000","last_fullsized":"2023-09-30T09:29:10.583109+0000","mapping_epoch":229,"log_start":"210'149","ondisk_log_start":"210'149","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5405,"log_dups_size":0,"ondisk_log_size":5405,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90570752,"num_objects":1382,"num_object_clones":0,"num_object_copies":4146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4450,"num_write_kb":186624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":129892352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","1","2","6","7"],"object_location_counts":[{"shards":"3,6","objects":1382}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.771398+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:30:40.771398+0000","last_peered":"2023-09-30T09:30:40.771398+0000","last_clean":"2023-09-30T09:30:40.771398+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:30:40.771398+0000","last_undegraded":"2023-09-30T09:30:40.771398+0000","last_fullsized":"2023-09-30T09:30:40.771398+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":172,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:48.720099+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:30:48.720099+0000","last_peered":"2023-09-30T09:30:48.720099+0000","last_clean":"2023-09-30T09:30:48.720099+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:30:48.720099+0000","last_undegraded":"2023-09-30T09:30:48.720099+0000","last_fullsized":"2023-09-30T09:30:48.720099+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"256'5472","reported_seq":9534,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-30T09:30:54.497932+0000","last_change":"2023-09-30T09:28:56.382198+0000","last_active":"2023-09-30T09:30:54.497932+0000","last_peered":"2023-09-30T09:30:54.497932+0000","last_clean":"2023-09-30T09:30:54.497932+0000","last_became_active":"2023-09-30T09:28:56.380752+0000","last_became_peered":"2023-09-30T09:28:56.380752+0000","last_unstale":"2023-09-30T09:30:54.497932+0000","last_undegraded":"2023-09-30T09:30:54.497932+0000","last_fullsized":"2023-09-30T09:30:54.497932+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86212608,"num_objects":1316,"num_object_clones":0,"num_object_copies":3948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4354,"num_write_kb":181408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1324,"num_bytes_recovered":86769664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":218,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673416+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:30:40.673416+0000","last_peered":"2023-09-30T09:30:40.673416+0000","last_clean":"2023-09-30T09:30:40.673416+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:30:40.673416+0000","last_undegraded":"2023-09-30T09:30:40.673416+0000","last_fullsized":"2023-09-30T09:30:40.673416+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":203,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.767331+0000","last_change":"2023-09-30T09:29:09.620809+0000","last_active":"2023-09-30T09:30:40.767331+0000","last_peered":"2023-09-30T09:30:40.767331+0000","last_clean":"2023-09-30T09:30:40.767331+0000","last_became_active":"2023-09-30T09:29:09.620497+0000","last_became_peered":"2023-09-30T09:29:09.620497+0000","last_unstale":"2023-09-30T09:30:40.767331+0000","last_undegraded":"2023-09-30T09:30:40.767331+0000","last_fullsized":"2023-09-30T09:30:40.767331+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:48.932180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"255'5659","reported_seq":11267,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:53.663400+0000","last_change":"2023-09-30T09:29:39.270828+0000","last_active":"2023-09-30T09:30:53.663400+0000","last_peered":"2023-09-30T09:30:53.663400+0000","last_clean":"2023-09-30T09:30:53.663400+0000","last_became_active":"2023-09-30T09:29:39.266753+0000","last_became_peered":"2023-09-30T09:29:39.266753+0000","last_unstale":"2023-09-30T09:30:53.663400+0000","last_undegraded":"2023-09-30T09:30:53.663400+0000","last_fullsized":"2023-09-30T09:30:53.663400+0000","mapping_epoch":242,"log_start":"210'242","ondisk_log_start":"210'242","created":213,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2608","last_scrub_stamp":"2023-09-30T09:28:45.597110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.597110+0000","objects_scrubbed":1407,"log_size":5417,"log_dups_size":0,"ondisk_log_size":5417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:06.120738+0000","scrub_duration":0.26278548800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85196800,"num_objects":1300,"num_object_clones":0,"num_object_copies":3900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3057,"num_write_kb":94400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2034,"num_bytes_recovered":133300224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":219,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671189+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:30:40.671189+0000","last_peered":"2023-09-30T09:30:40.671189+0000","last_clean":"2023-09-30T09:30:40.671189+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:30:40.671189+0000","last_undegraded":"2023-09-30T09:30:40.671189+0000","last_fullsized":"2023-09-30T09:30:40.671189+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":233,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670646+0000","last_change":"2023-09-30T09:29:09.612466+0000","last_active":"2023-09-30T09:30:40.670646+0000","last_peered":"2023-09-30T09:30:40.670646+0000","last_clean":"2023-09-30T09:30:40.670646+0000","last_became_active":"2023-09-30T09:29:09.612173+0000","last_became_peered":"2023-09-30T09:29:09.612173+0000","last_unstale":"2023-09-30T09:30:40.670646+0000","last_undegraded":"2023-09-30T09:30:40.670646+0000","last_fullsized":"2023-09-30T09:30:40.670646+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:15.457876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":686,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.770834+0000","last_change":"2023-09-30T09:30:28.370038+0000","last_active":"2023-09-30T09:30:40.770834+0000","last_peered":"2023-09-30T09:30:40.770834+0000","last_clean":"2023-09-30T09:30:40.770834+0000","last_became_active":"2023-09-30T09:30:28.369721+0000","last_became_peered":"2023-09-30T09:30:28.369721+0000","last_unstale":"2023-09-30T09:30:40.770834+0000","last_undegraded":"2023-09-30T09:30:40.770834+0000","last_fullsized":"2023-09-30T09:30:40.770834+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.16","version":"255'5692","reported_seq":5768,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:53.673622+0000","last_change":"2023-09-30T09:28:40.282669+0000","last_active":"2023-09-30T09:30:53.673622+0000","last_peered":"2023-09-30T09:30:53.673622+0000","last_clean":"2023-09-30T09:30:53.673622+0000","last_became_active":"2023-09-30T09:28:40.281544+0000","last_became_peered":"2023-09-30T09:28:40.281544+0000","last_unstale":"2023-09-30T09:30:53.673622+0000","last_undegraded":"2023-09-30T09:30:53.673622+0000","last_fullsized":"2023-09-30T09:30:53.673622+0000","mapping_epoch":213,"log_start":"210'400","ondisk_log_start":"210'400","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5292,"log_dups_size":0,"ondisk_log_size":5292,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87719936,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4555,"num_write_kb":188576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":233,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.672623+0000","last_change":"2023-09-30T09:29:09.626205+0000","last_active":"2023-09-30T09:30:40.672623+0000","last_peered":"2023-09-30T09:30:40.672623+0000","last_clean":"2023-09-30T09:30:40.672623+0000","last_became_active":"2023-09-30T09:29:09.618293+0000","last_became_peered":"2023-09-30T09:29:09.618293+0000","last_unstale":"2023-09-30T09:30:40.672623+0000","last_undegraded":"2023-09-30T09:30:40.672623+0000","last_fullsized":"2023-09-30T09:30:40.672623+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:04.337293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":243,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:48.721107+0000","last_change":"2023-09-30T09:29:09.606072+0000","last_active":"2023-09-30T09:30:48.721107+0000","last_peered":"2023-09-30T09:30:48.721107+0000","last_clean":"2023-09-30T09:30:48.721107+0000","last_became_active":"2023-09-30T09:29:09.605716+0000","last_became_peered":"2023-09-30T09:29:09.605716+0000","last_unstale":"2023-09-30T09:30:48.721107+0000","last_undegraded":"2023-09-30T09:30:48.721107+0000","last_fullsized":"2023-09-30T09:30:48.721107+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:50.972640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":704,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.672150+0000","last_change":"2023-09-30T09:30:28.376536+0000","last_active":"2023-09-30T09:30:40.672150+0000","last_peered":"2023-09-30T09:30:40.672150+0000","last_clean":"2023-09-30T09:30:40.672150+0000","last_became_active":"2023-09-30T09:30:28.376151+0000","last_became_peered":"2023-09-30T09:30:28.376151+0000","last_unstale":"2023-09-30T09:30:40.672150+0000","last_undegraded":"2023-09-30T09:30:40.672150+0000","last_fullsized":"2023-09-30T09:30:40.672150+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.17","version":"255'5651","reported_seq":5784,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:54.156468+0000","last_change":"2023-09-30T09:28:45.602319+0000","last_active":"2023-09-30T09:30:54.156468+0000","last_peered":"2023-09-30T09:30:54.156468+0000","last_clean":"2023-09-30T09:30:54.156468+0000","last_became_active":"2023-09-30T09:28:40.272792+0000","last_became_peered":"2023-09-30T09:28:40.272792+0000","last_unstale":"2023-09-30T09:30:54.156468+0000","last_undegraded":"2023-09-30T09:30:54.156468+0000","last_fullsized":"2023-09-30T09:30:54.156468+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2523","last_scrub_stamp":"2023-09-30T09:28:45.602248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.602248+0000","objects_scrubbed":1419,"log_size":5651,"log_dups_size":0,"ondisk_log_size":5651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:31:54.788871+0000","scrub_duration":0.27069680099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84148224,"num_objects":1284,"num_object_clones":0,"num_object_copies":3852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3135,"num_write_kb":96000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":149,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:49.708100+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:30:49.708100+0000","last_peered":"2023-09-30T09:30:49.708100+0000","last_clean":"2023-09-30T09:30:49.708100+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:30:49.708100+0000","last_undegraded":"2023-09-30T09:30:49.708100+0000","last_fullsized":"2023-09-30T09:30:49.708100+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":176,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673521+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:30:40.673521+0000","last_peered":"2023-09-30T09:30:40.673521+0000","last_clean":"2023-09-30T09:30:40.673521+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:30:40.673521+0000","last_undegraded":"2023-09-30T09:30:40.673521+0000","last_fullsized":"2023-09-30T09:30:40.673521+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":669,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.767461+0000","last_change":"2023-09-30T09:30:28.370488+0000","last_active":"2023-09-30T09:30:40.767461+0000","last_peered":"2023-09-30T09:30:40.767461+0000","last_clean":"2023-09-30T09:30:40.767461+0000","last_became_active":"2023-09-30T09:30:28.370202+0000","last_became_peered":"2023-09-30T09:30:28.370202+0000","last_unstale":"2023-09-30T09:30:40.767461+0000","last_undegraded":"2023-09-30T09:30:40.767461+0000","last_fullsized":"2023-09-30T09:30:40.767461+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.14","version":"255'5588","reported_seq":5687,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:51.980166+0000","last_change":"2023-09-30T09:28:44.596036+0000","last_active":"2023-09-30T09:30:51.980166+0000","last_peered":"2023-09-30T09:30:51.980166+0000","last_clean":"2023-09-30T09:30:51.980166+0000","last_became_active":"2023-09-30T09:28:40.272740+0000","last_became_peered":"2023-09-30T09:28:40.272740+0000","last_unstale":"2023-09-30T09:30:51.980166+0000","last_undegraded":"2023-09-30T09:30:51.980166+0000","last_fullsized":"2023-09-30T09:30:51.980166+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"214'2497","last_scrub_stamp":"2023-09-30T09:28:44.595809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:44.595809+0000","objects_scrubbed":1358,"log_size":5588,"log_dups_size":0,"ondisk_log_size":5588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:18:56.692373+0000","scrub_duration":0.31195125499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88539136,"num_objects":1351,"num_object_clones":0,"num_object_copies":4053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3099,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":198,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673767+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:30:40.673767+0000","last_peered":"2023-09-30T09:30:40.673767+0000","last_clean":"2023-09-30T09:30:40.673767+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:30:40.673767+0000","last_undegraded":"2023-09-30T09:30:40.673767+0000","last_fullsized":"2023-09-30T09:30:40.673767+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":149,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:49.707793+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:30:49.707793+0000","last_peered":"2023-09-30T09:30:49.707793+0000","last_clean":"2023-09-30T09:30:49.707793+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:30:49.707793+0000","last_undegraded":"2023-09-30T09:30:49.707793+0000","last_fullsized":"2023-09-30T09:30:49.707793+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":743,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.672346+0000","last_change":"2023-09-30T09:30:28.371325+0000","last_active":"2023-09-30T09:30:40.672346+0000","last_peered":"2023-09-30T09:30:40.672346+0000","last_clean":"2023-09-30T09:30:40.672346+0000","last_became_active":"2023-09-30T09:30:28.371035+0000","last_became_peered":"2023-09-30T09:30:28.371035+0000","last_unstale":"2023-09-30T09:30:40.672346+0000","last_undegraded":"2023-09-30T09:30:40.672346+0000","last_fullsized":"2023-09-30T09:30:40.672346+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.15","version":"255'5648","reported_seq":5714,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:54.185682+0000","last_change":"2023-09-30T09:28:40.288346+0000","last_active":"2023-09-30T09:30:54.185682+0000","last_peered":"2023-09-30T09:30:54.185682+0000","last_clean":"2023-09-30T09:30:54.185682+0000","last_became_active":"2023-09-30T09:28:40.286899+0000","last_became_peered":"2023-09-30T09:28:40.286899+0000","last_unstale":"2023-09-30T09:30:54.185682+0000","last_undegraded":"2023-09-30T09:30:54.185682+0000","last_fullsized":"2023-09-30T09:30:54.185682+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5648,"log_dups_size":0,"ondisk_log_size":5648,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83525632,"num_objects":1275,"num_object_clones":0,"num_object_copies":3825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4517,"num_write_kb":185312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":347,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.669800+0000","last_change":"2023-09-30T09:29:09.634829+0000","last_active":"2023-09-30T09:30:40.669800+0000","last_peered":"2023-09-30T09:30:40.669800+0000","last_clean":"2023-09-30T09:30:40.669800+0000","last_became_active":"2023-09-30T09:29:09.612183+0000","last_became_peered":"2023-09-30T09:29:09.612183+0000","last_unstale":"2023-09-30T09:30:40.669800+0000","last_undegraded":"2023-09-30T09:30:40.669800+0000","last_fullsized":"2023-09-30T09:30:40.669800+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:28.144436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":240,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.664096+0000","last_change":"2023-09-30T09:29:09.605918+0000","last_active":"2023-09-30T09:30:40.664096+0000","last_peered":"2023-09-30T09:30:40.664096+0000","last_clean":"2023-09-30T09:30:40.664096+0000","last_became_active":"2023-09-30T09:29:09.605521+0000","last_became_peered":"2023-09-30T09:29:09.605521+0000","last_unstale":"2023-09-30T09:30:40.664096+0000","last_undegraded":"2023-09-30T09:30:40.664096+0000","last_fullsized":"2023-09-30T09:30:40.664096+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:43.287337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":686,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671078+0000","last_change":"2023-09-30T09:30:28.373170+0000","last_active":"2023-09-30T09:30:40.671078+0000","last_peered":"2023-09-30T09:30:40.671078+0000","last_clean":"2023-09-30T09:30:40.671078+0000","last_became_active":"2023-09-30T09:30:28.372847+0000","last_became_peered":"2023-09-30T09:30:28.372847+0000","last_unstale":"2023-09-30T09:30:40.671078+0000","last_undegraded":"2023-09-30T09:30:40.671078+0000","last_fullsized":"2023-09-30T09:30:40.671078+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.a","version":"255'6921","reported_seq":6989,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:54.116360+0000","last_change":"2023-09-30T09:29:35.005933+0000","last_active":"2023-09-30T09:30:54.116360+0000","last_peered":"2023-09-30T09:30:54.116360+0000","last_clean":"2023-09-30T09:30:54.116360+0000","last_became_active":"2023-09-30T09:29:35.001538+0000","last_became_peered":"2023-09-30T09:29:35.001538+0000","last_unstale":"2023-09-30T09:30:54.116360+0000","last_undegraded":"2023-09-30T09:30:54.116360+0000","last_fullsized":"2023-09-30T09:30:54.116360+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6921,"log_dups_size":0,"ondisk_log_size":6921,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82313216,"num_objects":1256,"num_object_clones":0,"num_object_copies":3768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4474,"num_write_kb":183360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":258,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.674016+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:30:40.674016+0000","last_peered":"2023-09-30T09:30:40.674016+0000","last_clean":"2023-09-30T09:30:40.674016+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:30:40.674016+0000","last_undegraded":"2023-09-30T09:30:40.674016+0000","last_fullsized":"2023-09-30T09:30:40.674016+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":118,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673992+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:30:40.673992+0000","last_peered":"2023-09-30T09:30:40.673992+0000","last_clean":"2023-09-30T09:30:40.673992+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:30:40.673992+0000","last_undegraded":"2023-09-30T09:30:40.673992+0000","last_fullsized":"2023-09-30T09:30:40.673992+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":665,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.661032+0000","last_change":"2023-09-30T09:30:28.377243+0000","last_active":"2023-09-30T09:30:40.661032+0000","last_peered":"2023-09-30T09:30:40.661032+0000","last_clean":"2023-09-30T09:30:40.661032+0000","last_became_active":"2023-09-30T09:30:28.376810+0000","last_became_peered":"2023-09-30T09:30:28.376810+0000","last_unstale":"2023-09-30T09:30:40.661032+0000","last_undegraded":"2023-09-30T09:30:40.661032+0000","last_fullsized":"2023-09-30T09:30:40.661032+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"255'6965","reported_seq":7042,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:53.720708+0000","last_change":"2023-09-30T09:29:35.047558+0000","last_active":"2023-09-30T09:30:53.720708+0000","last_peered":"2023-09-30T09:30:53.720708+0000","last_clean":"2023-09-30T09:30:53.720708+0000","last_became_active":"2023-09-30T09:29:35.044991+0000","last_became_peered":"2023-09-30T09:29:35.044991+0000","last_unstale":"2023-09-30T09:30:53.720708+0000","last_undegraded":"2023-09-30T09:30:53.720708+0000","last_fullsized":"2023-09-30T09:30:53.720708+0000","mapping_epoch":206,"log_start":"210'284","ondisk_log_start":"210'284","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86999040,"num_objects":1327,"num_object_clones":0,"num_object_copies":3981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4497,"num_write_kb":186400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":259,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673822+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:30:40.673822+0000","last_peered":"2023-09-30T09:30:40.673822+0000","last_clean":"2023-09-30T09:30:40.673822+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:30:40.673822+0000","last_undegraded":"2023-09-30T09:30:40.673822+0000","last_fullsized":"2023-09-30T09:30:40.673822+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":230,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670044+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:30:40.670044+0000","last_peered":"2023-09-30T09:30:40.670044+0000","last_clean":"2023-09-30T09:30:40.670044+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:30:40.670044+0000","last_undegraded":"2023-09-30T09:30:40.670044+0000","last_fullsized":"2023-09-30T09:30:40.670044+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":508,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671024+0000","last_change":"2023-09-30T09:30:28.373928+0000","last_active":"2023-09-30T09:30:40.671024+0000","last_peered":"2023-09-30T09:30:40.671024+0000","last_clean":"2023-09-30T09:30:40.671024+0000","last_became_active":"2023-09-30T09:30:28.373178+0000","last_became_peered":"2023-09-30T09:30:28.373178+0000","last_unstale":"2023-09-30T09:30:40.671024+0000","last_undegraded":"2023-09-30T09:30:40.671024+0000","last_fullsized":"2023-09-30T09:30:40.671024+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"255'5529","reported_seq":5607,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:54.168557+0000","last_change":"2023-09-30T09:29:10.874488+0000","last_active":"2023-09-30T09:30:54.168557+0000","last_peered":"2023-09-30T09:30:54.168557+0000","last_clean":"2023-09-30T09:29:08.490081+0000","last_became_active":"2023-09-30T09:29:10.870401+0000","last_became_peered":"2023-09-30T09:29:10.870401+0000","last_unstale":"2023-09-30T09:30:54.168557+0000","last_undegraded":"2023-09-30T09:29:10.657673+0000","last_fullsized":"2023-09-30T09:29:10.593223+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85032960,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1975,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4415,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":416},{"shards":"3,7","objects":-678}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673269+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:30:40.673269+0000","last_peered":"2023-09-30T09:30:40.673269+0000","last_clean":"2023-09-30T09:30:40.673269+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:30:40.673269+0000","last_undegraded":"2023-09-30T09:30:40.673269+0000","last_fullsized":"2023-09-30T09:30:40.673269+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":224,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670725+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:30:40.670725+0000","last_peered":"2023-09-30T09:30:40.670725+0000","last_clean":"2023-09-30T09:30:40.670725+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:30:40.670725+0000","last_undegraded":"2023-09-30T09:30:40.670725+0000","last_fullsized":"2023-09-30T09:30:40.670725+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":508,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671223+0000","last_change":"2023-09-30T09:30:28.378002+0000","last_active":"2023-09-30T09:30:40.671223+0000","last_peered":"2023-09-30T09:30:40.671223+0000","last_clean":"2023-09-30T09:30:40.671223+0000","last_became_active":"2023-09-30T09:30:28.377648+0000","last_became_peered":"2023-09-30T09:30:28.377648+0000","last_unstale":"2023-09-30T09:30:40.671223+0000","last_undegraded":"2023-09-30T09:30:40.671223+0000","last_fullsized":"2023-09-30T09:30:40.671223+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"255'5518","reported_seq":5659,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:54.110989+0000","last_change":"2023-09-30T09:29:10.893105+0000","last_active":"2023-09-30T09:30:54.110989+0000","last_peered":"2023-09-30T09:30:54.110989+0000","last_clean":"2023-09-30T09:29:08.514351+0000","last_became_active":"2023-09-30T09:29:10.883964+0000","last_became_peered":"2023-09-30T09:29:10.883964+0000","last_unstale":"2023-09-30T09:30:54.110989+0000","last_undegraded":"2023-09-30T09:29:10.674378+0000","last_fullsized":"2023-09-30T09:29:10.596450+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2471","last_scrub_stamp":"2023-09-30T09:28:47.207706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:47.207706+0000","objects_scrubbed":1369,"log_size":5518,"log_dups_size":0,"ondisk_log_size":5518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:51.408261+0000","scrub_duration":0.14908584899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85065728,"num_objects":1298,"num_object_clones":0,"num_object_copies":3894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3047,"num_write_kb":95232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":438},{"shards":"3,7","objects":-699}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":214,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.772831+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:30:40.772831+0000","last_peered":"2023-09-30T09:30:40.772831+0000","last_clean":"2023-09-30T09:30:40.772831+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:30:40.772831+0000","last_undegraded":"2023-09-30T09:30:40.772831+0000","last_fullsized":"2023-09-30T09:30:40.772831+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":146,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:49.707530+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:30:49.707530+0000","last_peered":"2023-09-30T09:30:49.707530+0000","last_clean":"2023-09-30T09:30:49.707530+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:30:49.707530+0000","last_undegraded":"2023-09-30T09:30:49.707530+0000","last_fullsized":"2023-09-30T09:30:49.707530+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"255'6731","reported_seq":9767,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:53.975222+0000","last_change":"2023-09-30T09:29:36.032408+0000","last_active":"2023-09-30T09:30:53.975222+0000","last_peered":"2023-09-30T09:30:53.975222+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:36.028267+0000","last_became_peered":"2023-09-30T09:29:36.028267+0000","last_unstale":"2023-09-30T09:30:53.975222+0000","last_undegraded":"2023-09-30T09:29:36.007788+0000","last_fullsized":"2023-09-30T09:30:53.975222+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90177536,"num_objects":1376,"num_object_clones":0,"num_object_copies":4128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":443,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4332,"num_write_kb":182656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96894976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":443},{"shards":"1,4,7","objects":933}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":734,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673719+0000","last_change":"2023-09-30T09:29:12.637524+0000","last_active":"2023-09-30T09:30:40.673719+0000","last_peered":"2023-09-30T09:30:40.673719+0000","last_clean":"2023-09-30T09:30:40.673719+0000","last_became_active":"2023-09-30T09:29:12.637171+0000","last_became_peered":"2023-09-30T09:29:12.637171+0000","last_unstale":"2023-09-30T09:30:40.673719+0000","last_undegraded":"2023-09-30T09:30:40.673719+0000","last_fullsized":"2023-09-30T09:30:40.673719+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:14:09.735059+0000","scrub_duration":0.0079898150000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":255,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.664871+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:30:40.664871+0000","last_peered":"2023-09-30T09:30:40.664871+0000","last_clean":"2023-09-30T09:30:40.664871+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:30:40.664871+0000","last_undegraded":"2023-09-30T09:30:40.664871+0000","last_fullsized":"2023-09-30T09:30:40.664871+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":213,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671110+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:30:40.671110+0000","last_peered":"2023-09-30T09:30:40.671110+0000","last_clean":"2023-09-30T09:30:40.671110+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:30:40.671110+0000","last_undegraded":"2023-09-30T09:30:40.671110+0000","last_fullsized":"2023-09-30T09:30:40.671110+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"255'6825","reported_seq":9899,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:53.705395+0000","last_change":"2023-09-30T09:29:36.082560+0000","last_active":"2023-09-30T09:30:53.705395+0000","last_peered":"2023-09-30T09:30:53.705395+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:36.077849+0000","last_became_peered":"2023-09-30T09:29:36.077849+0000","last_unstale":"2023-09-30T09:30:53.705395+0000","last_undegraded":"2023-09-30T09:29:35.995233+0000","last_fullsized":"2023-09-30T09:30:53.705395+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6825,"log_dups_size":0,"ondisk_log_size":6825,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90767360,"num_objects":1385,"num_object_clones":0,"num_object_copies":4155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":444,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4377,"num_write_kb":184384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1497,"num_bytes_recovered":98107392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":444},{"shards":"3,4,6","objects":941}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":633,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.661684+0000","last_change":"2023-09-30T09:29:12.631999+0000","last_active":"2023-09-30T09:30:40.661684+0000","last_peered":"2023-09-30T09:30:40.661684+0000","last_clean":"2023-09-30T09:30:40.661684+0000","last_became_active":"2023-09-30T09:29:12.631620+0000","last_became_peered":"2023-09-30T09:29:12.631620+0000","last_unstale":"2023-09-30T09:30:40.661684+0000","last_undegraded":"2023-09-30T09:30:40.661684+0000","last_fullsized":"2023-09-30T09:30:40.661684+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.999760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.999760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":531037,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":215,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670657+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:30:40.670657+0000","last_peered":"2023-09-30T09:30:40.670657+0000","last_clean":"2023-09-30T09:30:40.670657+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:30:40.670657+0000","last_undegraded":"2023-09-30T09:30:40.670657+0000","last_fullsized":"2023-09-30T09:30:40.670657+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":139,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.768041+0000","last_change":"2023-09-30T09:29:09.612070+0000","last_active":"2023-09-30T09:30:40.768041+0000","last_peered":"2023-09-30T09:30:40.768041+0000","last_clean":"2023-09-30T09:30:40.768041+0000","last_became_active":"2023-09-30T09:29:09.611725+0000","last_became_peered":"2023-09-30T09:29:09.611725+0000","last_unstale":"2023-09-30T09:30:40.768041+0000","last_undegraded":"2023-09-30T09:30:40.768041+0000","last_fullsized":"2023-09-30T09:30:40.768041+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:55.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":669,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.767543+0000","last_change":"2023-09-30T09:30:28.370333+0000","last_active":"2023-09-30T09:30:40.767543+0000","last_peered":"2023-09-30T09:30:40.767543+0000","last_clean":"2023-09-30T09:30:40.767543+0000","last_became_active":"2023-09-30T09:30:28.370039+0000","last_became_peered":"2023-09-30T09:30:28.370039+0000","last_unstale":"2023-09-30T09:30:40.767543+0000","last_undegraded":"2023-09-30T09:30:40.767543+0000","last_fullsized":"2023-09-30T09:30:40.767543+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"255'6807","reported_seq":6884,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:53.674315+0000","last_change":"2023-09-30T09:29:35.074390+0000","last_active":"2023-09-30T09:30:53.674315+0000","last_peered":"2023-09-30T09:30:53.674315+0000","last_clean":"2023-09-30T09:30:53.674315+0000","last_became_active":"2023-09-30T09:29:35.072113+0000","last_became_peered":"2023-09-30T09:29:35.072113+0000","last_unstale":"2023-09-30T09:30:53.674315+0000","last_undegraded":"2023-09-30T09:30:53.674315+0000","last_fullsized":"2023-09-30T09:30:53.674315+0000","mapping_epoch":206,"log_start":"210'33","ondisk_log_start":"210'33","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6774,"log_dups_size":0,"ondisk_log_size":6774,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86704128,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4389,"num_write_kb":182784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":224,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.669970+0000","last_change":"2023-09-30T09:29:09.607621+0000","last_active":"2023-09-30T09:30:40.669970+0000","last_peered":"2023-09-30T09:30:40.669970+0000","last_clean":"2023-09-30T09:30:40.669970+0000","last_became_active":"2023-09-30T09:29:09.607245+0000","last_became_peered":"2023-09-30T09:29:09.607245+0000","last_unstale":"2023-09-30T09:30:40.669970+0000","last_undegraded":"2023-09-30T09:30:40.669970+0000","last_fullsized":"2023-09-30T09:30:40.669970+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:42:42.429421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":213,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671531+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:30:40.671531+0000","last_peered":"2023-09-30T09:30:40.671531+0000","last_clean":"2023-09-30T09:30:40.671531+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:30:40.671531+0000","last_undegraded":"2023-09-30T09:30:40.671531+0000","last_fullsized":"2023-09-30T09:30:40.671531+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"255'7059","reported_seq":10410,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:54.170413+0000","last_change":"2023-09-30T09:29:36.240193+0000","last_active":"2023-09-30T09:30:54.170413+0000","last_peered":"2023-09-30T09:30:54.170413+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:36.233690+0000","last_became_peered":"2023-09-30T09:29:36.233690+0000","last_unstale":"2023-09-30T09:30:54.170413+0000","last_undegraded":"2023-09-30T09:29:36.012206+0000","last_fullsized":"2023-09-30T09:30:54.170413+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":7059,"log_dups_size":0,"ondisk_log_size":7059,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84672512,"num_objects":1292,"num_object_clones":0,"num_object_copies":3876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":441,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4564,"num_write_kb":187392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1637,"num_bytes_recovered":107282432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":441},{"shards":"1,5,6","objects":851}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":462,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670607+0000","last_change":"2023-09-30T09:29:12.635044+0000","last_active":"2023-09-30T09:30:40.670607+0000","last_peered":"2023-09-30T09:30:40.670607+0000","last_clean":"2023-09-30T09:30:40.670607+0000","last_became_active":"2023-09-30T09:29:12.634703+0000","last_became_peered":"2023-09-30T09:29:12.634703+0000","last_unstale":"2023-09-30T09:30:40.670607+0000","last_undegraded":"2023-09-30T09:30:40.670607+0000","last_fullsized":"2023-09-30T09:30:40.670607+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.980385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.980385+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":222,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.661860+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:30:40.661860+0000","last_peered":"2023-09-30T09:30:40.661860+0000","last_clean":"2023-09-30T09:30:40.661860+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:30:40.661860+0000","last_undegraded":"2023-09-30T09:30:40.661860+0000","last_fullsized":"2023-09-30T09:30:40.661860+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":195,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.767677+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:30:40.767677+0000","last_peered":"2023-09-30T09:30:40.767677+0000","last_clean":"2023-09-30T09:30:40.767677+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:30:40.767677+0000","last_undegraded":"2023-09-30T09:30:40.767677+0000","last_fullsized":"2023-09-30T09:30:40.767677+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"256'4631","reported_seq":6417,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:54.489617+0000","last_change":"2023-09-30T09:29:36.014896+0000","last_active":"2023-09-30T09:30:54.489617+0000","last_peered":"2023-09-30T09:30:54.489617+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:36.012719+0000","last_became_peered":"2023-09-30T09:29:36.012719+0000","last_unstale":"2023-09-30T09:30:54.489617+0000","last_undegraded":"2023-09-30T09:29:35.992619+0000","last_fullsized":"2023-09-30T09:30:54.489617+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":4631,"log_dups_size":0,"ondisk_log_size":4631,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41484288,"num_objects":633,"num_object_clones":0,"num_object_copies":1899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":210,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1596,"num_write_kb":48736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":835,"num_bytes_recovered":54755328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":423},{"shards":"0,2,3","objects":210}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":509,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671172+0000","last_change":"2023-09-30T09:30:28.374178+0000","last_active":"2023-09-30T09:30:40.671172+0000","last_peered":"2023-09-30T09:30:40.671172+0000","last_clean":"2023-09-30T09:30:40.671172+0000","last_became_active":"2023-09-30T09:30:28.373607+0000","last_became_peered":"2023-09-30T09:30:28.373607+0000","last_unstale":"2023-09-30T09:30:40.671172+0000","last_undegraded":"2023-09-30T09:30:40.671172+0000","last_fullsized":"2023-09-30T09:30:40.671172+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":600,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:47.046414+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:30:47.046414+0000","last_peered":"2023-09-30T09:30:47.046414+0000","last_clean":"2023-09-30T09:30:47.046414+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:30:47.046414+0000","last_undegraded":"2023-09-30T09:30:47.046414+0000","last_fullsized":"2023-09-30T09:30:47.046414+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.771701+0000","last_change":"2023-09-30T09:29:09.611292+0000","last_active":"2023-09-30T09:30:40.771701+0000","last_peered":"2023-09-30T09:30:40.771701+0000","last_clean":"2023-09-30T09:30:40.771701+0000","last_became_active":"2023-09-30T09:29:09.611015+0000","last_became_peered":"2023-09-30T09:29:09.611015+0000","last_unstale":"2023-09-30T09:30:40.771701+0000","last_undegraded":"2023-09-30T09:30:40.771701+0000","last_fullsized":"2023-09-30T09:30:40.771701+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:48.124378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":215,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:47.046331+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:30:47.046331+0000","last_peered":"2023-09-30T09:30:47.046331+0000","last_clean":"2023-09-30T09:30:47.046331+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:30:47.046331+0000","last_undegraded":"2023-09-30T09:30:47.046331+0000","last_fullsized":"2023-09-30T09:30:47.046331+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"255'5686","reported_seq":5785,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:53.663543+0000","last_change":"2023-09-30T09:28:42.250412+0000","last_active":"2023-09-30T09:30:53.663543+0000","last_peered":"2023-09-30T09:30:53.663543+0000","last_clean":"2023-09-30T09:30:53.663543+0000","last_became_active":"2023-09-30T09:28:40.266645+0000","last_became_peered":"2023-09-30T09:28:40.266645+0000","last_unstale":"2023-09-30T09:30:53.663543+0000","last_undegraded":"2023-09-30T09:30:53.663543+0000","last_fullsized":"2023-09-30T09:30:53.663543+0000","mapping_epoch":206,"log_start":"210'411","ondisk_log_start":"210'411","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2368","last_scrub_stamp":"2023-09-30T09:28:42.250204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.250204+0000","objects_scrubbed":1224,"log_size":5275,"log_dups_size":0,"ondisk_log_size":5275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:50:26.843055+0000","scrub_duration":0.32304901400000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87031808,"num_objects":1328,"num_object_clones":0,"num_object_copies":3984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3324,"num_write_kb":109696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":687,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.771051+0000","last_change":"2023-09-30T09:30:28.364401+0000","last_active":"2023-09-30T09:30:40.771051+0000","last_peered":"2023-09-30T09:30:40.771051+0000","last_clean":"2023-09-30T09:30:40.771051+0000","last_became_active":"2023-09-30T09:30:28.364065+0000","last_became_peered":"2023-09-30T09:30:28.364065+0000","last_unstale":"2023-09-30T09:30:40.771051+0000","last_undegraded":"2023-09-30T09:30:40.771051+0000","last_fullsized":"2023-09-30T09:30:40.771051+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":403,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.767597+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:30:40.767597+0000","last_peered":"2023-09-30T09:30:40.767597+0000","last_clean":"2023-09-30T09:30:40.767597+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:30:40.767597+0000","last_undegraded":"2023-09-30T09:30:40.767597+0000","last_fullsized":"2023-09-30T09:30:40.767597+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":234,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670733+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:30:40.670733+0000","last_peered":"2023-09-30T09:30:40.670733+0000","last_clean":"2023-09-30T09:30:40.670733+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:30:40.670733+0000","last_undegraded":"2023-09-30T09:30:40.670733+0000","last_fullsized":"2023-09-30T09:30:40.670733+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":113,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.661809+0000","last_change":"2023-09-30T09:29:09.611507+0000","last_active":"2023-09-30T09:30:40.661809+0000","last_peered":"2023-09-30T09:30:40.661809+0000","last_clean":"2023-09-30T09:30:40.661809+0000","last_became_active":"2023-09-30T09:29:09.611120+0000","last_became_peered":"2023-09-30T09:29:09.611120+0000","last_unstale":"2023-09-30T09:30:40.661809+0000","last_undegraded":"2023-09-30T09:30:40.661809+0000","last_fullsized":"2023-09-30T09:30:40.661809+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:31.622647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.5","version":"255'5647","reported_seq":5721,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:54.157782+0000","last_change":"2023-09-30T09:28:46.022537+0000","last_active":"2023-09-30T09:30:54.157782+0000","last_peered":"2023-09-30T09:30:54.157782+0000","last_clean":"2023-09-30T09:30:54.157782+0000","last_became_active":"2023-09-30T09:28:40.276165+0000","last_became_peered":"2023-09-30T09:28:40.276165+0000","last_unstale":"2023-09-30T09:30:54.157782+0000","last_undegraded":"2023-09-30T09:30:54.157782+0000","last_fullsized":"2023-09-30T09:30:54.157782+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90013696,"num_objects":1373,"num_object_clones":0,"num_object_copies":4119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4515,"num_write_kb":188448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":744,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.672523+0000","last_change":"2023-09-30T09:30:28.371305+0000","last_active":"2023-09-30T09:30:40.672523+0000","last_peered":"2023-09-30T09:30:40.672523+0000","last_clean":"2023-09-30T09:30:40.672523+0000","last_became_active":"2023-09-30T09:30:28.370995+0000","last_became_peered":"2023-09-30T09:30:28.370995+0000","last_unstale":"2023-09-30T09:30:40.672523+0000","last_undegraded":"2023-09-30T09:30:40.672523+0000","last_fullsized":"2023-09-30T09:30:40.672523+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":428,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.662578+0000","last_change":"2023-09-30T09:29:36.005128+0000","last_active":"2023-09-30T09:30:40.662578+0000","last_peered":"2023-09-30T09:30:40.662578+0000","last_clean":"2023-09-30T09:30:40.662578+0000","last_became_active":"2023-09-30T09:29:36.004563+0000","last_became_peered":"2023-09-30T09:29:36.004563+0000","last_unstale":"2023-09-30T09:30:40.662578+0000","last_undegraded":"2023-09-30T09:30:40.662578+0000","last_fullsized":"2023-09-30T09:30:40.662578+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:59.839484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":203,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.768750+0000","last_change":"2023-09-30T09:29:09.607111+0000","last_active":"2023-09-30T09:30:40.768750+0000","last_peered":"2023-09-30T09:30:40.768750+0000","last_clean":"2023-09-30T09:30:40.768750+0000","last_became_active":"2023-09-30T09:29:09.606807+0000","last_became_peered":"2023-09-30T09:29:09.606807+0000","last_unstale":"2023-09-30T09:30:40.768750+0000","last_undegraded":"2023-09-30T09:30:40.768750+0000","last_fullsized":"2023-09-30T09:30:40.768750+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:49.380239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":250,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.767595+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:30:40.767595+0000","last_peered":"2023-09-30T09:30:40.767595+0000","last_clean":"2023-09-30T09:30:40.767595+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:30:40.767595+0000","last_undegraded":"2023-09-30T09:30:40.767595+0000","last_fullsized":"2023-09-30T09:30:40.767595+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"255'5559","reported_seq":5686,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:54.124703+0000","last_change":"2023-09-30T09:28:42.687918+0000","last_active":"2023-09-30T09:30:54.124703+0000","last_peered":"2023-09-30T09:30:54.124703+0000","last_clean":"2023-09-30T09:30:54.124703+0000","last_became_active":"2023-09-30T09:28:40.262566+0000","last_became_peered":"2023-09-30T09:28:40.262566+0000","last_unstale":"2023-09-30T09:30:54.124703+0000","last_undegraded":"2023-09-30T09:30:54.124703+0000","last_fullsized":"2023-09-30T09:30:54.124703+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2390","last_scrub_stamp":"2023-09-30T09:28:42.687821+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.687821+0000","objects_scrubbed":1235,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:07:56.313767+0000","scrub_duration":0.36697584500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84738048,"num_objects":1293,"num_object_clones":0,"num_object_copies":3879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3180,"num_write_kb":103616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":672,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:40.771169+0000","last_change":"2023-09-30T09:29:12.096468+0000","last_active":"2023-09-30T09:30:40.771169+0000","last_peered":"2023-09-30T09:30:40.771169+0000","last_clean":"2023-09-30T09:29:03.507390+0000","last_became_active":"2023-09-30T09:29:12.096051+0000","last_became_peered":"2023-09-30T09:29:12.096051+0000","last_unstale":"2023-09-30T09:30:40.771169+0000","last_undegraded":"2023-09-30T09:29:10.593445+0000","last_fullsized":"2023-09-30T09:29:10.592749+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:49.279327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":705,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673838+0000","last_change":"2023-09-30T09:30:28.372432+0000","last_active":"2023-09-30T09:30:40.673838+0000","last_peered":"2023-09-30T09:30:40.673838+0000","last_clean":"2023-09-30T09:30:40.673838+0000","last_became_active":"2023-09-30T09:30:28.372087+0000","last_became_peered":"2023-09-30T09:30:28.372087+0000","last_unstale":"2023-09-30T09:30:40.673838+0000","last_undegraded":"2023-09-30T09:30:40.673838+0000","last_fullsized":"2023-09-30T09:30:40.673838+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":171,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.660846+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:30:40.660846+0000","last_peered":"2023-09-30T09:30:40.660846+0000","last_clean":"2023-09-30T09:30:40.660846+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:30:40.660846+0000","last_undegraded":"2023-09-30T09:30:40.660846+0000","last_fullsized":"2023-09-30T09:30:40.660846+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":197,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670593+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:30:40.670593+0000","last_peered":"2023-09-30T09:30:40.670593+0000","last_clean":"2023-09-30T09:30:40.670593+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:30:40.670593+0000","last_undegraded":"2023-09-30T09:30:40.670593+0000","last_fullsized":"2023-09-30T09:30:40.670593+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"255'5674","reported_seq":5772,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:51.971750+0000","last_change":"2023-09-30T09:28:43.582313+0000","last_active":"2023-09-30T09:30:51.971750+0000","last_peered":"2023-09-30T09:30:51.971750+0000","last_clean":"2023-09-30T09:30:51.971750+0000","last_became_active":"2023-09-30T09:28:40.264204+0000","last_became_peered":"2023-09-30T09:28:40.264204+0000","last_unstale":"2023-09-30T09:30:51.971750+0000","last_undegraded":"2023-09-30T09:30:51.971750+0000","last_fullsized":"2023-09-30T09:30:51.971750+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2440","last_scrub_stamp":"2023-09-30T09:28:43.582001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:43.582001+0000","objects_scrubbed":1291,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:50.257550+0000","scrub_duration":0.30691791200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96010240,"num_objects":1465,"num_object_clones":0,"num_object_copies":4395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3244,"num_write_kb":109376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":669,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.767943+0000","last_change":"2023-09-30T09:30:28.369897+0000","last_active":"2023-09-30T09:30:40.767943+0000","last_peered":"2023-09-30T09:30:40.767943+0000","last_clean":"2023-09-30T09:30:40.767943+0000","last_became_active":"2023-09-30T09:30:28.369480+0000","last_became_peered":"2023-09-30T09:30:28.369480+0000","last_unstale":"2023-09-30T09:30:40.767943+0000","last_undegraded":"2023-09-30T09:30:40.767943+0000","last_fullsized":"2023-09-30T09:30:40.767943+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":815,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673748+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:30:40.673748+0000","last_peered":"2023-09-30T09:30:40.673748+0000","last_clean":"2023-09-30T09:30:40.673748+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:30:40.673748+0000","last_undegraded":"2023-09-30T09:30:40.673748+0000","last_fullsized":"2023-09-30T09:30:40.673748+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":173,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:49.707426+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:30:49.707426+0000","last_peered":"2023-09-30T09:30:49.707426+0000","last_clean":"2023-09-30T09:30:49.707426+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:30:49.707426+0000","last_undegraded":"2023-09-30T09:30:49.707426+0000","last_fullsized":"2023-09-30T09:30:49.707426+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":160,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.768033+0000","last_change":"2023-09-30T09:29:09.611376+0000","last_active":"2023-09-30T09:30:40.768033+0000","last_peered":"2023-09-30T09:30:40.768033+0000","last_clean":"2023-09-30T09:30:40.768033+0000","last_became_active":"2023-09-30T09:29:09.611090+0000","last_became_peered":"2023-09-30T09:29:09.611090+0000","last_unstale":"2023-09-30T09:30:40.768033+0000","last_undegraded":"2023-09-30T09:30:40.768033+0000","last_fullsized":"2023-09-30T09:30:40.768033+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:07:15.039397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"255'4721","reported_seq":6461,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:51.972009+0000","last_change":"2023-09-30T09:29:36.030348+0000","last_active":"2023-09-30T09:30:51.972009+0000","last_peered":"2023-09-30T09:30:51.972009+0000","last_clean":"2023-09-30T09:29:08.532768+0000","last_became_active":"2023-09-30T09:29:36.027562+0000","last_became_peered":"2023-09-30T09:29:36.027562+0000","last_unstale":"2023-09-30T09:30:51.972009+0000","last_undegraded":"2023-09-30T09:29:35.999894+0000","last_fullsized":"2023-09-30T09:30:51.972009+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":4721,"log_dups_size":0,"ondisk_log_size":4721,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44007424,"num_objects":671,"num_object_clones":0,"num_object_copies":2013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":228,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1672,"num_write_kb":54816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":816,"num_bytes_recovered":53477376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":227},{"shards":"1,6","objects":1},{"shards":"1,6,7","objects":443}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":509,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671095+0000","last_change":"2023-09-30T09:30:28.373013+0000","last_active":"2023-09-30T09:30:40.671095+0000","last_peered":"2023-09-30T09:30:40.671095+0000","last_clean":"2023-09-30T09:30:40.671095+0000","last_became_active":"2023-09-30T09:30:28.372690+0000","last_became_peered":"2023-09-30T09:30:28.372690+0000","last_unstale":"2023-09-30T09:30:40.671095+0000","last_undegraded":"2023-09-30T09:30:40.671095+0000","last_fullsized":"2023-09-30T09:30:40.671095+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":945,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.673335+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:30:40.673335+0000","last_peered":"2023-09-30T09:30:40.673335+0000","last_clean":"2023-09-30T09:30:40.673335+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:30:40.673335+0000","last_undegraded":"2023-09-30T09:30:40.673335+0000","last_fullsized":"2023-09-30T09:30:40.673335+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":219,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:47.046443+0000","last_change":"2023-09-30T09:29:09.611808+0000","last_active":"2023-09-30T09:30:47.046443+0000","last_peered":"2023-09-30T09:30:47.046443+0000","last_clean":"2023-09-30T09:30:47.046443+0000","last_became_active":"2023-09-30T09:29:09.611484+0000","last_became_peered":"2023-09-30T09:29:09.611484+0000","last_unstale":"2023-09-30T09:30:47.046443+0000","last_undegraded":"2023-09-30T09:30:47.046443+0000","last_fullsized":"2023-09-30T09:30:47.046443+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:19.562461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":145,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:49.707583+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:30:49.707583+0000","last_peered":"2023-09-30T09:30:49.707583+0000","last_clean":"2023-09-30T09:30:49.707583+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:30:49.707583+0000","last_undegraded":"2023-09-30T09:30:49.707583+0000","last_fullsized":"2023-09-30T09:30:49.707583+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"255'4449","reported_seq":7393,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:53.637843+0000","last_change":"2023-09-30T09:29:35.026309+0000","last_active":"2023-09-30T09:30:53.637843+0000","last_peered":"2023-09-30T09:30:53.637843+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:35.024657+0000","last_became_peered":"2023-09-30T09:29:35.024657+0000","last_unstale":"2023-09-30T09:30:53.637843+0000","last_undegraded":"2023-09-30T09:29:35.010418+0000","last_fullsized":"2023-09-30T09:29:35.009540+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":4449,"log_dups_size":0,"ondisk_log_size":4449,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40697856,"num_objects":621,"num_object_clones":0,"num_object_copies":1863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1451,"num_write_kb":43104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":195},{"shards":"3,6","objects":345}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":687,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671163+0000","last_change":"2023-09-30T09:30:28.370867+0000","last_active":"2023-09-30T09:30:40.671163+0000","last_peered":"2023-09-30T09:30:40.671163+0000","last_clean":"2023-09-30T09:30:40.671163+0000","last_became_active":"2023-09-30T09:30:28.370566+0000","last_became_peered":"2023-09-30T09:30:28.370566+0000","last_unstale":"2023-09-30T09:30:40.671163+0000","last_undegraded":"2023-09-30T09:30:40.671163+0000","last_fullsized":"2023-09-30T09:30:40.671163+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.664009+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:30:40.664009+0000","last_peered":"2023-09-30T09:30:40.664009+0000","last_clean":"2023-09-30T09:30:40.664009+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:30:40.664009+0000","last_undegraded":"2023-09-30T09:30:40.664009+0000","last_fullsized":"2023-09-30T09:30:40.664009+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1204,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.768042+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:30:40.768042+0000","last_peered":"2023-09-30T09:30:40.768042+0000","last_clean":"2023-09-30T09:30:40.768042+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:30:40.768042+0000","last_undegraded":"2023-09-30T09:30:40.768042+0000","last_fullsized":"2023-09-30T09:30:40.768042+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":240,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.661980+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:30:40.661980+0000","last_peered":"2023-09-30T09:30:40.661980+0000","last_clean":"2023-09-30T09:30:40.661980+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:30:40.661980+0000","last_undegraded":"2023-09-30T09:30:40.661980+0000","last_fullsized":"2023-09-30T09:30:40.661980+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"255'4537","reported_seq":8713,"reported_epoch":255,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:30:54.016409+0000","last_change":"2023-09-30T09:29:36.003902+0000","last_active":"2023-09-30T09:30:54.016409+0000","last_peered":"2023-09-30T09:30:54.016409+0000","last_clean":"2023-09-30T09:29:33.909482+0000","last_became_active":"2023-09-30T09:29:35.026292+0000","last_became_peered":"2023-09-30T09:29:35.026292+0000","last_unstale":"2023-09-30T09:30:54.016409+0000","last_undegraded":"2023-09-30T09:30:54.016409+0000","last_fullsized":"2023-09-30T09:30:54.016409+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":4537,"log_dups_size":0,"ondisk_log_size":4537,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40828928,"num_objects":623,"num_object_clones":0,"num_object_copies":1869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1538,"num_write_kb":46528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65765376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":["7","1","2","6"],"object_location_counts":[{"shards":"1,6,7","objects":623}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":664,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.661843+0000","last_change":"2023-09-30T09:30:28.372591+0000","last_active":"2023-09-30T09:30:40.661843+0000","last_peered":"2023-09-30T09:30:40.661843+0000","last_clean":"2023-09-30T09:30:40.661843+0000","last_became_active":"2023-09-30T09:30:28.372210+0000","last_became_peered":"2023-09-30T09:30:28.372210+0000","last_unstale":"2023-09-30T09:30:40.661843+0000","last_undegraded":"2023-09-30T09:30:40.661843+0000","last_fullsized":"2023-09-30T09:30:40.661843+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":532,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.671249+0000","last_change":"2023-09-30T09:29:09.643760+0000","last_active":"2023-09-30T09:30:40.671249+0000","last_peered":"2023-09-30T09:30:40.671249+0000","last_clean":"2023-09-30T09:30:40.671249+0000","last_became_active":"2023-09-30T09:29:09.620803+0000","last_became_peered":"2023-09-30T09:29:09.620803+0000","last_unstale":"2023-09-30T09:30:40.671249+0000","last_undegraded":"2023-09-30T09:30:40.671249+0000","last_fullsized":"2023-09-30T09:30:40.671249+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:54:49.883760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.768149+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:30:40.768149+0000","last_peered":"2023-09-30T09:30:40.768149+0000","last_clean":"2023-09-30T09:30:40.768149+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:30:40.768149+0000","last_undegraded":"2023-09-30T09:30:40.768149+0000","last_fullsized":"2023-09-30T09:30:40.768149+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":196,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670002+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:30:40.670002+0000","last_peered":"2023-09-30T09:30:40.670002+0000","last_clean":"2023-09-30T09:30:40.670002+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:30:40.670002+0000","last_undegraded":"2023-09-30T09:30:40.670002+0000","last_fullsized":"2023-09-30T09:30:40.670002+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1e","version":"255'6808","reported_seq":10007,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:30:53.974404+0000","last_change":"2023-09-30T09:29:35.080846+0000","last_active":"2023-09-30T09:30:53.974404+0000","last_peered":"2023-09-30T09:30:53.974404+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:35.061073+0000","last_became_peered":"2023-09-30T09:29:35.061073+0000","last_unstale":"2023-09-30T09:30:53.974404+0000","last_undegraded":"2023-09-30T09:29:35.049064+0000","last_fullsized":"2023-09-30T09:30:53.974404+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6808,"log_dups_size":0,"ondisk_log_size":6808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84344832,"num_objects":1287,"num_object_clones":0,"num_object_copies":3861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":416,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4409,"num_write_kb":182272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1562,"num_bytes_recovered":102334464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":416},{"shards":"1,4,7","objects":871}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":113,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.670825+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:30:40.670825+0000","last_peered":"2023-09-30T09:30:40.670825+0000","last_clean":"2023-09-30T09:30:40.670825+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:30:40.670825+0000","last_undegraded":"2023-09-30T09:30:40.670825+0000","last_fullsized":"2023-09-30T09:30:40.670825+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":166,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.770332+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:30:40.770332+0000","last_peered":"2023-09-30T09:30:40.770332+0000","last_clean":"2023-09-30T09:30:40.770332+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:30:40.770332+0000","last_undegraded":"2023-09-30T09:30:40.770332+0000","last_fullsized":"2023-09-30T09:30:40.770332+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"255'5581","reported_seq":5660,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:54.081056+0000","last_change":"2023-09-30T09:29:10.893253+0000","last_active":"2023-09-30T09:30:54.081056+0000","last_peered":"2023-09-30T09:30:54.081056+0000","last_clean":"2023-09-30T09:29:08.494599+0000","last_became_active":"2023-09-30T09:29:10.886737+0000","last_became_peered":"2023-09-30T09:29:10.886737+0000","last_unstale":"2023-09-30T09:30:54.081056+0000","last_undegraded":"2023-09-30T09:29:10.766629+0000","last_fullsized":"2023-09-30T09:29:10.598246+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5581,"log_dups_size":0,"ondisk_log_size":5581,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85327872,"num_objects":1302,"num_object_clones":0,"num_object_copies":3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2001,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4480,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":412},{"shards":"3,7","objects":-699}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":216,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.772695+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:30:40.772695+0000","last_peered":"2023-09-30T09:30:40.772695+0000","last_clean":"2023-09-30T09:30:40.772695+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:30:40.772695+0000","last_undegraded":"2023-09-30T09:30:40.772695+0000","last_fullsized":"2023-09-30T09:30:40.772695+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":197,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.664477+0000","last_change":"2023-09-30T09:29:09.609752+0000","last_active":"2023-09-30T09:30:40.664477+0000","last_peered":"2023-09-30T09:30:40.664477+0000","last_clean":"2023-09-30T09:30:40.664477+0000","last_became_active":"2023-09-30T09:29:09.609242+0000","last_became_peered":"2023-09-30T09:29:09.609242+0000","last_unstale":"2023-09-30T09:30:40.664477+0000","last_undegraded":"2023-09-30T09:30:40.664477+0000","last_fullsized":"2023-09-30T09:30:40.664477+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:31.366509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"255'5592","reported_seq":5669,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:30:54.169563+0000","last_change":"2023-09-30T09:29:10.864651+0000","last_active":"2023-09-30T09:30:54.169563+0000","last_peered":"2023-09-30T09:30:54.169563+0000","last_clean":"2023-09-30T09:29:08.532983+0000","last_became_active":"2023-09-30T09:29:10.860470+0000","last_became_peered":"2023-09-30T09:29:10.860470+0000","last_unstale":"2023-09-30T09:30:54.169563+0000","last_undegraded":"2023-09-30T09:29:10.638243+0000","last_fullsized":"2023-09-30T09:29:10.593564+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5592,"log_dups_size":0,"ondisk_log_size":5592,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81428480,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4479,"num_write_kb":183072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":439},{"shards":"3,7","objects":-814}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":199,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.771571+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:30:40.771571+0000","last_peered":"2023-09-30T09:30:40.771571+0000","last_clean":"2023-09-30T09:30:40.771571+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:30:40.771571+0000","last_undegraded":"2023-09-30T09:30:40.771571+0000","last_fullsized":"2023-09-30T09:30:40.771571+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":228,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.660942+0000","last_change":"2023-09-30T09:29:09.621218+0000","last_active":"2023-09-30T09:30:40.660942+0000","last_peered":"2023-09-30T09:30:40.660942+0000","last_clean":"2023-09-30T09:30:40.660942+0000","last_became_active":"2023-09-30T09:29:09.620972+0000","last_became_peered":"2023-09-30T09:29:09.620972+0000","last_unstale":"2023-09-30T09:30:40.660942+0000","last_undegraded":"2023-09-30T09:30:40.660942+0000","last_fullsized":"2023-09-30T09:30:40.660942+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:25.640288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1b","version":"255'6940","reported_seq":7016,"reported_epoch":255,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:30:53.649972+0000","last_change":"2023-09-30T09:29:35.093036+0000","last_active":"2023-09-30T09:30:53.649972+0000","last_peered":"2023-09-30T09:30:53.649972+0000","last_clean":"2023-09-30T09:29:33.992127+0000","last_became_active":"2023-09-30T09:29:35.070155+0000","last_became_peered":"2023-09-30T09:29:35.070155+0000","last_unstale":"2023-09-30T09:30:53.649972+0000","last_undegraded":"2023-09-30T09:30:53.649972+0000","last_fullsized":"2023-09-30T09:30:53.649972+0000","mapping_epoch":238,"log_start":"210'266","ondisk_log_start":"210'266","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85622784,"num_objects":1307,"num_object_clones":0,"num_object_copies":3921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4473,"num_write_kb":184928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":1307}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":235,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.660658+0000","last_change":"2023-09-30T09:29:09.615385+0000","last_active":"2023-09-30T09:30:40.660658+0000","last_peered":"2023-09-30T09:30:40.660658+0000","last_clean":"2023-09-30T09:30:40.660658+0000","last_became_active":"2023-09-30T09:29:09.614479+0000","last_became_peered":"2023-09-30T09:29:09.614479+0000","last_unstale":"2023-09-30T09:30:40.660658+0000","last_undegraded":"2023-09-30T09:30:40.660658+0000","last_fullsized":"2023-09-30T09:30:40.660658+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:14.671222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":265,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.662475+0000","last_change":"2023-09-30T09:29:09.600457+0000","last_active":"2023-09-30T09:30:40.662475+0000","last_peered":"2023-09-30T09:30:40.662475+0000","last_clean":"2023-09-30T09:30:40.662475+0000","last_became_active":"2023-09-30T09:29:09.600080+0000","last_became_peered":"2023-09-30T09:29:09.600080+0000","last_unstale":"2023-09-30T09:30:40.662475+0000","last_undegraded":"2023-09-30T09:30:40.662475+0000","last_fullsized":"2023-09-30T09:30:40.662475+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:10:02.250349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1a","version":"255'6966","reported_seq":7033,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-30T09:30:54.149511+0000","last_change":"2023-09-30T09:29:35.021535+0000","last_active":"2023-09-30T09:30:54.149511+0000","last_peered":"2023-09-30T09:30:54.149511+0000","last_clean":"2023-09-30T09:30:54.149511+0000","last_became_active":"2023-09-30T09:29:35.014808+0000","last_became_peered":"2023-09-30T09:29:35.014808+0000","last_unstale":"2023-09-30T09:30:54.149511+0000","last_undegraded":"2023-09-30T09:30:54.149511+0000","last_fullsized":"2023-09-30T09:30:54.149511+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6966,"log_dups_size":0,"ondisk_log_size":6966,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83558400,"num_objects":1275,"num_object_clones":0,"num_object_copies":3825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4519,"num_write_kb":185408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":138,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.771592+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:30:40.771592+0000","last_peered":"2023-09-30T09:30:40.771592+0000","last_clean":"2023-09-30T09:30:40.771592+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:30:40.771592+0000","last_undegraded":"2023-09-30T09:30:40.771592+0000","last_fullsized":"2023-09-30T09:30:40.771592+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":213,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-30T09:30:40.669623+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:30:40.669623+0000","last_peered":"2023-09-30T09:30:40.669623+0000","last_clean":"2023-09-30T09:30:40.669623+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:30:40.669623+0000","last_undegraded":"2023-09-30T09:30:40.669623+0000","last_fullsized":"2023-09-30T09:30:40.669623+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":36,"stat_sum":{"num_bytes":2741764120,"num_objects":41838,"num_object_clones":0,"num_object_copies":125514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12147,"num_objects_misplaced":1219,"num_objects_unfound":0,"num_objects_dirty":41838,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126729,"num_write_kb":4876353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22809,"num_bytes_recovered":1494810624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10147213312,"data_stored":10147201096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":206501,"ondisk_log_size":206501,"up":108,"acting":101,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":1418,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":435019,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":43,"num_write_kb":438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234231,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1509784,"kb_used_data":1322000,"kb_used_omap":0,"kb_used_meta":187776,"kb_avail":92235368,"statfs":{"total":95995035648,"available":94449016832,"internally_reserved":0,"allocated":1353728000,"data_stored":1352780999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":192282624},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:30 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.76,"15min":1.0580000000000001},"min":{"1min":0.25700000000000001,"5min":0.224,"15min":0.224},"max":{"1min":7.6180000000000003,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.746,"5min":1.538,"15min":0.94299999999999995},"min":{"1min":0.26600000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.5710000000000002,"5min":18.097999999999999,"15min":18.097999999999999},"last":4.3019999999999996}]},{"osd":1,"last update":"Sat Sep 30 09:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9039999999999999,"5min":4.4969999999999999,"15min":2.4910000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":63.616999999999997,"5min":70.566999999999993,"15min":82.025999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":3.7130000000000001,"5min":4.3860000000000001,"15min":2.4510000000000001},"min":{"1min":0.27700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":63.597000000000001,"5min":70.626000000000005,"15min":82.012},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:30:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":2.1469999999999998,"15min":1.2270000000000001},"min":{"1min":0.34599999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.6519999999999992,"5min":39.445,"15min":39.445},"last":2.3839999999999999},{"interface":"front","average":{"1min":1.2909999999999999,"5min":2.1309999999999998,"15min":1.2649999999999999},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.0049999999999999,"5min":40.988999999999997,"15min":40.988999999999997},"last":7.8479999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":3.5529999999999999,"15min":1.694},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.8140000000000001,"5min":76.909000000000006,"15min":76.909000000000006},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.99399999999999999,"5min":3.3359999999999999,"15min":1.575},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.2800000000000002,"5min":77.048000000000002,"15min":77.048000000000002},"last":7.9429999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":1.155,"15min":0.71899999999999997},"min":{"1min":0.318,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.75,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.996},{"interface":"front","average":{"1min":1.335,"5min":0.97599999999999998,"15min":0.69699999999999995},"min":{"1min":0.314,"5min":0.251,"15min":0.251},"max":{"1min":5.6879999999999997,"5min":6.282,"15min":6.282},"last":7.8890000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.2070000000000001,"15min":0.79100000000000004},"min":{"1min":0.38100000000000001,"5min":0.248,"15min":0.248},"max":{"1min":7.8490000000000002,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.579,"5min":1.325,"15min":0.84699999999999998},"min":{"1min":0.443,"5min":0.30199999999999999,"15min":0.28599999999999998},"max":{"1min":7.6909999999999998,"5min":11.022,"15min":11.022},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.234,"15min":0.82899999999999996},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33000000000000002},"max":{"1min":7.7210000000000001,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.492,"5min":1.0660000000000001,"15min":0.76900000000000002},"min":{"1min":0.34699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.4329999999999998,"5min":6.4329999999999998,"15min":6.4329999999999998},"last":7.915}]}]},{"osd":6,"up_from":46,"seq":197568495868,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1616792,"kb_used_data":1435472,"kb_used_omap":0,"kb_used_meta":181120,"kb_avail":92128360,"statfs":{"total":95995035648,"available":94339440640,"internally_reserved":0,"allocated":1469923328,"data_stored":1468976768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":185466880},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":2.2029999999999998,"15min":1.226},"min":{"1min":0.248,"5min":0.248,"15min":0.222},"max":{"1min":0.63800000000000001,"5min":45.927999999999997,"15min":45.927999999999997},"last":0.219},{"interface":"front","average":{"1min":1.175,"5min":2.657,"15min":1.3859999999999999},"min":{"1min":0.308,"5min":0.26800000000000002,"15min":0.23499999999999999},"max":{"1min":3.157,"5min":45.316000000000003,"15min":45.316000000000003},"last":2.617}]},{"osd":1,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":3.0630000000000002,"15min":1.51},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.249},"max":{"1min":17.629999999999999,"5min":69.150000000000006,"15min":69.150000000000006},"last":5.7350000000000003},{"interface":"front","average":{"1min":2.1800000000000002,"5min":3.5569999999999999,"15min":1.6679999999999999},"min":{"1min":0.41199999999999998,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":17.652999999999999,"5min":69.132000000000005,"15min":69.132000000000005},"last":2.5299999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":2.496,"15min":1.3400000000000001},"min":{"1min":0.313,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":3.871,"5min":37.978000000000002,"15min":37.978000000000002},"last":0.379},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.9359999999999999,"15min":1.167},"min":{"1min":0.377,"5min":0.31,"15min":0.31},"max":{"1min":1.4390000000000001,"5min":37.759999999999998,"15min":37.759999999999998},"last":0.28199999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5070000000000001,"5min":2.0699999999999998,"15min":1.181},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":23.411999999999999,"5min":23.411999999999999,"15min":23.411999999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":2.6190000000000002,"5min":2.294,"15min":1.304},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":23.271000000000001,"5min":23.271000000000001,"15min":23.271000000000001},"last":2.6400000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":2.0659999999999998,"15min":1.2589999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.2959999999999998,"5min":49.286999999999999,"15min":73.656000000000006},"last":2.5640000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.6339999999999999,"15min":1.1439999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.1559999999999997,"5min":49.252000000000002,"15min":73.882999999999996},"last":0.97899999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":9.3010000000000002,"5min":3.4289999999999998,"15min":1.5229999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28499999999999998},"max":{"1min":76.613,"5min":76.613,"15min":76.613},"last":0.29099999999999998},{"interface":"front","average":{"1min":8.7910000000000004,"5min":3.1960000000000002,"15min":1.49},"min":{"1min":0.36299999999999999,"5min":0.35399999999999998,"15min":0.311},"max":{"1min":76.329999999999998,"5min":76.329999999999998,"15min":76.329999999999998},"last":0.35299999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:30:10 2023","interfaces":[{"interface":"back","average":{"1min":11.069000000000001,"5min":4.0069999999999997,"15min":1.752},"min":{"1min":0.372,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":91.120999999999995,"5min":91.120999999999995,"15min":91.120999999999995},"last":2.5950000000000002},{"interface":"front","average":{"1min":10.925000000000001,"5min":3.851,"15min":1.7889999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":91.733999999999995,"5min":91.733999999999995,"15min":91.733999999999995},"last":5.774}]}]},{"osd":1,"up_from":99,"seq":425201762432,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1528260,"kb_used_data":1275320,"kb_used_omap":5,"kb_used_meta":252922,"kb_avail":92216892,"statfs":{"total":95995035648,"available":94430097408,"internally_reserved":0,"allocated":1305927680,"data_stored":1304484808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":258992310},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":4.4420000000000002,"15min":2.2999999999999998},"min":{"1min":0.20599999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.8340000000000001,"5min":150.93899999999999,"15min":150.93899999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.042,"5min":4.5250000000000004,"15min":2.3700000000000001},"min":{"1min":0.23899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":5.5380000000000003,"5min":156.74199999999999,"15min":156.74199999999999},"last":0.435}]},{"osd":2,"last update":"Sat Sep 30 09:30:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":7.3390000000000004,"15min":2.7959999999999998},"min":{"1min":0.36799999999999999,"5min":0.247,"15min":0.247},"max":{"1min":6.2569999999999997,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.974,"5min":7.2599999999999998,"15min":2.7959999999999998},"min":{"1min":0.32700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.0030000000000001,"5min":144.62,"15min":144.62},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":13.821,"5min":4.4210000000000003,"15min":1.8580000000000001},"min":{"1min":0.34499999999999997,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":60.743000000000002,"5min":60.743000000000002,"15min":60.743000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":13.981999999999999,"5min":4.2400000000000002,"15min":1.7849999999999999},"min":{"1min":0.32000000000000001,"5min":0.214,"15min":0.214},"max":{"1min":61.201999999999998,"5min":61.201999999999998,"15min":61.201999999999998},"last":0.55500000000000005}]},{"osd":4,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.421,"15min":0.95699999999999996},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.28299999999999997},"max":{"1min":5.383,"5min":10.173,"15min":10.173},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.397,"15min":0.96399999999999997},"min":{"1min":0.30499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.867,"5min":8.9299999999999997,"15min":8.9299999999999997},"last":0.40100000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":12.034000000000001,"5min":3.661,"15min":1.6719999999999999},"min":{"1min":0.38700000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":195.512,"5min":195.512,"15min":195.512},"last":2.5659999999999998},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.4630000000000001,"15min":0.98099999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4390000000000001,"5min":10.128,"15min":10.128},"last":1.3520000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.5069999999999999,"15min":0.98099999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":2.6899999999999999,"5min":14.356999999999999,"15min":14.356999999999999},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.6899999999999999,"15min":1.095},"min":{"1min":0.44500000000000001,"5min":0.313,"15min":0.313},"max":{"1min":4.5339999999999998,"5min":15.864000000000001,"15min":15.864000000000001},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.8540000000000001,"15min":1.1659999999999999},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.4790000000000001,"5min":14.403,"15min":14.403},"last":2.4159999999999999},{"interface":"front","average":{"1min":1.2430000000000001,"5min":3.0089999999999999,"15min":1.5740000000000001},"min":{"1min":0.44,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.1400000000000001,"5min":122.56699999999999,"15min":122.56699999999999},"last":3.0350000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925965,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":574912,"kb_used_data":337976,"kb_used_omap":5,"kb_used_meta":236922,"kb_avail":93170240,"statfs":{"total":95995035648,"available":95406325760,"internally_reserved":0,"allocated":346087424,"data_stored":344588653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":242608381},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:30:46 2023","interfaces":[{"interface":"back","average":{"1min":4.4059999999999997,"5min":3.6669999999999998,"15min":1.6140000000000001},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.188},"max":{"1min":54.314999999999998,"5min":125.024,"15min":125.024},"last":1.6200000000000001},{"interface":"front","average":{"1min":4.1440000000000001,"5min":3.3719999999999999,"15min":1.5600000000000001},"min":{"1min":0.224,"5min":0.192,"15min":0.192},"max":{"1min":53.536999999999999,"5min":124.901,"15min":124.901},"last":0.52200000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.968,"15min":1.1120000000000001},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":4.3460000000000001,"5min":55.823999999999998,"15min":55.823999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.74199999999999999,"5min":2.169,"15min":1.198},"min":{"1min":0.26200000000000001,"5min":0.245,"15min":0.245},"max":{"1min":2.855,"5min":55.790999999999997,"15min":55.790999999999997},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:30:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":2.6419999999999999,"15min":1.3700000000000001},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.8239999999999998,"5min":55.68,"15min":55.68},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.125,"5min":2.5179999999999998,"15min":1.3049999999999999},"min":{"1min":0.32200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.8609999999999998,"5min":55.244,"15min":55.244},"last":1.71}]},{"osd":4,"last update":"Sat Sep 30 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.4369999999999998,"15min":1.3480000000000001},"min":{"1min":0.32100000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.9129999999999998,"5min":121.486,"15min":121.486},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.446,"5min":2.9889999999999999,"15min":1.5820000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.31},"max":{"1min":5.1319999999999997,"5min":121.44499999999999,"15min":121.44499999999999},"last":0.622}]},{"osd":5,"last update":"Sat Sep 30 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":11.733000000000001,"5min":3.698,"15min":1.8999999999999999},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":135.34899999999999,"5min":135.34899999999999,"15min":135.34899999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":11.656000000000001,"5min":3.6509999999999998,"15min":1.8799999999999999},"min":{"1min":0.41599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":135.381,"5min":135.381,"15min":135.381},"last":1.661}]},{"osd":6,"last update":"Sat Sep 30 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.427,"15min":1.05},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.2770000000000001,"5min":13.811999999999999,"15min":13.811999999999999},"last":2.9300000000000002},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.2609999999999999,"15min":0.999},"min":{"1min":0.47699999999999998,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":1.5109999999999999,"5min":6.407,"15min":7.556},"last":0.81000000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":12.93,"5min":5.2809999999999997,"15min":2.3399999999999999},"min":{"1min":0.438,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":150.96899999999999,"5min":150.96899999999999,"15min":150.96899999999999},"last":2.891},{"interface":"front","average":{"1min":12.728,"5min":5.0129999999999999,"15min":2.2469999999999999},"min":{"1min":0.46100000000000002,"5min":0.371,"15min":0.371},"max":{"1min":150.328,"5min":150.328,"15min":150.328},"last":2.8199999999999998}]}]},{"osd":2,"up_from":0,"seq":446676598906,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435378,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2213400,"kb_used_data":1932432,"kb_used_omap":2,"kb_used_meta":280701,"kb_avail":91531752,"statfs":{"total":95995035648,"available":93728514048,"internally_reserved":0,"allocated":1978810368,"data_stored":1977342209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":287438183},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.544,"15min":1.5069999999999999},"min":{"1min":0.20200000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.0510000000000002,"5min":10.856,"15min":130.946},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.619,"5min":1.631,"15min":1.526},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.20899999999999999},"max":{"1min":7.3920000000000003,"5min":12.361000000000001,"15min":130.96899999999999},"last":0.23000000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":9.4280000000000008,"5min":3.4500000000000002,"15min":1.528},"min":{"1min":0.30399999999999999,"5min":0.23499999999999999,"15min":0.224},"max":{"1min":133.215,"5min":133.215,"15min":133.215},"last":0.35299999999999998},{"interface":"front","average":{"1min":9.3040000000000003,"5min":3.399,"15min":1.4830000000000001},"min":{"1min":0.24099999999999999,"5min":0.221,"15min":0.221},"max":{"1min":133.16,"5min":133.16,"15min":133.16},"last":0.27600000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":2.5630000000000002,"15min":1.95},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.6079999999999997,"5min":29.986000000000001,"15min":145.75899999999999},"last":0.315},{"interface":"front","average":{"1min":1.1850000000000001,"5min":2.319,"15min":1.847},"min":{"1min":0.33600000000000002,"5min":0.26200000000000001,"15min":0.22900000000000001},"max":{"1min":5.8780000000000001,"5min":30.242999999999999,"15min":150.358},"last":0.33600000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.399,"15min":0.90900000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.5549999999999997,"5min":15.960000000000001,"15min":15.960000000000001},"last":4.5949999999999998},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.6599999999999999,"15min":1.002},"min":{"1min":0.37,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.3490000000000002,"5min":15.638,"15min":15.638},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.732,"15min":1.0740000000000001},"min":{"1min":0.33100000000000002,"5min":0.314,"15min":0.308},"max":{"1min":2.7469999999999999,"5min":10.048999999999999,"15min":10.048999999999999},"last":1.6950000000000001},{"interface":"front","average":{"1min":1.669,"5min":1.9810000000000001,"15min":1.1839999999999999},"min":{"1min":0.42799999999999999,"5min":0.35299999999999998,"15min":0.33500000000000002},"max":{"1min":7.5419999999999998,"5min":12.785,"15min":12.785},"last":1.752}]},{"osd":6,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.542,"15min":0.99199999999999999},"min":{"1min":0.372,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.548,"5min":10.348000000000001,"15min":10.348000000000001},"last":4.548},{"interface":"front","average":{"1min":1.361,"5min":1.885,"15min":1.145},"min":{"1min":0.45600000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.8250000000000002,"5min":14.69,"15min":14.69},"last":2.8250000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":4.0529999999999999,"15min":1.847},"min":{"1min":0.40000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.3159999999999998,"5min":226.03999999999999,"15min":226.03999999999999},"last":2.2839999999999998},{"interface":"front","average":{"1min":1.165,"5min":3.625,"15min":1.6830000000000001},"min":{"1min":0.48499999999999999,"5min":0.377,"15min":0.372},"max":{"1min":4.1840000000000002,"5min":226.06100000000001,"15min":226.06100000000001},"last":2.3010000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019141,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":929940,"kb_used_data":766092,"kb_used_omap":0,"kb_used_meta":163840,"kb_avail":92815212,"statfs":{"total":95995035648,"available":95042777088,"internally_reserved":0,"allocated":784478208,"data_stored":783535801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":167772160},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":4.8890000000000002,"15min":2.5430000000000001},"min":{"1min":0.28599999999999998,"5min":0.22800000000000001,"15min":0.20599999999999999},"max":{"1min":5.0940000000000003,"5min":119.986,"15min":128.79300000000001},"last":5.2069999999999999},{"interface":"front","average":{"1min":1.2849999999999999,"5min":4.71,"15min":2.4140000000000001},"min":{"1min":0.255,"5min":0.253,"15min":0.248},"max":{"1min":8.7629999999999999,"5min":119.863,"15min":128.38800000000001},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":3.1659999999999999,"15min":1.599},"min":{"1min":0.28000000000000003,"5min":0.245,"15min":0.245},"max":{"1min":11.67,"5min":93.244,"15min":93.244},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.0389999999999999,"5min":3.1749999999999998,"15min":1.6279999999999999},"min":{"1min":0.34300000000000003,"5min":0.26800000000000002,"15min":0.23999999999999999},"max":{"1min":3.6949999999999998,"5min":93.465999999999994,"15min":93.465999999999994},"last":2.7069999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:30:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":5.4690000000000003,"15min":2.4380000000000002},"min":{"1min":0.32900000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.6639999999999997,"5min":140.601,"15min":140.601},"last":2.8929999999999998},{"interface":"front","average":{"1min":1.538,"5min":5.319,"15min":2.3900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.9450000000000003,"5min":140.54900000000001,"15min":140.54900000000001},"last":5.1669999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":5.3490000000000002,"15min":2.5179999999999998},"min":{"1min":0.40899999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.7800000000000002,"5min":281.42200000000003,"15min":281.42200000000003},"last":5.3250000000000002},{"interface":"front","average":{"1min":1.6579999999999999,"5min":5.351,"15min":2.5209999999999999},"min":{"1min":0.34899999999999998,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":11.715999999999999,"5min":281.44999999999999,"15min":281.44999999999999},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:30:24 2023","interfaces":[{"interface":"back","average":{"1min":9.2080000000000002,"5min":2.4990000000000001,"15min":1.4330000000000001},"min":{"1min":0.30399999999999999,"5min":0.26900000000000002,"15min":0.252},"max":{"1min":134.30600000000001,"5min":134.30600000000001,"15min":134.30600000000001},"last":4.875},{"interface":"front","average":{"1min":9.2949999999999999,"5min":2.75,"15min":1.5549999999999999},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":134.35300000000001,"5min":134.35300000000001,"15min":134.35300000000001},"last":5.109}]},{"osd":6,"last update":"Sat Sep 30 09:30:52 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.0649999999999999,"15min":0.71599999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":6.8440000000000003,"5min":6.8440000000000003,"15min":6.8440000000000003},"last":5.2380000000000004},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.212,"15min":0.79300000000000004},"min":{"1min":0.31,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":11.779999999999999,"5min":11.779999999999999,"15min":11.779999999999999},"last":0.438}]},{"osd":7,"last update":"Sat Sep 30 09:30:15 2023","interfaces":[{"interface":"back","average":{"1min":9.0370000000000008,"5min":2.8500000000000001,"15min":1.6399999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":136.27099999999999,"5min":136.27099999999999,"15min":136.27099999999999},"last":4.5},{"interface":"front","average":{"1min":8.5589999999999993,"5min":2.706,"15min":1.3300000000000001},"min":{"1min":0.42599999999999999,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":136.60400000000001,"5min":136.60400000000001,"15min":136.60400000000001},"last":5.2839999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757505,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1173636,"kb_used_data":1005180,"kb_used_omap":0,"kb_used_meta":168128,"kb_avail":92571516,"statfs":{"total":95995035648,"available":94793232384,"internally_reserved":0,"allocated":1029304320,"data_stored":1028373135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":172163072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":4.0279999999999996,"15min":1.885},"min":{"1min":0.23699999999999999,"5min":0.22,"15min":0.22},"max":{"1min":4.6600000000000001,"5min":148.79900000000001,"15min":148.79900000000001},"last":0.253},{"interface":"front","average":{"1min":0.64000000000000001,"5min":4.532,"15min":2.109},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.6340000000000003,"5min":148.779,"15min":148.779},"last":1.623}]},{"osd":1,"last update":"Sat Sep 30 09:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.9830000000000001,"5min":4.7809999999999997,"15min":2.431},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":37.268999999999998,"5min":166.708,"15min":166.708},"last":1.5620000000000001},{"interface":"front","average":{"1min":3.3900000000000001,"5min":4.7220000000000004,"15min":2.4140000000000001},"min":{"1min":0.307,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":37.307000000000002,"5min":166.607,"15min":166.607},"last":2.0169999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":4.931,"15min":2.601},"min":{"1min":0.372,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.4210000000000003,"5min":165.768,"15min":165.768},"last":1.675},{"interface":"front","average":{"1min":0.89500000000000002,"5min":6.202,"15min":3.105},"min":{"1min":0.33700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5779999999999998,"5min":166.24000000000001,"15min":166.24000000000001},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":1.9330000000000001,"15min":1.3400000000000001},"min":{"1min":0.35599999999999998,"5min":0.311,"15min":0.311},"max":{"1min":6.3040000000000003,"5min":20.079000000000001,"15min":28.015999999999998},"last":1.595},{"interface":"front","average":{"1min":1.9239999999999999,"5min":1.653,"15min":1.196},"min":{"1min":0.379,"5min":0.313,"15min":0.313},"max":{"1min":7.3959999999999999,"5min":20.021000000000001,"15min":28.039999999999999},"last":1.96}]},{"osd":4,"last update":"Sat Sep 30 09:30:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.0920000000000001,"15min":0.70799999999999996},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.222},"max":{"1min":6.2119999999999997,"5min":8.5359999999999996,"15min":8.5359999999999996},"last":1.9930000000000001},{"interface":"front","average":{"1min":0.86399999999999999,"5min":1.0349999999999999,"15min":0.74399999999999999},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.27000000000000002},"max":{"1min":4.2389999999999999,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":0.32300000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.45,"15min":0.85799999999999998},"min":{"1min":0.33100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.46,"5min":14.23,"15min":14.23},"last":2.04},{"interface":"front","average":{"1min":1.236,"5min":1.581,"15min":0.99399999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.6760000000000002,"5min":18.960000000000001,"15min":18.960000000000001},"last":1.655}]},{"osd":7,"last update":"Sat Sep 30 09:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.3660000000000001,"15min":0.90400000000000003},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.665,"5min":19.175999999999998,"15min":19.175999999999998},"last":1.694},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.141,"15min":0.81399999999999995},"min":{"1min":0.371,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.6449999999999996,"5min":8.8810000000000002,"15min":8.8810000000000002},"last":2.0630000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":333840384,"data_stored":333840384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1287393280,"data_stored":1287389208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1967456256,"data_stored":1967456256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1971781632,"data_stored":1971781632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":769916928,"data_stored":769916928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1017712640,"data_stored":1017708568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1456607232,"data_stored":1456603160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1342504960,"data_stored":1342504960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:30:59.178 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:30:59.178 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-09-30T09:30:59.198 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 11861/124410 objects degraded (9.534%), 16 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-30T09:30:59.207 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:59.321 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:30:59.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:59.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:30:59.437 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:59.506 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:59.658 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:59.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:59.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:30:59.769 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:30:59.840 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:30:59.903 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:30:59.973 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:30:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:30:59.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80446 5.7 5:e364f9e6:::benchmark_data_smithi019_287194_object80445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.619068+0000", 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 418.39840210300002, 2023-09-30T09:31:00.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1066015870000001, 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80446 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.619068+0000", 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.619068+0000", 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.619069+0000", 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6869999999999999e-06 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.619093+0000", 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4158000000000001e-05 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.619107+0000", 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3169999999999999e-05 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.619117+0000", 2023-09-30T09:31:00.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0061000000000001e-05 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.619215+0000", 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7992999999999995e-05 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.619227+0000", 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2036e-05 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606056+0000", 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98682901700000003 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606068+0000", 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1934e-05 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.696432+0000", 2023-09-30T09:31:00.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.090364876 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.696607+0000", 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000174678 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.696790+0000", 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018323999999999999 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.703108+0000", 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063172799999999998 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.706393+0000", 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003284941 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.723000+0000", 2023-09-30T09:31:00.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016607209000000001 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.723056+0000", 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5705000000000003e-05 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.723127+0000", 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1711999999999994e-05 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.725669+0000", 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025418900000000002 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80457 5.7 5:e4faaa77:::benchmark_data_smithi019_287194_object80456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.641721+0000", 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 418.37574815599999, 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0842669049999998, 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.032 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80457 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.641721+0000", 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.641721+0000", 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.641723+0000", 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.917e-06 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.641750+0000", 2023-09-30T09:31:00.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6364e-05 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.641772+0000", 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1950000000000002e-05 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.641782+0000", 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0209e-05 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.641878+0000", 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6062000000000003e-05 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.641891+0000", 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2856e-05 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606096+0000", 2023-09-30T09:31:00.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96420550299999996 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606106+0000", 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7999999999999993e-06 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.699746+0000", 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0936395029999999 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.699909+0000", 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016375999999999999 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.700089+0000", 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017922399999999999 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.706440+0000", 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.006351481 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.723018+0000", 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016577416000000001 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.725742+0000", 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027240849999999998 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.725792+0000", 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0540000000000001e-05 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.725866+0000", 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3818000000000004e-05 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.725988+0000", 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012241699999999999 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145653 5.5 5:ae5fdde1:::benchmark_data_smithi019_287194_object62875:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.313460+0000", 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 249.70400914499999, 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2244411880000001, 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145653 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313460+0000", 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313460+0000", 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313462+0000", 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.55e-06 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313469+0000", 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9240000000000001e-06 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313483+0000", 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4324999999999999e-05 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313493+0000", 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5810000000000003e-06 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313558+0000", 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5381000000000003e-05 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op must wait for map", 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313565+0000", 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.759e-06 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.324934+0000", 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011369086 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.324944+0000", 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.011e-05 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.311366+0000", 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98642155899999995 2023-09-30T09:31:00.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.311378+0000", 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2218000000000001e-05 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.537814+0000", 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2264360970000001 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.537902+0000", 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7597999999999997e-05 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145669 5.5 5:ad5f3322:::benchmark_data_smithi019_287194_object62891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.339481+0000", 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 249.67798814299999, 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.227797979, 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.048 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145669 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339481+0000", 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.049 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339481+0000", 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339483+0000", 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.217e-06 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339486+0000", 2023-09-30T09:31:00.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0570000000000001e-06 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339495+0000", 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9260000000000006e-06 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339501+0000", 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4110000000000001e-06 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339561+0000", 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0372999999999999e-05 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339572+0000", 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0998999999999999e-05 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.311409+0000", 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97183617899999997 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.311419+0000", 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0321e-05 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.538221+0000", 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.226802291 2023-09-30T09:31:00.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.538336+0000", 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011493399999999999 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,6,7", 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.538888+0000", 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00055175199999999997 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.558496+0000", 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.019607877999999999 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.558546+0000", 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0178e-05 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.558698+0000", 2023-09-30T09:31:00.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015172300000000001 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.567057+0000", 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0083589400000000005 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.567114+0000", 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7534000000000002e-05 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.567151+0000", 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6560999999999998e-05 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.567279+0000", 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012870500000000001 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.057 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145672 5.7 5:ebc96d08:::benchmark_data_smithi019_287194_object62894:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.339690+0000", 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 249.677779985, 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2260808810000001, 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145672 2023-09-30T09:31:00.058 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339690+0000", 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339690+0000", 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339690+0000", 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2399999999999997e-07 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339694+0000", 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5439999999999999e-06 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339707+0000", 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3226e-05 2023-09-30T09:31:00.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339714+0000", 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0199999999999997e-06 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339771+0000", 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7079e-05 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.339779+0000", 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8650000000000001e-06 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.313005+0000", 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97322630499999996 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.313015+0000", 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1509999999999993e-06 2023-09-30T09:31:00.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.535885+0000", 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.222870449 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.536036+0000", 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000151309 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:31:00.063 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.536605+0000", 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00056905499999999995 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.563500+0000", 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026894563999999999 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.563596+0000", 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5649999999999994e-05 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.064 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.563614+0000", 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7941999999999999e-05 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.565576+0000", 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001962264 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.565647+0000", 2023-09-30T09:31:00.065 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1619999999999995e-05 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.565771+0000", 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012311399999999999 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145682 5.7 5:ea4fbe4e:::benchmark_data_smithi019_287194_object62904:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:31:00.066 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.357755+0000", 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 249.65971488899999, 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2082295869999999, 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145682 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.067 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.357755+0000", 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.357755+0000", 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.357756+0000", 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0699999999999999e-06 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.068 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.357759+0000", 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3569999999999998e-06 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.357767+0000", 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1280000000000008e-06 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.357772+0000", 2023-09-30T09:31:00.069 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0989999999999998e-06 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.357822+0000", 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9314999999999998e-05 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.357830+0000", 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8259999999999993e-06 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.070 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.313029+0000", 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.955199087 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.313033+0000", 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3819999999999997e-06 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.554758+0000", 2023-09-30T09:31:00.071 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2417246040000001 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.554910+0000", 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015199899999999999 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.555073+0000", 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016354999999999999 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.072 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.563680+0000", 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086071959999999993 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.564637+0000", 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00095652300000000003 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.565284+0000", 2023-09-30T09:31:00.073 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00064752500000000003 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.565816+0000", 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00053139699999999997 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.565872+0000", 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6137999999999997e-05 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.565984+0000", 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000112391 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145689 5.5 5:a822bab3:::benchmark_data_smithi019_287194_object62911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.365899+0000", 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 249.65157043799999, 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2015801339999999, 2023-09-30T09:31:00.075 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145689 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.365899+0000", 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.076 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.365899+0000", 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.365900+0000", 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.184e-06 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.365905+0000", 2023-09-30T09:31:00.077 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7280000000000003e-06 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.365916+0000", 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1211e-05 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.365922+0000", 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1659999999999999e-06 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.078 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.365976+0000", 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3292999999999999e-05 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.365983+0000", 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4170000000000003e-06 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.079 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.311660+0000", 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94567657500000002 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.311670+0000", 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0719e-05 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.555195+0000", 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.243524082 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.080 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.555315+0000", 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012061999999999999 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,6,7", 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.555440+0000", 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012501400000000001 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.081 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.559035+0000", 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035950869999999998 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.559309+0000", 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027371600000000002 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.566994+0000", 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0076853019999999998 2023-09-30T09:31:00.082 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.567105+0000", 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000110985 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.567289+0000", 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000184048 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.083 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.567346+0000", 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6950000000000002e-05 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.567479+0000", 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013303699999999999 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.084 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145691 5.7 5:eeb1010e:::benchmark_data_smithi019_287194_object62913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.366985+0000", 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 249.650484471, 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.20249834, 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145691 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.085 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.366985+0000", 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.366985+0000", 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.086 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.366986+0000", 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8100000000000002e-07 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.366989+0000", 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4649999999999999e-06 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.366997+0000", 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6739999999999997e-06 2023-09-30T09:31:00.087 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.367002+0000", 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6280000000000004e-06 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.367048+0000", 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6020999999999999e-05 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.088 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.367054+0000", 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4699999999999999e-06 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.313053+0000", 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94599889699999995 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.313062+0000", 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5439999999999999e-06 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.558949+0000", 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2458878449999999 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.560066+0000", 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00111644 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:31:00.090 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.560231+0000", 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000165639 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.563726+0000", 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034946000000000001 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.564688+0000", 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00096199000000000002 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.091 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.565291+0000", 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00060314600000000004 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.569277+0000", 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0039856900000000001 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.569353+0000", 2023-09-30T09:31:00.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6124000000000005e-05 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.569484+0000", 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000130486 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148460 5.7 5:ea51fe56:::benchmark_data_smithi019_287194_object65682:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.210606+0000", 2023-09-30T09:31:00.093 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.80686409800001, 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.046356853, 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148460 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.210606+0000", 2023-09-30T09:31:00.094 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.210606+0000", 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.210607+0000", 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3799999999999999e-06 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.210611+0000", 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0799999999999999e-06 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.210621+0000", 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0284e-05 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.096 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.210627+0000", 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0360000000000003e-06 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.210932+0000", 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00030456599999999998 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.210943+0000", 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1021e-05 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.247344+0000", 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.036400937 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.247475+0000", 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013122000000000001 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.098 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.247571+0000", 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5909000000000006e-05 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.251729+0000", 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041577180000000004 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.254457+0000", 2023-09-30T09:31:00.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027280400000000002 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.256877+0000", 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002419933 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.256915+0000", 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8807000000000003e-05 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.256962+0000", 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6922000000000002e-05 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148465 5.7 5:e93153f0:::benchmark_data_smithi019_287194_object65687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.223316+0000", 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.79415379599999, 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.040803701, 2023-09-30T09:31:00.101 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148465 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.223316+0000", 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.223316+0000", 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.223317+0000", 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3340000000000001e-06 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.223323+0000", 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3419999999999998e-06 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.223337+0000", 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4185999999999999e-05 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.223346+0000", 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0100000000000001e-06 2023-09-30T09:31:00.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.223405+0000", 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8921000000000001e-05 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.223415+0000", 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0404999999999999e-05 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.251771+0000", 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0283562230000001 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.251890+0000", 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000118768 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.252017+0000", 2023-09-30T09:31:00.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012745599999999999 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.257010+0000", 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049926359999999999 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.260659+0000", 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036485620000000002 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264010+0000", 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003351124 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264078+0000", 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7799999999999995e-05 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264120+0000", 2023-09-30T09:31:00.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1934000000000002e-05 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148494 5.7 5:e0500fd4:::benchmark_data_smithi019_287194_object65716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.270470+0000", 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.74700012900001, 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99386993599999995, 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.109 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148494 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.270470+0000", 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.270470+0000", 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.270471+0000", 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3859999999999999e-06 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.270477+0000", 2023-09-30T09:31:00.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9390000000000002e-06 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.270494+0000", 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7249e-05 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.270503+0000", 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1670000000000005e-06 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.270579+0000", 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5782000000000001e-05 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.270589+0000", 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.007e-05 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.254498+0000", 2023-09-30T09:31:00.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98390923699999999 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.254615+0000", 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000116643 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.254685+0000", 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0365999999999997e-05 2023-09-30T09:31:00.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.257128+0000", 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024427099999999998 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.260668+0000", 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003539862 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264264+0000", 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035965049999999998 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264304+0000", 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9876000000000002e-05 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264339+0000", 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5144000000000001e-05 2023-09-30T09:31:00.116 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148500 5.7 5:e6b546ec:::benchmark_data_smithi019_287194_object65722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.286668+0000", 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.73080143799999, 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97780283099999998, 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.117 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148500 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286668+0000", 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.118 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286668+0000", 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286670+0000", 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9149999999999999e-06 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286677+0000", 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7499999999999997e-06 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286692+0000", 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.49e-05 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286701+0000", 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7949999999999996e-06 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.289369+0000", 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026684870000000002 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.289379+0000", 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.007e-05 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.257166+0000", 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96778660000000005 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.257267+0000", 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000101412 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.257351+0000", 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4287000000000001e-05 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264206+0000", 2023-09-30T09:31:00.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0068544560000000001 2023-09-30T09:31:00.123 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.123 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.123 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.123 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264350+0000", 2023-09-30T09:31:00.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014376300000000001 2023-09-30T09:31:00.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264402+0000", 2023-09-30T09:31:00.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2512999999999997e-05 2023-09-30T09:31:00.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264449+0000", 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7207999999999999e-05 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264471+0000", 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1675000000000002e-05 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.126 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148502 5.7 5:ec279eaf:::benchmark_data_smithi019_287194_object65724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.286726+0000", 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.73074406999999, 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98129615699999995, 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148502 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.127 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286726+0000", 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286726+0000", 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286726+0000", 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4999999999999998e-07 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286729+0000", 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.729e-06 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286736+0000", 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8469999999999998e-06 2023-09-30T09:31:00.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.286741+0000", 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5110000000000003e-06 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.289404+0000", 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026633719999999998 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.289411+0000", 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.156e-06 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.260710+0000", 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97129930600000003 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.260839+0000", 2023-09-30T09:31:00.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000129243 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.260960+0000", 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012034 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264355+0000", 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033951710000000002 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264380+0000", 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4919e-05 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.267919+0000", 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003539366 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.267979+0000", 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0127000000000002e-05 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.268022+0000", 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2620000000000002e-05 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.134 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148525 5.7 5:e602d8f8:::benchmark_data_smithi019_287194_object65747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.332995+0000", 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.68447413600001, 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94696762099999998, 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148525 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.332995+0000", 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.332995+0000", 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.332997+0000", 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.302e-06 2023-09-30T09:31:00.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.333002+0000", 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2329999999999997e-06 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.333015+0000", 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2557e-05 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.333024+0000", 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4350000000000003e-06 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.333093+0000", 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9102000000000003e-05 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.333102+0000", 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.268e-06 2023-09-30T09:31:00.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264518+0000", 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.931415294 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264665+0000", 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014690999999999999 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.264800+0000", 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013514000000000001 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.279770+0000", 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014970504000000001 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.279789+0000", 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9177e-05 2023-09-30T09:31:00.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.279859+0000", 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9961000000000003e-05 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.279919+0000", 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9348000000000002e-05 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.279963+0000", 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.439e-05 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148555 5.7 5:e03e3107:::benchmark_data_smithi019_287194_object65777:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.382584+0000", 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.63488533200001, 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90525092900000004, 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.142 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148555 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.382584+0000", 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.382584+0000", 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.382586+0000", 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6559999999999999e-06 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.382591+0000", 2023-09-30T09:31:00.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3040000000000004e-06 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.382604+0000", 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2374999999999999e-05 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.382611+0000", 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6690000000000006e-06 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.382680+0000", 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.86e-05 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.382691+0000", 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0875e-05 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.268060+0000", 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.88536933200000001 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.268214+0000", 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015341899999999999 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.268353+0000", 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013897900000000001 2023-09-30T09:31:00.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.279797+0000", 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011444517 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.280009+0000", 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021182500000000001 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.287722+0000", 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077130159999999996 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.287787+0000", 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4658000000000003e-05 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.287835+0000", 2023-09-30T09:31:00.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8704e-05 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148567 5.7 5:e958c7c0:::benchmark_data_smithi019_287194_object65789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.402374+0000", 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.61509531199999, 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.88563787299999996, 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.150 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148567 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.402374+0000", 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.151 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.402374+0000", 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.402376+0000", 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7039999999999999e-06 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.402383+0000", 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5180000000000001e-06 2023-09-30T09:31:00.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.402398+0000", 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.554e-05 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.402410+0000", 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1467e-05 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.402492+0000", 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2097999999999999e-05 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.402502+0000", 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0475000000000001e-05 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.271659+0000", 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86915652099999996 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.271805+0000", 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014598500000000001 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.271941+0000", 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000136391 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.280051+0000", 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0081100410000000001 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.283685+0000", 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003633687 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.287892+0000", 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042070129999999999 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.287950+0000", 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8483999999999998e-05 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.288012+0000", 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1948999999999999e-05 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148641 5.7 5:efdd0df9:::benchmark_data_smithi019_287194_object65863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.541614+0000", 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.47585518599999, 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.75383889400000004, 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148641 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.158 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.541614+0000", 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.541614+0000", 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.541616+0000", 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6530000000000001e-06 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.541623+0000", 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0280000000000003e-06 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.541643+0000", 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9720000000000001e-05 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.541653+0000", 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0013000000000001e-05 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.541732+0000", 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9041000000000004e-05 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.541741+0000", 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4970000000000007e-06 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.275819+0000", 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.73407807599999997 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.162 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.275962+0000", 2023-09-30T09:31:00.163 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:00.163 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000142156 2023-09-30T09:31:00.163 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.276093+0000", 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013129999999999999 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.287654+0000", 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011560777 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.295208+0000", 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0075538239999999998 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.295380+0000", 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000172542 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.295426+0000", 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5475000000000001e-05 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.295453+0000", 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7792000000000001e-05 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.166 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:148652 5.7 5:e7cad2a9:::benchmark_data_smithi019_287194_object65874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:58.564070+0000", 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 241.45339926400001, 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.73154433699999999, 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 148652 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.167 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.564070+0000", 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.564070+0000", 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.564072+0000", 2023-09-30T09:31:00.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7770000000000001e-06 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.564079+0000", 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7410000000000003e-06 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.564102+0000", 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3213e-05 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.564112+0000", 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9299999999999998e-06 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.564193+0000", 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1088999999999996e-05 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:58.564203+0000", 2023-09-30T09:31:00.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4250000000000004e-06 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.280242+0000", 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.71603898499999996 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.280379+0000", 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013735400000000001 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.280525+0000", 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014637999999999999 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.291962+0000", 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011437161 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.172 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.295258+0000", 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0032955459999999999 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.295517+0000", 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000258659 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.295575+0000", 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8662999999999997e-05 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:59.295615+0000", 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9413999999999999e-05 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35939 6.1 6:85c88f7a:::benchmark_data_smithi019_355417_object35938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.259152+0000", 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 140.75831733499999, 2023-09-30T09:31:00.174 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0077918690000001, 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35939 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259152+0000", 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.175 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259152+0000", 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259155+0000", 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4169999999999999e-06 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259183+0000", 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8271000000000001e-05 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259211+0000", 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7766000000000001e-05 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259222+0000", 2023-09-30T09:31:00.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1435e-05 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259295+0000", 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2458000000000003e-05 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op must wait for map", 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259300+0000", 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9200000000000003e-06 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.178 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.458323+0000", 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.199023175 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.458331+0000", 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6119999999999999e-06 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.266840+0000", 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.80850840599999996 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.266944+0000", 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000104409 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35952 6.1 6:82cbd2c5:::benchmark_data_smithi019_355417_object35951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:31:00.180 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.442045+0000", 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 140.57542422500001, 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.83208114399999999, 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35952 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.181 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.442045+0000", 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.442045+0000", 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.442047+0000", 2023-09-30T09:31:00.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5370000000000001e-06 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.442079+0000", 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2506999999999999e-05 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.442092+0000", 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2573e-05 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.183 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.442104+0000", 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2354000000000001e-05 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.458359+0000", 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016254873999999999 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.458366+0000", 2023-09-30T09:31:00.184 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5089999999999999e-06 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.267178+0000", 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.80881183999999995 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.267331+0000", 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015293800000000001 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.267505+0000", 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017416799999999999 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268743+0000", 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012383800000000001 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.269030+0000", 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000287142 2023-09-30T09:31:00.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274024+0000", 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049939219999999996 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274086+0000", 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2339000000000001e-05 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:00.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274127+0000", 2023-09-30T09:31:00.188 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0061000000000002e-05 2023-09-30T09:31:00.188 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.188 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.188 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.188 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:00.188 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:00.188 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:31:00.188 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:00.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:30:59 smithi019 ceph-mon[224043]: pgmap v618: 129 pgs: 9 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 11384/119418 objects degraded (9.533%); 1219/119418 objects misplaced (1.021%); 20 MiB/s, 319 objects/s recovering 2023-09-30T09:31:00.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:30:59 smithi019 ceph-mon[216815]: pgmap v618: 129 pgs: 9 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 11384/119418 objects degraded (9.533%); 1219/119418 objects misplaced (1.021%); 20 MiB/s, 319 objects/s recovering 2023-09-30T09:31:00.227 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:00.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:30:59 smithi167 ceph-mon[192880]: pgmap v618: 129 pgs: 9 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 11384/119418 objects degraded (9.533%); 1219/119418 objects misplaced (1.021%); 20 MiB/s, 319 objects/s recovering 2023-09-30T09:31:00.379 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:00.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:00.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:00.483 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:00.527 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:00.830 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:31:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1208516096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1208516096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:01.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1208516096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:01.318 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:01.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:01.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:01.426 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:01.461 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:31:01.495 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:01.592 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:01.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:01.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:01.699 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:01.770 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:02.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:02.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1208516096' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:02.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[216815]: failed to encode map e258 with expected crc 2023-09-30T09:31:02.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[216815]: osdmap e258: 8 total, 8 up, 7 in 2023-09-30T09:31:02.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[216815]: failed to encode map e258 with expected crc 2023-09-30T09:31:02.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[216815]: pgmap v620: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 10562/113169 objects degraded (9.333%); 1106/113169 objects misplaced (0.977%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:31:02.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:02.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1208516096' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:02.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[224043]: failed to encode map e258 with expected crc 2023-09-30T09:31:02.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[224043]: osdmap e258: 8 total, 8 up, 7 in 2023-09-30T09:31:02.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[224043]: failed to encode map e258 with expected crc 2023-09-30T09:31:02.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:01 smithi019 ceph-mon[224043]: pgmap v620: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 10562/113169 objects degraded (9.333%); 1106/113169 objects misplaced (0.977%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:31:02.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:01 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:02.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1208516096' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:02.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:01 smithi167 ceph-mon[192880]: failed to encode map e258 with expected crc 2023-09-30T09:31:02.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:01 smithi167 ceph-mon[192880]: osdmap e258: 8 total, 8 up, 7 in 2023-09-30T09:31:02.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:01 smithi167 ceph-mon[192880]: failed to encode map e258 with expected crc 2023-09-30T09:31:02.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:01 smithi167 ceph-mon[192880]: pgmap v620: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 10562/113169 objects degraded (9.333%); 1106/113169 objects misplaced (0.977%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:31:02.423 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:02.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:02.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:02.528 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:02.600 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:02.699 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:02.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:02.804 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:02.847 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:02.893 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:02.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:02.997 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:03.076 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:03.494 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:03.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:03.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:03.600 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:03.628 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:03.969 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:31:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:31:03] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-30T09:31:04.019 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:04.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:04.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:04.112 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:04.125 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:04.180 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:04.238 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:04.238 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:04.238 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:03 smithi019 ceph-mon[216815]: pgmap v621: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 10495/111774 objects degraded (9.389%); 1106/111774 objects misplaced (0.989%); 13 MiB/s, 207 objects/s recovering 2023-09-30T09:31:04.239 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 10562/113169 objects degraded (9.333%), 16 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:04.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:04.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:04.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:03 smithi019 ceph-mon[224043]: pgmap v621: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 10495/111774 objects degraded (9.389%); 1106/111774 objects misplaced (0.989%); 13 MiB/s, 207 objects/s recovering 2023-09-30T09:31:04.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 10562/113169 objects degraded (9.333%), 16 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:03 smithi167 ceph-mon[192880]: pgmap v621: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 10495/111774 objects degraded (9.389%); 1106/111774 objects misplaced (0.989%); 13 MiB/s, 207 objects/s recovering 2023-09-30T09:31:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 10562/113169 objects degraded (9.333%), 16 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:04.311 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:04.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:04.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:04.424 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:04.478 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:04.589 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:31:04.623 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:31:04.653 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:31:04.669 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:04.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:04.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:04.689 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:31:04.723 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:31:04.769 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:31:04.778 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:04.810 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:31:04.845 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:04.851 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:31:04.858 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:31:04.858 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:31:04.858 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:31:04.858 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:31:04.859 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:31:04.859 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:31:04.859 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:31:04.859 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:31:04.973 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/436444981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:05.053 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:05.059 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:31:05.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:05.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:05.164 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:05.233 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/436444981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:06.112 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/436444981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:06.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:06.113 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:06.113 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:06.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:06.114 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 1 pg_num to 31 2023-09-30T09:31:06.215 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:06.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:05 smithi019 ceph-mon[216815]: pgmap v622: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 880 op/s; 8942/99438 objects degraded (8.993%); 965/99438 objects misplaced (0.970%); 17 MiB/s, 325 objects/s recovering 2023-09-30T09:31:06.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1131731871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-30T09:31:06.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:05 smithi019 ceph-mon[224043]: pgmap v622: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 880 op/s; 8942/99438 objects degraded (8.993%); 965/99438 objects misplaced (0.970%); 17 MiB/s, 325 objects/s recovering 2023-09-30T09:31:06.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1131731871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-30T09:31:06.360 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:05 smithi167 ceph-mon[192880]: pgmap v622: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 880 op/s; 8942/99438 objects degraded (8.993%); 965/99438 objects misplaced (0.970%); 17 MiB/s, 325 objects/s recovering 2023-09-30T09:31:06.360 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1131731871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-30T09:31:06.430 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:06.530 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:06.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:06.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:06.640 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:06.683 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:07.137 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:07.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:07.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:07.246 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1131731871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-30T09:31:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:06 smithi167 ceph-mon[192880]: osdmap e259: 8 total, 8 up, 7 in 2023-09-30T09:31:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:06 smithi167 ceph-mon[192880]: failed to encode map e259 with expected crc 2023-09-30T09:31:07.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1131731871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-30T09:31:07.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:06 smithi019 ceph-mon[216815]: osdmap e259: 8 total, 8 up, 7 in 2023-09-30T09:31:07.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:06 smithi019 ceph-mon[216815]: failed to encode map e259 with expected crc 2023-09-30T09:31:07.290 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1131731871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-30T09:31:07.290 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:06 smithi019 ceph-mon[224043]: osdmap e259: 8 total, 8 up, 7 in 2023-09-30T09:31:07.290 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:06 smithi019 ceph-mon[224043]: failed to encode map e259 with expected crc 2023-09-30T09:31:07.293 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:07.849 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:07.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:07.954 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:08.031 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:08.148 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:08.160 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:07 smithi019 ceph-mon[216815]: failed to encode map e259 with expected crc 2023-09-30T09:31:08.160 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:07 smithi019 ceph-mon[216815]: pgmap v624: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 885 op/s; 8879/98157 objects degraded (9.046%); 965/98157 objects misplaced (0.983%); 16 MiB/s, 312 objects/s recovering 2023-09-30T09:31:08.160 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-30T09:31:08.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:07 smithi019 ceph-mon[224043]: failed to encode map e259 with expected crc 2023-09-30T09:31:08.161 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:07 smithi019 ceph-mon[224043]: pgmap v624: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 885 op/s; 8879/98157 objects degraded (9.046%); 965/98157 objects misplaced (0.983%); 16 MiB/s, 312 objects/s recovering 2023-09-30T09:31:08.161 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-30T09:31:08.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:08.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:08.264 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:07 smithi167 ceph-mon[192880]: failed to encode map e259 with expected crc 2023-09-30T09:31:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:07 smithi167 ceph-mon[192880]: pgmap v624: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 2.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 885 op/s; 8879/98157 objects degraded (9.046%); 965/98157 objects misplaced (0.983%); 16 MiB/s, 312 objects/s recovering 2023-09-30T09:31:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-30T09:31:08.338 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:08.343 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:08.459 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:31:08.459 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:31:08.469 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:08.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:08.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:08.583 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:08.630 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:08.865 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:08.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:08.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:08.971 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:09.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-30T09:31:09.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[216815]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[216815]: osdmap e260: 8 total, 8 up, 7 in 2023-09-30T09:31:09.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[216815]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[216815]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[216815]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[216815]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 8879/98157 objects degraded (9.046%), 16 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:09.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-30T09:31:09.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[224043]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[224043]: osdmap e260: 8 total, 8 up, 7 in 2023-09-30T09:31:09.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[224043]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[224043]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[224043]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[224043]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 8879/98157 objects degraded (9.046%), 16 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:09.028 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:09.094 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:31:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-30T09:31:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:08 smithi167 ceph-mon[192880]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:08 smithi167 ceph-mon[192880]: osdmap e260: 8 total, 8 up, 7 in 2023-09-30T09:31:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:08 smithi167 ceph-mon[192880]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:08 smithi167 ceph-mon[192880]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:08 smithi167 ceph-mon[192880]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:08 smithi167 ceph-mon[192880]: failed to encode map e260 with expected crc 2023-09-30T09:31:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 8879/98157 objects degraded (9.046%), 16 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:09.497 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:09.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:09.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:09.601 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:09.655 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:10.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: osdmap e261: 8 total, 8 up, 7 in 2023-09-30T09:31:10.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:10 smithi167 ceph-mon[192880]: pgmap v627: 139 pgs: 8 peering, 10 unknown, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 1.9 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 8258/92064 objects degraded (8.970%); 965/92064 objects misplaced (1.048%); 18 MiB/s, 374 objects/s recovering 2023-09-30T09:31:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: osdmap e261: 8 total, 8 up, 7 in 2023-09-30T09:31:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[216815]: pgmap v627: 139 pgs: 8 peering, 10 unknown, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 1.9 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 8258/92064 objects degraded (8.970%); 965/92064 objects misplaced (1.048%); 18 MiB/s, 374 objects/s recovering 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: osdmap e261: 8 total, 8 up, 7 in 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: failed to encode map e261 with expected crc 2023-09-30T09:31:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:10 smithi019 ceph-mon[224043]: pgmap v627: 139 pgs: 8 peering, 10 unknown, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 1.9 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 8258/92064 objects degraded (8.970%); 965/92064 objects misplaced (1.048%); 18 MiB/s, 374 objects/s recovering 2023-09-30T09:31:10.609 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:31:10.711 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:10.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:10.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:10.824 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:10.855 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:10.887 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:11.006 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:11.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:11.127 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:11.198 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:11.395 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:11.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:11.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:11.499 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:11.569 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:11.733 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:11.819 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:11.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:11.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:11.932 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:12.002 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:12.095 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:12.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:12.205 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:12.270 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:12.271 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:31:12.372 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:12.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:12.483 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:12.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:12 smithi167 ceph-mon[192880]: pgmap v628: 139 pgs: 8 peering, 9 unknown, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 7308/84843 objects degraded (8.614%); 839/84843 objects misplaced (0.989%); 12 MiB/s, 272 objects/s recovering 2023-09-30T09:31:12.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2596622156' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:12.545 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:12.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:12 smithi019 ceph-mon[216815]: pgmap v628: 139 pgs: 8 peering, 9 unknown, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 7308/84843 objects degraded (8.614%); 839/84843 objects misplaced (0.989%); 12 MiB/s, 272 objects/s recovering 2023-09-30T09:31:12.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2596622156' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:12 smithi019 ceph-mon[224043]: pgmap v628: 139 pgs: 8 peering, 9 unknown, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 7308/84843 objects degraded (8.614%); 839/84843 objects misplaced (0.989%); 12 MiB/s, 272 objects/s recovering 2023-09-30T09:31:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2596622156' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:12.601 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:12.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:12.709 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:12.772 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:12.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:12.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:12.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:12.986 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:13.045 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:13.047 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:31:13.047 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:31:13.048 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-30T09:31:13.048 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:31:13.048 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:31:13.048 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-30T09:31:13.125 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:13.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:13.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:13.234 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:13.257 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:13 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:13.257 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2596622156' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:31:13.257 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:13 smithi167 ceph-mon[192880]: osdmap e262: 8 total, 8 up, 7 in 2023-09-30T09:31:13.269 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2596622156' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:31:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[216815]: osdmap e262: 8 total, 8 up, 7 in 2023-09-30T09:31:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[216815]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[216815]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[216815]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[216815]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[216815]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2596622156' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[224043]: osdmap e262: 8 total, 8 up, 7 in 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[224043]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[224043]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[224043]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[224043]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:13 smithi019 ceph-mon[224043]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.394 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:13.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:13.509 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:13 smithi167 ceph-mon[192880]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:13 smithi167 ceph-mon[192880]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:13 smithi167 ceph-mon[192880]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:13 smithi167 ceph-mon[192880]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:13 smithi167 ceph-mon[192880]: failed to encode map e262 with expected crc 2023-09-30T09:31:13.578 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:13.688 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:13.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:13.798 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:13.858 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:31:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:31:13] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-30T09:31:13.865 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:13.992 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:13.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:13.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:14.097 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:14.126 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:14 smithi167 ceph-mon[192880]: pgmap v630: 139 pgs: 8 peering, 8 unknown, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 7277/83370 objects degraded (8.729%); 839/83370 objects misplaced (1.006%); 12 MiB/s, 266 objects/s recovering 2023-09-30T09:31:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7277/83370 objects degraded (8.729%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:14.576 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:14.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:14 smithi019 ceph-mon[224043]: pgmap v630: 139 pgs: 8 peering, 8 unknown, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 7277/83370 objects degraded (8.729%); 839/83370 objects misplaced (1.006%); 12 MiB/s, 266 objects/s recovering 2023-09-30T09:31:14.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7277/83370 objects degraded (8.729%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:14.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:14.583 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:14 smithi019 ceph-mon[216815]: pgmap v630: 139 pgs: 8 peering, 8 unknown, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 7277/83370 objects degraded (8.729%); 839/83370 objects misplaced (1.006%); 12 MiB/s, 266 objects/s recovering 2023-09-30T09:31:14.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7277/83370 objects degraded (8.729%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:14.684 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:14.731 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:14.850 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:14.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:14.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:14.956 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:14.982 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:15.038 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:15.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:15.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:15.046 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:15.132 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:31:15.142 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:15.152 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:31:15.169 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:15.175 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:31:15.198 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:31:15.212 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:15.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:15.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:15.219 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:31:15.238 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:31:15.258 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:31:15.278 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:31:15.279 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:31:15.279 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:31:15.280 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:31:15.280 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:31:15.280 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:31:15.280 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:31:15.280 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:31:15.280 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:31:15.316 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:15.346 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:15.412 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:31:15.413 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:31:15.413 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:31:15.413 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:31:15.413 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:31:15.433 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:15.442 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[216815]: osdmap e263: 8 total, 8 up, 7 in 2023-09-30T09:31:15.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[216815]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[216815]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[216815]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[216815]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[216815]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1337078821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:15.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[224043]: osdmap e263: 8 total, 8 up, 7 in 2023-09-30T09:31:15.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[224043]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[224043]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[224043]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[224043]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[224043]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.445 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1337078821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:15.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:15.546 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:15.607 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:15 smithi167 ceph-mon[192880]: osdmap e263: 8 total, 8 up, 7 in 2023-09-30T09:31:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:15 smithi167 ceph-mon[192880]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:15 smithi167 ceph-mon[192880]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:15 smithi167 ceph-mon[192880]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:15 smithi167 ceph-mon[192880]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:15 smithi167 ceph-mon[192880]: failed to encode map e263 with expected crc 2023-09-30T09:31:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1337078821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:15.786 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:15.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:15.858 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:31:15.890 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:15.948 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: pgmap v632: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 1.24k op/s; 6553/69171 objects degraded (9.474%); 698/69171 objects misplaced (1.009%); 5.6 MiB/s, 274 objects/s recovering 2023-09-30T09:31:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: osdmap e264: 8 total, 8 up, 7 in 2023-09-30T09:31:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[224043]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: pgmap v632: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 1.24k op/s; 6553/69171 objects degraded (9.474%); 698/69171 objects misplaced (1.009%); 5.6 MiB/s, 274 objects/s recovering 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: osdmap e264: 8 total, 8 up, 7 in 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:16 smithi019 ceph-mon[216815]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: pgmap v632: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 1.24k op/s; 6553/69171 objects degraded (9.474%); 698/69171 objects misplaced (1.009%); 5.6 MiB/s, 274 objects/s recovering 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: osdmap e264: 8 total, 8 up, 7 in 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: failed to encode map e264 with expected crc 2023-09-30T09:31:16.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:16 smithi167 ceph-mon[192880]: failed to encode map e264 with expected crc 2023-09-30T09:31:17.464 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:31:17.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[224043]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[224043]: osdmap e265: 8 total, 8 up, 7 in 2023-09-30T09:31:17.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[224043]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[224043]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[224043]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[216815]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[216815]: osdmap e265: 8 total, 8 up, 7 in 2023-09-30T09:31:17.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[216815]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[216815]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:17 smithi019 ceph-mon[216815]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.590 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:17.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:17.697 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:17.760 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:17 smithi167 ceph-mon[192880]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.761 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:17 smithi167 ceph-mon[192880]: osdmap e265: 8 total, 8 up, 7 in 2023-09-30T09:31:17.761 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:17 smithi167 ceph-mon[192880]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.761 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:17 smithi167 ceph-mon[192880]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.761 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:17 smithi167 ceph-mon[192880]: failed to encode map e265 with expected crc 2023-09-30T09:31:17.767 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:17.870 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:17.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:17.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:17.980 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:18.008 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:18.052 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:18.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:18.156 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:18.197 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:18.476 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:18.476 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:18 smithi019 ceph-mon[224043]: pgmap v635: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 1.4 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 1.05k op/s; 6553/67935 objects degraded (9.646%); 698/67935 objects misplaced (1.027%); 784 KiB/s, 145 objects/s recovering 2023-09-30T09:31:18.476 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:18.476 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6553/67935 objects degraded (9.646%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:18.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:18 smithi019 ceph-mon[216815]: pgmap v635: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 1.4 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 1.05k op/s; 6553/67935 objects degraded (9.646%); 698/67935 objects misplaced (1.027%); 784 KiB/s, 145 objects/s recovering 2023-09-30T09:31:18.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:18.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6553/67935 objects degraded (9.646%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:18.540 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:18.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:18.646 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:18.702 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:18.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:18 smithi167 ceph-mon[192880]: pgmap v635: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 1.4 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 1.05k op/s; 6553/67935 objects degraded (9.646%); 698/67935 objects misplaced (1.027%); 784 KiB/s, 145 objects/s recovering 2023-09-30T09:31:18.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:18.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6553/67935 objects degraded (9.646%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:18.817 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:18.830 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:18.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:18.937 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:18.992 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:19.090 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:19.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:19.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:19.205 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:19.265 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:19.378 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:19.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:19.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:19.498 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:31:19.499 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:19.562 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:20.400 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:31:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:31:20.401 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:31:20.420 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:20.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:20.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:20.528 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:20 smithi019 ceph-mon[216815]: pgmap v636: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 1.16k op/s; 6277/62562 objects degraded (10.033%); 698/62562 objects misplaced (1.116%); 642 KiB/s, 165 objects/s recovering 2023-09-30T09:31:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:20 smithi019 ceph-mon[224043]: pgmap v636: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 1.16k op/s; 6277/62562 objects degraded (10.033%); 698/62562 objects misplaced (1.116%); 642 KiB/s, 165 objects/s recovering 2023-09-30T09:31:20.590 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:20 smithi167 ceph-mon[192880]: pgmap v636: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 1.16k op/s; 6277/62562 objects degraded (10.033%); 698/62562 objects misplaced (1.116%); 642 KiB/s, 165 objects/s recovering 2023-09-30T09:31:21.059 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:21.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:21.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:21.162 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:21.213 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:21.986 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:21.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:21.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:22.092 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:22.155 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:22.259 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:22.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:22.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:22.368 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:22.412 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:22.431 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:22.538 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:22.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:22.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:22.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:22 smithi019 ceph-mon[224043]: pgmap v637: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 5853/55833 objects degraded (10.483%); 547/55833 objects misplaced (0.980%); 0 B/s, 102 objects/s recovering 2023-09-30T09:31:22.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:22 smithi019 ceph-mon[216815]: pgmap v637: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 5853/55833 objects degraded (10.483%); 547/55833 objects misplaced (0.980%); 0 B/s, 102 objects/s recovering 2023-09-30T09:31:22.647 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:22.721 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:22.776 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:22 smithi167 ceph-mon[192880]: pgmap v637: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 5853/55833 objects degraded (10.483%); 547/55833 objects misplaced (0.980%); 0 B/s, 102 objects/s recovering 2023-09-30T09:31:22.881 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:31:22.881 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:31:22.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80421 5.a 5:536b7ad8:::benchmark_data_smithi019_287194_object80420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.575313+0000", 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 441.29474701499998, 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3514731449999999, 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:22.882 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80421 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575313+0000", 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575313+0000", 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575314+0000", 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1510000000000001e-06 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575338+0000", 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3546000000000001e-05 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575355+0000", 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7292e-05 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575363+0000", 2023-09-30T09:31:22.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5979999999999999e-06 2023-09-30T09:31:22.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.886 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.582558+0000", 2023-09-30T09:31:22.886 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0071956959999999997 2023-09-30T09:31:22.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.582681+0000", 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000122588 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.582821+0000", 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013951 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.591425+0000", 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086042619999999997 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.591442+0000", 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7652e-05 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620803+0000", 2023-09-30T09:31:22.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.029360377999999999 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620815+0000", 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172e-05 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606293+0000", 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98547855500000003 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606306+0000", 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2862e-05 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.926699+0000", 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3203928170000001 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.926786+0000", 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7517999999999998e-05 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80431 5.a 5:539a1d6a:::benchmark_data_smithi019_287194_object80430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.592490+0000", 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 441.27757028899998, 2023-09-30T09:31:22.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3383307100000001, 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80431 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.892 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592490+0000", 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592490+0000", 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592490+0000", 2023-09-30T09:31:22.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1199999999999998e-07 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592513+0000", 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2864999999999999e-05 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592520+0000", 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3910000000000003e-06 2023-09-30T09:31:22.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592525+0000", 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0100000000000003e-06 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592804+0000", 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027989900000000001 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592953+0000", 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014838199999999999 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.593107+0000", 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015438 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599129+0000", 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0060221859999999997 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.604596+0000", 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0054663929999999999 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620852+0000", 2023-09-30T09:31:22.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.01625619 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620861+0000", 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5620000000000004e-06 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606349+0000", 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98548850600000004 2023-09-30T09:31:22.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606360+0000", 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0954e-05 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.930701+0000", 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3243411780000001 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.930821+0000", 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011940200000000001 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80432 5.a 5:5bdae413:::benchmark_data_smithi019_287194_object80431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.592531+0000", 2023-09-30T09:31:22.900 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 441.27752876199997, 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.338405018, 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80432 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592531+0000", 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592531+0000", 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592532+0000", 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.84e-07 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592551+0000", 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8927999999999999e-05 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592557+0000", 2023-09-30T09:31:22.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0920000000000003e-06 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592562+0000", 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8010000000000003e-06 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.595900+0000", 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033379730000000002 2023-09-30T09:31:22.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.596033+0000", 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013348799999999999 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.596167+0000", 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013392100000000001 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.604602+0000", 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0084350959999999996 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.607643+0000", 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003040423 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620894+0000", 2023-09-30T09:31:22.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013251639000000001 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620903+0000", 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4479999999999994e-06 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606397+0000", 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98549406399999995 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606407+0000", 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0677e-05 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.930857+0000", 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.324449378 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.930936+0000", 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9705999999999999e-05 2023-09-30T09:31:22.908 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80436 5.d 5:bc14734e:::benchmark_data_smithi019_287194_object80435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.599675+0000", 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 441.27038483400003, 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3736680059999999, 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.909 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80436 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599675+0000", 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599675+0000", 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599676+0000", 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4099999999999999e-07 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599700+0000", 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4502e-05 2023-09-30T09:31:22.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599708+0000", 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5730000000000002e-06 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599714+0000", 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2149999999999998e-06 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.601660+0000", 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019455449999999999 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.601801+0000", 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000141008 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.601953+0000", 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015205700000000001 2023-09-30T09:31:22.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.610590+0000", 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086373960000000003 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.614958+0000", 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043675220000000004 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621354+0000", 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063966839999999997 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621364+0000", 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8810000000000008e-06 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.607174+0000", 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98580998200000003 2023-09-30T09:31:22.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.607192+0000", 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8068999999999999e-05 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.973241+0000", 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3660484550000001 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.973343+0000", 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000102576 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119587 5.d 5:b6587ad4:::benchmark_data_smithi019_287194_object36809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.240740+0000", 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.62932005499999, 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0487021569999999, 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.917 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119587 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240740+0000", 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240740+0000", 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240741+0000", 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172e-06 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240745+0000", 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1949999999999996e-06 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240753+0000", 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4649999999999997e-06 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240758+0000", 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7060000000000003e-06 2023-09-30T09:31:22.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.306657+0000", 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.065899345999999998 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.306831+0000", 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017438600000000001 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.306966+0000", 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013437099999999999 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.315190+0000", 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0082239329999999992 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.315204+0000", 2023-09-30T09:31:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4064e-05 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319784+0000", 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0045798760000000001 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319794+0000", 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0005e-05 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289378+0000", 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96958487100000001 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289442+0000", 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3767000000000001e-05 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119589 5.d 5:b4212818:::benchmark_data_smithi019_287194_object36811:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.240788+0000", 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.629272302, 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0491740350000001, 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119589 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.925 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240788+0000", 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240788+0000", 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240788+0000", 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4400000000000001e-07 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240789+0000", 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-06 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240794+0000", 2023-09-30T09:31:22.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6020000000000002e-06 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.306657+0000", 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.06586322 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.310985+0000", 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043277230000000003 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.311112+0000", 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012651399999999999 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.311238+0000", 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000126062 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.320082+0000", 2023-09-30T09:31:22.929 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088445490000000002 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.320091+0000", 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6920000000000008e-06 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289911+0000", 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96982005599999999 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289962+0000", 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0992000000000002e-05 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119593 5.d 5:bec5f115:::benchmark_data_smithi019_287194_object36815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.319460+0000", 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.55060042100001, 2023-09-30T09:31:22.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98468280600000002, 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119593 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319460+0000", 2023-09-30T09:31:22.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319460+0000", 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319461+0000", 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.212e-06 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.933 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319465+0000", 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1450000000000001e-06 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.323330+0000", 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038647619999999999 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.323344+0000", 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3753e-05 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.323455+0000", 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011108900000000001 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.323466+0000", 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1452e-05 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289993+0000", 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96652676000000004 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.290096+0000", 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010355499999999999 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.290179+0000", 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2276999999999998e-05 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.303990+0000", 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013811742 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304056+0000", 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5294000000000002e-05 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304067+0000", 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1251e-05 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304103+0000", 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.608e-05 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304143+0000", 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9434000000000002e-05 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119597 5.d 5:b8437bdd:::benchmark_data_smithi019_287194_object36819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.324156+0000", 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.545904506, 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98024695500000003, 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:22.939 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119597 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324156+0000", 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324156+0000", 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324157+0000", 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2109999999999999e-06 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324162+0000", 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9960000000000001e-06 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324174+0000", 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.167e-05 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324182+0000", 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0590000000000003e-06 2023-09-30T09:31:22.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324312+0000", 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013004 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324320+0000", 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0819999999999999e-06 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.293198+0000", 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96887842199999996 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.293327+0000", 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000128944 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.293460+0000", 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013269700000000001 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304156+0000", 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010696056000000001 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304247+0000", 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1582999999999994e-05 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304311+0000", 2023-09-30T09:31:22.945 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3157999999999994e-05 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304365+0000", 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4490000000000002e-05 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304403+0000", 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7546999999999998e-05 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.946 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119612 5.d 5:bd3bda5c:::benchmark_data_smithi019_287194_object36834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.384780+0000", 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.48528020499998, 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.91984285700000001, 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.947 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119612 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384780+0000", 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384780+0000", 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384781+0000", 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.043e-06 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384786+0000", 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.259e-06 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.949 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384802+0000", 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5756e-05 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384810+0000", 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1540000000000002e-06 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384940+0000", 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000129352 2023-09-30T09:31:22.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384949+0000", 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7880000000000001e-06 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.296569+0000", 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.911620082 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.296679+0000", 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010973500000000001 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.296787+0000", 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000108106 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304172+0000", 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0073844059999999996 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304482+0000", 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00031017600000000001 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304533+0000", 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1066999999999997e-05 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304584+0000", 2023-09-30T09:31:22.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0881999999999998e-05 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304623+0000", 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9051000000000003e-05 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119631 5.d 5:b005cdcf:::benchmark_data_smithi019_287194_object36853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:31:22.954 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.406119+0000", 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.46394148600001, 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90989635000000002, 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119631 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406119+0000", 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406119+0000", 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406120+0000", 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.249e-06 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406125+0000", 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7559999999999999e-06 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406135+0000", 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0463e-05 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406142+0000", 2023-09-30T09:31:22.957 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3300000000000001e-06 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406237+0000", 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4462999999999993e-05 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406248+0000", 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0614999999999999e-05 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.300293+0000", 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89404568500000003 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.300433+0000", 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013951 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.300571+0000", 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000137923 2023-09-30T09:31:22.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304417+0000", 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038467610000000002 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304672+0000", 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025463200000000002 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.315933+0000", 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011261353 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.315976+0000", 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2250999999999999e-05 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316015+0000", 2023-09-30T09:31:22.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9359e-05 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119646 5.d 5:b288c6e9:::benchmark_data_smithi019_287194_object36868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.427954+0000", 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.44210586299999, 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89682026999999997, 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:22.962 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119646 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427954+0000", 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.963 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427954+0000", 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427956+0000", 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.217e-06 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427960+0000", 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6090000000000003e-06 2023-09-30T09:31:22.964 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427976+0000", 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5390999999999999e-05 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427985+0000", 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2990000000000002e-06 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.965 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428087+0000", 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000102631 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428102+0000", 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4069e-05 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304713+0000", 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87661197499999999 2023-09-30T09:31:22.966 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304820+0000", 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010655 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304934+0000", 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113917 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316036+0000", 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011102153 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316076+0000", 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0252e-05 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324680+0000", 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086032560000000001 2023-09-30T09:31:22.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324736+0000", 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5887999999999998e-05 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324775+0000", 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9063000000000002e-05 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.969 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119649 5.d 5:b97acc98:::benchmark_data_smithi019_287194_object36871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.428009+0000", 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.44205123400002, 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.896981166, 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119649 2023-09-30T09:31:22.970 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428009+0000", 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428009+0000", 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.971 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428009+0000", 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6499999999999999e-07 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428013+0000", 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0809999999999998e-06 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428020+0000", 2023-09-30T09:31:22.972 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3300000000000001e-06 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428026+0000", 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3099999999999997e-06 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428132+0000", 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010583500000000001 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.973 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428141+0000", 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9949999999999994e-06 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.308041+0000", 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87989993799999999 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.974 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.308172+0000", 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013134999999999999 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.308274+0000", 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010208 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316086+0000", 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.007811162 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.975 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324840+0000", 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087541049999999999 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324896+0000", 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5889e-05 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:22.976 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324954+0000", 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8155000000000001e-05 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324990+0000", 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6371000000000001e-05 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.977 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119659 5.d 5:b8082ea3:::benchmark_data_smithi019_287194_object36881:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.436320+0000", 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.43374044199999, 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90387485099999998, 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119659 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.978 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436320+0000", 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436320+0000", 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436320+0000", 2023-09-30T09:31:22.979 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4900000000000001e-07 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436323+0000", 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3410000000000001e-06 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436336+0000", 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3641e-05 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.980 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436342+0000", 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9510000000000003e-06 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436407+0000", 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4412000000000006e-05 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436416+0000", 2023-09-30T09:31:22.981 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4970000000000007e-06 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316129+0000", 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87971309900000005 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316274+0000", 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014522499999999999 2023-09-30T09:31:22.982 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316389+0000", 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000114208 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.325003+0000", 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086141789999999996 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.983 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.332748+0000", 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077455029999999999 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340107+0000", 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0073584699999999998 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340143+0000", 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6634999999999997e-05 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.984 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340195+0000", 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1240999999999999e-05 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119661 5.d 5:b77bcc20:::benchmark_data_smithi019_287194_object36883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.441103+0000", 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 312.42895676000001, 2023-09-30T09:31:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89930644500000001, 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119661 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441103+0000", 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441103+0000", 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441105+0000", 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4139999999999999e-06 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441111+0000", 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.049e-06 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441125+0000", 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4153e-05 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441133+0000", 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0450000000000001e-06 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441245+0000", 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000111558 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441256+0000", 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1579999999999999e-05 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.320272+0000", 2023-09-30T09:31:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87901538099999998 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.320371+0000", 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9270000000000003e-05 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.320476+0000", 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000104944 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.336739+0000", 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016263357999999999 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340293+0000", 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035540900000000002 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340339+0000", 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5862000000000002e-05 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340377+0000", 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8192999999999998e-05 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340410+0000", 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2548000000000001e-05 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145642 5.a 5:5d87f834:::benchmark_data_smithi019_287194_object62864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.286985+0000", 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 272.58307511300001, 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6001666220000001, 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145642 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.286985+0000", 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.286985+0000", 2023-09-30T09:31:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.286987+0000", 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.657e-06 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.286993+0000", 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9259999999999997e-06 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287007+0000", 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3898e-05 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287015+0000", 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2199999999999992e-06 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.289744+0000", 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027293949999999999 2023-09-30T09:31:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.289891+0000", 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014728899999999999 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.290029+0000", 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000137758 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.308124+0000", 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018094566999999999 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313660+0000", 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055359249999999997 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.324923+0000", 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011262853 2023-09-30T09:31:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.324938+0000", 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5201999999999999e-05 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308729+0000", 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98379169399999999 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308743+0000", 2023-09-30T09:31:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3352e-05 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.800834+0000", 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.492090793 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.800989+0000", 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015555500000000001 2023-09-30T09:31:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.887074+0000", 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.086085150999999999 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.887152+0000", 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7386999999999996e-05 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145643 5.a 5:52928c9d:::benchmark_data_smithi019_287194_object62865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.287030+0000", 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 272.58303048699997, 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6018917200000002, 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145643 2023-09-30T09:31:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287030+0000", 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287030+0000", 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287030+0000", 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3000000000000001e-07 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287033+0000", 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9610000000000001e-06 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287041+0000", 2023-09-30T09:31:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3520000000000001e-06 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287048+0000", 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9979999999999997e-06 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.294960+0000", 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0079122800000000007 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.295082+0000", 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000122559 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.295203+0000", 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012053900000000001 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313664+0000", 2023-09-30T09:31:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018460982000000001 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.315555+0000", 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018911620000000001 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325177+0000", 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0096223430000000002 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325203+0000", 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6016999999999998e-05 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308838+0000", 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98363487100000002 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308854+0000", 2023-09-30T09:31:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6166000000000001e-05 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.802671+0000", 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4938168599999999 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.802866+0000", 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019441700000000001 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.888843+0000", 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.085977656999999999 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.888921+0000", 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8026000000000001e-05 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145646 5.a 5:58abbb4b:::benchmark_data_smithi019_287194_object62868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.291309+0000", 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 272.57875152299999, 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6126824850000001, 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145646 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291309+0000", 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291309+0000", 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291310+0000", 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3999999999999999e-06 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291315+0000", 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1950000000000002e-06 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291332+0000", 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7005000000000001e-05 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291341+0000", 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1370000000000008e-06 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.303167+0000", 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011826066 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.303285+0000", 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000117348 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.303390+0000", 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000104719 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313667+0000", 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010277873999999999 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.315592+0000", 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001924697 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325238+0000", 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0096459260000000008 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325246+0000", 2023-09-30T09:31:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1550000000000007e-06 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308908+0000", 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98366193300000004 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308919+0000", 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1124000000000001e-05 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.802923+0000", 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.494003881 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803085+0000", 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016144 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.903883+0000", 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.100798417 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.903991+0000", 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000108168 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145647 5.a 5:5b66dd4d:::benchmark_data_smithi019_287194_object62869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.291355+0000", 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 272.57870557500001, 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6146902910000001, 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145647 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291355+0000", 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291355+0000", 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291355+0000", 2023-09-30T09:31:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7599999999999997e-07 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291358+0000", 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7690000000000001e-06 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291368+0000", 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7799999999999995e-06 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291376+0000", 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5210000000000002e-06 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.308715+0000", 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017338610000000001 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.308876+0000", 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016106900000000001 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.309012+0000", 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000136067 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.315628+0000", 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0066166080000000004 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.315959+0000", 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033090799999999998 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325316+0000", 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0093571009999999996 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325325+0000", 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3189999999999995e-06 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308955+0000", 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98363034999999999 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308980+0000", 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4943000000000002e-05 2023-09-30T09:31:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803130+0000", 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4941497420000001 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803279+0000", 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000149476 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.905950+0000", 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.10267042 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.906045+0000", 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5131999999999997e-05 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145680 5.a 5:5166d668:::benchmark_data_smithi019_287194_object62902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.355901+0000", 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 272.51415931100001, 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5249182000000001, 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145680 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355901+0000", 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355901+0000", 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355903+0000", 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.655e-06 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355909+0000", 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.596e-06 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355921+0000", 2023-09-30T09:31:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1505000000000001e-05 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355928+0000", 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9720000000000003e-06 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355981+0000", 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3405000000000001e-05 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355989+0000", 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6729999999999992e-06 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.309011+0000", 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95302198100000002 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.309020+0000", 2023-09-30T09:31:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0340000000000002e-06 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803332+0000", 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4943121939999999 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803503+0000", 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017102799999999999 2023-09-30T09:31:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803685+0000", 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018205900000000001 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.832506+0000", 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028821060999999999 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.833422+0000", 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00091556299999999999 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.879313+0000", 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.045890959000000002 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.879377+0000", 2023-09-30T09:31:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4225000000000003e-05 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.880819+0000", 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0014422899999999999 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145685 5.b 5:deae4cba:::benchmark_data_smithi019_287194_object62907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.359013+0000", 2023-09-30T09:31:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 272.51104665899999, 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.420992553, 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145685 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359013+0000", 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359013+0000", 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359014+0000", 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2399999999999999e-07 2023-09-30T09:31:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359016+0000", 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.379e-06 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359022+0000", 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.626e-06 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359027+0000", 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3639999999999998e-06 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359084+0000", 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7089000000000002e-05 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359091+0000", 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.516e-06 2023-09-30T09:31:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308905+0000", 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94981431199999999 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308916+0000", 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0776e-05 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.627874+0000", 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.318958147 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.627957+0000", 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2960999999999994e-05 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,7", 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.628052+0000", 2023-09-30T09:31:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5212999999999998e-05 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.777472+0000", 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14941965300000001 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.778998+0000", 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001525649 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.779072+0000", 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4690999999999994e-05 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.779145+0000", 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2857999999999999e-05 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.780006+0000", 2023-09-30T09:31:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00086089499999999998 2023-09-30T09:31:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:23.046 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:31:23.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:23.047 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:23.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:23.150 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:23.214 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:23.435 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:31:23.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1515503949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:23.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5853/55833 objects degraded (10.483%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:23.488 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1515503949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:23.488 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5853/55833 objects degraded (10.483%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:23.514 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:23.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:23.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:23.621 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:23.685 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1515503949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:23.685 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5853/55833 objects degraded (10.483%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:23.694 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:23.814 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:23.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:23.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:23.920 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:23.954 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:31:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:31:23] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.43.0" 2023-09-30T09:31:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: pgmap v638: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 5853/54240 objects degraded (10.791%); 547/54240 objects misplaced (1.008%); 0 B/s, 88 objects/s recovering 2023-09-30T09:31:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:31:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1515503949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: osdmap e266: 8 total, 8 up, 7 in 2023-09-30T09:31:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[216815]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: pgmap v638: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 5853/54240 objects degraded (10.791%); 547/54240 objects misplaced (1.008%); 0 B/s, 88 objects/s recovering 2023-09-30T09:31:24.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:31:24.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1515503949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:24.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: osdmap e266: 8 total, 8 up, 7 in 2023-09-30T09:31:24.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:24 smithi019 ceph-mon[224043]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: pgmap v638: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 5853/54240 objects degraded (10.791%); 547/54240 objects misplaced (1.008%); 0 B/s, 88 objects/s recovering 2023-09-30T09:31:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:31:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1515503949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: osdmap e266: 8 total, 8 up, 7 in 2023-09-30T09:31:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:24 smithi167 ceph-mon[192880]: failed to encode map e266 with expected crc 2023-09-30T09:31:24.932 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:31:25.081 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:25.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:25.187 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:25.260 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:25.322 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:25.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:25.427 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:25.472 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:25.833 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:25.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:25.941 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:25.997 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:26.104 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:26.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:26.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:26.133 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:26.222 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:26.244 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:31:26.278 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:26.283 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:31:26.317 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:31:26.345 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:31:26.377 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:31:26.380 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:26.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:26.422 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:31:26.460 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:31:26.490 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:26.491 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:31:26.496 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:31:26.496 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:31:26.496 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:31:26.496 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:31:26.496 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:31:26.496 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:31:26.496 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:31:26.497 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:31:26.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:26 smithi167 ceph-mon[192880]: pgmap v640: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 840 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 5279/40203 objects degraded (13.131%); 389/40203 objects misplaced (0.968%); 0 B/s, 144 objects/s recovering 2023-09-30T09:31:26.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/358531779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:26.555 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:26.665 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:26.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:26.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:26.776 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:26.833 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:26 smithi019 ceph-mon[216815]: pgmap v640: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 840 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 5279/40203 objects degraded (13.131%); 389/40203 objects misplaced (0.968%); 0 B/s, 144 objects/s recovering 2023-09-30T09:31:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/358531779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:26 smithi019 ceph-mon[224043]: pgmap v640: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 840 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 5279/40203 objects degraded (13.131%); 389/40203 objects misplaced (0.968%); 0 B/s, 144 objects/s recovering 2023-09-30T09:31:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/358531779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:26.883 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:31:26.883 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:31:26.883 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-30T09:31:26.884 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:31:26.925 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:26.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:26.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:27.038 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:27.653 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:27.717 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:27.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:27.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:27.822 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:27.850 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:27.854 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:31:28.332 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:28.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:28.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:28.446 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:28.450 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:28.505 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:28 smithi019 ceph-mon[224043]: pgmap v641: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 806 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 978 op/s; 5279/38556 objects degraded (13.692%); 389/38556 objects misplaced (1.009%); 0 B/s, 128 objects/s recovering 2023-09-30T09:31:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5279/38556 objects degraded (13.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:28.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:28 smithi019 ceph-mon[216815]: pgmap v641: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 806 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 978 op/s; 5279/38556 objects degraded (13.692%); 389/38556 objects misplaced (1.009%); 0 B/s, 128 objects/s recovering 2023-09-30T09:31:28.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5279/38556 objects degraded (13.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:28.636 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:28.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:28.649 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:28 smithi167 ceph-mon[192880]: pgmap v641: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 806 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 978 op/s; 5279/38556 objects degraded (13.692%); 389/38556 objects misplaced (1.009%); 0 B/s, 128 objects/s recovering 2023-09-30T09:31:28.649 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5279/38556 objects degraded (13.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:28.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:28.751 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:28.806 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:28.919 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:28.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:28.939 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:29.026 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:29.040 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:31:29.040 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:31:29.122 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:29.762 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:29.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:29.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:29.866 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:29.920 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:29.964 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:29.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:29.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:30.068 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:30.119 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:30.218 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:30.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:30.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:30.299 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:31:30.323 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:30.377 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:30 smithi167 ceph-mon[192880]: pgmap v642: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 678 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 5279/32403 objects degraded (16.292%); 389/32403 objects misplaced (1.201%); 0 B/s, 100 objects/s recovering 2023-09-30T09:31:30.393 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:30.493 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:30.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:30.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:30 smithi019 ceph-mon[216815]: pgmap v642: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 678 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 5279/32403 objects degraded (16.292%); 389/32403 objects misplaced (1.201%); 0 B/s, 100 objects/s recovering 2023-09-30T09:31:30.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:30 smithi019 ceph-mon[224043]: pgmap v642: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 678 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 5279/32403 objects degraded (16.292%); 389/32403 objects misplaced (1.201%); 0 B/s, 100 objects/s recovering 2023-09-30T09:31:30.604 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:30.649 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:30.827 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:31:30.870 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:30.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:30.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:30.979 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:31.034 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:31.151 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:31.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:31.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:31.259 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:31.323 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:31.699 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:31.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:31.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:31.806 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:31.862 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:31.916 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:31:31.927 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:31:31.933 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:31.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:31.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:32.039 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:32.116 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:32.209 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:32.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:32.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:32.313 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:32.358 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:32 smithi167 ceph-mon[192880]: pgmap v643: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 509 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 4706/24306 objects degraded (19.361%); 232/24306 objects misplaced (0.954%); 0 B/s, 114 objects/s recovering 2023-09-30T09:31:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:32 smithi019 ceph-mon[216815]: pgmap v643: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 509 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 4706/24306 objects degraded (19.361%); 232/24306 objects misplaced (0.954%); 0 B/s, 114 objects/s recovering 2023-09-30T09:31:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:32 smithi019 ceph-mon[224043]: pgmap v643: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 509 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 4706/24306 objects degraded (19.361%); 232/24306 objects misplaced (0.954%); 0 B/s, 114 objects/s recovering 2023-09-30T09:31:33.177 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:33.211 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:33.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:33.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:33.243 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":643,"stamp":"2023-09-30T09:31:31.195293+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":533706986,"num_objects":8102,"num_object_clones":0,"num_object_copies":24306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":232,"num_objects_unfound":0,"num_objects_dirty":8102,"num_whiteouts":0,"num_read":6032,"num_read_kb":12314,"num_write":161442,"num_write_kb":4889516,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30407,"num_bytes_recovered":1763874008,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":201381,"ondisk_log_size":201381,"up":417,"acting":413,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":420,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3775540,"kb_used_data":2080188,"kb_used_omap":14,"kb_used_meta":1694833,"kb_avail":652440524,"statfs":{"total":671965249536,"available":668099096576,"internally_reserved":0,"allocated":2130112512,"data_stored":2121515765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14566,"internal_metadata":1735509786},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-688717824,"num_objects":-10509,"num_object_clones":0,"num_object_copies":-31527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1147,"num_objects_misplaced":-315,"num_objects_unfound":0,"num_objects_dirty":-10509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10509,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1147,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009034"},"pg_stats":[{"pgid":"6.23","version":"266'5003","reported_seq":9188,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.212804+0000","last_change":"2023-09-30T09:29:35.062393+0000","last_active":"2023-09-30T09:31:29.212804+0000","last_peered":"2023-09-30T09:31:29.212804+0000","last_clean":"2023-09-30T09:31:29.212804+0000","last_became_active":"2023-09-30T09:29:35.061339+0000","last_became_peered":"2023-09-30T09:29:35.061339+0000","last_unstale":"2023-09-30T09:31:29.212804+0000","last_undegraded":"2023-09-30T09:31:29.212804+0000","last_fullsized":"2023-09-30T09:31:29.212804+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9895936,"num_objects":151,"num_object_clones":0,"num_object_copies":453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2004,"num_write_kb":46336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65699840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.22","version":"266'4962","reported_seq":7916,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:31:28.525333+0000","last_change":"2023-09-30T09:29:37.013899+0000","last_active":"2023-09-30T09:31:28.525333+0000","last_peered":"2023-09-30T09:31:28.525333+0000","last_clean":"2023-09-30T09:29:08.552432+0000","last_became_active":"2023-09-30T09:29:37.011820+0000","last_became_peered":"2023-09-30T09:29:37.011820+0000","last_unstale":"2023-09-30T09:31:28.525333+0000","last_undegraded":"2023-09-30T09:29:36.997644+0000","last_fullsized":"2023-09-30T09:29:36.996855+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":6,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":4962,"log_dups_size":0,"ondisk_log_size":4962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5767168,"num_objects":88,"num_object_clones":0,"num_object_copies":264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":304,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1965,"num_write_kb":42464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,6","objects":-216}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.21","version":"266'5245","reported_seq":7453,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:27.039328+0000","last_change":"2023-09-30T09:30:54.703073+0000","last_active":"2023-09-30T09:31:27.039328+0000","last_peered":"2023-09-30T09:31:27.039328+0000","last_clean":"2023-09-30T09:31:27.039328+0000","last_became_active":"2023-09-30T09:29:35.075606+0000","last_became_peered":"2023-09-30T09:29:35.075606+0000","last_unstale":"2023-09-30T09:31:27.039328+0000","last_undegraded":"2023-09-30T09:31:27.039328+0000","last_fullsized":"2023-09-30T09:31:27.039328+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":6,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":5245,"log_dups_size":0,"ondisk_log_size":5245,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7700480,"num_objects":118,"num_object_clones":0,"num_object_copies":354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2197,"num_write_kb":53856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1043,"num_bytes_recovered":68354048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.20","version":"266'5204","reported_seq":7441,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.496798+0000","last_change":"2023-09-30T09:31:14.316519+0000","last_active":"2023-09-30T09:31:29.496798+0000","last_peered":"2023-09-30T09:31:29.496798+0000","last_clean":"2023-09-30T09:31:29.496798+0000","last_became_active":"2023-09-30T09:29:35.036894+0000","last_became_peered":"2023-09-30T09:29:35.036894+0000","last_unstale":"2023-09-30T09:31:29.496798+0000","last_undegraded":"2023-09-30T09:31:29.496798+0000","last_fullsized":"2023-09-30T09:31:29.496798+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":6,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":5204,"log_dups_size":0,"ondisk_log_size":5204,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7340032,"num_objects":112,"num_object_clones":0,"num_object_copies":336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2170,"num_write_kb":50400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1055,"num_bytes_recovered":69107712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":654,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296465+0000","last_change":"2023-09-30T09:31:09.040934+0000","last_active":"2023-09-30T09:31:16.296465+0000","last_peered":"2023-09-30T09:31:16.296465+0000","last_clean":"2023-09-30T09:31:16.296465+0000","last_became_active":"2023-09-30T09:31:09.040581+0000","last_became_peered":"2023-09-30T09:31:09.040581+0000","last_unstale":"2023-09-30T09:31:16.296465+0000","last_undegraded":"2023-09-30T09:31:16.296465+0000","last_fullsized":"2023-09-30T09:31:16.296465+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.999760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.999760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":265519,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1f","version":"266'8065","reported_seq":12344,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:28.739260+0000","last_change":"2023-09-30T09:31:09.249453+0000","last_active":"2023-09-30T09:31:28.739260+0000","last_peered":"2023-09-30T09:31:28.739260+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:35.087685+0000","last_became_peered":"2023-09-30T09:29:35.087685+0000","last_unstale":"2023-09-30T09:31:28.739260+0000","last_undegraded":"2023-09-30T09:31:28.739260+0000","last_fullsized":"2023-09-30T09:31:28.739260+0000","mapping_epoch":238,"log_start":"225'3250","ondisk_log_start":"225'3250","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4815,"log_dups_size":0,"ondisk_log_size":4815,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12517376,"num_objects":191,"num_object_clones":0,"num_object_copies":573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5617,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2094,"num_bytes_recovered":137232384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":191}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":253,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312286+0000","last_change":"2023-09-30T09:29:09.598494+0000","last_active":"2023-09-30T09:31:16.312286+0000","last_peered":"2023-09-30T09:31:16.312286+0000","last_clean":"2023-09-30T09:31:16.312286+0000","last_became_active":"2023-09-30T09:29:09.598158+0000","last_became_peered":"2023-09-30T09:29:09.598158+0000","last_unstale":"2023-09-30T09:31:16.312286+0000","last_undegraded":"2023-09-30T09:31:16.312286+0000","last_fullsized":"2023-09-30T09:31:16.312286+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:03.225682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":232,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:23.440329+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:31:23.440329+0000","last_peered":"2023-09-30T09:31:23.440329+0000","last_clean":"2023-09-30T09:31:23.440329+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:31:23.440329+0000","last_undegraded":"2023-09-30T09:31:23.440329+0000","last_fullsized":"2023-09-30T09:31:23.440329+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":689,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312721+0000","last_change":"2023-09-30T09:31:09.036888+0000","last_active":"2023-09-30T09:31:16.312721+0000","last_peered":"2023-09-30T09:31:16.312721+0000","last_clean":"2023-09-30T09:31:16.312721+0000","last_became_active":"2023-09-30T09:31:09.036525+0000","last_became_peered":"2023-09-30T09:31:09.036525+0000","last_unstale":"2023-09-30T09:31:16.312721+0000","last_undegraded":"2023-09-30T09:31:16.312721+0000","last_fullsized":"2023-09-30T09:31:16.312721+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1c","version":"266'7857","reported_seq":7944,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:28.726688+0000","last_change":"2023-09-30T09:29:35.102559+0000","last_active":"2023-09-30T09:31:28.726688+0000","last_peered":"2023-09-30T09:31:28.726688+0000","last_clean":"2023-09-30T09:29:33.948392+0000","last_became_active":"2023-09-30T09:29:35.089045+0000","last_became_peered":"2023-09-30T09:29:35.089045+0000","last_unstale":"2023-09-30T09:31:28.726688+0000","last_undegraded":"2023-09-30T09:31:28.726688+0000","last_fullsized":"2023-09-30T09:31:28.726688+0000","mapping_epoch":238,"log_start":"223'2906","ondisk_log_start":"223'2906","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4951,"log_dups_size":0,"ondisk_log_size":4951,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13697024,"num_objects":209,"num_object_clones":0,"num_object_copies":627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5439,"num_write_kb":180736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":209}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":229,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312117+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:31:16.312117+0000","last_peered":"2023-09-30T09:31:16.312117+0000","last_clean":"2023-09-30T09:31:16.312117+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:31:16.312117+0000","last_undegraded":"2023-09-30T09:31:16.312117+0000","last_fullsized":"2023-09-30T09:31:16.312117+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":160,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.293744+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:31:16.293744+0000","last_peered":"2023-09-30T09:31:16.293744+0000","last_clean":"2023-09-30T09:31:16.293744+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:31:16.293744+0000","last_undegraded":"2023-09-30T09:31:16.293744+0000","last_fullsized":"2023-09-30T09:31:16.293744+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":482,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.304409+0000","last_change":"2023-09-30T09:31:09.043341+0000","last_active":"2023-09-30T09:31:16.304409+0000","last_peered":"2023-09-30T09:31:16.304409+0000","last_clean":"2023-09-30T09:31:16.304409+0000","last_became_active":"2023-09-30T09:31:09.043024+0000","last_became_peered":"2023-09-30T09:31:09.043024+0000","last_unstale":"2023-09-30T09:31:16.304409+0000","last_undegraded":"2023-09-30T09:31:16.304409+0000","last_fullsized":"2023-09-30T09:31:16.304409+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.980385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.980385+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.1d","version":"266'8022","reported_seq":12176,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:29.121161+0000","last_change":"2023-09-30T09:31:09.278583+0000","last_active":"2023-09-30T09:31:29.121161+0000","last_peered":"2023-09-30T09:31:29.121161+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:35.073573+0000","last_became_peered":"2023-09-30T09:29:35.073573+0000","last_unstale":"2023-09-30T09:31:29.121161+0000","last_undegraded":"2023-09-30T09:31:29.121161+0000","last_fullsized":"2023-09-30T09:31:29.121161+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":8022,"log_dups_size":0,"ondisk_log_size":8022,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16384024,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5527,"num_write_kb":184833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2034,"num_bytes_recovered":133300224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2","6"],"object_location_counts":[{"shards":"1,5,6","objects":251}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":282,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296892+0000","last_change":"2023-09-30T09:29:09.606288+0000","last_active":"2023-09-30T09:31:16.296892+0000","last_peered":"2023-09-30T09:31:16.296892+0000","last_clean":"2023-09-30T09:31:16.296892+0000","last_became_active":"2023-09-30T09:29:09.605776+0000","last_became_peered":"2023-09-30T09:29:09.605776+0000","last_unstale":"2023-09-30T09:31:16.296892+0000","last_undegraded":"2023-09-30T09:31:16.296892+0000","last_fullsized":"2023-09-30T09:31:16.296892+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:07.935982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":225,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305769+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:31:16.305769+0000","last_peered":"2023-09-30T09:31:16.305769+0000","last_clean":"2023-09-30T09:31:16.305769+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:31:16.305769+0000","last_undegraded":"2023-09-30T09:31:16.305769+0000","last_fullsized":"2023-09-30T09:31:16.305769+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":707,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311427+0000","last_change":"2023-09-30T09:31:09.045110+0000","last_active":"2023-09-30T09:31:16.311427+0000","last_peered":"2023-09-30T09:31:16.311427+0000","last_clean":"2023-09-30T09:31:16.311427+0000","last_became_active":"2023-09-30T09:31:09.044721+0000","last_became_peered":"2023-09-30T09:31:09.044721+0000","last_unstale":"2023-09-30T09:31:16.311427+0000","last_undegraded":"2023-09-30T09:31:16.311427+0000","last_fullsized":"2023-09-30T09:31:16.311427+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.12","version":"266'6429","reported_seq":10601,"reported_epoch":266,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:31:29.177553+0000","last_change":"2023-09-30T09:29:10.766595+0000","last_active":"2023-09-30T09:31:29.177553+0000","last_peered":"2023-09-30T09:31:29.177553+0000","last_clean":"2023-09-30T09:29:08.564047+0000","last_became_active":"2023-09-30T09:29:10.763967+0000","last_became_peered":"2023-09-30T09:29:10.763967+0000","last_unstale":"2023-09-30T09:31:29.177553+0000","last_undegraded":"2023-09-30T09:31:29.177553+0000","last_fullsized":"2023-09-30T09:31:29.177553+0000","mapping_epoch":229,"log_start":"211'1776","ondisk_log_start":"211'1776","created":213,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2497","last_scrub_stamp":"2023-09-30T09:28:46.026181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:46.026181+0000","objects_scrubbed":1334,"log_size":4653,"log_dups_size":0,"ondisk_log_size":4653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:02.322482+0000","scrub_duration":0.15695645899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15204352,"num_objects":232,"num_object_clones":0,"num_object_copies":696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":232,"num_objects_unfound":0,"num_objects_dirty":232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3932,"num_write_kb":90560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1347,"num_bytes_recovered":88276992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,2],"avail_no_missing":["7","2","4","6"],"object_location_counts":[{"shards":"2,4,7","objects":232}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":159,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.293801+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:31:16.293801+0000","last_peered":"2023-09-30T09:31:16.293801+0000","last_clean":"2023-09-30T09:31:16.293801+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:31:16.293801+0000","last_undegraded":"2023-09-30T09:31:16.293801+0000","last_fullsized":"2023-09-30T09:31:16.293801+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":232,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296977+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:31:16.296977+0000","last_peered":"2023-09-30T09:31:16.296977+0000","last_clean":"2023-09-30T09:31:16.296977+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:31:16.296977+0000","last_undegraded":"2023-09-30T09:31:16.296977+0000","last_fullsized":"2023-09-30T09:31:16.296977+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":677,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296900+0000","last_change":"2023-09-30T09:30:28.376570+0000","last_active":"2023-09-30T09:31:16.296900+0000","last_peered":"2023-09-30T09:31:16.296900+0000","last_clean":"2023-09-30T09:31:16.296900+0000","last_became_active":"2023-09-30T09:30:28.376194+0000","last_became_peered":"2023-09-30T09:30:28.376194+0000","last_unstale":"2023-09-30T09:31:16.296900+0000","last_undegraded":"2023-09-30T09:31:16.296900+0000","last_fullsized":"2023-09-30T09:31:16.296900+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"266'6593","reported_seq":12052,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:28.717013+0000","last_change":"2023-09-30T09:30:55.322970+0000","last_active":"2023-09-30T09:31:28.717013+0000","last_peered":"2023-09-30T09:31:28.717013+0000","last_clean":"2023-09-30T09:28:50.280641+0000","last_became_active":"2023-09-30T09:30:55.307289+0000","last_became_peered":"2023-09-30T09:30:55.307289+0000","last_unstale":"2023-09-30T09:31:28.717013+0000","last_undegraded":"2023-09-30T09:31:28.717013+0000","last_fullsized":"2023-09-30T09:31:28.717013+0000","mapping_epoch":256,"log_start":"223'2545","ondisk_log_start":"223'2545","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4048,"log_dups_size":0,"ondisk_log_size":4048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22478848,"num_objects":343,"num_object_clones":0,"num_object_copies":1029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5489,"num_write_kb":186624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":129892352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":["3","1","2","6","7"],"object_location_counts":[{"shards":"1,3,6","objects":343}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":117,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301232+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:31:16.301232+0000","last_peered":"2023-09-30T09:31:16.301232+0000","last_clean":"2023-09-30T09:31:16.301232+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:31:16.301232+0000","last_undegraded":"2023-09-30T09:31:16.301232+0000","last_fullsized":"2023-09-30T09:31:16.301232+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":183,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.298289+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:31:16.298289+0000","last_peered":"2023-09-30T09:31:16.298289+0000","last_clean":"2023-09-30T09:31:16.298289+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:31:16.298289+0000","last_undegraded":"2023-09-30T09:31:16.298289+0000","last_fullsized":"2023-09-30T09:31:16.298289+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":529,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305920+0000","last_change":"2023-09-30T09:31:09.057587+0000","last_active":"2023-09-30T09:31:16.305920+0000","last_peered":"2023-09-30T09:31:16.305920+0000","last_clean":"2023-09-30T09:31:16.305920+0000","last_became_active":"2023-09-30T09:31:09.057239+0000","last_became_peered":"2023-09-30T09:31:09.057239+0000","last_unstale":"2023-09-30T09:31:16.305920+0000","last_undegraded":"2023-09-30T09:31:16.305920+0000","last_fullsized":"2023-09-30T09:31:16.305920+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.10","version":"266'6546","reported_seq":10618,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.478750+0000","last_change":"2023-09-30T09:28:56.382198+0000","last_active":"2023-09-30T09:31:29.478750+0000","last_peered":"2023-09-30T09:31:29.478750+0000","last_clean":"2023-09-30T09:31:29.478750+0000","last_became_active":"2023-09-30T09:28:56.380752+0000","last_became_peered":"2023-09-30T09:28:56.380752+0000","last_unstale":"2023-09-30T09:31:29.478750+0000","last_undegraded":"2023-09-30T09:31:29.478750+0000","last_fullsized":"2023-09-30T09:31:29.478750+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":6546,"log_dups_size":0,"ondisk_log_size":6546,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15826944,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5428,"num_write_kb":181408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1324,"num_bytes_recovered":86769664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":229,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.299565+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:31:16.299565+0000","last_peered":"2023-09-30T09:31:16.299565+0000","last_clean":"2023-09-30T09:31:16.299565+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:31:16.299565+0000","last_undegraded":"2023-09-30T09:31:16.299565+0000","last_fullsized":"2023-09-30T09:31:16.299565+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":214,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312318+0000","last_change":"2023-09-30T09:29:09.620809+0000","last_active":"2023-09-30T09:31:16.312318+0000","last_peered":"2023-09-30T09:31:16.312318+0000","last_clean":"2023-09-30T09:31:16.312318+0000","last_became_active":"2023-09-30T09:29:09.620497+0000","last_became_peered":"2023-09-30T09:29:09.620497+0000","last_unstale":"2023-09-30T09:31:16.312318+0000","last_undegraded":"2023-09-30T09:31:16.312318+0000","last_fullsized":"2023-09-30T09:31:16.312318+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:48.932180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":529,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305115+0000","last_change":"2023-09-30T09:31:09.036952+0000","last_active":"2023-09-30T09:31:16.305115+0000","last_peered":"2023-09-30T09:31:16.305115+0000","last_clean":"2023-09-30T09:31:16.305115+0000","last_became_active":"2023-09-30T09:31:09.036302+0000","last_became_peered":"2023-09-30T09:31:09.036302+0000","last_unstale":"2023-09-30T09:31:16.305115+0000","last_undegraded":"2023-09-30T09:31:16.305115+0000","last_fullsized":"2023-09-30T09:31:16.305115+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.11","version":"266'6703","reported_seq":12322,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:28.692627+0000","last_change":"2023-09-30T09:29:39.270828+0000","last_active":"2023-09-30T09:31:28.692627+0000","last_peered":"2023-09-30T09:31:28.692627+0000","last_clean":"2023-09-30T09:31:28.692627+0000","last_became_active":"2023-09-30T09:29:39.266753+0000","last_became_peered":"2023-09-30T09:29:39.266753+0000","last_unstale":"2023-09-30T09:31:28.692627+0000","last_undegraded":"2023-09-30T09:31:28.692627+0000","last_fullsized":"2023-09-30T09:31:28.692627+0000","mapping_epoch":242,"log_start":"215'2607","ondisk_log_start":"215'2607","created":213,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2608","last_scrub_stamp":"2023-09-30T09:28:45.597110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.597110+0000","objects_scrubbed":1407,"log_size":4096,"log_dups_size":0,"ondisk_log_size":4096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:06.120738+0000","scrub_duration":0.26278548800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16777216,"num_objects":256,"num_object_clones":0,"num_object_copies":768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4101,"num_write_kb":94400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2034,"num_bytes_recovered":133300224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":230,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312025+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:31:16.312025+0000","last_peered":"2023-09-30T09:31:16.312025+0000","last_clean":"2023-09-30T09:31:16.312025+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:31:16.312025+0000","last_undegraded":"2023-09-30T09:31:16.312025+0000","last_fullsized":"2023-09-30T09:31:16.312025+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":244,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311433+0000","last_change":"2023-09-30T09:29:09.612466+0000","last_active":"2023-09-30T09:31:16.311433+0000","last_peered":"2023-09-30T09:31:16.311433+0000","last_clean":"2023-09-30T09:31:16.311433+0000","last_became_active":"2023-09-30T09:29:09.612173+0000","last_became_peered":"2023-09-30T09:29:09.612173+0000","last_unstale":"2023-09-30T09:31:16.311433+0000","last_undegraded":"2023-09-30T09:31:16.311433+0000","last_fullsized":"2023-09-30T09:31:16.311433+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:15.457876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":697,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301440+0000","last_change":"2023-09-30T09:30:28.370038+0000","last_active":"2023-09-30T09:31:16.301440+0000","last_peered":"2023-09-30T09:31:16.301440+0000","last_clean":"2023-09-30T09:31:16.301440+0000","last_became_active":"2023-09-30T09:30:28.369721+0000","last_became_peered":"2023-09-30T09:30:28.369721+0000","last_unstale":"2023-09-30T09:31:16.301440+0000","last_undegraded":"2023-09-30T09:31:16.301440+0000","last_fullsized":"2023-09-30T09:31:16.301440+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.16","version":"266'6763","reported_seq":6851,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:28.746058+0000","last_change":"2023-09-30T09:28:40.282669+0000","last_active":"2023-09-30T09:31:28.746058+0000","last_peered":"2023-09-30T09:31:28.746058+0000","last_clean":"2023-09-30T09:31:28.746058+0000","last_became_active":"2023-09-30T09:28:40.281544+0000","last_became_peered":"2023-09-30T09:28:40.281544+0000","last_unstale":"2023-09-30T09:31:28.746058+0000","last_undegraded":"2023-09-30T09:31:28.746058+0000","last_fullsized":"2023-09-30T09:31:28.746058+0000","mapping_epoch":213,"log_start":"223'2685","ondisk_log_start":"223'2685","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4078,"log_dups_size":0,"ondisk_log_size":4078,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17530880,"num_objects":268,"num_object_clones":0,"num_object_copies":804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5626,"num_write_kb":188576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":244,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.299915+0000","last_change":"2023-09-30T09:29:09.626205+0000","last_active":"2023-09-30T09:31:16.299915+0000","last_peered":"2023-09-30T09:31:16.299915+0000","last_clean":"2023-09-30T09:31:16.299915+0000","last_became_active":"2023-09-30T09:29:09.618293+0000","last_became_peered":"2023-09-30T09:29:09.618293+0000","last_unstale":"2023-09-30T09:31:16.299915+0000","last_undegraded":"2023-09-30T09:31:16.299915+0000","last_fullsized":"2023-09-30T09:31:16.299915+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:04.337293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":254,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.297510+0000","last_change":"2023-09-30T09:29:09.606072+0000","last_active":"2023-09-30T09:31:16.297510+0000","last_peered":"2023-09-30T09:31:16.297510+0000","last_clean":"2023-09-30T09:31:16.297510+0000","last_became_active":"2023-09-30T09:29:09.605716+0000","last_became_peered":"2023-09-30T09:29:09.605716+0000","last_unstale":"2023-09-30T09:31:16.297510+0000","last_undegraded":"2023-09-30T09:31:16.297510+0000","last_fullsized":"2023-09-30T09:31:16.297510+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:50.972640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":715,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296555+0000","last_change":"2023-09-30T09:30:28.376536+0000","last_active":"2023-09-30T09:31:16.296555+0000","last_peered":"2023-09-30T09:31:16.296555+0000","last_clean":"2023-09-30T09:31:16.296555+0000","last_became_active":"2023-09-30T09:30:28.376151+0000","last_became_peered":"2023-09-30T09:30:28.376151+0000","last_unstale":"2023-09-30T09:31:16.296555+0000","last_undegraded":"2023-09-30T09:31:16.296555+0000","last_fullsized":"2023-09-30T09:31:16.296555+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.17","version":"266'6669","reported_seq":6814,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.174266+0000","last_change":"2023-09-30T09:28:45.602319+0000","last_active":"2023-09-30T09:31:29.174266+0000","last_peered":"2023-09-30T09:31:29.174266+0000","last_clean":"2023-09-30T09:31:29.174266+0000","last_became_active":"2023-09-30T09:28:40.272792+0000","last_became_peered":"2023-09-30T09:28:40.272792+0000","last_unstale":"2023-09-30T09:31:29.174266+0000","last_undegraded":"2023-09-30T09:31:29.174266+0000","last_fullsized":"2023-09-30T09:31:29.174266+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"215'2523","last_scrub_stamp":"2023-09-30T09:28:45.602248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.602248+0000","objects_scrubbed":1419,"log_size":6669,"log_dups_size":0,"ondisk_log_size":6669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:31:54.788871+0000","scrub_duration":0.27069680099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17432576,"num_objects":266,"num_object_clones":0,"num_object_copies":798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4153,"num_write_kb":96000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":159,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.293297+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:31:16.293297+0000","last_peered":"2023-09-30T09:31:16.293297+0000","last_clean":"2023-09-30T09:31:16.293297+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:31:16.293297+0000","last_undegraded":"2023-09-30T09:31:16.293297+0000","last_fullsized":"2023-09-30T09:31:16.293297+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":187,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.299636+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:31:16.299636+0000","last_peered":"2023-09-30T09:31:16.299636+0000","last_clean":"2023-09-30T09:31:16.299636+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:31:16.299636+0000","last_undegraded":"2023-09-30T09:31:16.299636+0000","last_fullsized":"2023-09-30T09:31:16.299636+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":682,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312735+0000","last_change":"2023-09-30T09:30:28.370488+0000","last_active":"2023-09-30T09:31:16.312735+0000","last_peered":"2023-09-30T09:31:16.312735+0000","last_clean":"2023-09-30T09:31:16.312735+0000","last_became_active":"2023-09-30T09:30:28.370202+0000","last_became_peered":"2023-09-30T09:30:28.370202+0000","last_unstale":"2023-09-30T09:31:16.312735+0000","last_undegraded":"2023-09-30T09:31:16.312735+0000","last_fullsized":"2023-09-30T09:31:16.312735+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.14","version":"266'6678","reported_seq":6789,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:27.065919+0000","last_change":"2023-09-30T09:28:44.596036+0000","last_active":"2023-09-30T09:31:27.065919+0000","last_peered":"2023-09-30T09:31:27.065919+0000","last_clean":"2023-09-30T09:31:27.065919+0000","last_became_active":"2023-09-30T09:28:40.272740+0000","last_became_peered":"2023-09-30T09:28:40.272740+0000","last_unstale":"2023-09-30T09:31:27.065919+0000","last_undegraded":"2023-09-30T09:31:27.065919+0000","last_fullsized":"2023-09-30T09:31:27.065919+0000","mapping_epoch":213,"log_start":"211'1864","ondisk_log_start":"211'1864","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"214'2497","last_scrub_stamp":"2023-09-30T09:28:44.595809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:44.595809+0000","objects_scrubbed":1358,"log_size":4814,"log_dups_size":0,"ondisk_log_size":4814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:18:56.692373+0000","scrub_duration":0.31195125499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17104896,"num_objects":261,"num_object_clones":0,"num_object_copies":783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4189,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":209,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.300508+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:31:16.300508+0000","last_peered":"2023-09-30T09:31:16.300508+0000","last_clean":"2023-09-30T09:31:16.300508+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:31:16.300508+0000","last_undegraded":"2023-09-30T09:31:16.300508+0000","last_fullsized":"2023-09-30T09:31:16.300508+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":159,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.293705+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:31:16.293705+0000","last_peered":"2023-09-30T09:31:16.293705+0000","last_clean":"2023-09-30T09:31:16.293705+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:31:16.293705+0000","last_undegraded":"2023-09-30T09:31:16.293705+0000","last_fullsized":"2023-09-30T09:31:16.293705+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":754,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296606+0000","last_change":"2023-09-30T09:30:28.371325+0000","last_active":"2023-09-30T09:31:16.296606+0000","last_peered":"2023-09-30T09:31:16.296606+0000","last_clean":"2023-09-30T09:31:16.296606+0000","last_became_active":"2023-09-30T09:30:28.371035+0000","last_became_peered":"2023-09-30T09:30:28.371035+0000","last_unstale":"2023-09-30T09:31:16.296606+0000","last_undegraded":"2023-09-30T09:31:16.296606+0000","last_fullsized":"2023-09-30T09:31:16.296606+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.15","version":"266'6705","reported_seq":6782,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.189860+0000","last_change":"2023-09-30T09:28:40.288346+0000","last_active":"2023-09-30T09:31:29.189860+0000","last_peered":"2023-09-30T09:31:29.189860+0000","last_clean":"2023-09-30T09:31:29.189860+0000","last_became_active":"2023-09-30T09:28:40.286899+0000","last_became_peered":"2023-09-30T09:28:40.286899+0000","last_unstale":"2023-09-30T09:31:29.189860+0000","last_undegraded":"2023-09-30T09:31:29.189860+0000","last_fullsized":"2023-09-30T09:31:29.189860+0000","mapping_epoch":213,"log_start":"212'2207","ondisk_log_start":"212'2207","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4498,"log_dups_size":0,"ondisk_log_size":4498,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14254080,"num_objects":218,"num_object_clones":0,"num_object_copies":654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5574,"num_write_kb":185312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":358,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.304889+0000","last_change":"2023-09-30T09:29:09.634829+0000","last_active":"2023-09-30T09:31:16.304889+0000","last_peered":"2023-09-30T09:31:16.304889+0000","last_clean":"2023-09-30T09:31:16.304889+0000","last_became_active":"2023-09-30T09:29:09.612183+0000","last_became_peered":"2023-09-30T09:29:09.612183+0000","last_unstale":"2023-09-30T09:31:16.304889+0000","last_undegraded":"2023-09-30T09:31:16.304889+0000","last_fullsized":"2023-09-30T09:31:16.304889+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:28.144436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":252,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.299246+0000","last_change":"2023-09-30T09:29:09.605918+0000","last_active":"2023-09-30T09:31:16.299246+0000","last_peered":"2023-09-30T09:31:16.299246+0000","last_clean":"2023-09-30T09:31:16.299246+0000","last_became_active":"2023-09-30T09:29:09.605521+0000","last_became_peered":"2023-09-30T09:29:09.605521+0000","last_unstale":"2023-09-30T09:31:16.299246+0000","last_undegraded":"2023-09-30T09:31:16.299246+0000","last_fullsized":"2023-09-30T09:31:16.299246+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:43.287337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":707,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311472+0000","last_change":"2023-09-30T09:31:09.043132+0000","last_active":"2023-09-30T09:31:16.311472+0000","last_peered":"2023-09-30T09:31:16.311472+0000","last_clean":"2023-09-30T09:31:16.311472+0000","last_became_active":"2023-09-30T09:31:09.042743+0000","last_became_peered":"2023-09-30T09:31:09.042743+0000","last_unstale":"2023-09-30T09:31:16.311472+0000","last_undegraded":"2023-09-30T09:31:16.311472+0000","last_fullsized":"2023-09-30T09:31:16.311472+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.a","version":"266'7926","reported_seq":8005,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.189676+0000","last_change":"2023-09-30T09:29:35.005933+0000","last_active":"2023-09-30T09:31:29.189676+0000","last_peered":"2023-09-30T09:31:29.189676+0000","last_clean":"2023-09-30T09:31:29.189676+0000","last_became_active":"2023-09-30T09:29:35.001538+0000","last_became_peered":"2023-09-30T09:29:35.001538+0000","last_unstale":"2023-09-30T09:31:29.189676+0000","last_undegraded":"2023-09-30T09:31:29.189676+0000","last_fullsized":"2023-09-30T09:31:29.189676+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":7926,"log_dups_size":0,"ondisk_log_size":7926,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16449536,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5479,"num_write_kb":183360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":269,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.297208+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:31:16.297208+0000","last_peered":"2023-09-30T09:31:16.297208+0000","last_clean":"2023-09-30T09:31:16.297208+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:31:16.297208+0000","last_undegraded":"2023-09-30T09:31:16.297208+0000","last_fullsized":"2023-09-30T09:31:16.297208+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.302701+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:31:16.302701+0000","last_peered":"2023-09-30T09:31:16.302701+0000","last_clean":"2023-09-30T09:31:16.302701+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:31:16.302701+0000","last_undegraded":"2023-09-30T09:31:16.302701+0000","last_fullsized":"2023-09-30T09:31:16.302701+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":686,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296999+0000","last_change":"2023-09-30T09:31:09.039244+0000","last_active":"2023-09-30T09:31:16.296999+0000","last_peered":"2023-09-30T09:31:16.296999+0000","last_clean":"2023-09-30T09:31:16.296999+0000","last_became_active":"2023-09-30T09:31:09.038831+0000","last_became_peered":"2023-09-30T09:31:09.038831+0000","last_unstale":"2023-09-30T09:31:16.296999+0000","last_undegraded":"2023-09-30T09:31:16.296999+0000","last_fullsized":"2023-09-30T09:31:16.296999+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"266'8026","reported_seq":8114,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:28.754566+0000","last_change":"2023-09-30T09:29:35.047558+0000","last_active":"2023-09-30T09:31:28.754566+0000","last_peered":"2023-09-30T09:31:28.754566+0000","last_clean":"2023-09-30T09:31:28.754566+0000","last_became_active":"2023-09-30T09:29:35.044991+0000","last_became_peered":"2023-09-30T09:29:35.044991+0000","last_unstale":"2023-09-30T09:31:28.754566+0000","last_undegraded":"2023-09-30T09:31:28.754566+0000","last_fullsized":"2023-09-30T09:31:28.754566+0000","mapping_epoch":206,"log_start":"225'3111","ondisk_log_start":"225'3111","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4915,"log_dups_size":0,"ondisk_log_size":4915,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17465344,"num_objects":266,"num_object_clones":0,"num_object_copies":798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5558,"num_write_kb":186400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":270,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.298878+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:31:16.298878+0000","last_peered":"2023-09-30T09:31:16.298878+0000","last_clean":"2023-09-30T09:31:16.298878+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:31:16.298878+0000","last_undegraded":"2023-09-30T09:31:16.298878+0000","last_fullsized":"2023-09-30T09:31:16.298878+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":241,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311055+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:31:16.311055+0000","last_peered":"2023-09-30T09:31:16.311055+0000","last_clean":"2023-09-30T09:31:16.311055+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:31:16.311055+0000","last_undegraded":"2023-09-30T09:31:16.311055+0000","last_fullsized":"2023-09-30T09:31:16.311055+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":519,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305623+0000","last_change":"2023-09-30T09:30:28.373928+0000","last_active":"2023-09-30T09:31:16.305623+0000","last_peered":"2023-09-30T09:31:16.305623+0000","last_clean":"2023-09-30T09:31:16.305623+0000","last_became_active":"2023-09-30T09:30:28.373178+0000","last_became_peered":"2023-09-30T09:30:28.373178+0000","last_unstale":"2023-09-30T09:31:16.305623+0000","last_undegraded":"2023-09-30T09:31:16.305623+0000","last_fullsized":"2023-09-30T09:31:16.305623+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"266'6578","reported_seq":9810,"reported_epoch":266,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:31:29.225971+0000","last_change":"2023-09-30T09:31:15.084340+0000","last_active":"2023-09-30T09:31:29.225971+0000","last_peered":"2023-09-30T09:31:29.225971+0000","last_clean":"2023-09-30T09:29:08.490081+0000","last_became_active":"2023-09-30T09:29:10.870401+0000","last_became_peered":"2023-09-30T09:29:10.870401+0000","last_unstale":"2023-09-30T09:31:29.225971+0000","last_undegraded":"2023-09-30T09:29:10.657673+0000","last_fullsized":"2023-09-30T09:29:10.593223+0000","mapping_epoch":229,"log_start":"211'2072","ondisk_log_start":"211'2072","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4506,"log_dups_size":0,"ondisk_log_size":4506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16285696,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":404,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5464,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1571,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"3,7","objects":-156}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":252,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.299492+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:31:16.299492+0000","last_peered":"2023-09-30T09:31:16.299492+0000","last_clean":"2023-09-30T09:31:16.299492+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:31:16.299492+0000","last_undegraded":"2023-09-30T09:31:16.299492+0000","last_fullsized":"2023-09-30T09:31:16.299492+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":235,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305771+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:31:16.305771+0000","last_peered":"2023-09-30T09:31:16.305771+0000","last_clean":"2023-09-30T09:31:16.305771+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:31:16.305771+0000","last_undegraded":"2023-09-30T09:31:16.305771+0000","last_fullsized":"2023-09-30T09:31:16.305771+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":529,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305308+0000","last_change":"2023-09-30T09:31:09.036075+0000","last_active":"2023-09-30T09:31:16.305308+0000","last_peered":"2023-09-30T09:31:16.305308+0000","last_clean":"2023-09-30T09:31:16.305308+0000","last_became_active":"2023-09-30T09:31:09.035778+0000","last_became_peered":"2023-09-30T09:31:09.035778+0000","last_unstale":"2023-09-30T09:31:16.305308+0000","last_undegraded":"2023-09-30T09:31:16.305308+0000","last_fullsized":"2023-09-30T09:31:16.305308+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"266'6579","reported_seq":6731,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:31:29.210716+0000","last_change":"2023-09-30T09:29:10.893105+0000","last_active":"2023-09-30T09:31:29.210716+0000","last_peered":"2023-09-30T09:31:29.210716+0000","last_clean":"2023-09-30T09:29:08.514351+0000","last_became_active":"2023-09-30T09:29:10.883964+0000","last_became_peered":"2023-09-30T09:29:10.883964+0000","last_unstale":"2023-09-30T09:31:29.210716+0000","last_undegraded":"2023-09-30T09:29:10.674378+0000","last_fullsized":"2023-09-30T09:29:10.596450+0000","mapping_epoch":229,"log_start":"212'2143","ondisk_log_start":"212'2143","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2471","last_scrub_stamp":"2023-09-30T09:28:47.207706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:47.207706+0000","objects_scrubbed":1369,"log_size":4436,"log_dups_size":0,"ondisk_log_size":4436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:51.408261+0000","scrub_duration":0.14908584899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15532032,"num_objects":237,"num_object_clones":0,"num_object_copies":711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4108,"num_write_kb":95232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"3,7","objects":-1760}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":225,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301277+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:31:16.301277+0000","last_peered":"2023-09-30T09:31:16.301277+0000","last_clean":"2023-09-30T09:31:16.301277+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:31:16.301277+0000","last_undegraded":"2023-09-30T09:31:16.301277+0000","last_fullsized":"2023-09-30T09:31:16.301277+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":156,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.293419+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:31:16.293419+0000","last_peered":"2023-09-30T09:31:16.293419+0000","last_clean":"2023-09-30T09:31:16.293419+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:31:16.293419+0000","last_undegraded":"2023-09-30T09:31:16.293419+0000","last_fullsized":"2023-09-30T09:31:16.293419+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"266'7805","reported_seq":11738,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:28.975599+0000","last_change":"2023-09-30T09:31:09.239446+0000","last_active":"2023-09-30T09:31:28.975599+0000","last_peered":"2023-09-30T09:31:28.975599+0000","last_clean":"2023-09-30T09:29:08.539130+0000","last_became_active":"2023-09-30T09:29:36.028267+0000","last_became_peered":"2023-09-30T09:29:36.028267+0000","last_unstale":"2023-09-30T09:31:28.975599+0000","last_undegraded":"2023-09-30T09:31:28.975599+0000","last_fullsized":"2023-09-30T09:31:28.975599+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":7805,"log_dups_size":0,"ondisk_log_size":7805,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19791872,"num_objects":302,"num_object_clones":0,"num_object_copies":906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5406,"num_write_kb":182656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":125927424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":["4","1","2","7"],"object_location_counts":[{"shards":"1,4,7","objects":302}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":755,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.300440+0000","last_change":"2023-09-30T09:31:09.042736+0000","last_active":"2023-09-30T09:31:16.300440+0000","last_peered":"2023-09-30T09:31:16.300440+0000","last_clean":"2023-09-30T09:31:16.300440+0000","last_became_active":"2023-09-30T09:31:09.042107+0000","last_became_peered":"2023-09-30T09:31:09.042107+0000","last_unstale":"2023-09-30T09:31:16.300440+0000","last_undegraded":"2023-09-30T09:31:16.300440+0000","last_fullsized":"2023-09-30T09:31:16.300440+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0079898150000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":266,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.297392+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:31:16.297392+0000","last_peered":"2023-09-30T09:31:16.297392+0000","last_clean":"2023-09-30T09:31:16.297392+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:31:16.297392+0000","last_undegraded":"2023-09-30T09:31:16.297392+0000","last_fullsized":"2023-09-30T09:31:16.297392+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":224,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311944+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:31:16.311944+0000","last_peered":"2023-09-30T09:31:16.311944+0000","last_clean":"2023-09-30T09:31:16.311944+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:31:16.311944+0000","last_undegraded":"2023-09-30T09:31:16.311944+0000","last_fullsized":"2023-09-30T09:31:16.311944+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"266'7884","reported_seq":11857,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:28.759409+0000","last_change":"2023-09-30T09:31:09.249386+0000","last_active":"2023-09-30T09:31:28.759409+0000","last_peered":"2023-09-30T09:31:28.759409+0000","last_clean":"2023-09-30T09:29:08.549482+0000","last_became_active":"2023-09-30T09:29:36.077849+0000","last_became_peered":"2023-09-30T09:29:36.077849+0000","last_unstale":"2023-09-30T09:31:28.759409+0000","last_undegraded":"2023-09-30T09:31:28.759409+0000","last_fullsized":"2023-09-30T09:31:28.759409+0000","mapping_epoch":239,"log_start":"219'2847","ondisk_log_start":"219'2847","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5037,"log_dups_size":0,"ondisk_log_size":5037,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21364736,"num_objects":326,"num_object_clones":0,"num_object_copies":978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5436,"num_write_kb":184384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1941,"num_bytes_recovered":127205376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","2","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":326}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":654,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:23.440535+0000","last_change":"2023-09-30T09:31:09.038339+0000","last_active":"2023-09-30T09:31:23.440535+0000","last_peered":"2023-09-30T09:31:23.440535+0000","last_clean":"2023-09-30T09:31:23.440535+0000","last_became_active":"2023-09-30T09:31:09.037968+0000","last_became_peered":"2023-09-30T09:31:09.037968+0000","last_unstale":"2023-09-30T09:31:23.440535+0000","last_undegraded":"2023-09-30T09:31:23.440535+0000","last_fullsized":"2023-09-30T09:31:23.440535+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.999760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.999760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":265518,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":226,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305688+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:31:16.305688+0000","last_peered":"2023-09-30T09:31:16.305688+0000","last_clean":"2023-09-30T09:31:16.305688+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:31:16.305688+0000","last_undegraded":"2023-09-30T09:31:16.305688+0000","last_fullsized":"2023-09-30T09:31:16.305688+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":151,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:23.440080+0000","last_change":"2023-09-30T09:29:09.612070+0000","last_active":"2023-09-30T09:31:23.440080+0000","last_peered":"2023-09-30T09:31:23.440080+0000","last_clean":"2023-09-30T09:31:23.440080+0000","last_became_active":"2023-09-30T09:29:09.611725+0000","last_became_peered":"2023-09-30T09:29:09.611725+0000","last_unstale":"2023-09-30T09:31:23.440080+0000","last_undegraded":"2023-09-30T09:31:23.440080+0000","last_fullsized":"2023-09-30T09:31:23.440080+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:55.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":689,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312840+0000","last_change":"2023-09-30T09:31:09.035152+0000","last_active":"2023-09-30T09:31:16.312840+0000","last_peered":"2023-09-30T09:31:16.312840+0000","last_clean":"2023-09-30T09:31:16.312840+0000","last_became_active":"2023-09-30T09:31:09.034770+0000","last_became_peered":"2023-09-30T09:31:09.034770+0000","last_unstale":"2023-09-30T09:31:16.312840+0000","last_undegraded":"2023-09-30T09:31:16.312840+0000","last_fullsized":"2023-09-30T09:31:16.312840+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"266'7848","reported_seq":7936,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:28.720753+0000","last_change":"2023-09-30T09:29:35.074390+0000","last_active":"2023-09-30T09:31:28.720753+0000","last_peered":"2023-09-30T09:31:28.720753+0000","last_clean":"2023-09-30T09:31:28.720753+0000","last_became_active":"2023-09-30T09:29:35.072113+0000","last_became_peered":"2023-09-30T09:29:35.072113+0000","last_unstale":"2023-09-30T09:31:28.720753+0000","last_undegraded":"2023-09-30T09:31:28.720753+0000","last_fullsized":"2023-09-30T09:31:28.720753+0000","mapping_epoch":206,"log_start":"221'2818","ondisk_log_start":"221'2818","created":206,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":5030,"log_dups_size":0,"ondisk_log_size":5030,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18481152,"num_objects":282,"num_object_clones":0,"num_object_copies":846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5430,"num_write_kb":182784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":235,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.310901+0000","last_change":"2023-09-30T09:29:09.607621+0000","last_active":"2023-09-30T09:31:16.310901+0000","last_peered":"2023-09-30T09:31:16.310901+0000","last_clean":"2023-09-30T09:31:16.310901+0000","last_became_active":"2023-09-30T09:29:09.607245+0000","last_became_peered":"2023-09-30T09:29:09.607245+0000","last_unstale":"2023-09-30T09:31:16.310901+0000","last_undegraded":"2023-09-30T09:31:16.310901+0000","last_fullsized":"2023-09-30T09:31:16.310901+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:42:42.429421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":224,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312869+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:31:16.312869+0000","last_peered":"2023-09-30T09:31:16.312869+0000","last_clean":"2023-09-30T09:31:16.312869+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:31:16.312869+0000","last_undegraded":"2023-09-30T09:31:16.312869+0000","last_fullsized":"2023-09-30T09:31:16.312869+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"266'8158","reported_seq":12402,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:29.165848+0000","last_change":"2023-09-30T09:31:09.260081+0000","last_active":"2023-09-30T09:31:29.165848+0000","last_peered":"2023-09-30T09:31:29.165848+0000","last_clean":"2023-09-30T09:29:08.551510+0000","last_became_active":"2023-09-30T09:29:36.233690+0000","last_became_peered":"2023-09-30T09:29:36.233690+0000","last_unstale":"2023-09-30T09:31:29.165848+0000","last_undegraded":"2023-09-30T09:31:29.165848+0000","last_fullsized":"2023-09-30T09:31:29.165848+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":8158,"log_dups_size":0,"ondisk_log_size":8158,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12648448,"num_objects":193,"num_object_clones":0,"num_object_copies":579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5663,"num_write_kb":187392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2078,"num_bytes_recovered":136183808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","1","2","6"],"object_location_counts":[{"shards":"1,5,6","objects":193}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":483,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305605+0000","last_change":"2023-09-30T09:31:09.035655+0000","last_active":"2023-09-30T09:31:16.305605+0000","last_peered":"2023-09-30T09:31:16.305605+0000","last_clean":"2023-09-30T09:31:16.305605+0000","last_became_active":"2023-09-30T09:31:09.035325+0000","last_became_peered":"2023-09-30T09:31:09.035325+0000","last_unstale":"2023-09-30T09:31:16.305605+0000","last_undegraded":"2023-09-30T09:31:16.305605+0000","last_fullsized":"2023-09-30T09:31:16.305605+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.980385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.980385+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":233,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296758+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:31:16.296758+0000","last_peered":"2023-09-30T09:31:16.296758+0000","last_clean":"2023-09-30T09:31:16.296758+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:31:16.296758+0000","last_undegraded":"2023-09-30T09:31:16.296758+0000","last_fullsized":"2023-09-30T09:31:16.296758+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":206,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312890+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:31:16.312890+0000","last_peered":"2023-09-30T09:31:16.312890+0000","last_clean":"2023-09-30T09:31:16.312890+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:31:16.312890+0000","last_undegraded":"2023-09-30T09:31:16.312890+0000","last_fullsized":"2023-09-30T09:31:16.312890+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"266'5132","reported_seq":7348,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:29.450243+0000","last_change":"2023-09-30T09:31:09.195322+0000","last_active":"2023-09-30T09:31:29.450243+0000","last_peered":"2023-09-30T09:31:29.450243+0000","last_clean":"2023-09-30T09:29:08.514302+0000","last_became_active":"2023-09-30T09:29:36.012719+0000","last_became_peered":"2023-09-30T09:29:36.012719+0000","last_unstale":"2023-09-30T09:31:29.450243+0000","last_undegraded":"2023-09-30T09:31:29.450243+0000","last_fullsized":"2023-09-30T09:31:29.450243+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"215'2502","last_scrub_stamp":"2023-09-30T09:28:45.406287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:45.406287+0000","objects_scrubbed":1413,"log_size":5132,"log_dups_size":0,"ondisk_log_size":5132,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.34991823999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8650752,"num_objects":132,"num_object_clones":0,"num_object_copies":396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2097,"num_write_kb":48736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1045,"num_bytes_recovered":68517888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","2","3"],"object_location_counts":[{"shards":"0,1,3","objects":132}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":530,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305850+0000","last_change":"2023-09-30T09:31:09.046568+0000","last_active":"2023-09-30T09:31:16.305850+0000","last_peered":"2023-09-30T09:31:16.305850+0000","last_clean":"2023-09-30T09:31:16.305850+0000","last_became_active":"2023-09-30T09:31:09.046256+0000","last_became_peered":"2023-09-30T09:31:09.046256+0000","last_unstale":"2023-09-30T09:31:16.305850+0000","last_undegraded":"2023-09-30T09:31:16.305850+0000","last_fullsized":"2023-09-30T09:31:16.305850+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":611,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312680+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:31:16.312680+0000","last_peered":"2023-09-30T09:31:16.312680+0000","last_clean":"2023-09-30T09:31:16.312680+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:31:16.312680+0000","last_undegraded":"2023-09-30T09:31:16.312680+0000","last_fullsized":"2023-09-30T09:31:16.312680+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":252,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301752+0000","last_change":"2023-09-30T09:29:09.611292+0000","last_active":"2023-09-30T09:31:16.301752+0000","last_peered":"2023-09-30T09:31:16.301752+0000","last_clean":"2023-09-30T09:31:16.301752+0000","last_became_active":"2023-09-30T09:29:09.611015+0000","last_became_peered":"2023-09-30T09:29:09.611015+0000","last_unstale":"2023-09-30T09:31:16.301752+0000","last_undegraded":"2023-09-30T09:31:16.301752+0000","last_fullsized":"2023-09-30T09:31:16.301752+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:48.124378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":226,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312399+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:31:16.312399+0000","last_peered":"2023-09-30T09:31:16.312399+0000","last_clean":"2023-09-30T09:31:16.312399+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:31:16.312399+0000","last_undegraded":"2023-09-30T09:31:16.312399+0000","last_fullsized":"2023-09-30T09:31:16.312399+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"266'6743","reported_seq":6853,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:28.607296+0000","last_change":"2023-09-30T09:28:42.250412+0000","last_active":"2023-09-30T09:31:28.607296+0000","last_peered":"2023-09-30T09:31:28.607296+0000","last_clean":"2023-09-30T09:31:28.607296+0000","last_became_active":"2023-09-30T09:28:40.266645+0000","last_became_peered":"2023-09-30T09:28:40.266645+0000","last_unstale":"2023-09-30T09:31:28.607296+0000","last_undegraded":"2023-09-30T09:31:28.607296+0000","last_fullsized":"2023-09-30T09:31:28.607296+0000","mapping_epoch":206,"log_start":"223'2687","ondisk_log_start":"223'2687","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2368","last_scrub_stamp":"2023-09-30T09:28:42.250204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.250204+0000","objects_scrubbed":1224,"log_size":4056,"log_dups_size":0,"ondisk_log_size":4056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:50:26.843055+0000","scrub_duration":0.32304901400000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17760256,"num_objects":271,"num_object_clones":0,"num_object_copies":813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4381,"num_write_kb":109696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":698,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301731+0000","last_change":"2023-09-30T09:30:28.364401+0000","last_active":"2023-09-30T09:31:16.301731+0000","last_peered":"2023-09-30T09:31:16.301731+0000","last_clean":"2023-09-30T09:31:16.301731+0000","last_became_active":"2023-09-30T09:30:28.364065+0000","last_became_peered":"2023-09-30T09:30:28.364065+0000","last_unstale":"2023-09-30T09:31:16.301731+0000","last_undegraded":"2023-09-30T09:31:16.301731+0000","last_fullsized":"2023-09-30T09:31:16.301731+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0077623639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":414,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312529+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:31:16.312529+0000","last_peered":"2023-09-30T09:31:16.312529+0000","last_clean":"2023-09-30T09:31:16.312529+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:31:16.312529+0000","last_undegraded":"2023-09-30T09:31:16.312529+0000","last_fullsized":"2023-09-30T09:31:16.312529+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":245,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311509+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:31:16.311509+0000","last_peered":"2023-09-30T09:31:16.311509+0000","last_clean":"2023-09-30T09:31:16.311509+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:31:16.311509+0000","last_undegraded":"2023-09-30T09:31:16.311509+0000","last_fullsized":"2023-09-30T09:31:16.311509+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":125,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:23.445287+0000","last_change":"2023-09-30T09:29:09.611507+0000","last_active":"2023-09-30T09:31:23.445287+0000","last_peered":"2023-09-30T09:31:23.445287+0000","last_clean":"2023-09-30T09:31:23.445287+0000","last_became_active":"2023-09-30T09:29:09.611120+0000","last_became_peered":"2023-09-30T09:29:09.611120+0000","last_unstale":"2023-09-30T09:31:23.445287+0000","last_undegraded":"2023-09-30T09:31:23.445287+0000","last_fullsized":"2023-09-30T09:31:23.445287+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:31.622647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.5","version":"266'6769","reported_seq":6854,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.166248+0000","last_change":"2023-09-30T09:28:46.022537+0000","last_active":"2023-09-30T09:31:29.166248+0000","last_peered":"2023-09-30T09:31:29.166248+0000","last_clean":"2023-09-30T09:31:29.166248+0000","last_became_active":"2023-09-30T09:28:40.276165+0000","last_became_peered":"2023-09-30T09:28:40.276165+0000","last_unstale":"2023-09-30T09:31:29.166248+0000","last_undegraded":"2023-09-30T09:31:29.166248+0000","last_fullsized":"2023-09-30T09:31:29.166248+0000","mapping_epoch":206,"log_start":"212'2263","ondisk_log_start":"212'2263","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4506,"log_dups_size":0,"ondisk_log_size":4506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16482304,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5637,"num_write_kb":188448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":755,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296962+0000","last_change":"2023-09-30T09:30:28.371305+0000","last_active":"2023-09-30T09:31:16.296962+0000","last_peered":"2023-09-30T09:31:16.296962+0000","last_clean":"2023-09-30T09:31:16.296962+0000","last_became_active":"2023-09-30T09:30:28.370995+0000","last_became_peered":"2023-09-30T09:30:28.370995+0000","last_unstale":"2023-09-30T09:31:16.296962+0000","last_undegraded":"2023-09-30T09:31:16.296962+0000","last_fullsized":"2023-09-30T09:31:16.296962+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.036355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.036355+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0030804579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":440,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:23.442201+0000","last_change":"2023-09-30T09:29:36.005128+0000","last_active":"2023-09-30T09:31:23.442201+0000","last_peered":"2023-09-30T09:31:23.442201+0000","last_clean":"2023-09-30T09:31:23.442201+0000","last_became_active":"2023-09-30T09:29:36.004563+0000","last_became_peered":"2023-09-30T09:29:36.004563+0000","last_unstale":"2023-09-30T09:31:23.442201+0000","last_undegraded":"2023-09-30T09:31:23.442201+0000","last_fullsized":"2023-09-30T09:31:23.442201+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:59.839484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":214,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312483+0000","last_change":"2023-09-30T09:29:09.607111+0000","last_active":"2023-09-30T09:31:16.312483+0000","last_peered":"2023-09-30T09:31:16.312483+0000","last_clean":"2023-09-30T09:31:16.312483+0000","last_became_active":"2023-09-30T09:29:09.606807+0000","last_became_peered":"2023-09-30T09:29:09.606807+0000","last_unstale":"2023-09-30T09:31:16.312483+0000","last_undegraded":"2023-09-30T09:31:16.312483+0000","last_fullsized":"2023-09-30T09:31:16.312483+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:49.380239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":262,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312785+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:31:16.312785+0000","last_peered":"2023-09-30T09:31:16.312785+0000","last_clean":"2023-09-30T09:31:16.312785+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:31:16.312785+0000","last_undegraded":"2023-09-30T09:31:16.312785+0000","last_fullsized":"2023-09-30T09:31:16.312785+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"266'6637","reported_seq":6775,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.157684+0000","last_change":"2023-09-30T09:28:42.687918+0000","last_active":"2023-09-30T09:31:29.157684+0000","last_peered":"2023-09-30T09:31:29.157684+0000","last_clean":"2023-09-30T09:31:29.157684+0000","last_became_active":"2023-09-30T09:28:40.262566+0000","last_became_peered":"2023-09-30T09:28:40.262566+0000","last_unstale":"2023-09-30T09:31:29.157684+0000","last_undegraded":"2023-09-30T09:31:29.157684+0000","last_fullsized":"2023-09-30T09:31:29.157684+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2390","last_scrub_stamp":"2023-09-30T09:28:42.687821+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:42.687821+0000","objects_scrubbed":1235,"log_size":6637,"log_dups_size":0,"ondisk_log_size":6637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:07:56.313767+0000","scrub_duration":0.36697584500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14090240,"num_objects":215,"num_object_clones":0,"num_object_copies":645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4258,"num_write_kb":103616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":727,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.307519+0000","last_change":"2023-09-30T09:31:16.303084+0000","last_active":"2023-09-30T09:31:16.307519+0000","last_peered":"2023-09-30T09:31:16.307519+0000","last_clean":"2023-09-30T09:31:16.307519+0000","last_became_active":"2023-09-30T09:31:16.302581+0000","last_became_peered":"2023-09-30T09:31:16.302581+0000","last_unstale":"2023-09-30T09:31:16.307519+0000","last_undegraded":"2023-09-30T09:31:16.307519+0000","last_fullsized":"2023-09-30T09:31:16.307519+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:52:26.865953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":716,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.302468+0000","last_change":"2023-09-30T09:30:28.372432+0000","last_active":"2023-09-30T09:31:16.302468+0000","last_peered":"2023-09-30T09:31:16.302468+0000","last_clean":"2023-09-30T09:31:16.302468+0000","last_became_active":"2023-09-30T09:30:28.372087+0000","last_became_peered":"2023-09-30T09:30:28.372087+0000","last_unstale":"2023-09-30T09:31:16.302468+0000","last_undegraded":"2023-09-30T09:31:16.302468+0000","last_fullsized":"2023-09-30T09:31:16.302468+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:27:54.119176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001866876,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.297282+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:31:16.297282+0000","last_peered":"2023-09-30T09:31:16.297282+0000","last_clean":"2023-09-30T09:31:16.297282+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:31:16.297282+0000","last_undegraded":"2023-09-30T09:31:16.297282+0000","last_fullsized":"2023-09-30T09:31:16.297282+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":208,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305901+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:31:16.305901+0000","last_peered":"2023-09-30T09:31:16.305901+0000","last_clean":"2023-09-30T09:31:16.305901+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:31:16.305901+0000","last_undegraded":"2023-09-30T09:31:16.305901+0000","last_fullsized":"2023-09-30T09:31:16.305901+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"266'6812","reported_seq":6921,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:27.072820+0000","last_change":"2023-09-30T09:28:43.582313+0000","last_active":"2023-09-30T09:31:27.072820+0000","last_peered":"2023-09-30T09:31:27.072820+0000","last_clean":"2023-09-30T09:31:27.072820+0000","last_became_active":"2023-09-30T09:28:40.264204+0000","last_became_peered":"2023-09-30T09:28:40.264204+0000","last_unstale":"2023-09-30T09:31:27.072820+0000","last_undegraded":"2023-09-30T09:31:27.072820+0000","last_fullsized":"2023-09-30T09:31:27.072820+0000","mapping_epoch":206,"log_start":"211'2104","ondisk_log_start":"211'2104","created":206,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2440","last_scrub_stamp":"2023-09-30T09:28:43.582001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:43.582001+0000","objects_scrubbed":1291,"log_size":4708,"log_dups_size":0,"ondisk_log_size":4708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:50.257550+0000","scrub_duration":0.30691791200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21430272,"num_objects":327,"num_object_clones":0,"num_object_copies":981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4382,"num_write_kb":109376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":682,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.313094+0000","last_change":"2023-09-30T09:30:28.369897+0000","last_active":"2023-09-30T09:31:16.313094+0000","last_peered":"2023-09-30T09:31:16.313094+0000","last_clean":"2023-09-30T09:31:16.313094+0000","last_became_active":"2023-09-30T09:30:28.369480+0000","last_became_peered":"2023-09-30T09:30:28.369480+0000","last_unstale":"2023-09-30T09:31:16.313094+0000","last_undegraded":"2023-09-30T09:31:16.313094+0000","last_fullsized":"2023-09-30T09:31:16.313094+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:24.276525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:24.276525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":826,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301469+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:31:16.301469+0000","last_peered":"2023-09-30T09:31:16.301469+0000","last_clean":"2023-09-30T09:31:16.301469+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:31:16.301469+0000","last_undegraded":"2023-09-30T09:31:16.301469+0000","last_fullsized":"2023-09-30T09:31:16.301469+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":184,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:23.369610+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:31:23.369610+0000","last_peered":"2023-09-30T09:31:23.369610+0000","last_clean":"2023-09-30T09:31:23.369610+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:31:23.369610+0000","last_undegraded":"2023-09-30T09:31:23.369610+0000","last_fullsized":"2023-09-30T09:31:23.369610+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":172,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.313513+0000","last_change":"2023-09-30T09:29:09.611376+0000","last_active":"2023-09-30T09:31:16.313513+0000","last_peered":"2023-09-30T09:31:16.313513+0000","last_clean":"2023-09-30T09:31:16.313513+0000","last_became_active":"2023-09-30T09:29:09.611090+0000","last_became_peered":"2023-09-30T09:29:09.611090+0000","last_unstale":"2023-09-30T09:31:16.313513+0000","last_undegraded":"2023-09-30T09:31:16.313513+0000","last_fullsized":"2023-09-30T09:31:16.313513+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:07:15.039397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"266'5216","reported_seq":7427,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:26.653075+0000","last_change":"2023-09-30T09:31:15.084181+0000","last_active":"2023-09-30T09:31:26.653075+0000","last_peered":"2023-09-30T09:31:26.653075+0000","last_clean":"2023-09-30T09:31:26.653075+0000","last_became_active":"2023-09-30T09:29:36.027562+0000","last_became_peered":"2023-09-30T09:29:36.027562+0000","last_unstale":"2023-09-30T09:31:26.653075+0000","last_undegraded":"2023-09-30T09:31:26.653075+0000","last_fullsized":"2023-09-30T09:31:26.653075+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"214'2420","last_scrub_stamp":"2023-09-30T09:28:41.531661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:41.531661+0000","objects_scrubbed":1262,"log_size":5216,"log_dups_size":0,"ondisk_log_size":5216,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.28554934199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11567104,"num_objects":176,"num_object_clones":0,"num_object_copies":528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2167,"num_write_kb":54816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1044,"num_bytes_recovered":68419584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":530,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305458+0000","last_change":"2023-09-30T09:31:09.035034+0000","last_active":"2023-09-30T09:31:16.305458+0000","last_peered":"2023-09-30T09:31:16.305458+0000","last_clean":"2023-09-30T09:31:16.305458+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:31:16.305458+0000","last_undegraded":"2023-09-30T09:31:16.305458+0000","last_fullsized":"2023-09-30T09:31:16.305458+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":956,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.299480+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:31:16.299480+0000","last_peered":"2023-09-30T09:31:16.299480+0000","last_clean":"2023-09-30T09:31:16.299480+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:31:16.299480+0000","last_undegraded":"2023-09-30T09:31:16.299480+0000","last_fullsized":"2023-09-30T09:31:16.299480+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":230,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312895+0000","last_change":"2023-09-30T09:29:09.611808+0000","last_active":"2023-09-30T09:31:16.312895+0000","last_peered":"2023-09-30T09:31:16.312895+0000","last_clean":"2023-09-30T09:31:16.312895+0000","last_became_active":"2023-09-30T09:29:09.611484+0000","last_became_peered":"2023-09-30T09:29:09.611484+0000","last_unstale":"2023-09-30T09:31:16.312895+0000","last_undegraded":"2023-09-30T09:31:16.312895+0000","last_fullsized":"2023-09-30T09:31:16.312895+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:19.562461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":155,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.293483+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:31:16.293483+0000","last_peered":"2023-09-30T09:31:16.293483+0000","last_clean":"2023-09-30T09:31:16.293483+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:31:16.293483+0000","last_undegraded":"2023-09-30T09:31:16.293483+0000","last_fullsized":"2023-09-30T09:31:16.293483+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"266'4908","reported_seq":8425,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:28.722384+0000","last_change":"2023-09-30T09:31:16.315959+0000","last_active":"2023-09-30T09:31:28.722384+0000","last_peered":"2023-09-30T09:31:28.722384+0000","last_clean":"2023-09-30T09:31:28.722384+0000","last_became_active":"2023-09-30T09:31:16.307100+0000","last_became_peered":"2023-09-30T09:31:16.307100+0000","last_unstale":"2023-09-30T09:31:28.722384+0000","last_undegraded":"2023-09-30T09:31:28.722384+0000","last_fullsized":"2023-09-30T09:31:28.722384+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2525","last_scrub_stamp":"2023-09-30T09:28:48.115363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:48.115363+0000","objects_scrubbed":1451,"log_size":4908,"log_dups_size":0,"ondisk_log_size":4908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.21953820299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10616832,"num_objects":162,"num_object_clones":0,"num_object_copies":486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1910,"num_write_kb":43104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":984,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":708,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311517+0000","last_change":"2023-09-30T09:31:09.043194+0000","last_active":"2023-09-30T09:31:16.311517+0000","last_peered":"2023-09-30T09:31:16.311517+0000","last_clean":"2023-09-30T09:31:16.311517+0000","last_became_active":"2023-09-30T09:31:09.042735+0000","last_became_peered":"2023-09-30T09:31:09.042735+0000","last_unstale":"2023-09-30T09:31:16.311517+0000","last_undegraded":"2023-09-30T09:31:16.311517+0000","last_fullsized":"2023-09-30T09:31:16.311517+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":252,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.299241+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:31:16.299241+0000","last_peered":"2023-09-30T09:31:16.299241+0000","last_clean":"2023-09-30T09:31:16.299241+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:31:16.299241+0000","last_undegraded":"2023-09-30T09:31:16.299241+0000","last_fullsized":"2023-09-30T09:31:16.299241+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1216,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.313596+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:31:16.313596+0000","last_peered":"2023-09-30T09:31:16.313596+0000","last_clean":"2023-09-30T09:31:16.313596+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:31:16.313596+0000","last_undegraded":"2023-09-30T09:31:16.313596+0000","last_fullsized":"2023-09-30T09:31:16.313596+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":252,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:23.441530+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:31:23.441530+0000","last_peered":"2023-09-30T09:31:23.441530+0000","last_clean":"2023-09-30T09:31:23.441530+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:31:23.441530+0000","last_undegraded":"2023-09-30T09:31:23.441530+0000","last_fullsized":"2023-09-30T09:31:23.441530+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"266'5071","reported_seq":9258,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:29.106286+0000","last_change":"2023-09-30T09:29:36.003902+0000","last_active":"2023-09-30T09:31:29.106286+0000","last_peered":"2023-09-30T09:31:29.106286+0000","last_clean":"2023-09-30T09:29:33.909482+0000","last_became_active":"2023-09-30T09:29:35.026292+0000","last_became_peered":"2023-09-30T09:29:35.026292+0000","last_unstale":"2023-09-30T09:31:29.106286+0000","last_undegraded":"2023-09-30T09:31:29.106286+0000","last_fullsized":"2023-09-30T09:31:29.106286+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":206,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"217'2488","last_scrub_stamp":"2023-09-30T09:28:49.183514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:49.183514+0000","objects_scrubbed":1414,"log_size":5071,"log_dups_size":0,"ondisk_log_size":5071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.16423574599999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5832704,"num_objects":89,"num_object_clones":0,"num_object_copies":267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2072,"num_write_kb":46528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65765376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":["7","1","2","6"],"object_location_counts":[{"shards":"1,6,7","objects":89}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":677,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.297696+0000","last_change":"2023-09-30T09:30:28.372591+0000","last_active":"2023-09-30T09:31:16.297696+0000","last_peered":"2023-09-30T09:31:16.297696+0000","last_clean":"2023-09-30T09:31:16.297696+0000","last_became_active":"2023-09-30T09:30:28.372210+0000","last_became_peered":"2023-09-30T09:30:28.372210+0000","last_unstale":"2023-09-30T09:31:16.297696+0000","last_undegraded":"2023-09-30T09:31:16.297696+0000","last_fullsized":"2023-09-30T09:31:16.297696+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":543,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305932+0000","last_change":"2023-09-30T09:29:09.643760+0000","last_active":"2023-09-30T09:31:16.305932+0000","last_peered":"2023-09-30T09:31:16.305932+0000","last_clean":"2023-09-30T09:31:16.305932+0000","last_became_active":"2023-09-30T09:29:09.620803+0000","last_became_peered":"2023-09-30T09:29:09.620803+0000","last_unstale":"2023-09-30T09:31:16.305932+0000","last_undegraded":"2023-09-30T09:31:16.305932+0000","last_fullsized":"2023-09-30T09:31:16.305932+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:54:49.883760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":144,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.313664+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:31:16.313664+0000","last_peered":"2023-09-30T09:31:16.313664+0000","last_clean":"2023-09-30T09:31:16.313664+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:31:16.313664+0000","last_undegraded":"2023-09-30T09:31:16.313664+0000","last_fullsized":"2023-09-30T09:31:16.313664+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":207,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.310981+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:31:16.310981+0000","last_peered":"2023-09-30T09:31:16.310981+0000","last_clean":"2023-09-30T09:31:16.310981+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:31:16.310981+0000","last_undegraded":"2023-09-30T09:31:16.310981+0000","last_fullsized":"2023-09-30T09:31:16.310981+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":754,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.299769+0000","last_change":"2023-09-30T09:31:09.042555+0000","last_active":"2023-09-30T09:31:16.299769+0000","last_peered":"2023-09-30T09:31:16.299769+0000","last_clean":"2023-09-30T09:31:16.299769+0000","last_became_active":"2023-09-30T09:31:09.042144+0000","last_became_peered":"2023-09-30T09:31:09.042144+0000","last_unstale":"2023-09-30T09:31:16.299769+0000","last_undegraded":"2023-09-30T09:31:16.299769+0000","last_fullsized":"2023-09-30T09:31:16.299769+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0079898150000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.1e","version":"266'7896","reported_seq":11942,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:28.940518+0000","last_change":"2023-09-30T09:31:15.083442+0000","last_active":"2023-09-30T09:31:28.940518+0000","last_peered":"2023-09-30T09:31:28.940518+0000","last_clean":"2023-09-30T09:31:28.940518+0000","last_became_active":"2023-09-30T09:29:35.061073+0000","last_became_peered":"2023-09-30T09:29:35.061073+0000","last_unstale":"2023-09-30T09:31:28.940518+0000","last_undegraded":"2023-09-30T09:31:28.940518+0000","last_fullsized":"2023-09-30T09:31:28.940518+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":7896,"log_dups_size":0,"ondisk_log_size":7896,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13041664,"num_objects":199,"num_object_clones":0,"num_object_copies":597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5497,"num_write_kb":182272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1978,"num_bytes_recovered":129597440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":124,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311594+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:31:16.311594+0000","last_peered":"2023-09-30T09:31:16.311594+0000","last_clean":"2023-09-30T09:31:16.311594+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:31:16.311594+0000","last_undegraded":"2023-09-30T09:31:16.311594+0000","last_fullsized":"2023-09-30T09:31:16.311594+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":177,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.312767+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:31:16.312767+0000","last_peered":"2023-09-30T09:31:16.312767+0000","last_clean":"2023-09-30T09:31:16.312767+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:31:16.312767+0000","last_undegraded":"2023-09-30T09:31:16.312767+0000","last_fullsized":"2023-09-30T09:31:16.312767+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":528,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.305020+0000","last_change":"2023-09-30T09:31:09.037079+0000","last_active":"2023-09-30T09:31:16.305020+0000","last_peered":"2023-09-30T09:31:16.305020+0000","last_clean":"2023-09-30T09:31:16.305020+0000","last_became_active":"2023-09-30T09:31:09.036716+0000","last_became_peered":"2023-09-30T09:31:09.036716+0000","last_unstale":"2023-09-30T09:31:16.305020+0000","last_undegraded":"2023-09-30T09:31:16.305020+0000","last_fullsized":"2023-09-30T09:31:16.305020+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.19","version":"266'6635","reported_seq":6725,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:31:29.121144+0000","last_change":"2023-09-30T09:29:10.893253+0000","last_active":"2023-09-30T09:31:29.121144+0000","last_peered":"2023-09-30T09:31:29.121144+0000","last_clean":"2023-09-30T09:29:08.494599+0000","last_became_active":"2023-09-30T09:29:10.886737+0000","last_became_peered":"2023-09-30T09:29:10.886737+0000","last_unstale":"2023-09-30T09:31:29.121144+0000","last_undegraded":"2023-09-30T09:29:10.766629+0000","last_fullsized":"2023-09-30T09:29:10.598246+0000","mapping_epoch":229,"log_start":"214'2203","ondisk_log_start":"214'2203","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4432,"log_dups_size":0,"ondisk_log_size":4432,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2001,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5534,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"3,7","objects":-1753}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301124+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:31:16.301124+0000","last_peered":"2023-09-30T09:31:16.301124+0000","last_clean":"2023-09-30T09:31:16.301124+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:31:16.301124+0000","last_undegraded":"2023-09-30T09:31:16.301124+0000","last_fullsized":"2023-09-30T09:31:16.301124+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":208,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296917+0000","last_change":"2023-09-30T09:29:09.609752+0000","last_active":"2023-09-30T09:31:16.296917+0000","last_peered":"2023-09-30T09:31:16.296917+0000","last_clean":"2023-09-30T09:31:16.296917+0000","last_became_active":"2023-09-30T09:29:09.609242+0000","last_became_peered":"2023-09-30T09:29:09.609242+0000","last_unstale":"2023-09-30T09:31:16.296917+0000","last_undegraded":"2023-09-30T09:31:16.296917+0000","last_fullsized":"2023-09-30T09:31:16.296917+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:31.366509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"266'6631","reported_seq":10844,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.231949+0000","last_change":"2023-09-30T09:31:15.355025+0000","last_active":"2023-09-30T09:31:29.231949+0000","last_peered":"2023-09-30T09:31:29.231949+0000","last_clean":"2023-09-30T09:31:29.231949+0000","last_became_active":"2023-09-30T09:31:15.311944+0000","last_became_peered":"2023-09-30T09:31:15.311944+0000","last_unstale":"2023-09-30T09:31:29.231949+0000","last_undegraded":"2023-09-30T09:31:29.231949+0000","last_fullsized":"2023-09-30T09:31:29.231949+0000","mapping_epoch":263,"log_start":"212'2190","ondisk_log_start":"212'2190","created":213,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4441,"log_dups_size":0,"ondisk_log_size":4441,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13336576,"num_objects":204,"num_object_clones":0,"num_object_copies":612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5518,"num_write_kb":183072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2057,"num_bytes_recovered":28180480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":210,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301659+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:31:16.301659+0000","last_peered":"2023-09-30T09:31:16.301659+0000","last_clean":"2023-09-30T09:31:16.301659+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:31:16.301659+0000","last_undegraded":"2023-09-30T09:31:16.301659+0000","last_fullsized":"2023-09-30T09:31:16.301659+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":239,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.297375+0000","last_change":"2023-09-30T09:29:09.621218+0000","last_active":"2023-09-30T09:31:16.297375+0000","last_peered":"2023-09-30T09:31:16.297375+0000","last_clean":"2023-09-30T09:31:16.297375+0000","last_became_active":"2023-09-30T09:29:09.620972+0000","last_became_peered":"2023-09-30T09:29:09.620972+0000","last_unstale":"2023-09-30T09:31:16.297375+0000","last_undegraded":"2023-09-30T09:31:16.297375+0000","last_fullsized":"2023-09-30T09:31:16.297375+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:25.640288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":685,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.296803+0000","last_change":"2023-09-30T09:31:09.055286+0000","last_active":"2023-09-30T09:31:16.296803+0000","last_peered":"2023-09-30T09:31:16.296803+0000","last_clean":"2023-09-30T09:31:16.296803+0000","last_became_active":"2023-09-30T09:31:09.054907+0000","last_became_peered":"2023-09-30T09:31:09.054907+0000","last_unstale":"2023-09-30T09:31:16.296803+0000","last_undegraded":"2023-09-30T09:31:16.296803+0000","last_fullsized":"2023-09-30T09:31:16.296803+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1b","version":"266'7998","reported_seq":8085,"reported_epoch":266,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:31:28.726926+0000","last_change":"2023-09-30T09:29:35.093036+0000","last_active":"2023-09-30T09:31:28.726926+0000","last_peered":"2023-09-30T09:31:28.726926+0000","last_clean":"2023-09-30T09:29:33.992127+0000","last_became_active":"2023-09-30T09:29:35.070155+0000","last_became_peered":"2023-09-30T09:29:35.070155+0000","last_unstale":"2023-09-30T09:31:28.726926+0000","last_undegraded":"2023-09-30T09:31:28.726926+0000","last_fullsized":"2023-09-30T09:31:28.726926+0000","mapping_epoch":238,"log_start":"223'3059","ondisk_log_start":"223'3059","created":238,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":4939,"log_dups_size":0,"ondisk_log_size":4939,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16285696,"num_objects":249,"num_object_clones":0,"num_object_copies":747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5531,"num_write_kb":184928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":249}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":247,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:23.440476+0000","last_change":"2023-09-30T09:29:09.615385+0000","last_active":"2023-09-30T09:31:23.440476+0000","last_peered":"2023-09-30T09:31:23.440476+0000","last_clean":"2023-09-30T09:31:23.440476+0000","last_became_active":"2023-09-30T09:29:09.614479+0000","last_became_peered":"2023-09-30T09:29:09.614479+0000","last_unstale":"2023-09-30T09:31:23.440476+0000","last_undegraded":"2023-09-30T09:31:23.440476+0000","last_fullsized":"2023-09-30T09:31:23.440476+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:14.671222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":277,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.298272+0000","last_change":"2023-09-30T09:29:09.600457+0000","last_active":"2023-09-30T09:31:16.298272+0000","last_peered":"2023-09-30T09:31:16.298272+0000","last_clean":"2023-09-30T09:31:16.298272+0000","last_became_active":"2023-09-30T09:29:09.600080+0000","last_became_peered":"2023-09-30T09:29:09.600080+0000","last_unstale":"2023-09-30T09:31:16.298272+0000","last_undegraded":"2023-09-30T09:31:16.298272+0000","last_fullsized":"2023-09-30T09:31:16.298272+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:10:02.250349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":706,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.311369+0000","last_change":"2023-09-30T09:31:09.044630+0000","last_active":"2023-09-30T09:31:16.311369+0000","last_peered":"2023-09-30T09:31:16.311369+0000","last_clean":"2023-09-30T09:31:16.311369+0000","last_became_active":"2023-09-30T09:31:09.044286+0000","last_became_peered":"2023-09-30T09:31:09.044286+0000","last_unstale":"2023-09-30T09:31:16.311369+0000","last_undegraded":"2023-09-30T09:31:16.311369+0000","last_fullsized":"2023-09-30T09:31:16.311369+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1a","version":"266'8000","reported_seq":8078,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-30T09:31:29.117742+0000","last_change":"2023-09-30T09:29:35.021535+0000","last_active":"2023-09-30T09:31:29.117742+0000","last_peered":"2023-09-30T09:31:29.117742+0000","last_clean":"2023-09-30T09:31:29.117742+0000","last_became_active":"2023-09-30T09:29:35.014808+0000","last_became_peered":"2023-09-30T09:29:35.014808+0000","last_unstale":"2023-09-30T09:31:29.117742+0000","last_undegraded":"2023-09-30T09:31:29.117742+0000","last_fullsized":"2023-09-30T09:31:29.117742+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:28:11.533602+0000","last_clean_scrub_stamp":"2023-09-30T09:28:11.533602+0000","objects_scrubbed":0,"log_size":8000,"log_dups_size":0,"ondisk_log_size":8000,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15794176,"num_objects":241,"num_object_clones":0,"num_object_copies":723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5553,"num_write_kb":185408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":149,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.301642+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:31:16.301642+0000","last_peered":"2023-09-30T09:31:16.301642+0000","last_clean":"2023-09-30T09:31:16.301642+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:31:16.301642+0000","last_undegraded":"2023-09-30T09:31:16.301642+0000","last_fullsized":"2023-09-30T09:31:16.301642+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":224,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-30T09:31:16.310858+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:31:16.310858+0000","last_peered":"2023-09-30T09:31:16.310858+0000","last_clean":"2023-09-30T09:31:16.310858+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:31:16.310858+0000","last_undegraded":"2023-09-30T09:31:16.310858+0000","last_fullsized":"2023-09-30T09:31:16.310858+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":36,"stat_sum":{"num_bytes":529072152,"num_objects":8075,"num_object_clones":0,"num_object_copies":24225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":232,"num_objects_unfound":0,"num_objects_dirty":8075,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":160492,"num_write_kb":4876353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30251,"num_bytes_recovered":1754791936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3659083776,"data_stored":3659071560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195359,"ondisk_log_size":195359,"up":108,"acting":104,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":1418,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":435019,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":43,"num_write_kb":438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234238,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":552604,"kb_used_data":349012,"kb_used_omap":0,"kb_used_meta":203264,"kb_avail":93192548,"statfs":{"total":95995035648,"available":95429169152,"internally_reserved":0,"allocated":357388288,"data_stored":356397236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":208142336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:30 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.76,"15min":1.0580000000000001},"min":{"1min":0.25700000000000001,"5min":0.224,"15min":0.224},"max":{"1min":7.6180000000000003,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.746,"5min":1.538,"15min":0.94299999999999995},"min":{"1min":0.26600000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.5710000000000002,"5min":18.097999999999999,"15min":18.097999999999999},"last":0.47599999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9039999999999999,"5min":4.4969999999999999,"15min":2.4910000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":63.616999999999997,"5min":70.566999999999993,"15min":82.025999999999996},"last":0.55900000000000005},{"interface":"front","average":{"1min":3.7130000000000001,"5min":4.3860000000000001,"15min":2.4510000000000001},"min":{"1min":0.27700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":63.597000000000001,"5min":70.626000000000005,"15min":82.012},"last":0.496}]},{"osd":2,"last update":"Sat Sep 30 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":2.0249999999999999,"15min":1.306},"min":{"1min":0.34499999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.6159999999999997,"5min":39.445,"15min":39.445},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.9510000000000001,"15min":1.3400000000000001},"min":{"1min":0.38300000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.8479999999999999,"5min":40.988999999999997,"15min":40.988999999999997},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":3.5529999999999999,"15min":1.694},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.8140000000000001,"5min":76.909000000000006,"15min":76.909000000000006},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.99399999999999999,"5min":3.3359999999999999,"15min":1.575},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.2800000000000002,"5min":77.048000000000002,"15min":77.048000000000002},"last":0.53600000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.2170000000000001,"15min":0.75},"min":{"1min":0.23899999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.8849999999999998,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0029999999999999,"15min":0.73499999999999999},"min":{"1min":0.25700000000000001,"5min":0.251,"15min":0.251},"max":{"1min":7.8890000000000002,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":0.63900000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.4139999999999999,"15min":0.87},"min":{"1min":0.27700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":6.6500000000000004,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":3.6389999999999998},{"interface":"front","average":{"1min":1.103,"5min":1.3660000000000001,"15min":0.88600000000000001},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.28599999999999998},"max":{"1min":3.79,"5min":11.022,"15min":11.022},"last":11.4}]},{"osd":6,"last update":"Sat Sep 30 09:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.2769999999999999,"15min":0.85499999999999998},"min":{"1min":0.36899999999999999,"5min":0.33300000000000002,"15min":0.33000000000000002},"max":{"1min":2.9670000000000001,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.1220000000000001,"15min":0.81899999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.915,"5min":7.915,"15min":7.915},"last":0.57499999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495875,"num_pgs":84,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":622044,"kb_used_data":403924,"kb_used_omap":0,"kb_used_meta":218112,"kb_avail":93123108,"statfs":{"total":95995035648,"available":95358062592,"internally_reserved":0,"allocated":413618176,"data_stored":412627565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":223346688},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":2.2029999999999998,"15min":1.226},"min":{"1min":0.248,"5min":0.248,"15min":0.222},"max":{"1min":0.63800000000000001,"5min":45.927999999999997,"15min":45.927999999999997},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.175,"5min":2.657,"15min":1.3859999999999999},"min":{"1min":0.308,"5min":0.26800000000000002,"15min":0.23499999999999999},"max":{"1min":3.157,"5min":45.316000000000003,"15min":45.316000000000003},"last":0.55000000000000004}]},{"osd":1,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":3.0630000000000002,"15min":1.51},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.249},"max":{"1min":17.629999999999999,"5min":69.150000000000006,"15min":69.150000000000006},"last":76.433000000000007},{"interface":"front","average":{"1min":2.1800000000000002,"5min":3.5569999999999999,"15min":1.6679999999999999},"min":{"1min":0.41199999999999998,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":17.652999999999999,"5min":69.132000000000005,"15min":69.132000000000005},"last":76.399000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:31:11 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.738,"15min":1.3540000000000001},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":2.5409999999999999,"5min":30.638999999999999,"15min":37.978000000000002},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.454,"15min":1.216},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.0810000000000004,"5min":29.759,"15min":37.759999999999998},"last":0.503}]},{"osd":3,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5070000000000001,"5min":2.0699999999999998,"15min":1.181},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":23.411999999999999,"5min":23.411999999999999,"15min":23.411999999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.6190000000000002,"5min":2.294,"15min":1.304},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":23.271000000000001,"5min":23.271000000000001,"15min":23.271000000000001},"last":0.52400000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":2.0659999999999998,"15min":1.2589999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.2959999999999998,"5min":49.286999999999999,"15min":73.656000000000006},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.6339999999999999,"15min":1.1439999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.1559999999999997,"5min":49.252000000000002,"15min":73.882999999999996},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":9.3010000000000002,"5min":3.4289999999999998,"15min":1.5229999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28499999999999998},"max":{"1min":76.613,"5min":76.613,"15min":76.613},"last":0.46300000000000002},{"interface":"front","average":{"1min":8.7910000000000004,"5min":3.1960000000000002,"15min":1.49},"min":{"1min":0.36299999999999999,"5min":0.35399999999999998,"15min":0.311},"max":{"1min":76.329999999999998,"5min":76.329999999999998,"15min":76.329999999999998},"last":0.37}]},{"osd":7,"last update":"Sat Sep 30 09:31:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":4.173,"15min":1.851},"min":{"1min":0.38700000000000001,"5min":0.34699999999999998,"15min":0.32100000000000001},"max":{"1min":8.0649999999999995,"5min":91.120999999999995,"15min":91.120999999999995},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.3220000000000001,"5min":4.0279999999999996,"15min":1.9059999999999999},"min":{"1min":0.39500000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":6.0490000000000004,"5min":91.733999999999995,"15min":91.733999999999995},"last":0.44500000000000001}]}]},{"osd":1,"up_from":99,"seq":425201762439,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":695716,"kb_used_data":420572,"kb_used_omap":5,"kb_used_meta":275130,"kb_avail":93049436,"statfs":{"total":95995035648,"available":95282622464,"internally_reserved":0,"allocated":430665728,"data_stored":429148982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":281733302},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":4.4420000000000002,"15min":2.2999999999999998},"min":{"1min":0.20599999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.8340000000000001,"5min":150.93899999999999,"15min":150.93899999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.042,"5min":4.5250000000000004,"15min":2.3700000000000001},"min":{"1min":0.23899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":5.5380000000000003,"5min":156.74199999999999,"15min":156.74199999999999},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:31:10 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":5.2649999999999997,"15min":2.823},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":3.2650000000000001,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":5.5270000000000001,"15min":2.839},"min":{"1min":0.28499999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.4199999999999999,"5min":144.62,"15min":144.62},"last":0.93300000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":13.821,"5min":4.4210000000000003,"15min":1.8580000000000001},"min":{"1min":0.34499999999999997,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":60.743000000000002,"5min":60.743000000000002,"15min":60.743000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":13.981999999999999,"5min":4.2400000000000002,"15min":1.7849999999999999},"min":{"1min":0.32000000000000001,"5min":0.214,"15min":0.214},"max":{"1min":61.201999999999998,"5min":61.201999999999998,"15min":61.201999999999998},"last":0.96299999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.421,"15min":0.95699999999999996},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.28299999999999997},"max":{"1min":5.383,"5min":10.173,"15min":10.173},"last":0.438},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.397,"15min":0.96399999999999997},"min":{"1min":0.30499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.867,"5min":8.9299999999999997,"15min":8.9299999999999997},"last":2.3780000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":12.034000000000001,"5min":3.661,"15min":1.6719999999999999},"min":{"1min":0.38700000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":195.512,"5min":195.512,"15min":195.512},"last":0.379},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.4630000000000001,"15min":0.98099999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4390000000000001,"5min":10.128,"15min":10.128},"last":2.262}]},{"osd":6,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.5069999999999999,"15min":0.98099999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":2.6899999999999999,"5min":14.356999999999999,"15min":14.356999999999999},"last":2.3530000000000002},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.6899999999999999,"15min":1.095},"min":{"1min":0.44500000000000001,"5min":0.313,"15min":0.313},"max":{"1min":4.5339999999999998,"5min":15.864000000000001,"15min":15.864000000000001},"last":0.499}]},{"osd":7,"last update":"Sat Sep 30 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.8540000000000001,"15min":1.1659999999999999},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.4790000000000001,"5min":14.403,"15min":14.403},"last":2.3199999999999998},{"interface":"front","average":{"1min":1.2430000000000001,"5min":3.0089999999999999,"15min":1.5740000000000001},"min":{"1min":0.44,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.1400000000000001,"5min":122.56699999999999,"15min":122.56699999999999},"last":0.46400000000000002}]}]},{"osd":0,"up_from":94,"seq":403726925972,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325040,"kb_used_data":70056,"kb_used_omap":5,"kb_used_meta":254970,"kb_avail":93420112,"statfs":{"total":95995035648,"available":95662194688,"internally_reserved":0,"allocated":71737344,"data_stored":70153716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":261089533},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:30:46 2023","interfaces":[{"interface":"back","average":{"1min":4.4059999999999997,"5min":3.6669999999999998,"15min":1.6140000000000001},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.188},"max":{"1min":54.314999999999998,"5min":125.024,"15min":125.024},"last":0.24399999999999999},{"interface":"front","average":{"1min":4.1440000000000001,"5min":3.3719999999999999,"15min":1.5600000000000001},"min":{"1min":0.224,"5min":0.192,"15min":0.192},"max":{"1min":53.536999999999999,"5min":124.901,"15min":124.901},"last":1.2330000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.9670000000000001,"15min":1.161},"min":{"1min":0.26400000000000001,"5min":0.23200000000000001,"15min":0.218},"max":{"1min":3.4260000000000002,"5min":55.823999999999998,"15min":55.823999999999998},"last":1.2030000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":2.0950000000000002,"15min":1.232},"min":{"1min":0.27300000000000002,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":4.7249999999999996,"5min":55.790999999999997,"15min":55.790999999999997},"last":1.2989999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:30:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":2.6419999999999999,"15min":1.3700000000000001},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.8239999999999998,"5min":55.68,"15min":55.68},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.125,"5min":2.5179999999999998,"15min":1.3049999999999999},"min":{"1min":0.32200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.8609999999999998,"5min":55.244,"15min":55.244},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.4369999999999998,"15min":1.3480000000000001},"min":{"1min":0.32100000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.9129999999999998,"5min":121.486,"15min":121.486},"last":1.2669999999999999},{"interface":"front","average":{"1min":1.446,"5min":2.9889999999999999,"15min":1.5820000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.31},"max":{"1min":5.1319999999999997,"5min":121.44499999999999,"15min":121.44499999999999},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":11.733000000000001,"5min":3.698,"15min":1.8999999999999999},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":135.34899999999999,"5min":135.34899999999999,"15min":135.34899999999999},"last":3.3029999999999999},{"interface":"front","average":{"1min":11.656000000000001,"5min":3.6509999999999998,"15min":1.8799999999999999},"min":{"1min":0.41599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":135.381,"5min":135.381,"15min":135.381},"last":3.2879999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.427,"15min":1.05},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.2770000000000001,"5min":13.811999999999999,"15min":13.811999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.2609999999999999,"15min":0.999},"min":{"1min":0.47699999999999998,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":1.5109999999999999,"5min":6.407,"15min":7.556},"last":1.165}]},{"osd":7,"last update":"Sat Sep 30 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":12.93,"5min":5.2809999999999997,"15min":2.3399999999999999},"min":{"1min":0.438,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":150.96899999999999,"5min":150.96899999999999,"15min":150.96899999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":12.728,"5min":5.0129999999999999,"15min":2.2469999999999999},"min":{"1min":0.46100000000000002,"5min":0.371,"15min":0.371},"max":{"1min":150.328,"5min":150.328,"15min":150.328},"last":5.3810000000000002}]}]},{"osd":2,"up_from":0,"seq":446676598913,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435385,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856712,"kb_used_data":485696,"kb_used_omap":2,"kb_used_meta":370941,"kb_avail":92888440,"statfs":{"total":95995035648,"available":95117762560,"internally_reserved":0,"allocated":497352704,"data_stored":495799688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":379843943},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.544,"15min":1.5069999999999999},"min":{"1min":0.20200000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.0510000000000002,"5min":10.856,"15min":130.946},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.619,"5min":1.631,"15min":1.526},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.20899999999999999},"max":{"1min":7.3920000000000003,"5min":12.361000000000001,"15min":130.96899999999999},"last":2.371}]},{"osd":1,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":9.4280000000000008,"5min":3.4500000000000002,"15min":1.528},"min":{"1min":0.30399999999999999,"5min":0.23499999999999999,"15min":0.224},"max":{"1min":133.215,"5min":133.215,"15min":133.215},"last":0.36299999999999999},{"interface":"front","average":{"1min":9.3040000000000003,"5min":3.399,"15min":1.4830000000000001},"min":{"1min":0.24099999999999999,"5min":0.221,"15min":0.221},"max":{"1min":133.16,"5min":133.16,"15min":133.16},"last":5.6859999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":2.5630000000000002,"15min":1.95},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.6079999999999997,"5min":29.986000000000001,"15min":145.75899999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.1850000000000001,"5min":2.319,"15min":1.847},"min":{"1min":0.33600000000000002,"5min":0.26200000000000001,"15min":0.22900000000000001},"max":{"1min":5.8780000000000001,"5min":30.242999999999999,"15min":150.358},"last":0.623}]},{"osd":4,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.399,"15min":0.90900000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.5549999999999997,"5min":15.960000000000001,"15min":15.960000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.6599999999999999,"15min":1.002},"min":{"1min":0.37,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.3490000000000002,"5min":15.638,"15min":15.638},"last":2.4510000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.732,"15min":1.0740000000000001},"min":{"1min":0.33100000000000002,"5min":0.314,"15min":0.308},"max":{"1min":2.7469999999999999,"5min":10.048999999999999,"15min":10.048999999999999},"last":5.7240000000000002},{"interface":"front","average":{"1min":1.669,"5min":1.9810000000000001,"15min":1.1839999999999999},"min":{"1min":0.42799999999999999,"5min":0.35299999999999998,"15min":0.33500000000000002},"max":{"1min":7.5419999999999998,"5min":12.785,"15min":12.785},"last":4.7380000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.542,"15min":0.99199999999999999},"min":{"1min":0.372,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.548,"5min":10.348000000000001,"15min":10.348000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.361,"5min":1.885,"15min":1.145},"min":{"1min":0.45600000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.8250000000000002,"5min":14.69,"15min":14.69},"last":2.4049999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":4.0529999999999999,"15min":1.847},"min":{"1min":0.40000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.3159999999999998,"5min":226.03999999999999,"15min":226.03999999999999},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.165,"5min":3.625,"15min":1.6830000000000001},"min":{"1min":0.48499999999999999,"5min":0.377,"15min":0.372},"max":{"1min":4.1840000000000002,"5min":226.06100000000001,"15min":226.06100000000001},"last":1.1439999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019148,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333656,"kb_used_data":154768,"kb_used_omap":0,"kb_used_meta":178816,"kb_avail":93411496,"statfs":{"total":95995035648,"available":95653371904,"internally_reserved":0,"allocated":158482432,"data_stored":157495974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":183107584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:31:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":2.6930000000000001,"15min":2.6070000000000002},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":5.5599999999999996,"5min":45.884999999999998,"15min":128.79300000000001},"last":1.946},{"interface":"front","average":{"1min":1.3240000000000001,"5min":2.411,"15min":2.4590000000000001},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.248},"max":{"1min":11.744,"5min":45.871000000000002,"15min":128.38800000000001},"last":0.48999999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":3.1659999999999999,"15min":1.599},"min":{"1min":0.28000000000000003,"5min":0.245,"15min":0.245},"max":{"1min":11.67,"5min":93.244,"15min":93.244},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.0389999999999999,"5min":3.1749999999999998,"15min":1.6279999999999999},"min":{"1min":0.34300000000000003,"5min":0.26800000000000002,"15min":0.23999999999999999},"max":{"1min":3.6949999999999998,"5min":93.465999999999994,"15min":93.465999999999994},"last":1.5820000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:31:19 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":3.2719999999999998,"15min":2.4820000000000002},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.29799999999999999},"max":{"1min":4.4340000000000002,"5min":77.620000000000005,"15min":140.601},"last":1.617},{"interface":"front","average":{"1min":1.5820000000000001,"5min":3.2120000000000002,"15min":2.456},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.484,"5min":77.638999999999996,"15min":140.54900000000001},"last":1.9810000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":5.3490000000000002,"15min":2.5179999999999998},"min":{"1min":0.40899999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.7800000000000002,"5min":281.42200000000003,"15min":281.42200000000003},"last":3.9889999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":5.351,"15min":2.5209999999999999},"min":{"1min":0.34899999999999998,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":11.715999999999999,"5min":281.44999999999999,"15min":281.44999999999999},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:31:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.8090000000000002,"15min":1.5609999999999999},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.252},"max":{"1min":11.808,"5min":134.30600000000001,"15min":134.30600000000001},"last":3.1379999999999999},{"interface":"front","average":{"1min":1.4430000000000001,"5min":2.839,"15min":1.619},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":5.7999999999999998,"5min":134.35300000000001,"15min":134.35300000000001},"last":1.6639999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:30:52 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.0649999999999999,"15min":0.71599999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":6.8440000000000003,"5min":6.8440000000000003,"15min":6.8440000000000003},"last":1.919},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.212,"15min":0.79300000000000004},"min":{"1min":0.31,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":11.779999999999999,"5min":11.779999999999999,"15min":11.779999999999999},"last":4.5650000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:31:19 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":3.0680000000000001,"15min":1.722},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":11.444000000000001,"5min":136.27099999999999,"15min":136.27099999999999},"last":1.643},{"interface":"front","average":{"1min":1.3600000000000001,"5min":2.8460000000000001,"15min":1.3859999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":5.5279999999999996,"5min":136.60400000000001,"15min":136.60400000000001},"last":4.9009999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757512,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389768,"kb_used_data":196160,"kb_used_omap":0,"kb_used_meta":193600,"kb_avail":93355384,"statfs":{"total":95995035648,"available":95595913216,"internally_reserved":0,"allocated":200867840,"data_stored":199892604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":198246400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":4.0279999999999996,"15min":1.885},"min":{"1min":0.23699999999999999,"5min":0.22,"15min":0.22},"max":{"1min":4.6600000000000001,"5min":148.79900000000001,"15min":148.79900000000001},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.64000000000000001,"5min":4.532,"15min":2.109},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.6340000000000003,"5min":148.779,"15min":148.779},"last":0.90200000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.9830000000000001,"5min":4.7809999999999997,"15min":2.431},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":37.268999999999998,"5min":166.708,"15min":166.708},"last":0.92200000000000004},{"interface":"front","average":{"1min":3.3900000000000001,"5min":4.7220000000000004,"15min":2.4140000000000001},"min":{"1min":0.307,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":37.307000000000002,"5min":166.607,"15min":166.607},"last":0.26600000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:31:11 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":3.2570000000000001,"15min":2.6850000000000001},"min":{"1min":0.35299999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.6669999999999998,"5min":131.655,"15min":165.768},"last":1.008},{"interface":"front","average":{"1min":1.2130000000000001,"5min":2.8100000000000001,"15min":3.1429999999999998},"min":{"1min":0.35499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.0129999999999999,"5min":131.77600000000001,"15min":166.24000000000001},"last":1.0009999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":1.9330000000000001,"15min":1.3400000000000001},"min":{"1min":0.35599999999999998,"5min":0.311,"15min":0.311},"max":{"1min":6.3040000000000003,"5min":20.079000000000001,"15min":28.015999999999998},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.9239999999999999,"5min":1.653,"15min":1.196},"min":{"1min":0.379,"5min":0.313,"15min":0.313},"max":{"1min":7.3959999999999999,"5min":20.021000000000001,"15min":28.039999999999999},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:31:22 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.131,"15min":0.75900000000000001},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.222},"max":{"1min":7.4299999999999997,"5min":7.4299999999999997,"15min":8.5359999999999996},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.121,"15min":0.79600000000000004},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":2.9830000000000001,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":0.76700000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.45,"15min":0.85799999999999998},"min":{"1min":0.33100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.46,"5min":14.23,"15min":14.23},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.236,"5min":1.581,"15min":0.99399999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.6760000000000002,"5min":18.960000000000001,"15min":18.960000000000001},"last":0.82399999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:30:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.597,"15min":0.98399999999999999},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.609,"5min":19.175999999999998,"15min":19.175999999999998},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.421,"5min":1.3069999999999999,"15min":0.874},"min":{"1min":0.38,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":7.4900000000000002,"5min":8.8810000000000002,"15min":8.8810000000000002},"last":0.44900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":59572224,"data_stored":59572224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":403050496,"data_stored":403046424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1687879680,"data_stored":1687879680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":446824448,"data_stored":446824448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":147456000,"data_stored":147456000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":184291328,"data_stored":184287256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":389550080,"data_stored":389546008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":340459520,"data_stored":340459520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:31:33.289 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 31 2023-09-30T09:31:33.321 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:33.363 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:33.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:31:33.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: osdmap e267: 8 total, 8 up, 7 in 2023-09-30T09:31:33.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: 6.1e scrub starts 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:33.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4706/24306 objects degraded (19.361%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: osdmap e267: 8 total, 8 up, 7 in 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: 6.1e scrub starts 2023-09-30T09:31:33.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:33.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:33.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4706/24306 objects degraded (19.361%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:33.492 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:33.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:33.602 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:33.664 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:31:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: osdmap e267: 8 total, 8 up, 7 in 2023-09-30T09:31:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: failed to encode map e267 with expected crc 2023-09-30T09:31:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: 6.1e scrub starts 2023-09-30T09:31:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4706/24306 objects degraded (19.361%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:33.972 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:33.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:34.077 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:34.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:31:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:31:33] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-30T09:31:34.135 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:34.226 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:34.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:34.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:34.337 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:34.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[216815]: 6.20 scrub starts 2023-09-30T09:31:34.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[216815]: pgmap v645: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 480 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 4706/22932 objects degraded (20.522%); 232/22932 objects misplaced (1.012%); 0 B/s, 116 objects/s recovering 2023-09-30T09:31:34.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[216815]: failed to encode map e268 with expected crc 2023-09-30T09:31:34.405 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[216815]: osdmap e268: 8 total, 8 up, 7 in 2023-09-30T09:31:34.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:34 smithi167 ceph-mon[192880]: 6.20 scrub starts 2023-09-30T09:31:34.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:34 smithi167 ceph-mon[192880]: pgmap v645: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 480 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 4706/22932 objects degraded (20.522%); 232/22932 objects misplaced (1.012%); 0 B/s, 116 objects/s recovering 2023-09-30T09:31:34.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:34 smithi167 ceph-mon[192880]: failed to encode map e268 with expected crc 2023-09-30T09:31:34.408 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:34 smithi167 ceph-mon[192880]: osdmap e268: 8 total, 8 up, 7 in 2023-09-30T09:31:34.408 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:34 smithi167 ceph-mon[192880]: failed to encode map e268 with expected crc 2023-09-30T09:31:34.408 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4191648188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:31:34.408 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:34 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:31:34.421 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:34.555 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:34.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:34.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:34.665 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:34.734 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:34.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[216815]: failed to encode map e268 with expected crc 2023-09-30T09:31:34.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4191648188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:31:34.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:31:34.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[224043]: 6.20 scrub starts 2023-09-30T09:31:34.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[224043]: pgmap v645: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 480 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 4706/22932 objects degraded (20.522%); 232/22932 objects misplaced (1.012%); 0 B/s, 116 objects/s recovering 2023-09-30T09:31:34.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[224043]: failed to encode map e268 with expected crc 2023-09-30T09:31:34.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[224043]: osdmap e268: 8 total, 8 up, 7 in 2023-09-30T09:31:34.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[224043]: failed to encode map e268 with expected crc 2023-09-30T09:31:34.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4191648188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:31:34.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:34 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:31:34.865 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:34.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:34.978 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:34.979 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:34.992 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:31:35.040 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:35.148 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:35.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:35.260 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:35.339 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:35.443 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:35.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:35.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:35.550 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:35 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:35 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:31:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:35 smithi019 ceph-mon[224043]: osdmap e269: 8 total, 8 up, 7 in 2023-09-30T09:31:35.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:35 smithi019 ceph-mon[224043]: failed to encode map e269 with expected crc 2023-09-30T09:31:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:35 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:35 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:31:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:35 smithi019 ceph-mon[216815]: osdmap e269: 8 total, 8 up, 7 in 2023-09-30T09:31:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:35 smithi019 ceph-mon[216815]: failed to encode map e269 with expected crc 2023-09-30T09:31:35.600 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:35 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:35 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:31:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:35 smithi167 ceph-mon[192880]: osdmap e269: 8 total, 8 up, 7 in 2023-09-30T09:31:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:35 smithi167 ceph-mon[192880]: failed to encode map e269 with expected crc 2023-09-30T09:31:35.992 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:35.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:36.100 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:36.157 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:36.253 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:36.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:36.351 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:36.361 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:36.420 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:36.541 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:36.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:36.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:36 smithi019 ceph-mon[224043]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:36 smithi019 ceph-mon[224043]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:36 smithi019 ceph-mon[224043]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:36 smithi019 ceph-mon[224043]: pgmap v648: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 199 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 4058/9435 objects degraded (43.010%); 72/9435 objects misplaced (0.763%); 0 B/s, 203 objects/s recovering 2023-09-30T09:31:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:36 smithi019 ceph-mon[216815]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:36 smithi019 ceph-mon[216815]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:36 smithi019 ceph-mon[216815]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:36 smithi019 ceph-mon[216815]: pgmap v648: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 199 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 4058/9435 objects degraded (43.010%); 72/9435 objects misplaced (0.763%); 0 B/s, 203 objects/s recovering 2023-09-30T09:31:36.593 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Removed 92301 objects 2023-09-30T09:31:36.593 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Clean up completed and total clean up time :104.648 2023-09-30T09:31:36.596 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:31:36.616 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:31:36.636 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:31:36.645 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:36.656 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:31:36.670 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:36 smithi167 ceph-mon[192880]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.670 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:36 smithi167 ceph-mon[192880]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.671 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:36 smithi167 ceph-mon[192880]: failed to encode map e269 with expected crc 2023-09-30T09:31:36.671 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:36 smithi167 ceph-mon[192880]: pgmap v648: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 199 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 4058/9435 objects degraded (43.010%); 72/9435 objects misplaced (0.763%); 0 B/s, 203 objects/s recovering 2023-09-30T09:31:36.672 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:36.676 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:31:36.696 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:31:36.715 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:31:36.719 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:36.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:36.734 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:31:36.736 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:31:36.736 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:31:36.736 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:31:36.736 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:31:36.736 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:31:36.736 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:31:36.736 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:31:36.736 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:31:36.827 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:36.853 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:36.898 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:36.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:37.004 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:37.036 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:37.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1193390581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:37.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1193390581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:37.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1193390581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:37.880 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:37.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:37.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:37.983 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:38.017 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:38.121 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:38.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:38.185 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:31:38.226 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:38.254 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:38.300 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:38.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:38.405 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:38.439 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:38.487 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:38.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:38 smithi019 ceph-mon[224043]: pgmap v649: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 174 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 896 op/s; 4058/8202 objects degraded (49.476%); 72/8202 objects misplaced (0.878%); 0 B/s, 107 objects/s recovering 2023-09-30T09:31:38.489 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4058/8202 objects degraded (49.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:38.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:38 smithi019 ceph-mon[216815]: pgmap v649: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 174 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 896 op/s; 4058/8202 objects degraded (49.476%); 72/8202 objects misplaced (0.878%); 0 B/s, 107 objects/s recovering 2023-09-30T09:31:38.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4058/8202 objects degraded (49.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:38.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:38.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:38 smithi167 ceph-mon[192880]: pgmap v649: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 174 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 896 op/s; 4058/8202 objects degraded (49.476%); 72/8202 objects misplaced (0.878%); 0 B/s, 107 objects/s recovering 2023-09-30T09:31:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4058/8202 objects degraded (49.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:38.592 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:38.621 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:39.070 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:39.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:39.173 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:39.206 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:39.344 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:31:39.344 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:31:39.344 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:31:39.344 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:31:39.344 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:31:39.373 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:39.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:39.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:39.476 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:39.503 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:39.548 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:39.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:39.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:39.653 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:39.680 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:39.725 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:39.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:39.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:39.830 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:39.857 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:39.903 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:39.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:40.010 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:40.042 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:40.050 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:31:40.130 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:40.157 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:31:40.252 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:40.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:40.357 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:40.383 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:40.429 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:40.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:40.534 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:40.563 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:40.662 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:40.666 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:40 smithi019 ceph-mon[224043]: pgmap v650: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 104 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 4058/4845 objects degraded (83.756%); 72/4845 objects misplaced (1.486%); 0 B/s, 95 objects/s recovering 2023-09-30T09:31:40.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:40 smithi019 ceph-mon[216815]: pgmap v650: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 104 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 4058/4845 objects degraded (83.756%); 72/4845 objects misplaced (1.486%); 0 B/s, 95 objects/s recovering 2023-09-30T09:31:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:40.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:40.768 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:40.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:40 smithi167 ceph-mon[192880]: pgmap v650: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 104 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 4058/4845 objects degraded (83.756%); 72/4845 objects misplaced (1.486%); 0 B/s, 95 objects/s recovering 2023-09-30T09:31:40.796 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:40.840 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:40.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:40.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:40.858 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:40.944 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:40.972 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:41.017 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:41.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:41.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:41.121 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:41.150 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:41.195 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:41.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:41.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:41.298 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:41.325 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:41.442 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 1 pgp_num to 31 2023-09-30T09:31:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1128646743' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-30T09:31:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:41 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-30T09:31:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:41 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:31:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1128646743' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-30T09:31:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:41 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-30T09:31:41.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:41 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:31:41.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1128646743' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-30T09:31:41.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:41 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-30T09:31:41.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:41 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:31:42.279 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:42.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:42.336 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-30T09:31:42.337 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-30T09:31:42.349 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:42.385 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:42.413 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[216815]: pgmap v651: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 943 op/s; 2953/1275 objects degraded (231.608%); 0 B/s, 219 objects/s recovering 2023-09-30T09:31:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-30T09:31:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[216815]: failed to encode map e270 with expected crc 2023-09-30T09:31:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[216815]: osdmap e270: 8 total, 8 up, 7 in 2023-09-30T09:31:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[216815]: failed to encode map e270 with expected crc 2023-09-30T09:31:42.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[224043]: pgmap v651: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 943 op/s; 2953/1275 objects degraded (231.608%); 0 B/s, 219 objects/s recovering 2023-09-30T09:31:42.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-30T09:31:42.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[224043]: failed to encode map e270 with expected crc 2023-09-30T09:31:42.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[224043]: osdmap e270: 8 total, 8 up, 7 in 2023-09-30T09:31:42.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:42 smithi019 ceph-mon[224043]: failed to encode map e270 with expected crc 2023-09-30T09:31:42.759 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:42 smithi167 ceph-mon[192880]: pgmap v651: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 943 op/s; 2953/1275 objects degraded (231.608%); 0 B/s, 219 objects/s recovering 2023-09-30T09:31:42.759 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:42 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-30T09:31:42.759 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:42 smithi167 ceph-mon[192880]: failed to encode map e270 with expected crc 2023-09-30T09:31:42.759 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:42 smithi167 ceph-mon[192880]: osdmap e270: 8 total, 8 up, 7 in 2023-09-30T09:31:42.759 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:42 smithi167 ceph-mon[192880]: failed to encode map e270 with expected crc 2023-09-30T09:31:42.821 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:42.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:42.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:42.926 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:42.956 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:43.004 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:43.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:43.110 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:43.139 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:43.454 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:31:43.470 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:43.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:43.572 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:43.602 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:43 smithi167 ceph-mon[192880]: osdmap e271: 8 total, 8 up, 7 in 2023-09-30T09:31:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:43 smithi167 ceph-mon[192880]: failed to encode map e271 with expected crc 2023-09-30T09:31:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:43 smithi167 ceph-mon[192880]: failed to encode map e271 with expected crc 2023-09-30T09:31:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/91319551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:43 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 2953/1275 objects degraded (231.608%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:43.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[216815]: osdmap e271: 8 total, 8 up, 7 in 2023-09-30T09:31:43.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[216815]: failed to encode map e271 with expected crc 2023-09-30T09:31:43.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[216815]: failed to encode map e271 with expected crc 2023-09-30T09:31:43.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/91319551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:43.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:43.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 2953/1275 objects degraded (231.608%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:43.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[224043]: osdmap e271: 8 total, 8 up, 7 in 2023-09-30T09:31:43.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[224043]: failed to encode map e271 with expected crc 2023-09-30T09:31:43.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[224043]: failed to encode map e271 with expected crc 2023-09-30T09:31:43.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/91319551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:43.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:43.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 2953/1275 objects degraded (231.608%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:44.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:31:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:31:43] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-30T09:31:44.202 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:44.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:44.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:44.306 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:44.339 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:44.394 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:44.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:44.503 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:44.538 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:31:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:31:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:31:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35934 6.16 6:6a0d4ce1:::benchmark_data_smithi019_355417_object35933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 185.275006236, 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.022920797, 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35934 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259900+0000", 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.573e-06 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259928+0000", 2023-09-30T09:31:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8756999999999999e-05 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259941+0000", 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3066e-05 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259951+0000", 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8050000000000001e-06 2023-09-30T09:31:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311357+0000", 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.051405992999999997 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311380+0000", 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.283e-05 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.282746+0000", 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97136601 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.282819+0000", 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2762999999999994e-05 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35936 6.2 6:47063778:::benchmark_data_smithi019_355417_object35935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 185.27493825400001, 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0116794739999999, 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35936 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:31:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0800000000000005e-07 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259989+0000", 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2908000000000002e-05 2023-09-30T09:31:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259996+0000", 2023-09-30T09:31:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8650000000000003e-06 2023-09-30T09:31:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260001+0000", 2023-09-30T09:31:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2310000000000004e-06 2023-09-30T09:31:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.310543+0000", 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.050541517000000001 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.310552+0000", 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2320000000000007e-06 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271569+0000", 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96101677600000002 2023-09-30T09:31:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271645+0000", 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6437e-05 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35941 6.6 6:62f338d5:::benchmark_data_smithi019_355417_object35940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:31:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 185.274893185, 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0179260409999999, 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35941 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:31:44.553 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8299999999999998e-07 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260036+0000", 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4766999999999999e-05 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260043+0000", 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.387e-06 2023-09-30T09:31:44.554 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260047+0000", 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9030000000000003e-06 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311366+0000", 2023-09-30T09:31:44.555 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.051318726000000002 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311377+0000", 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0621999999999999e-05 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268227+0000", 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95684991799999997 2023-09-30T09:31:44.556 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268372+0000", 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014511 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268552+0000", 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017976400000000001 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.557 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271663+0000", 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031116619999999998 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271672+0000", 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3180000000000007e-06 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.558 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277851+0000", 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061798599999999997 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277902+0000", 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0611e-05 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.559 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277937+0000", 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5009999999999999e-05 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35951 6.6 6:612781fc:::benchmark_data_smithi019_355417_object35950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 185.20360586000001, 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94995752200000005, 2023-09-30T09:31:44.560 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35951 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:31:44.561 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331301+0000", 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9739999999999998e-06 2023-09-30T09:31:44.562 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331348+0000", 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7113000000000001e-05 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331384+0000", 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5253000000000003e-05 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.563 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331402+0000", 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8756000000000001e-05 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331491+0000", 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8213000000000004e-05 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331502+0000", 2023-09-30T09:31:44.564 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1328999999999999e-05 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270400+0000", 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.93889793200000005 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270562+0000", 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016258000000000001 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.565 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270732+0000", 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016932199999999999 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271969+0000", 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00123719 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.566 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274748+0000", 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002778689 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281163+0000", 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0064157759999999998 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281223+0000", 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9302000000000002e-05 2023-09-30T09:31:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281256+0000", 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3093000000000003e-05 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:99314 6.13 6:cb94d029:::benchmark_data_smithi019_355417_object7010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:29:59.583400+0000", 2023-09-30T09:31:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 104.951504071, 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94488777800000001, 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 99314 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583400+0000", 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583400+0000", 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583403+0000", 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.542e-06 2023-09-30T09:31:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583414+0000", 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1259e-05 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583434+0000", 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0562999999999999e-05 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583447+0000", 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2894999999999999e-05 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.624839+0000", 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.041391322000000001 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.626266+0000", 2023-09-30T09:31:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00142702 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.811282+0000", 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.18501577399999999 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.813318+0000", 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020367900000000001 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.396757+0000", 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.58343899600000004 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.398448+0000", 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001690951 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.401393+0000", 2023-09-30T09:31:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029450679999999999 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.406617+0000", 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0052237079999999997 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.407985+0000", 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001367769 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.523096+0000", 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.11511101999999999 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.524223+0000", 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011267029999999999 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.528288+0000", 2023-09-30T09:31:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040653979999999996 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:108354 6.13 6:c97625f7:::benchmark_data_smithi019_355417_object16050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:12.894778+0000", 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 91.640126252000002, 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.88049378899999997, 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 108354 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:12.894778+0000", 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:12.894778+0000", 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:12.894780+0000", 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8080000000000001e-06 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:12.894785+0000", 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6380000000000001e-06 2023-09-30T09:31:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:12.894798+0000", 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2972999999999999e-05 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:12.894809+0000", 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1069e-05 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:12.921332+0000", 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026522332999999999 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:12.922331+0000", 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00099967200000000006 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.191632+0000", 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.26930021300000001 2023-09-30T09:31:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.193736+0000", 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021038530000000001 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.754967+0000", 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.56123104899999998 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.756393+0000", 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00142684 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.758858+0000", 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024645969999999998 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.768569+0000", 2023-09-30T09:31:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0097105999999999998 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.769094+0000", 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00052534200000000004 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.769769+0000", 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00067542100000000005 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.771004+0000", 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012351269999999999 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.775272+0000", 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042672539999999998 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:109115 6.13 6:c9d5c360:::benchmark_data_smithi019_355417_object16811:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:13.841444+0000", 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 90.693459806000007, 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87746232899999999, 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 109115 2023-09-30T09:31:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.841444+0000", 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.841444+0000", 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.841446+0000", 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.204e-06 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.841449+0000", 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.89e-06 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.841462+0000", 2023-09-30T09:31:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2255e-05 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.841468+0000", 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4999999999999996e-06 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.982315+0000", 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14084703400000001 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:13.983288+0000", 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000972414 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.103510+0000", 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.120222017 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:31:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.105752+0000", 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002242441 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.673046+0000", 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.56729408000000003 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.674628+0000", 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015814100000000001 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.677476+0000", 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028487460000000001 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.697421+0000", 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.019944198 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.713021+0000", 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015600395 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.713976+0000", 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00095501500000000003 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.715087+0000", 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011107490000000001 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.718907+0000", 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038199810000000001 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:109327 6.13 6:c816e4cd:::benchmark_data_smithi019_355417_object17023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:14.171381+0000", 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 90.363523267000005, 2023-09-30T09:31:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.942496942, 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 109327 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.171381+0000", 2023-09-30T09:31:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.171381+0000", 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.171382+0000", 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8299999999999996e-07 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.171385+0000", 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4470000000000002e-06 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.171396+0000", 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0778000000000001e-05 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.171401+0000", 2023-09-30T09:31:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4179999999999996e-06 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.179749+0000", 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0083480389999999998 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.180795+0000", 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0010460630000000001 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.448918+0000", 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.26812295800000002 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.450717+0000", 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001798322 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.968671+0000", 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.51795389199999997 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.970128+0000", 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0014578519999999999 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.972932+0000", 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002803767 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.989426+0000", 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016493652000000001 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:14.989905+0000", 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00047910999999999999 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:15.108467+0000", 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.118562111 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:15.109751+0000", 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012839069999999999 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:15.113878+0000", 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041269430000000001 2023-09-30T09:31:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:112907 6.13 6:cc36a634:::benchmark_data_smithi019_355417_object20603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:19.317462+0000", 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.217441766999997, 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.030684653, 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 112907 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317462+0000", 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317462+0000", 2023-09-30T09:31:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317464+0000", 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2759999999999999e-06 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317469+0000", 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1490000000000002e-06 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317482+0000", 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2897e-05 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317490+0000", 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7099999999999996e-06 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.321704+0000", 2023-09-30T09:31:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042135260000000004 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.322413+0000", 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0007092 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.704196+0000", 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.38178283899999998 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.706325+0000", 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002128984 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.285269+0000", 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.57894403699999997 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.287195+0000", 2023-09-30T09:31:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001926072 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.290526+0000", 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033311479999999999 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.298483+0000", 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0079569210000000005 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.316366+0000", 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017883202000000001 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.342631+0000", 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026264167000000001 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.343876+0000", 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012450569999999999 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.348147+0000", 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042714679999999996 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143205 6.13 6:ca286b4d:::benchmark_data_smithi019_355417_object50901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.317267+0000", 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.217637394, 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0055789479999999, 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143205 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317267+0000", 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317267+0000", 2023-09-30T09:31:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317267+0000", 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5600000000000001e-07 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317270+0000", 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4930000000000001e-06 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317278+0000", 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1939999999999998e-06 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317282+0000", 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.639e-06 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317574+0000", 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00029113799999999998 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317687+0000", 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113625 2023-09-30T09:31:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.318221+0000", 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00053415699999999995 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.320841+0000", 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026193520000000001 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.324853+0000", 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040123479999999998 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.324865+0000", 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1793999999999999e-05 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316130+0000", 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99126465200000002 2023-09-30T09:31:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316278+0000", 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000148326 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322784+0000", 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0065062449999999999 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322846+0000", 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1629000000000002e-05 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143210 6.13 6:cd7c9817:::benchmark_data_smithi019_355417_object50906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.322320+0000", 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.212584182999997, 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0007609209999999, 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143210 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322320+0000", 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322320+0000", 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322321+0000", 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1999999999999998e-07 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322325+0000", 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2100000000000003e-06 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322732+0000", 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000407291 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322744+0000", 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.182e-05 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325773+0000", 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030286169999999999 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325782+0000", 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4979999999999995e-06 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316559+0000", 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99077679100000005 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316643+0000", 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3658000000000004e-05 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316998+0000", 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035565800000000002 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322459+0000", 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0054604650000000003 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322886+0000", 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00042728800000000001 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322934+0000", 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7895000000000001e-05 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322969+0000", 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5264e-05 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323081+0000", 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011154600000000001 2023-09-30T09:31:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143213 6.13 6:cf52ab7f:::benchmark_data_smithi019_355417_object50909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.325783+0000", 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.209121660000001, 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99744182599999998, 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143213 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325783+0000", 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325783+0000", 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325784+0000", 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3209999999999999e-06 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325788+0000", 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.459e-06 2023-09-30T09:31:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325798+0000", 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6709999999999994e-06 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325805+0000", 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0520000000000004e-06 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.326702+0000", 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00089691900000000004 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.326712+0000", 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0101e-05 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.318705+0000", 2023-09-30T09:31:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99199274900000001 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.318821+0000", 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115704 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.319064+0000", 2023-09-30T09:31:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000243847 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322593+0000", 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003528659 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322913+0000", 2023-09-30T09:31:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000319957 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323109+0000", 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019595999999999999 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323144+0000", 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4684000000000002e-05 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323224+0000", 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0742999999999996e-05 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143232 6.13 6:ce26f66a:::benchmark_data_smithi019_355417_object50928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.351507+0000", 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.183397274999997, 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97190067099999999, 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143232 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351507+0000", 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351507+0000", 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351508+0000", 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.75e-07 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351510+0000", 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6230000000000001e-06 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351522+0000", 2023-09-30T09:31:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2160999999999999e-05 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351528+0000", 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0889999999999996e-06 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351658+0000", 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000129782 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351667+0000", 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1439999999999992e-06 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.320634+0000", 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96896631099999997 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.320745+0000", 2023-09-30T09:31:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000111035 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.320999+0000", 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000253983 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322763+0000", 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001764568 2023-09-30T09:31:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323255+0000", 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000492122 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323302+0000", 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7076999999999997e-05 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323345+0000", 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2654000000000002e-05 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323408+0000", 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2546999999999996e-05 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143245 6.13 6:c9599472:::benchmark_data_smithi019_355417_object50941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.361939+0000", 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.172965009000002, 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97745515400000005, 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143245 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361939+0000", 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361939+0000", 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361940+0000", 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5099999999999999e-07 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361943+0000", 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1750000000000001e-06 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361951+0000", 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8939999999999993e-06 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361956+0000", 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5840000000000002e-06 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.362030+0000", 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3905000000000005e-05 2023-09-30T09:31:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.362042+0000", 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1392999999999999e-05 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323462+0000", 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96141999300000003 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323546+0000", 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4419000000000006e-05 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323803+0000", 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025670000000000001 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330037+0000", 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062342020000000003 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330102+0000", 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4901999999999996e-05 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339211+0000", 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091090440000000002 2023-09-30T09:31:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339279+0000", 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8442000000000006e-05 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339394+0000", 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011485 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143266 6.13 6:cd9d4c46:::benchmark_data_smithi019_355417_object50962:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.376173+0000", 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.158731422999999, 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96345502800000005, 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143266 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376173+0000", 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376173+0000", 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376174+0000", 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0559999999999999e-06 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376178+0000", 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.357e-06 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376202+0000", 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3513e-05 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376209+0000", 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6660000000000008e-06 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376277+0000", 2023-09-30T09:31:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7251999999999999e-05 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376289+0000", 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2873e-05 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.325866+0000", 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94957660499999996 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.325992+0000", 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012573700000000001 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.326318+0000", 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00032582100000000001 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330166+0000", 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038479159999999998 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330176+0000", 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0284e-05 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339447+0000", 2023-09-30T09:31:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0092713829999999994 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339506+0000", 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8841999999999997e-05 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339628+0000", 2023-09-30T09:31:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000121723 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143340 6.13 6:cc508b91:::benchmark_data_smithi019_355417_object51036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.445987+0000", 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.088916883000003, 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.893826605, 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143340 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.445987+0000", 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.445987+0000", 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.445988+0000", 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8400000000000002e-07 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.445992+0000", 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3620000000000002e-06 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.446002+0000", 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9629999999999993e-06 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.446007+0000", 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1499999999999998e-06 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.446056+0000", 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9496999999999997e-05 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.446063+0000", 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4729999999999997e-06 2023-09-30T09:31:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.328157+0000", 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.882094567 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.328325+0000", 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016778499999999999 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.328597+0000", 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027214699999999998 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.334621+0000", 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0060234090000000004 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.334751+0000", 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013027899999999999 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339669+0000", 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049184099999999998 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339717+0000", 2023-09-30T09:31:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7608000000000002e-05 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339814+0000", 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6971000000000004e-05 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143349 6.13 6:ce5d0213:::benchmark_data_smithi019_355417_object51045:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:31:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.452868+0000", 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.082036686000002, 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.88716028400000002, 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143349 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.670 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.452868+0000", 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.452868+0000", 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.452869+0000", 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.173e-06 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.452872+0000", 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6950000000000002e-06 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.672 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.452883+0000", 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0869999999999999e-05 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.452891+0000", 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.469e-06 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.452950+0000", 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9231000000000003e-05 2023-09-30T09:31:44.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.452960+0000", 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6059999999999992e-06 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330250+0000", 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87729033499999998 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.674 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330396+0000", 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000145723 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330712+0000", 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000316817 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.334729+0000", 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040164229999999999 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.334758+0000", 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8646999999999999e-05 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339857+0000", 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050990000000000002 2023-09-30T09:31:44.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339912+0000", 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5389000000000001e-05 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.340028+0000", 2023-09-30T09:31:44.677 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115906 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143363 6.13 6:cb92da3c:::benchmark_data_smithi019_355417_object51059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.464992+0000", 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 50.069911894000001, 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89040021199999997, 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.678 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143363 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.464992+0000", 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.464992+0000", 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.464993+0000", 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0999999999999997e-07 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.464998+0000", 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5979999999999999e-06 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.465008+0000", 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0206e-05 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.465015+0000", 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1940000000000001e-06 2023-09-30T09:31:44.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.465080+0000", 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4744000000000002e-05 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.465090+0000", 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8900000000000002e-06 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.682 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.332392+0000", 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86730193799999999 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.332522+0000", 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000130362 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:31:44.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.332793+0000", 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027142600000000001 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339158+0000", 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.006364115 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.344880+0000", 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0057223040000000001 2023-09-30T09:31:44.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.355251+0000", 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010371436 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.355299+0000", 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7883000000000002e-05 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.685 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.355393+0000", 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3306000000000004e-05 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:162579 6.2 6:42108c99:::benchmark_data_smithi019_355417_object70275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:31:15.355717+0000", 2023-09-30T09:31:44.686 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 29.179187697, 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96398006300000005, 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 162579 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355717+0000", 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355717+0000", 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355718+0000", 2023-09-30T09:31:44.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6789999999999999e-06 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355723+0000", 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3510000000000004e-06 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355739+0000", 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6198e-05 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355747+0000", 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8299999999999996e-06 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355829+0000", 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2921000000000001e-05 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.690 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355842+0000", 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2018000000000001e-05 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.314498+0000", 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95865608899999999 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.314578+0000", 2023-09-30T09:31:44.691 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0556999999999995e-05 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.314779+0000", 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020123799999999999 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.317791+0000", 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030113269999999998 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.692 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.317833+0000", 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2048000000000001e-05 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319642+0000", 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00180915 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.693 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319673+0000", 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1548999999999999e-05 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319697+0000", 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3108e-05 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.694 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:162650 6.2 6:401f1452:::benchmark_data_smithi019_355417_object70346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:31:15.422924+0000", 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 29.111980666000001, 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89687057000000003, 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 162650 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:31:44.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.422924+0000", 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.422924+0000", 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.422925+0000", 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8700000000000001e-06 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.422931+0000", 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8200000000000002e-06 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.422949+0000", 2023-09-30T09:31:44.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7861999999999999e-05 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.422958+0000", 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6570000000000003e-06 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.423150+0000", 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019216900000000001 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.423161+0000", 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1412e-05 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.316037+0000", 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89287558099999997 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.316120+0000", 2023-09-30T09:31:44.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3460000000000004e-05 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.316261+0000", 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000140822 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.318743+0000", 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024821589999999998 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.700 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319601+0000", 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00085776899999999998 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319737+0000", 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000135549 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.701 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319765+0000", 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8303999999999999e-05 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319794+0000", 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9136e-05 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.702 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:31:44.703 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:31:44.703 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:31:44.703 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[216815]: failed to encode map e271 with expected crc 2023-09-30T09:31:44.704 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[216815]: pgmap v654: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 2953/1275 objects degraded (231.608%); 0 B/s, 138 objects/s recovering 2023-09-30T09:31:44.704 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:44.704 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:44.704 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[216815]: failed to encode map e272 with expected crc 2023-09-30T09:31:44.704 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[216815]: osdmap e272: 8 total, 8 up, 7 in 2023-09-30T09:31:44.704 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[216815]: failed to encode map e272 with expected crc 2023-09-30T09:31:44.704 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:44.704 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:44.705 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[224043]: failed to encode map e271 with expected crc 2023-09-30T09:31:44.705 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[224043]: pgmap v654: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 2953/1275 objects degraded (231.608%); 0 B/s, 138 objects/s recovering 2023-09-30T09:31:44.705 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:44.705 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:44.705 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[224043]: failed to encode map e272 with expected crc 2023-09-30T09:31:44.705 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[224043]: osdmap e272: 8 total, 8 up, 7 in 2023-09-30T09:31:44.705 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:44 smithi019 ceph-mon[224043]: failed to encode map e272 with expected crc 2023-09-30T09:31:44.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:44 smithi167 ceph-mon[192880]: failed to encode map e271 with expected crc 2023-09-30T09:31:44.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:44 smithi167 ceph-mon[192880]: pgmap v654: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 129 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 2953/1275 objects degraded (231.608%); 0 B/s, 138 objects/s recovering 2023-09-30T09:31:44.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:44 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:44.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:44 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:31:44.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:44 smithi167 ceph-mon[192880]: failed to encode map e272 with expected crc 2023-09-30T09:31:44.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:44 smithi167 ceph-mon[192880]: osdmap e272: 8 total, 8 up, 7 in 2023-09-30T09:31:44.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:44 smithi167 ceph-mon[192880]: failed to encode map e272 with expected crc 2023-09-30T09:31:44.706 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:44.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:44.808 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:44.848 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:45.096 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:45.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:45.199 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:45.229 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:45.331 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:45.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:45.438 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:45.466 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:45.515 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:45.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:45.621 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:45.651 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:45.710 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:31:45.753 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:45.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:45.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:45.858 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:45.885 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:45.932 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:45.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:45.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:46.039 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:46.070 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:46.090 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:31:46.220 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:46.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:46.250 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:31:46.325 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:46.353 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:46.397 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:46.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:46.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:46.501 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:46.536 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:46.558 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:31:46.585 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:31:46.594 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:46.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:46 smithi019 ceph-mon[224043]: pgmap v656: 139 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 132 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 1137/1275 objects degraded (89.176%); 0 B/s, 486 objects/s recovering 2023-09-30T09:31:46.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:46.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:46.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:46 smithi019 ceph-mon[216815]: pgmap v656: 139 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 132 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 1137/1275 objects degraded (89.176%); 0 B/s, 486 objects/s recovering 2023-09-30T09:31:46.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:46.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:46.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:46.619 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:31:46.646 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:31:46.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:31:46.670 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:46 smithi167 ceph-mon[192880]: pgmap v656: 139 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 132 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 1137/1275 objects degraded (89.176%); 0 B/s, 486 objects/s recovering 2023-09-30T09:31:46.670 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:46.670 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:46.685 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:31:46.699 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:46.704 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:31:46.723 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:31:46.724 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:31:46.724 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:31:46.724 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:31:46.725 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:31:46.725 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:31:46.725 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:31:46.725 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:31:46.725 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:31:46.726 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:46.771 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:46.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:46.874 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:46.902 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:47.337 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:31:47.337 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:31:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:47 smithi167 ceph-mon[192880]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:47 smithi167 ceph-mon[192880]: osdmap e273: 8 total, 8 up, 7 in 2023-09-30T09:31:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:47 smithi167 ceph-mon[192880]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:47 smithi167 ceph-mon[192880]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3411407377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:47.791 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:47.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[216815]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[216815]: osdmap e273: 8 total, 8 up, 7 in 2023-09-30T09:31:47.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[216815]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[216815]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3411407377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:47.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[224043]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[224043]: osdmap e273: 8 total, 8 up, 7 in 2023-09-30T09:31:47.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[224043]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[224043]: failed to encode map e273 with expected crc 2023-09-30T09:31:47.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3411407377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:47.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:47.899 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:47.932 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:47.981 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:47.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:47.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:48.010 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:48.085 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:48.113 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:48.159 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:48.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:48.708 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:48.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[216815]: pgmap v658: 139 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 132 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 1137/1275 objects degraded (89.176%); 0 B/s, 314 objects/s recovering 2023-09-30T09:31:48.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[216815]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[216815]: osdmap e274: 8 total, 8 up, 7 in 2023-09-30T09:31:48.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[216815]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[216815]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1137/1275 objects degraded (89.176%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[224043]: pgmap v658: 139 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 132 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 1137/1275 objects degraded (89.176%); 0 B/s, 314 objects/s recovering 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[224043]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[224043]: osdmap e274: 8 total, 8 up, 7 in 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[224043]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[224043]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:48.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:48.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1137/1275 objects degraded (89.176%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:48.740 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:48 smithi167 ceph-mon[192880]: pgmap v658: 139 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait, 132 active+clean; 29 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 1137/1275 objects degraded (89.176%); 0 B/s, 314 objects/s recovering 2023-09-30T09:31:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:48 smithi167 ceph-mon[192880]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:48 smithi167 ceph-mon[192880]: osdmap e274: 8 total, 8 up, 7 in 2023-09-30T09:31:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:48 smithi167 ceph-mon[192880]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:48 smithi167 ceph-mon[192880]: failed to encode map e274 with expected crc 2023-09-30T09:31:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:31:48.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1137/1275 objects degraded (89.176%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:31:49.190 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:49.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:49.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:49.294 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:49.322 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:49.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:49 smithi019 ceph-mon[216815]: failed to encode map e274 with expected crc 2023-09-30T09:31:49.474 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_1' removed 2023-09-30T09:31:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:49 smithi167 ceph-mon[192880]: failed to encode map e274 with expected crc 2023-09-30T09:31:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:49 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2154541183' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:31:49.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:31:49.788 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:49 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2154541183' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:31:49.788 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:31:49.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:49 smithi019 ceph-mon[224043]: failed to encode map e274 with expected crc 2023-09-30T09:31:49.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:49 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2154541183' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:31:49.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:31:49.822 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:49.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:49.856 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:31:49.926 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:49.954 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:50.241 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:50.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:50.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:50.345 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:50.379 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:50.396 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-30T09:31:50.396 INFO:tasks.radosbench:Beginning radosbench... 2023-09-30T09:31:50.397 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-30T09:31:50.397 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-30T09:31:50.486 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:50.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:50.520 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:50 smithi019 ceph-mon[216815]: pgmap v660: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 136 active+clean; 29 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 833/1275 objects degraded (65.333%); 0 B/s, 353 objects/s recovering 2023-09-30T09:31:50.521 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:50 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:31:50.521 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:50 smithi019 ceph-mon[216815]: failed to encode map e275 with expected crc 2023-09-30T09:31:50.521 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:50 smithi019 ceph-mon[216815]: osdmap e275: 8 total, 8 up, 7 in 2023-09-30T09:31:50.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:50 smithi019 ceph-mon[224043]: pgmap v660: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 136 active+clean; 29 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 833/1275 objects degraded (65.333%); 0 B/s, 353 objects/s recovering 2023-09-30T09:31:50.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:50 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:31:50.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:50 smithi019 ceph-mon[224043]: failed to encode map e275 with expected crc 2023-09-30T09:31:50.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:50 smithi019 ceph-mon[224043]: osdmap e275: 8 total, 8 up, 7 in 2023-09-30T09:31:50.592 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:50.631 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:50 smithi167 ceph-mon[192880]: pgmap v660: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 136 active+clean; 29 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 833/1275 objects degraded (65.333%); 0 B/s, 353 objects/s recovering 2023-09-30T09:31:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:50 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:31:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:50 smithi167 ceph-mon[192880]: failed to encode map e275 with expected crc 2023-09-30T09:31:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:50 smithi167 ceph-mon[192880]: osdmap e275: 8 total, 8 up, 7 in 2023-09-30T09:31:50.938 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:50.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:50.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:50.985 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:51.041 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:51.078 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:51.127 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:51.286 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:51.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:51.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:51.390 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:51.419 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:51.485 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:31:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[216815]: failed to encode map e275 with expected crc 2023-09-30T09:31:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[216815]: failed to encode map e275 with expected crc 2023-09-30T09:31:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1542538871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:51.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-30T09:31:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[224043]: failed to encode map e275 with expected crc 2023-09-30T09:31:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[224043]: failed to encode map e275 with expected crc 2023-09-30T09:31:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1542538871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:51.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:51 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-30T09:31:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:51 smithi167 ceph-mon[192880]: failed to encode map e275 with expected crc 2023-09-30T09:31:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:51 smithi167 ceph-mon[192880]: failed to encode map e275 with expected crc 2023-09-30T09:31:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1542538871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:51.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:51 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:31:51.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:51 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:51.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:51.914 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:51.949 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:52.001 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:52.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:52.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:52.105 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:52.134 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:52.179 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:52.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:52.283 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:52.311 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:52.358 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:52.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:52.461 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:52.499 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: pgmap v662: 103 pgs: 103 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:31:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 833/1275 objects degraded (65.333%), 1 pg degraded, 1 pg undersized) 2023-09-30T09:31:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:31:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-30T09:31:52.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: osdmap e276: 8 total, 8 up, 7 in 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:52.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:52 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:52.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: pgmap v662: 103 pgs: 103 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:31:52.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 833/1275 objects degraded (65.333%), 1 pg degraded, 1 pg undersized) 2023-09-30T09:31:52.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:52.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:31:52.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-30T09:31:52.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.796 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: osdmap e276: 8 total, 8 up, 7 in 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:52.797 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: pgmap v662: 103 pgs: 103 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 833/1275 objects degraded (65.333%), 1 pg degraded, 1 pg undersized) 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: failed to encode map e276 with expected crc 2023-09-30T09:31:52.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: osdmap e276: 8 total, 8 up, 7 in 2023-09-30T09:31:52.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:52.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:52 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:53.202 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:53.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:53.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:53.306 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:53.337 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:53.537 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:53.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:53.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:53.641 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:53.644 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: 1.2 scrub starts 2023-09-30T09:31:53.644 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: 1.2 scrub ok 2023-09-30T09:31:53.644 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: osdmap e277: 8 total, 8 up, 7 in 2023-09-30T09:31:53.644 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.644 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.644 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.644 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:31:53.645 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:31:53.646 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:53.646 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:53 smithi167 ceph-mon[192880]: osdmap e278: 8 total, 8 up, 7 in 2023-09-30T09:31:53.669 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:53.717 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:53.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:53.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:53.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: 1.2 scrub starts 2023-09-30T09:31:53.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: 1.2 scrub ok 2023-09-30T09:31:53.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: osdmap e277: 8 total, 8 up, 7 in 2023-09-30T09:31:53.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[224043]: osdmap e278: 8 total, 8 up, 7 in 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: 1.2 scrub starts 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: 1.2 scrub ok 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: osdmap e277: 8 total, 8 up, 7 in 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.807 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: failed to encode map e277 with expected crc 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:31:53.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:31:53.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:31:53.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:53 smithi019 ceph-mon[216815]: osdmap e278: 8 total, 8 up, 7 in 2023-09-30T09:31:53.822 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:53.849 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:53.994 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:53.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:53.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:31:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:31:53] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-30T09:31:54.097 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:54.131 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:54.148 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:31:54.148 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:31:54.233 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:54.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:54.337 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:54.367 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:54.422 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:54.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:54.447 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:31:54.470 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:31:54.492 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:31:54.512 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:31:54.526 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:54.532 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:31:54.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:31:54.558 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:54.587 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:31:54.608 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:31:54.615 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:54.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:54.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:54.643 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:31:54.644 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:31:54.645 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:31:54.645 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:31:54.645 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:31:54.645 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:31:54.645 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:31:54.645 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:31:54.645 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:31:54.719 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:54.747 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:54.906 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:54.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:54.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:54.917 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_2' created 2023-09-30T09:31:55.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: pgmap v665: 103 pgs: 2 unknown, 2 peering, 99 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:31:55.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: 1.1 scrub starts 2023-09-30T09:31:55.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: 1.1 scrub ok 2023-09-30T09:31:55.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: 1.5 scrub starts 2023-09-30T09:31:55.002 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:54 smithi167 ceph-mon[192880]: 1.5 scrub ok 2023-09-30T09:31:55.003 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:31:55.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:55.039 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:55.083 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:31:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:55.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:31:55.187 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:31:55.215 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:55.262 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:31:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:55.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:31:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: pgmap v665: 103 pgs: 2 unknown, 2 peering, 99 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:31:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: 1.1 scrub starts 2023-09-30T09:31:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: 1.1 scrub ok 2023-09-30T09:31:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: 1.5 scrub starts 2023-09-30T09:31:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[216815]: 1.5 scrub ok 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: pgmap v665: 103 pgs: 2 unknown, 2 peering, 99 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: 1.1 scrub starts 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: 1.1 scrub ok 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: failed to encode map e278 with expected crc 2023-09-30T09:31:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: 1.5 scrub starts 2023-09-30T09:31:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:54 smithi019 ceph-mon[224043]: 1.5 scrub ok 2023-09-30T09:31:55.351 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:31:55.369 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:55.396 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:55.671 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:55.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:55.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:55.775 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:55.801 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:55.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: 1.0 scrub starts 2023-09-30T09:31:55.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: 1.0 scrub ok 2023-09-30T09:31:55.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: 1.3 scrub starts 2023-09-30T09:31:55.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: 1.3 scrub ok 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3550536500' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/317922061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3550536500' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: osdmap e279: 8 total, 8 up, 7 in 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.927 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.927 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.927 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: failed to encode map e279 with expected crc 2023-09-30T09:31:55.927 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: pgmap v668: 119 pgs: 16 unknown, 16 active+undersized+remapped, 2 peering, 85 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s rd, 137 KiB/s wr, 42 op/s; 295 KiB/s, 1 objects/s recovering 2023-09-30T09:31:55.927 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[216815]: osdmap e280: 8 total, 8 up, 7 in 2023-09-30T09:31:55.949 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:55.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:56.054 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:56.081 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:56.147 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:56.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:56.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: 1.0 scrub starts 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: 1.0 scrub ok 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: 1.3 scrub starts 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: 1.3 scrub ok 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3550536500' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/317922061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3550536500' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: osdmap e279: 8 total, 8 up, 7 in 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: pgmap v668: 119 pgs: 16 unknown, 16 active+undersized+remapped, 2 peering, 85 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s rd, 137 KiB/s wr, 42 op/s; 295 KiB/s, 1 objects/s recovering 2023-09-30T09:31:56.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:55 smithi019 ceph-mon[224043]: osdmap e280: 8 total, 8 up, 7 in 2023-09-30T09:31:56.202 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-30T09:31:56.250 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:56.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: 1.0 scrub starts 2023-09-30T09:31:56.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: 1.0 scrub ok 2023-09-30T09:31:56.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: 1.3 scrub starts 2023-09-30T09:31:56.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: 1.3 scrub ok 2023-09-30T09:31:56.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3550536500' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-30T09:31:56.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/317922061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:31:56.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3550536500' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-30T09:31:56.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: osdmap e279: 8 total, 8 up, 7 in 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: failed to encode map e279 with expected crc 2023-09-30T09:31:56.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: pgmap v668: 119 pgs: 16 unknown, 16 active+undersized+remapped, 2 peering, 85 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s rd, 137 KiB/s wr, 42 op/s; 295 KiB/s, 1 objects/s recovering 2023-09-30T09:31:56.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:55 smithi167 ceph-mon[192880]: osdmap e280: 8 total, 8 up, 7 in 2023-09-30T09:31:56.279 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:56.297 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:56.524 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:56.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:56.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:56.627 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:31:56.656 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:56.956 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:31:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:56.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:31:57.060 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:31:57.089 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:57.122 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:31:57.122 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:31:57.122 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:31:57.123 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:31:57.123 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:31:57.134 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:31:57.136 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: 1.6 scrub starts 2023-09-30T09:31:57.136 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: 1.6 scrub ok 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:31:57.137 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:56 smithi167 ceph-mon[192880]: osdmap e281: 8 total, 8 up, 7 in 2023-09-30T09:31:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:57.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:31:57.239 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:31:57.266 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: 1.6 scrub starts 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: 1.6 scrub ok 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:31:57.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[224043]: osdmap e281: 8 total, 8 up, 7 in 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: 1.6 scrub starts 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: 1.6 scrub ok 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: failed to encode map e280 with expected crc 2023-09-30T09:31:57.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:31:57.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:56 smithi019 ceph-mon[216815]: osdmap e281: 8 total, 8 up, 7 in 2023-09-30T09:31:57.269 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:57.500 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:57.527 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:31:57.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:57.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:31:57.633 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:57.661 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:57.849 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:31:57.864 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:57.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:57.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:57.960 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:31:57.971 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:31:58.034 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:58.162 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: 1.7 scrub starts 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: 1.7 scrub ok 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: 1.8 scrub starts 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: 1.8 scrub ok 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: 1.1a scrub starts 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: 1.1a scrub ok 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: failed to encode map e281 with expected crc 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: pgmap v671: 119 pgs: 13 unknown, 14 active+undersized+remapped, 92 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s rd, 137 KiB/s wr, 42 op/s; 295 KiB/s, 1 objects/s recovering 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: 1.12 scrub starts 2023-09-30T09:31:58.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[216815]: 1.12 scrub ok 2023-09-30T09:31:58.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: 1.7 scrub starts 2023-09-30T09:31:58.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: 1.7 scrub ok 2023-09-30T09:31:58.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: 1.8 scrub starts 2023-09-30T09:31:58.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: 1.8 scrub ok 2023-09-30T09:31:58.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: 1.1a scrub starts 2023-09-30T09:31:58.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: 1.1a scrub ok 2023-09-30T09:31:58.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: failed to encode map e281 with expected crc 2023-09-30T09:31:58.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: pgmap v671: 119 pgs: 13 unknown, 14 active+undersized+remapped, 92 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s rd, 137 KiB/s wr, 42 op/s; 295 KiB/s, 1 objects/s recovering 2023-09-30T09:31:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: 1.12 scrub starts 2023-09-30T09:31:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:57 smithi019 ceph-mon[224043]: 1.12 scrub ok 2023-09-30T09:31:58.213 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: 1.7 scrub starts 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: 1.7 scrub ok 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: 1.8 scrub starts 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: 1.8 scrub ok 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: 1.1a scrub starts 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: 1.1a scrub ok 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: failed to encode map e281 with expected crc 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: pgmap v671: 119 pgs: 13 unknown, 14 active+undersized+remapped, 92 active+clean; 4.4 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s rd, 137 KiB/s wr, 42 op/s; 295 KiB/s, 1 objects/s recovering 2023-09-30T09:31:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: 1.12 scrub starts 2023-09-30T09:31:58.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:57 smithi167 ceph-mon[192880]: 1.12 scrub ok 2023-09-30T09:31:58.811 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:31:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:58.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:31:58.894 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:31:58.916 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:58.944 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: 1.9 scrub starts 2023-09-30T09:31:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: 1.9 scrub ok 2023-09-30T09:31:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: 1.b scrub starts 2023-09-30T09:31:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: 1.b scrub ok 2023-09-30T09:31:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: 1.10 scrub starts 2023-09-30T09:31:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: 1.10 scrub ok 2023-09-30T09:31:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:58 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/402306826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:59.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: 1.9 scrub starts 2023-09-30T09:31:59.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: 1.9 scrub ok 2023-09-30T09:31:59.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: 1.b scrub starts 2023-09-30T09:31:59.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: 1.b scrub ok 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: 1.10 scrub starts 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: 1.10 scrub ok 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/402306826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: 1.9 scrub starts 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: failed to encode map e281 with expected crc 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: 1.9 scrub ok 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: 1.b scrub starts 2023-09-30T09:31:59.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: 1.b scrub ok 2023-09-30T09:31:59.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: 1.10 scrub starts 2023-09-30T09:31:59.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: 1.10 scrub ok 2023-09-30T09:31:59.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:58 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/402306826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:31:59.143 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:59.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:59.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:59.247 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:31:59.275 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:59.300 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:31:59.377 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:31:59.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:59.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:31:59.480 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:31:59.509 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:31:59.659 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:31:59.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:31:59.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:31:59.697 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:31:59.697 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:43714 5.e 5:72d8c250:::benchmark_data_smithi019_287194_object43713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:24.030348+0000", 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 515.66381925500002, 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.27711378199999998, 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.698 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 43714 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030348+0000", 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030348+0000", 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030347+0000", 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030402+0000", 2023-09-30T09:31:59.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5665999999999999e-05 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030417+0000", 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4938e-05 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030423+0000", 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1550000000000004e-06 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.701 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030607+0000", 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000184124 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030726+0000", 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000118941 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.030863+0000", 2023-09-30T09:31:59.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013704799999999999 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.033947+0000", 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030832379999999999 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.036858+0000", 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029110360000000001 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.306593+0000", 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.269735748 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.306631+0000", 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7755e-05 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.704 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.307462+0000", 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00083068400000000004 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:43715 5.e 5:7ce53f20:::benchmark_data_smithi019_287194_object43714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:24.032308+0000", 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 515.66185892500005, 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.27575674100000003, 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.705 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 43715 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032308+0000", 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032308+0000", 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032307+0000", 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032335+0000", 2023-09-30T09:31:59.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7787e-05 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032342+0000", 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9199999999999998e-06 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032346+0000", 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9330000000000003e-06 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032374+0000", 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7908000000000001e-05 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032497+0000", 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012358200000000001 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.032601+0000", 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103354 2023-09-30T09:31:59.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.034103+0000", 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001501947 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.036915+0000", 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028126710000000001 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.307509+0000", 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.270593955 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.307565+0000", 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5124999999999998e-05 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.308065+0000", 2023-09-30T09:31:59.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00050067500000000001 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:43723 5.e 5:79a80065:::benchmark_data_smithi019_287194_object43722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:24.037126+0000", 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 515.65704180199998, 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.27156727600000002, 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.712 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 43723 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037126+0000", 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.713 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037126+0000", 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037124+0000", 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037166+0000", 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2116999999999997e-05 2023-09-30T09:31:59.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037182+0000", 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5327000000000001e-05 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037188+0000", 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8299999999999998e-06 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037230+0000", 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1869000000000002e-05 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037352+0000", 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012168900000000001 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.037478+0000", 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012598700000000001 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.038785+0000", 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013066989999999999 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.039088+0000", 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00030367399999999998 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.308114+0000", 2023-09-30T09:31:59.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26902532200000001 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.308171+0000", 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7241999999999998e-05 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.308693+0000", 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052196000000000002 2023-09-30T09:31:59.718 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:43726 5.e 5:7b00ed25:::benchmark_data_smithi019_287194_object43725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:23:24.040706+0000", 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 515.65346117800004, 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26866345400000002, 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.719 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 43726 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040706+0000", 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040706+0000", 2023-09-30T09:31:59.720 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040705+0000", 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040729+0000", 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3558999999999999e-05 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040741+0000", 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1977000000000001e-05 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040745+0000", 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4460000000000003e-06 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040779+0000", 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4057999999999998e-05 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040881+0000", 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101288 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,5", 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.040994+0000", 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000113991 2023-09-30T09:31:59.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.042125+0000", 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011310020000000001 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.042389+0000", 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00026325200000000001 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.308749+0000", 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26636068200000002 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.308814+0000", 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4669e-05 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:23:24.309370+0000", 2023-09-30T09:31:59.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00055553999999999998 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:135244 5.1 5:87738a27:::benchmark_data_smithi019_287194_object52466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.114532+0000", 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 325.57963540399999, 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21996616699999999, 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.726 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 135244 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114532+0000", 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.727 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114532+0000", 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114531+0000", 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114536+0000", 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0969999999999996e-06 2023-09-30T09:31:59.728 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114546+0000", 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6520000000000001e-06 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114551+0000", 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1529999999999996e-06 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114585+0000", 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4465999999999998e-05 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114665+0000", 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9425000000000006e-05 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114738+0000", 2023-09-30T09:31:59.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3571000000000004e-05 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.117493+0000", 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027542759999999999 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.118451+0000", 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00095841000000000003 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334392+0000", 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21594123600000001 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334449+0000", 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.664e-05 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334498+0000", 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9209999999999998e-05 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:147190 5.1 5:8f06d857:::benchmark_data_smithi019_287194_object64412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:55.553920+0000", 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 304.14024699100003, 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.196767887, 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.733 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 147190 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553920+0000", 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553920+0000", 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553920+0000", 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553925+0000", 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.361e-06 2023-09-30T09:31:59.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553934+0000", 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7739999999999993e-06 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553939+0000", 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4940000000000003e-06 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.736 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.557550+0000", 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003610458 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.557685+0000", 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013476499999999999 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.557805+0000", 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012047199999999999 2023-09-30T09:31:59.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.561618+0000", 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0038124930000000001 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.750555+0000", 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.18893771200000001 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.750596+0000", 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0507999999999999e-05 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.750646+0000", 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0364000000000001e-05 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.750688+0000", 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2097e-05 2023-09-30T09:31:59.739 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71271 6.e 6:780eb166:::benchmark_data_smithi019_355417_object71270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 153.68779687899999, 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.228409374, 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.740 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71271 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006369+0000", 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006424+0000", 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5114000000000001e-05 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006439+0000", 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5241e-05 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006447+0000", 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6529999999999994e-06 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006492+0000", 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5022999999999999e-05 2023-09-30T09:31:59.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006590+0000", 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7985000000000005e-05 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006695+0000", 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000104984 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.744 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.010237+0000", 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003542271 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.013024+0000", 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027868979999999999 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.231353+0000", 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.218328895 2023-09-30T09:31:59.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.231406+0000", 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3362000000000002e-05 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234780+0000", 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003373572 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.746 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71277 6.e 6:746a3c45:::benchmark_data_smithi019_355417_object71276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 153.68671513199999, 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.230281654, 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.747 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71277 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007451+0000", 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007503+0000", 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2033999999999997e-05 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007515+0000", 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1418e-05 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007520+0000", 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.938e-06 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008550+0000", 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010299840000000001 2023-09-30T09:31:59.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008653+0000", 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103688 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008752+0000", 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8879e-05 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.010372+0000", 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016195960000000001 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.013054+0000", 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026824940000000001 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.752 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.754 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234915+0000", 2023-09-30T09:31:59.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22186042 2023-09-30T09:31:59.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.754 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234978+0000", 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3447000000000004e-05 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237734+0000", 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027555909999999999 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71282 6.e 6:707e64e4:::benchmark_data_smithi019_355417_object71281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 153.67850328, 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22500474600000001, 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71282 2023-09-30T09:31:59.756 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015663+0000", 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015715+0000", 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1873999999999998e-05 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015725+0000", 2023-09-30T09:31:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0387000000000001e-05 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015730+0000", 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.814e-06 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015768+0000", 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7673000000000003e-05 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015871+0000", 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103274 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015979+0000", 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108563 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.018931+0000", 2023-09-30T09:31:59.760 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029516809999999998 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020250+0000", 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013193289999999999 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237789+0000", 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21753858700000001 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237850+0000", 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0448999999999997e-05 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240669+0000", 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028193039999999999 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71283 6.e 6:7093bd40:::benchmark_data_smithi019_355417_object71282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 153.67816300699999, 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22746534299999999, 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71283 2023-09-30T09:31:59.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016074+0000", 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0246999999999993e-05 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016082+0000", 2023-09-30T09:31:59.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4610000000000002e-06 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016085+0000", 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5489999999999998e-06 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017714+0000", 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016290549999999999 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017801+0000", 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7076000000000005e-05 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017904+0000", 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000102876 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020212+0000", 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023077570000000001 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.022334+0000", 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021222160000000001 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240736+0000", 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.218401652 2023-09-30T09:31:59.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240791+0000", 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.554e-05 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243470+0000", 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026784019999999999 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71284 6.e 6:7ce92b46:::benchmark_data_smithi019_355417_object71283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 153.67767462399999, 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22906620999999999, 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.770 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71284 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016492+0000", 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016557+0000", 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4941e-05 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016565+0000", 2023-09-30T09:31:59.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3669999999999997e-06 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016570+0000", 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2390000000000004e-06 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.018977+0000", 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024078609999999999 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.019059+0000", 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1386999999999999e-05 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.019149+0000", 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9938000000000005e-05 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020349+0000", 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012002530000000001 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.022546+0000", 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021970819999999999 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243549+0000", 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22100276599999999 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243617+0000", 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8547999999999995e-05 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.245559+0000", 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019414879999999999 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:96725 6.e 6:743dbf6c:::benchmark_data_smithi019_355417_object4421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 123.21098697799999, 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.205057722, 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 96725 2023-09-30T09:31:59.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483179+0000", 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483187+0000", 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3750000000000003e-06 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483203+0000", 2023-09-30T09:31:59.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6215999999999999e-05 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483210+0000", 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0180000000000004e-06 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483254+0000", 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3426000000000001e-05 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.780 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.485269+0000", 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002015644 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488763+0000", 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034939120000000001 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488819+0000", 2023-09-30T09:31:59.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6135999999999999e-05 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488893+0000", 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3734999999999998e-05 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.490570+0000", 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001677015 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.490591+0000", 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0973000000000001e-05 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.686691+0000", 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.196100203 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.686739+0000", 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7395000000000003e-05 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.688238+0000", 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001499301 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.784 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:98164 6.1e 6:7b1cf70c:::benchmark_data_smithi019_355417_object5860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 121.633618712, 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21486691799999999, 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 98164 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060547+0000", 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060554+0000", 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4069999999999998e-06 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060565+0000", 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1336e-05 2023-09-30T09:31:59.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060571+0000", 2023-09-30T09:31:59.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2219999999999999e-06 2023-09-30T09:31:59.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.074775+0000", 2023-09-30T09:31:59.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014203604999999999 2023-09-30T09:31:59.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.075990+0000", 2023-09-30T09:31:59.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012149319999999999 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.092854+0000", 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016863878999999998 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.092955+0000", 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101431 2023-09-30T09:31:59.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.093073+0000", 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000117923 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.097188+0000", 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041146919999999997 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.105705+0000", 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0085167469999999999 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.273965+0000", 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16826055000000001 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.274028+0000", 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2502999999999994e-05 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.275416+0000", 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013878479999999999 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:98166 6.e 6:75ad8793:::benchmark_data_smithi019_355417_object5862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:31:59.793 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 121.633465198, 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.214711604, 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 98166 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.794 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060701+0000", 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060712+0000", 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0445000000000001e-05 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060723+0000", 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1114999999999999e-05 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060730+0000", 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1130000000000003e-06 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.076100+0000", 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015369651999999999 2023-09-30T09:31:59.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.076669+0000", 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00056970499999999999 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095225+0000", 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.018555250999999998 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095303+0000", 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8380999999999994e-05 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095431+0000", 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012828399999999999 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.098950+0000", 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035189520000000001 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.105689+0000", 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0067384180000000004 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.273979+0000", 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.168290369 2023-09-30T09:31:59.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.274032+0000", 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2456000000000001e-05 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.275414+0000", 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001382203 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.801 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:99939 6.1e 6:78215d60:::benchmark_data_smithi019_355417_object7635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 119.1759757, 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.201779021, 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.802 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 99939 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518190+0000", 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518198+0000", 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3209999999999999e-06 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518211+0000", 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3689000000000001e-05 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518219+0000", 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0320000000000003e-06 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518265+0000", 2023-09-30T09:31:59.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5704999999999997e-05 2023-09-30T09:31:59.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518919+0000", 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00065372300000000004 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.707844+0000", 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.18892562099999999 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.707924+0000", 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0043999999999995e-05 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.708033+0000", 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108694 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.711930+0000", 2023-09-30T09:31:59.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003896708 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.714999+0000", 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030689319999999999 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.718299+0000", 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033004290000000001 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.718348+0000", 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8959999999999999e-05 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.719971+0000", 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001622472 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:99940 6.e 6:70189e37:::benchmark_data_smithi019_355417_object7636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 119.174130327, 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20653424000000001, 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 99940 2023-09-30T09:31:59.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520036+0000", 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520043+0000", 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9550000000000003e-06 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520056+0000", 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3206e-05 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520061+0000", 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.242e-06 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520098+0000", 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6636e-05 2023-09-30T09:31:59.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520966+0000", 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00086822099999999997 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716327+0000", 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.19536082900000001 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716416+0000", 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9463999999999996e-05 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716529+0000", 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112768 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.721106+0000", 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004577378 2023-09-30T09:31:59.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.723760+0000", 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026536900000000002 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.725227+0000", 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001467199 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.725281+0000", 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3758999999999998e-05 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.726571+0000", 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001290177 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:101192 6.e 6:74a2e37f:::benchmark_data_smithi019_355417_object8888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 117.344596243, 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.198305908, 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101192 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349570+0000", 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349577+0000", 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1400000000000002e-06 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349594+0000", 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7044000000000001e-05 2023-09-30T09:31:59.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349601+0000", 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3339999999999996e-06 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349774+0000", 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017279799999999999 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.350391+0000", 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00061680600000000002 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534413+0000", 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.184022306 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534506+0000", 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3183000000000006e-05 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534635+0000", 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012914900000000001 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.537971+0000", 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033357170000000002 2023-09-30T09:31:59.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.540045+0000", 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020742669999999999 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.546502+0000", 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006457043 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.546557+0000", 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4824999999999998e-05 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.547877+0000", 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013197580000000001 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107544 6.e 6:73fbd947:::benchmark_data_smithi019_355417_object15240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 107.96061828400001, 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26128712500000001, 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107544 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.827 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733548+0000", 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733555+0000", 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6210000000000004e-06 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733572+0000", 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7017999999999999e-05 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733580+0000", 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1799999999999996e-06 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733633+0000", 2023-09-30T09:31:59.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2781999999999997e-05 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734357+0000", 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00072381799999999999 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739005+0000", 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046474890000000003 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739086+0000", 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1181999999999996e-05 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739168+0000", 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2346000000000001e-05 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745745+0000", 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0065768140000000003 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747298+0000", 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001552994 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.993625+0000", 2023-09-30T09:31:59.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.24632679199999999 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.993663+0000", 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8668000000000003e-05 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994836+0000", 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001172841 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.834 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107545 6.e 6:77969f61:::benchmark_data_smithi019_355417_object15241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 107.959423051, 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26112492199999998, 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.835 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107545 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:31:59.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734751+0000", 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9609999999999999e-06 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734761+0000", 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0096e-05 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734765+0000", 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5199999999999999e-06 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734813+0000", 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7290000000000003e-05 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735419+0000", 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00060657400000000002 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742024+0000", 2023-09-30T09:31:59.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0066044190000000003 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742129+0000", 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010515599999999999 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742280+0000", 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015174899999999999 2023-09-30T09:31:59.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747335+0000", 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050547270000000002 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747342+0000", 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4069999999999998e-06 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994874+0000", 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.24753244799999999 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994933+0000", 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8959000000000003e-05 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995869+0000", 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00093624799999999998 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107547 6.e 6:7322664d:::benchmark_data_smithi019_355417_object15243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 107.95861458900001, 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26146208700000001, 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:31:59.843 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107547 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735552+0000", 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:31:59.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735558+0000", 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6540000000000004e-06 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735569+0000", 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1029000000000001e-05 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735575+0000", 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4320000000000004e-06 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735624+0000", 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9038e-05 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.736267+0000", 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00064271499999999997 2023-09-30T09:31:59.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745778+0000", 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0095116769999999996 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745840+0000", 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1790000000000003e-05 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745925+0000", 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5045999999999999e-05 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747718+0000", 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017924869999999999 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.748063+0000", 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00034562999999999998 2023-09-30T09:31:59.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995923+0000", 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.247859211 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995976+0000", 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3766e-05 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:31:59.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:31:59.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.997015+0000", 2023-09-30T09:31:59.851 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001038436 2023-09-30T09:31:59.851 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.851 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.851 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.851 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:31:59.851 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:31:59.851 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:31:59.851 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:31:59.883 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:00.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: pgmap v672: 119 pgs: 4 unknown, 8 active+undersized+remapped, 107 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 22 KiB/s rd, 58 KiB/s wr, 17 op/s; 147 KiB/s, 0 objects/s recovering 2023-09-30T09:32:00.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:32:00.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/402306826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:00.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: failed to encode map e282 with expected crc 2023-09-30T09:32:00.242 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: osdmap e282: 8 total, 8 up, 7 in 2023-09-30T09:32:00.242 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: 1.1d scrub starts 2023-09-30T09:32:00.242 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: 1.1d scrub ok 2023-09-30T09:32:00.242 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3467037930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:32:00.242 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:32:00.242 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: 1.17 scrub starts 2023-09-30T09:32:00.242 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[224043]: 1.17 scrub ok 2023-09-30T09:32:00.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: pgmap v672: 119 pgs: 4 unknown, 8 active+undersized+remapped, 107 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 22 KiB/s rd, 58 KiB/s wr, 17 op/s; 147 KiB/s, 0 objects/s recovering 2023-09-30T09:32:00.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:32:00.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/402306826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:00.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: failed to encode map e282 with expected crc 2023-09-30T09:32:00.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: osdmap e282: 8 total, 8 up, 7 in 2023-09-30T09:32:00.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: 1.1d scrub starts 2023-09-30T09:32:00.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: 1.1d scrub ok 2023-09-30T09:32:00.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3467037930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:32:00.244 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:32:00.244 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: 1.17 scrub starts 2023-09-30T09:32:00.244 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:31:59 smithi019 ceph-mon[216815]: 1.17 scrub ok 2023-09-30T09:32:00.248 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:00.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: pgmap v672: 119 pgs: 4 unknown, 8 active+undersized+remapped, 107 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 22 KiB/s rd, 58 KiB/s wr, 17 op/s; 147 KiB/s, 0 objects/s recovering 2023-09-30T09:32:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:32:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/402306826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: failed to encode map e282 with expected crc 2023-09-30T09:32:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: osdmap e282: 8 total, 8 up, 7 in 2023-09-30T09:32:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: 1.1d scrub starts 2023-09-30T09:32:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: 1.1d scrub ok 2023-09-30T09:32:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3467037930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:32:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:32:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: 1.17 scrub starts 2023-09-30T09:32:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:31:59 smithi167 ceph-mon[192880]: 1.17 scrub ok 2023-09-30T09:32:00.300 INFO:teuthology.orchestra.run.smithi019.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-30T09:32:00.354 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:00.367 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:32:00.384 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:00.428 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:00.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:00.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:00.534 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:00.563 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:00.607 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:00.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:00.714 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:00.734 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:00.741 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:00.785 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:00.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:00.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:00.888 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:00.915 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:00.963 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:00.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:00.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:01.067 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:01.094 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:00 smithi167 ceph-mon[192880]: failed to encode map e282 with expected crc 2023-09-30T09:32:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:00 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:32:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:00 smithi167 ceph-mon[192880]: 1.1c scrub ok 2023-09-30T09:32:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:00 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:32:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:00 smithi167 ceph-mon[192880]: osdmap e283: 8 total, 8 up, 7 in 2023-09-30T09:32:01.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[216815]: failed to encode map e282 with expected crc 2023-09-30T09:32:01.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:32:01.275 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[216815]: 1.1c scrub ok 2023-09-30T09:32:01.275 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:32:01.275 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[216815]: osdmap e283: 8 total, 8 up, 7 in 2023-09-30T09:32:01.275 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[224043]: failed to encode map e282 with expected crc 2023-09-30T09:32:01.275 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:32:01.275 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[224043]: 1.1c scrub ok 2023-09-30T09:32:01.275 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:32:01.275 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:00 smithi019 ceph-mon[224043]: osdmap e283: 8 total, 8 up, 7 in 2023-09-30T09:32:01.288 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:01.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:01.393 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:01.419 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:01.462 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:01.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:01.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:01.565 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:01.592 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:01.794 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:32:01.813 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:32:01.832 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:32:01.852 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:32:01.871 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:32:01.891 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:32:01.904 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:01.910 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:32:01.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:01.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:01.928 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:32:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:32:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:32:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:32:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:32:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:32:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:32:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:32:01.930 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:32:02.012 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:02.039 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:02.161 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: failed to encode map e283 with expected crc 2023-09-30T09:32:02.161 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:32:02.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: 1.a scrub ok 2023-09-30T09:32:02.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: 1.14 scrub starts 2023-09-30T09:32:02.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: 1.14 scrub ok 2023-09-30T09:32:02.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: pgmap v675: 119 pgs: 119 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:32:02.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: 1.1b scrub starts 2023-09-30T09:32:02.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: 1.1b scrub ok 2023-09-30T09:32:02.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1408121697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:02.184 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:02.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:02.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:02.259 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: failed to encode map e283 with expected crc 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: 1.a scrub ok 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: 1.14 scrub starts 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: 1.14 scrub ok 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: pgmap v675: 119 pgs: 119 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: 1.1b scrub starts 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: 1.1b scrub ok 2023-09-30T09:32:02.287 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1408121697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: failed to encode map e283 with expected crc 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: 1.a scrub ok 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: 1.14 scrub starts 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: 1.14 scrub ok 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: pgmap v675: 119 pgs: 119 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: 1.1b scrub starts 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: 1.1b scrub ok 2023-09-30T09:32:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1408121697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:02.289 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:02.317 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:02.356 DEBUG:teuthology.orchestra.run.smithi019:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-30T09:32:02.357 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-30T09:32:02.357 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:32:02.359 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-30T09:32:02.359 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-30T09:32:02.372 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:02.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:02.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:02.383 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:32:02.383 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:32:02.412 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:hints = 1 2023-09-30T09:32:02.412 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-30T09:32:02.412 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object prefix: benchmark_data_smithi019_404601 2023-09-30T09:32:02.413 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:32:02.413 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 0 0 0 0 0 0 - 0 2023-09-30T09:32:02.480 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:02.521 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:02.643 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:02.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:02.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:02.747 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:02.781 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:02.830 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:02.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:02.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:02.859 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:02.923 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:02.936 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:02.951 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:02 smithi019 ceph-mon[216815]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:32:02.952 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:02 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:32:02.953 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:02 smithi019 ceph-mon[224043]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:32:02.953 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:02 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:32:03.009 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:03.216 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:03.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:03.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:03.229 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:02 smithi167 ceph-mon[192880]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:32:03.229 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:02 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:32:03.262 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:32:03.321 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:03.368 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:03.413 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 1 16 2348 2332 145.742 145.75 0.00385059 0.0068413 2023-09-30T09:32:03.453 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:03.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:03.559 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:03.603 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:03.694 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:03.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:03.802 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:03.806 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:32:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:32:03] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2023-09-30T09:32:03.851 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:03.860 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[224043]: pgmap v676: 119 pgs: 119 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:32:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[224043]: 1.13 scrub starts 2023-09-30T09:32:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[224043]: 1.13 scrub ok 2023-09-30T09:32:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[224043]: 1.18 scrub starts 2023-09-30T09:32:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[224043]: 1.18 scrub ok 2023-09-30T09:32:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[216815]: pgmap v676: 119 pgs: 119 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:32:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[216815]: 1.13 scrub starts 2023-09-30T09:32:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[216815]: 1.13 scrub ok 2023-09-30T09:32:04.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[216815]: 1.18 scrub starts 2023-09-30T09:32:04.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:03 smithi019 ceph-mon[216815]: 1.18 scrub ok 2023-09-30T09:32:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:03 smithi167 ceph-mon[192880]: pgmap v676: 119 pgs: 119 active+clean; 4.3 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-09-30T09:32:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:03 smithi167 ceph-mon[192880]: 1.13 scrub starts 2023-09-30T09:32:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:03 smithi167 ceph-mon[192880]: 1.13 scrub ok 2023-09-30T09:32:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:03 smithi167 ceph-mon[192880]: 1.18 scrub starts 2023-09-30T09:32:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:03 smithi167 ceph-mon[192880]: 1.18 scrub ok 2023-09-30T09:32:04.413 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 2 16 5674 5658 176.8 207.875 0.00599782 0.00564476 2023-09-30T09:32:04.666 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:32:05.025 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:05.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:05.130 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:05.185 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:04 smithi167 ceph-mon[192880]: 1.4 scrub starts 2023-09-30T09:32:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:04 smithi167 ceph-mon[192880]: 1.4 scrub ok 2023-09-30T09:32:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:04 smithi167 ceph-mon[192880]: 1.16 scrub starts 2023-09-30T09:32:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:04 smithi167 ceph-mon[192880]: 1.16 scrub ok 2023-09-30T09:32:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:04 smithi019 ceph-mon[224043]: 1.4 scrub starts 2023-09-30T09:32:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:04 smithi019 ceph-mon[224043]: 1.4 scrub ok 2023-09-30T09:32:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:04 smithi019 ceph-mon[224043]: 1.16 scrub starts 2023-09-30T09:32:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:04 smithi019 ceph-mon[224043]: 1.16 scrub ok 2023-09-30T09:32:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:04 smithi019 ceph-mon[216815]: 1.4 scrub starts 2023-09-30T09:32:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:04 smithi019 ceph-mon[216815]: 1.4 scrub ok 2023-09-30T09:32:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:04 smithi019 ceph-mon[216815]: 1.16 scrub starts 2023-09-30T09:32:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:04 smithi019 ceph-mon[216815]: 1.16 scrub ok 2023-09-30T09:32:05.415 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 3 16 8702 8686 180.84 189.25 0.0061062 0.00552019 2023-09-30T09:32:06.135 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:06.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:06.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:06.241 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:06.270 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:05 smithi167 ceph-mon[192880]: 1.11 scrub starts 2023-09-30T09:32:06.270 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:05 smithi167 ceph-mon[192880]: 1.11 scrub ok 2023-09-30T09:32:06.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:05 smithi167 ceph-mon[192880]: pgmap v677: 119 pgs: 119 active+clean; 217 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 26 MiB/s wr, 417 op/s 2023-09-30T09:32:06.273 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:05 smithi019 ceph-mon[216815]: 1.11 scrub starts 2023-09-30T09:32:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:05 smithi019 ceph-mon[216815]: 1.11 scrub ok 2023-09-30T09:32:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:05 smithi019 ceph-mon[216815]: pgmap v677: 119 pgs: 119 active+clean; 217 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 26 MiB/s wr, 417 op/s 2023-09-30T09:32:06.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:05 smithi019 ceph-mon[224043]: 1.11 scrub starts 2023-09-30T09:32:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:05 smithi019 ceph-mon[224043]: 1.11 scrub ok 2023-09-30T09:32:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:05 smithi019 ceph-mon[224043]: pgmap v677: 119 pgs: 119 active+clean; 217 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 26 MiB/s wr, 417 op/s 2023-09-30T09:32:06.405 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:06.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:06.415 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 4 16 10873 10857 169.551 135.688 0.00164423 0.0056332 2023-09-30T09:32:06.509 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:06.564 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:06.783 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:32:06.783 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:32:06.962 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:06.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:07.077 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:07.090 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:07.152 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:06 smithi167 ceph-mon[192880]: 1.f scrub starts 2023-09-30T09:32:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:06 smithi167 ceph-mon[192880]: 1.f scrub ok 2023-09-30T09:32:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:06 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:32:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:06 smithi167 ceph-mon[192880]: 1.e scrub ok 2023-09-30T09:32:07.319 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:07.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:06 smithi019 ceph-mon[216815]: 1.f scrub starts 2023-09-30T09:32:07.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:06 smithi019 ceph-mon[216815]: 1.f scrub ok 2023-09-30T09:32:07.323 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:06 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:32:07.323 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:06 smithi019 ceph-mon[216815]: 1.e scrub ok 2023-09-30T09:32:07.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:06 smithi019 ceph-mon[224043]: 1.f scrub starts 2023-09-30T09:32:07.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:06 smithi019 ceph-mon[224043]: 1.f scrub ok 2023-09-30T09:32:07.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:06 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:32:07.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:06 smithi019 ceph-mon[224043]: 1.e scrub ok 2023-09-30T09:32:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:07.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:07.415 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 5 16 12807 12791 159.816 120.875 0.00321783 0.00624996 2023-09-30T09:32:07.425 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:07.457 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:07.501 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:07.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:07.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:07.604 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:07.633 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:07.675 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:07.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:07.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:07.778 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:07.862 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:08.059 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:32:08.062 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:32:08.081 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:32:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:07 smithi167 ceph-mon[192880]: 1.15 scrub starts 2023-09-30T09:32:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:07 smithi167 ceph-mon[192880]: 1.15 scrub ok 2023-09-30T09:32:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:07 smithi167 ceph-mon[192880]: pgmap v678: 119 pgs: 119 active+clean; 351 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s 2023-09-30T09:32:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:07 smithi019 ceph-mon[216815]: 1.15 scrub starts 2023-09-30T09:32:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:07 smithi019 ceph-mon[216815]: 1.15 scrub ok 2023-09-30T09:32:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:07 smithi019 ceph-mon[216815]: pgmap v678: 119 pgs: 119 active+clean; 351 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s 2023-09-30T09:32:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:07 smithi019 ceph-mon[224043]: 1.15 scrub starts 2023-09-30T09:32:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:07 smithi019 ceph-mon[224043]: 1.15 scrub ok 2023-09-30T09:32:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:07 smithi019 ceph-mon[224043]: pgmap v678: 119 pgs: 119 active+clean; 351 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s 2023-09-30T09:32:08.415 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 6 16 14488 14472 150.687 105.062 0.00474718 0.00663036 2023-09-30T09:32:08.677 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:08.861 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:08.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:08.966 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:09.024 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:09.209 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:09.748 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:32:09.894 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 7 16 17161 17145 152.96 167.062 0.00694095 0.0065319 2023-09-30T09:32:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:09.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[224043]: 1.19 scrub starts 2023-09-30T09:32:09.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[224043]: 1.19 scrub ok 2023-09-30T09:32:09.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[224043]: 1.d deep-scrub starts 2023-09-30T09:32:09.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[224043]: 1.d deep-scrub ok 2023-09-30T09:32:09.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[224043]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:32:09.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/620651737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[216815]: 1.19 scrub starts 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[216815]: 1.19 scrub ok 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[216815]: 1.d deep-scrub starts 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[216815]: 1.d deep-scrub ok 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[216815]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/620651737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:09.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:09.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:09.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:09.903 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:09 smithi167 ceph-mon[192880]: 1.19 scrub starts 2023-09-30T09:32:09.903 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:09 smithi167 ceph-mon[192880]: 1.19 scrub ok 2023-09-30T09:32:09.903 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:09 smithi167 ceph-mon[192880]: 1.d deep-scrub starts 2023-09-30T09:32:09.903 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:09 smithi167 ceph-mon[192880]: 1.d deep-scrub ok 2023-09-30T09:32:09.903 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:09 smithi167 ceph-mon[192880]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:32:09.904 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/620651737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:09.904 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:09 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:09.904 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:10.004 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:10.065 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: pgmap v679: 119 pgs: 119 active+clean; 776 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-30T09:32:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:32:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: osdmap e284: 8 total, 8 up, 7 in 2023-09-30T09:32:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[216815]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: pgmap v679: 119 pgs: 119 active+clean; 776 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-30T09:32:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:32:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: osdmap e284: 8 total, 8 up, 7 in 2023-09-30T09:32:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:10 smithi019 ceph-mon[224043]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.419 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 8 16 19062 19046 148.693 118.812 0.00433999 0.00672023 2023-09-30T09:32:10.739 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: pgmap v679: 119 pgs: 119 active+clean; 776 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-30T09:32:10.739 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:10.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:32:10.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: osdmap e284: 8 total, 8 up, 7 in 2023-09-30T09:32:10.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: failed to encode map e284 with expected crc 2023-09-30T09:32:10.740 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:10 smithi167 ceph-mon[192880]: failed to encode map e284 with expected crc 2023-09-30T09:32:11.390 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:32:11.391 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:32:11.418 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 9 16 21090 21074 146.255 126.75 0.00536301 0.00683015 2023-09-30T09:32:11.528 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:11.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:11.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:11.629 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-09-30T09:32:07.206076+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":368191349,"num_objects":5576,"num_object_clones":0,"num_object_copies":16728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5576,"num_whiteouts":0,"num_read":6010,"num_read_kb":12272,"num_write":6488,"num_write_kb":368127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":158,"num_bytes_recovered":9093396,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":11570,"ondisk_log_size":11570,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2918328,"kb_used_data":1136000,"kb_used_omap":14,"kb_used_meta":1782257,"kb_avail":653297736,"statfs":{"total":671965249536,"available":668976881664,"internally_reserved":0,"allocated":1163264000,"data_stored":1154339748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14566,"internal_metadata":1825031962},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":363639850,"num_objects":5549,"num_object_clones":0,"num_object_copies":16647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5549,"num_whiteouts":0,"num_read":-90,"num_read_kb":-167,"num_write":5504,"num_write_kb":354633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3,"num_bytes_recovered":-850225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002252"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":692,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.542814+0000","last_change":"2023-09-30T09:32:03.542762+0000","last_active":"2023-09-30T09:32:03.542814+0000","last_peered":"2023-09-30T09:32:03.542814+0000","last_clean":"2023-09-30T09:32:03.542814+0000","last_became_active":"2023-09-30T09:31:55.917059+0000","last_became_peered":"2023-09-30T09:31:55.917059+0000","last_unstale":"2023-09-30T09:32:03.542814+0000","last_undegraded":"2023-09-30T09:32:03.542814+0000","last_fullsized":"2023-09-30T09:32:03.542814+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:50.452781+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":270,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345990+0000","last_change":"2023-09-30T09:29:09.598494+0000","last_active":"2023-09-30T09:31:59.345990+0000","last_peered":"2023-09-30T09:31:59.345990+0000","last_clean":"2023-09-30T09:31:59.345990+0000","last_became_active":"2023-09-30T09:29:09.598158+0000","last_became_peered":"2023-09-30T09:29:09.598158+0000","last_unstale":"2023-09-30T09:31:59.345990+0000","last_undegraded":"2023-09-30T09:31:59.345990+0000","last_fullsized":"2023-09-30T09:31:59.345990+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:26:03.225682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":249,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297310+0000","last_change":"2023-09-30T09:21:29.093480+0000","last_active":"2023-09-30T09:31:59.297310+0000","last_peered":"2023-09-30T09:31:59.297310+0000","last_clean":"2023-09-30T09:31:59.297310+0000","last_became_active":"2023-09-30T09:21:29.092950+0000","last_became_peered":"2023-09-30T09:21:29.092950+0000","last_unstale":"2023-09-30T09:31:59.297310+0000","last_undegraded":"2023-09-30T09:31:59.297310+0000","last_fullsized":"2023-09-30T09:31:59.297310+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.876307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":727,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:01.514190+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:32:01.514190+0000","last_peered":"2023-09-30T09:32:01.514190+0000","last_clean":"2023-09-30T09:32:01.514190+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:32:01.514190+0000","last_undegraded":"2023-09-30T09:32:01.514190+0000","last_fullsized":"2023-09-30T09:32:01.514190+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":246,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345223+0000","last_change":"2023-09-30T09:26:51.325562+0000","last_active":"2023-09-30T09:31:59.345223+0000","last_peered":"2023-09-30T09:31:59.345223+0000","last_clean":"2023-09-30T09:31:59.345223+0000","last_became_active":"2023-09-30T09:26:51.325230+0000","last_became_peered":"2023-09-30T09:26:51.325230+0000","last_unstale":"2023-09-30T09:31:59.345223+0000","last_undegraded":"2023-09-30T09:31:59.345223+0000","last_fullsized":"2023-09-30T09:31:59.345223+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:24.037149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":177,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.294764+0000","last_change":"2023-09-30T09:23:08.023727+0000","last_active":"2023-09-30T09:31:59.294764+0000","last_peered":"2023-09-30T09:31:59.294764+0000","last_clean":"2023-09-30T09:31:59.294764+0000","last_became_active":"2023-09-30T09:23:08.023275+0000","last_became_peered":"2023-09-30T09:23:08.023275+0000","last_unstale":"2023-09-30T09:31:59.294764+0000","last_undegraded":"2023-09-30T09:31:59.294764+0000","last_fullsized":"2023-09-30T09:31:59.294764+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:44.872213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":459,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345897+0000","last_change":"2023-09-30T09:31:56.857511+0000","last_active":"2023-09-30T09:31:59.345897+0000","last_peered":"2023-09-30T09:31:59.345897+0000","last_clean":"2023-09-30T09:31:59.345897+0000","last_became_active":"2023-09-30T09:31:55.916101+0000","last_became_peered":"2023-09-30T09:31:55.916101+0000","last_unstale":"2023-09-30T09:31:59.345897+0000","last_undegraded":"2023-09-30T09:31:59.345897+0000","last_fullsized":"2023-09-30T09:31:59.345897+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:08:16.703524+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":299,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296846+0000","last_change":"2023-09-30T09:29:09.606288+0000","last_active":"2023-09-30T09:31:59.296846+0000","last_peered":"2023-09-30T09:31:59.296846+0000","last_clean":"2023-09-30T09:31:59.296846+0000","last_became_active":"2023-09-30T09:29:09.605776+0000","last_became_peered":"2023-09-30T09:29:09.605776+0000","last_unstale":"2023-09-30T09:31:59.296846+0000","last_undegraded":"2023-09-30T09:31:59.296846+0000","last_fullsized":"2023-09-30T09:31:59.296846+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:58:07.935982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":242,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.521738+0000","last_change":"2023-09-30T09:20:23.580029+0000","last_active":"2023-09-30T09:31:59.521738+0000","last_peered":"2023-09-30T09:31:59.521738+0000","last_clean":"2023-09-30T09:31:59.521738+0000","last_became_active":"2023-09-30T09:20:23.579636+0000","last_became_peered":"2023-09-30T09:20:23.579636+0000","last_unstale":"2023-09-30T09:31:59.521738+0000","last_undegraded":"2023-09-30T09:31:59.521738+0000","last_fullsized":"2023-09-30T09:31:59.521738+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:37:04.350730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":736,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542061+0000","last_change":"2023-09-30T09:31:55.927835+0000","last_active":"2023-09-30T09:31:59.542061+0000","last_peered":"2023-09-30T09:31:59.542061+0000","last_clean":"2023-09-30T09:31:59.542061+0000","last_became_active":"2023-09-30T09:31:55.927488+0000","last_became_peered":"2023-09-30T09:31:55.927488+0000","last_unstale":"2023-09-30T09:31:59.542061+0000","last_undegraded":"2023-09-30T09:31:59.542061+0000","last_fullsized":"2023-09-30T09:31:59.542061+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":176,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.294286+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:31:59.294286+0000","last_peered":"2023-09-30T09:31:59.294286+0000","last_clean":"2023-09-30T09:31:59.294286+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:31:59.294286+0000","last_undegraded":"2023-09-30T09:31:59.294286+0000","last_fullsized":"2023-09-30T09:31:59.294286+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":249,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296870+0000","last_change":"2023-09-30T09:21:29.104648+0000","last_active":"2023-09-30T09:31:59.296870+0000","last_peered":"2023-09-30T09:31:59.296870+0000","last_clean":"2023-09-30T09:31:59.296870+0000","last_became_active":"2023-09-30T09:21:29.104287+0000","last_became_peered":"2023-09-30T09:21:29.104287+0000","last_unstale":"2023-09-30T09:31:59.296870+0000","last_undegraded":"2023-09-30T09:31:59.296870+0000","last_fullsized":"2023-09-30T09:31:59.296870+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:54:34.920300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":667,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:01.001064+0000","last_change":"2023-09-30T09:32:01.001027+0000","last_active":"2023-09-30T09:32:01.001064+0000","last_peered":"2023-09-30T09:32:01.001064+0000","last_clean":"2023-09-30T09:32:01.001064+0000","last_became_active":"2023-09-30T09:31:55.930313+0000","last_became_peered":"2023-09-30T09:31:55.930313+0000","last_unstale":"2023-09-30T09:32:01.001064+0000","last_undegraded":"2023-09-30T09:32:01.001064+0000","last_fullsized":"2023-09-30T09:32:01.001064+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:58:35.620412+0000","scrub_duration":0.0014706299999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":134,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300494+0000","last_change":"2023-09-30T09:25:14.358921+0000","last_active":"2023-09-30T09:31:59.300494+0000","last_peered":"2023-09-30T09:31:59.300494+0000","last_clean":"2023-09-30T09:31:59.300494+0000","last_became_active":"2023-09-30T09:25:14.358549+0000","last_became_peered":"2023-09-30T09:25:14.358549+0000","last_unstale":"2023-09-30T09:31:59.300494+0000","last_undegraded":"2023-09-30T09:31:59.300494+0000","last_fullsized":"2023-09-30T09:31:59.300494+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:28.888088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":200,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296953+0000","last_change":"2023-09-30T09:21:29.104391+0000","last_active":"2023-09-30T09:31:59.296953+0000","last_peered":"2023-09-30T09:31:59.296953+0000","last_clean":"2023-09-30T09:31:59.296953+0000","last_became_active":"2023-09-30T09:21:29.104056+0000","last_became_peered":"2023-09-30T09:21:29.104056+0000","last_unstale":"2023-09-30T09:31:59.296953+0000","last_undegraded":"2023-09-30T09:31:59.296953+0000","last_fullsized":"2023-09-30T09:31:59.296953+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:21.788610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":449,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.523607+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:31:59.523607+0000","last_peered":"2023-09-30T09:31:59.523607+0000","last_clean":"2023-09-30T09:31:59.523607+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:31:59.523607+0000","last_undegraded":"2023-09-30T09:31:59.523607+0000","last_fullsized":"2023-09-30T09:31:59.523607+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":246,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542195+0000","last_change":"2023-09-30T09:21:29.092097+0000","last_active":"2023-09-30T09:31:59.542195+0000","last_peered":"2023-09-30T09:31:59.542195+0000","last_clean":"2023-09-30T09:31:59.542195+0000","last_became_active":"2023-09-30T09:21:29.091747+0000","last_became_peered":"2023-09-30T09:21:29.091747+0000","last_unstale":"2023-09-30T09:31:59.542195+0000","last_undegraded":"2023-09-30T09:31:59.542195+0000","last_fullsized":"2023-09-30T09:31:59.542195+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:42.627180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":232,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:00.676154+0000","last_change":"2023-09-30T09:29:09.620809+0000","last_active":"2023-09-30T09:32:00.676154+0000","last_peered":"2023-09-30T09:32:00.676154+0000","last_clean":"2023-09-30T09:32:00.676154+0000","last_became_active":"2023-09-30T09:29:09.620497+0000","last_became_peered":"2023-09-30T09:29:09.620497+0000","last_unstale":"2023-09-30T09:32:00.676154+0000","last_undegraded":"2023-09-30T09:32:00.676154+0000","last_fullsized":"2023-09-30T09:32:00.676154+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:48.932180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":445,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:04.392263+0000","last_change":"2023-09-30T09:32:04.392342+0000","last_active":"2023-09-30T09:32:04.392263+0000","last_peered":"2023-09-30T09:32:04.392263+0000","last_clean":"2023-09-30T09:32:04.392263+0000","last_became_active":"2023-09-30T09:31:55.913712+0000","last_became_peered":"2023-09-30T09:31:55.913712+0000","last_unstale":"2023-09-30T09:32:04.392263+0000","last_undegraded":"2023-09-30T09:32:04.392263+0000","last_fullsized":"2023-09-30T09:32:04.392263+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:29.350261+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":247,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346165+0000","last_change":"2023-09-30T09:12:07.944788+0000","last_active":"2023-09-30T09:31:59.346165+0000","last_peered":"2023-09-30T09:31:59.346165+0000","last_clean":"2023-09-30T09:31:59.346165+0000","last_became_active":"2023-09-30T09:12:07.944252+0000","last_became_peered":"2023-09-30T09:12:07.944252+0000","last_unstale":"2023-09-30T09:31:59.346165+0000","last_undegraded":"2023-09-30T09:31:59.346165+0000","last_fullsized":"2023-09-30T09:31:59.346165+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:11.115178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":261,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346209+0000","last_change":"2023-09-30T09:29:09.612466+0000","last_active":"2023-09-30T09:31:59.346209+0000","last_peered":"2023-09-30T09:31:59.346209+0000","last_clean":"2023-09-30T09:31:59.346209+0000","last_became_active":"2023-09-30T09:29:09.612173+0000","last_became_peered":"2023-09-30T09:29:09.612173+0000","last_unstale":"2023-09-30T09:31:59.346209+0000","last_undegraded":"2023-09-30T09:31:59.346209+0000","last_fullsized":"2023-09-30T09:31:59.346209+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:15.457876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":601,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296686+0000","last_change":"2023-09-30T09:31:55.916684+0000","last_active":"2023-09-30T09:31:59.296686+0000","last_peered":"2023-09-30T09:31:59.296686+0000","last_clean":"2023-09-30T09:31:59.296686+0000","last_became_active":"2023-09-30T09:31:55.915993+0000","last_became_peered":"2023-09-30T09:31:55.915993+0000","last_unstale":"2023-09-30T09:31:59.296686+0000","last_undegraded":"2023-09-30T09:31:59.296686+0000","last_fullsized":"2023-09-30T09:31:59.296686+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:27.323455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:27.323455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":261,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.541638+0000","last_change":"2023-09-30T09:29:09.626205+0000","last_active":"2023-09-30T09:31:59.541638+0000","last_peered":"2023-09-30T09:31:59.541638+0000","last_clean":"2023-09-30T09:31:59.541638+0000","last_became_active":"2023-09-30T09:29:09.618293+0000","last_became_peered":"2023-09-30T09:29:09.618293+0000","last_unstale":"2023-09-30T09:31:59.541638+0000","last_undegraded":"2023-09-30T09:31:59.541638+0000","last_fullsized":"2023-09-30T09:31:59.541638+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:04.337293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":271,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297125+0000","last_change":"2023-09-30T09:29:09.606072+0000","last_active":"2023-09-30T09:31:59.297125+0000","last_peered":"2023-09-30T09:31:59.297125+0000","last_clean":"2023-09-30T09:31:59.297125+0000","last_became_active":"2023-09-30T09:29:09.605716+0000","last_became_peered":"2023-09-30T09:29:09.605716+0000","last_unstale":"2023-09-30T09:31:59.297125+0000","last_undegraded":"2023-09-30T09:31:59.297125+0000","last_fullsized":"2023-09-30T09:31:59.297125+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:50.972640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":609,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.299994+0000","last_change":"2023-09-30T09:31:58.499461+0000","last_active":"2023-09-30T09:31:59.299994+0000","last_peered":"2023-09-30T09:31:59.299994+0000","last_clean":"2023-09-30T09:31:59.299994+0000","last_became_active":"2023-09-30T09:31:55.921823+0000","last_became_peered":"2023-09-30T09:31:55.921823+0000","last_unstale":"2023-09-30T09:31:59.299994+0000","last_undegraded":"2023-09-30T09:31:59.299994+0000","last_fullsized":"2023-09-30T09:31:59.299994+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:34.223063+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.295106+0000","last_change":"2023-09-30T09:23:08.024272+0000","last_active":"2023-09-30T09:31:59.295106+0000","last_peered":"2023-09-30T09:31:59.295106+0000","last_clean":"2023-09-30T09:31:59.295106+0000","last_became_active":"2023-09-30T09:23:08.023327+0000","last_became_peered":"2023-09-30T09:23:08.023327+0000","last_unstale":"2023-09-30T09:31:59.295106+0000","last_undegraded":"2023-09-30T09:31:59.295106+0000","last_fullsized":"2023-09-30T09:31:59.295106+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:28:05.687819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":204,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542254+0000","last_change":"2023-09-30T09:26:51.325120+0000","last_active":"2023-09-30T09:31:59.542254+0000","last_peered":"2023-09-30T09:31:59.542254+0000","last_clean":"2023-09-30T09:31:59.542254+0000","last_became_active":"2023-09-30T09:26:51.324631+0000","last_became_peered":"2023-09-30T09:26:51.324631+0000","last_unstale":"2023-09-30T09:31:59.542254+0000","last_undegraded":"2023-09-30T09:31:59.542254+0000","last_fullsized":"2023-09-30T09:31:59.542254+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:02.738554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":722,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.362769+0000","last_change":"2023-09-30T09:32:03.362840+0000","last_active":"2023-09-30T09:32:03.362769+0000","last_peered":"2023-09-30T09:32:03.362769+0000","last_clean":"2023-09-30T09:32:03.362769+0000","last_became_active":"2023-09-30T09:31:55.913374+0000","last_became_peered":"2023-09-30T09:31:55.913374+0000","last_unstale":"2023-09-30T09:32:03.362769+0000","last_undegraded":"2023-09-30T09:32:03.362769+0000","last_fullsized":"2023-09-30T09:32:03.362769+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:45:03.598052+0000","scrub_duration":0.010869449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":226,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.541372+0000","last_change":"2023-09-30T09:12:07.947490+0000","last_active":"2023-09-30T09:31:59.541372+0000","last_peered":"2023-09-30T09:31:59.541372+0000","last_clean":"2023-09-30T09:31:59.541372+0000","last_became_active":"2023-09-30T09:12:07.947218+0000","last_became_peered":"2023-09-30T09:12:07.947218+0000","last_unstale":"2023-09-30T09:31:59.541372+0000","last_undegraded":"2023-09-30T09:31:59.541372+0000","last_fullsized":"2023-09-30T09:31:59.541372+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:00.808461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.294895+0000","last_change":"2023-09-30T09:23:08.024803+0000","last_active":"2023-09-30T09:31:59.294895+0000","last_peered":"2023-09-30T09:31:59.294895+0000","last_clean":"2023-09-30T09:31:59.294895+0000","last_became_active":"2023-09-30T09:23:08.023374+0000","last_became_peered":"2023-09-30T09:23:08.023374+0000","last_unstale":"2023-09-30T09:31:59.294895+0000","last_undegraded":"2023-09-30T09:31:59.294895+0000","last_fullsized":"2023-09-30T09:31:59.294895+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:09.904109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":731,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300154+0000","last_change":"2023-09-30T09:31:57.465093+0000","last_active":"2023-09-30T09:31:59.300154+0000","last_peered":"2023-09-30T09:31:59.300154+0000","last_clean":"2023-09-30T09:31:59.300154+0000","last_became_active":"2023-09-30T09:31:55.922422+0000","last_became_peered":"2023-09-30T09:31:55.922422+0000","last_unstale":"2023-09-30T09:31:59.300154+0000","last_undegraded":"2023-09-30T09:31:59.300154+0000","last_fullsized":"2023-09-30T09:31:59.300154+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:28:50.527071+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":375,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.520839+0000","last_change":"2023-09-30T09:29:09.634829+0000","last_active":"2023-09-30T09:31:59.520839+0000","last_peered":"2023-09-30T09:31:59.520839+0000","last_clean":"2023-09-30T09:31:59.520839+0000","last_became_active":"2023-09-30T09:29:09.612183+0000","last_became_peered":"2023-09-30T09:29:09.612183+0000","last_unstale":"2023-09-30T09:31:59.520839+0000","last_undegraded":"2023-09-30T09:31:59.520839+0000","last_fullsized":"2023-09-30T09:31:59.520839+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:28.144436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":269,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297662+0000","last_change":"2023-09-30T09:29:09.605918+0000","last_active":"2023-09-30T09:31:59.297662+0000","last_peered":"2023-09-30T09:31:59.297662+0000","last_clean":"2023-09-30T09:31:59.297662+0000","last_became_active":"2023-09-30T09:29:09.605521+0000","last_became_peered":"2023-09-30T09:29:09.605521+0000","last_unstale":"2023-09-30T09:31:59.297662+0000","last_undegraded":"2023-09-30T09:31:59.297662+0000","last_fullsized":"2023-09-30T09:31:59.297662+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:43.287337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"283'734","reported_seq":749,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:07.101356+0000","last_change":"2023-09-30T09:31:55.918720+0000","last_active":"2023-09-30T09:32:07.101356+0000","last_peered":"2023-09-30T09:32:07.101356+0000","last_clean":"2023-09-30T09:32:07.101356+0000","last_became_active":"2023-09-30T09:31:55.918454+0000","last_became_peered":"2023-09-30T09:31:55.918454+0000","last_unstale":"2023-09-30T09:32:07.101356+0000","last_undegraded":"2023-09-30T09:32:07.101356+0000","last_fullsized":"2023-09-30T09:32:07.101356+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":734,"log_dups_size":0,"ondisk_log_size":734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:12:13.310720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48103424,"num_objects":734,"num_object_clones":0,"num_object_copies":2202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":734,"num_write_kb":46976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":736,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542000+0000","last_change":"2023-09-30T09:31:55.926414+0000","last_active":"2023-09-30T09:31:59.542000+0000","last_peered":"2023-09-30T09:31:59.542000+0000","last_clean":"2023-09-30T09:31:59.542000+0000","last_became_active":"2023-09-30T09:31:55.926041+0000","last_became_peered":"2023-09-30T09:31:55.926041+0000","last_unstale":"2023-09-30T09:31:59.542000+0000","last_undegraded":"2023-09-30T09:31:59.542000+0000","last_fullsized":"2023-09-30T09:31:59.542000+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:40.435935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:40.435935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":286,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542225+0000","last_change":"2023-09-30T09:26:51.339020+0000","last_active":"2023-09-30T09:31:59.542225+0000","last_peered":"2023-09-30T09:31:59.542225+0000","last_clean":"2023-09-30T09:31:59.542225+0000","last_became_active":"2023-09-30T09:26:51.320609+0000","last_became_peered":"2023-09-30T09:26:51.320609+0000","last_unstale":"2023-09-30T09:31:59.542225+0000","last_undegraded":"2023-09-30T09:31:59.542225+0000","last_fullsized":"2023-09-30T09:31:59.542225+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:04.786746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":146,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.541497+0000","last_change":"2023-09-30T09:26:51.325290+0000","last_active":"2023-09-30T09:31:59.541497+0000","last_peered":"2023-09-30T09:31:59.541497+0000","last_clean":"2023-09-30T09:31:59.541497+0000","last_became_active":"2023-09-30T09:26:51.325006+0000","last_became_peered":"2023-09-30T09:26:51.325006+0000","last_unstale":"2023-09-30T09:31:59.541497+0000","last_undegraded":"2023-09-30T09:31:59.541497+0000","last_fullsized":"2023-09-30T09:31:59.541497+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:23.026972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"283'226","reported_seq":241,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.786366+0000","last_change":"2023-09-30T09:31:55.921830+0000","last_active":"2023-09-30T09:32:03.786366+0000","last_peered":"2023-09-30T09:32:03.786366+0000","last_clean":"2023-09-30T09:32:03.786366+0000","last_became_active":"2023-09-30T09:31:55.921572+0000","last_became_peered":"2023-09-30T09:31:55.921572+0000","last_unstale":"2023-09-30T09:32:03.786366+0000","last_undegraded":"2023-09-30T09:32:03.786366+0000","last_fullsized":"2023-09-30T09:32:03.786366+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":226,"log_dups_size":0,"ondisk_log_size":226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:16.946044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14811136,"num_objects":226,"num_object_clones":0,"num_object_copies":678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":226,"num_write_kb":14464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":722,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297031+0000","last_change":"2023-09-30T09:31:55.917022+0000","last_active":"2023-09-30T09:31:59.297031+0000","last_peered":"2023-09-30T09:31:59.297031+0000","last_clean":"2023-09-30T09:31:59.297031+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:31:59.297031+0000","last_undegraded":"2023-09-30T09:31:59.297031+0000","last_fullsized":"2023-09-30T09:31:59.297031+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:26.894134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:26.894134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":287,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542426+0000","last_change":"2023-09-30T09:26:51.341060+0000","last_active":"2023-09-30T09:31:59.542426+0000","last_peered":"2023-09-30T09:31:59.542426+0000","last_clean":"2023-09-30T09:31:59.542426+0000","last_became_active":"2023-09-30T09:26:51.324249+0000","last_became_peered":"2023-09-30T09:26:51.324249+0000","last_unstale":"2023-09-30T09:31:59.542426+0000","last_undegraded":"2023-09-30T09:31:59.542426+0000","last_fullsized":"2023-09-30T09:31:59.542426+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:57.953676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":258,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345968+0000","last_change":"2023-09-30T09:21:29.097658+0000","last_active":"2023-09-30T09:31:59.345968+0000","last_peered":"2023-09-30T09:31:59.345968+0000","last_clean":"2023-09-30T09:31:59.345968+0000","last_became_active":"2023-09-30T09:21:29.097303+0000","last_became_peered":"2023-09-30T09:21:29.097303+0000","last_unstale":"2023-09-30T09:31:59.345968+0000","last_undegraded":"2023-09-30T09:31:59.345968+0000","last_fullsized":"2023-09-30T09:31:59.345968+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:55:00.552151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"283'721","reported_seq":736,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:07.097519+0000","last_change":"2023-09-30T09:31:55.915143+0000","last_active":"2023-09-30T09:32:07.097519+0000","last_peered":"2023-09-30T09:32:07.097519+0000","last_clean":"2023-09-30T09:32:07.097519+0000","last_became_active":"2023-09-30T09:31:55.914825+0000","last_became_peered":"2023-09-30T09:31:55.914825+0000","last_unstale":"2023-09-30T09:32:07.097519+0000","last_undegraded":"2023-09-30T09:32:07.097519+0000","last_fullsized":"2023-09-30T09:32:07.097519+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":721,"log_dups_size":0,"ondisk_log_size":721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:59.312161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47251456,"num_objects":721,"num_object_clones":0,"num_object_copies":2163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":721,"num_write_kb":46144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":556,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.521645+0000","last_change":"2023-09-30T09:31:55.930336+0000","last_active":"2023-09-30T09:31:59.521645+0000","last_peered":"2023-09-30T09:31:59.521645+0000","last_clean":"2023-09-30T09:31:59.521645+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:31:59.521645+0000","last_undegraded":"2023-09-30T09:31:59.521645+0000","last_fullsized":"2023-09-30T09:31:59.521645+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:30.495619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:30.495619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":269,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542098+0000","last_change":"2023-09-30T09:26:51.342837+0000","last_active":"2023-09-30T09:31:59.542098+0000","last_peered":"2023-09-30T09:31:59.542098+0000","last_clean":"2023-09-30T09:31:59.542098+0000","last_became_active":"2023-09-30T09:26:51.325577+0000","last_became_peered":"2023-09-30T09:26:51.325577+0000","last_unstale":"2023-09-30T09:31:59.542098+0000","last_undegraded":"2023-09-30T09:31:59.542098+0000","last_fullsized":"2023-09-30T09:31:59.542098+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:06.098713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":252,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.521205+0000","last_change":"2023-09-30T09:23:03.985209+0000","last_active":"2023-09-30T09:31:59.521205+0000","last_peered":"2023-09-30T09:31:59.521205+0000","last_clean":"2023-09-30T09:31:59.521205+0000","last_became_active":"2023-09-30T09:23:03.984926+0000","last_became_peered":"2023-09-30T09:23:03.984926+0000","last_unstale":"2023-09-30T09:31:59.521205+0000","last_undegraded":"2023-09-30T09:31:59.521205+0000","last_fullsized":"2023-09-30T09:31:59.521205+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:33.305792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"283'332","reported_seq":343,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:04.245297+0000","last_change":"2023-09-30T09:31:55.932572+0000","last_active":"2023-09-30T09:32:04.245297+0000","last_peered":"2023-09-30T09:32:04.245297+0000","last_clean":"2023-09-30T09:32:04.245297+0000","last_became_active":"2023-09-30T09:31:55.932277+0000","last_became_peered":"2023-09-30T09:31:55.932277+0000","last_unstale":"2023-09-30T09:32:04.245297+0000","last_undegraded":"2023-09-30T09:32:04.245297+0000","last_fullsized":"2023-09-30T09:32:04.245297+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":332,"log_dups_size":0,"ondisk_log_size":332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:06.871416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21757952,"num_objects":332,"num_object_clones":0,"num_object_copies":996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":332,"num_write_kb":21248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":439,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542092+0000","last_change":"2023-09-30T09:31:55.932891+0000","last_active":"2023-09-30T09:31:59.542092+0000","last_peered":"2023-09-30T09:31:59.542092+0000","last_clean":"2023-09-30T09:31:59.542092+0000","last_became_active":"2023-09-30T09:31:55.932448+0000","last_became_peered":"2023-09-30T09:31:55.932448+0000","last_unstale":"2023-09-30T09:31:59.542092+0000","last_undegraded":"2023-09-30T09:31:59.542092+0000","last_fullsized":"2023-09-30T09:31:59.542092+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":242,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300441+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:31:59.300441+0000","last_peered":"2023-09-30T09:31:59.300441+0000","last_clean":"2023-09-30T09:31:59.300441+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:31:59.300441+0000","last_undegraded":"2023-09-30T09:31:59.300441+0000","last_fullsized":"2023-09-30T09:31:59.300441+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":173,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.294266+0000","last_change":"2023-09-30T09:23:08.016677+0000","last_active":"2023-09-30T09:31:59.294266+0000","last_peered":"2023-09-30T09:31:59.294266+0000","last_clean":"2023-09-30T09:31:59.294266+0000","last_became_active":"2023-09-30T09:23:08.016034+0000","last_became_peered":"2023-09-30T09:23:08.016034+0000","last_unstale":"2023-09-30T09:31:59.294266+0000","last_undegraded":"2023-09-30T09:31:59.294266+0000","last_fullsized":"2023-09-30T09:31:59.294266+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:59.808793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"283'211","reported_seq":226,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.778006+0000","last_change":"2023-09-30T09:31:55.927410+0000","last_active":"2023-09-30T09:32:03.778006+0000","last_peered":"2023-09-30T09:32:03.778006+0000","last_clean":"2023-09-30T09:32:03.778006+0000","last_became_active":"2023-09-30T09:31:55.926993+0000","last_became_peered":"2023-09-30T09:31:55.926993+0000","last_unstale":"2023-09-30T09:32:03.778006+0000","last_undegraded":"2023-09-30T09:32:03.778006+0000","last_fullsized":"2023-09-30T09:32:03.778006+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:58:31.020413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13828096,"num_objects":211,"num_object_clones":0,"num_object_copies":633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":211,"num_write_kb":13504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":782,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.541361+0000","last_change":"2023-09-30T09:31:58.035150+0000","last_active":"2023-09-30T09:31:59.541361+0000","last_peered":"2023-09-30T09:31:59.541361+0000","last_clean":"2023-09-30T09:31:59.541361+0000","last_became_active":"2023-09-30T09:31:09.042107+0000","last_became_peered":"2023-09-30T09:31:09.042107+0000","last_unstale":"2023-09-30T09:31:59.541361+0000","last_undegraded":"2023-09-30T09:31:59.541361+0000","last_fullsized":"2023-09-30T09:31:59.541361+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:18:19.217557+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":283,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297324+0000","last_change":"2023-09-30T09:26:51.333883+0000","last_active":"2023-09-30T09:31:59.297324+0000","last_peered":"2023-09-30T09:31:59.297324+0000","last_clean":"2023-09-30T09:31:59.297324+0000","last_became_active":"2023-09-30T09:26:51.315940+0000","last_became_peered":"2023-09-30T09:26:51.315940+0000","last_unstale":"2023-09-30T09:31:59.297324+0000","last_undegraded":"2023-09-30T09:31:59.297324+0000","last_fullsized":"2023-09-30T09:31:59.297324+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:34.300761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":241,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346063+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:31:59.346063+0000","last_peered":"2023-09-30T09:31:59.346063+0000","last_clean":"2023-09-30T09:31:59.346063+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:31:59.346063+0000","last_undegraded":"2023-09-30T09:31:59.346063+0000","last_fullsized":"2023-09-30T09:31:59.346063+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"283'208","reported_seq":223,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.779935+0000","last_change":"2023-09-30T09:31:55.928200+0000","last_active":"2023-09-30T09:32:03.779935+0000","last_peered":"2023-09-30T09:32:03.779935+0000","last_clean":"2023-09-30T09:32:03.779935+0000","last_became_active":"2023-09-30T09:31:55.927147+0000","last_became_peered":"2023-09-30T09:31:55.927147+0000","last_unstale":"2023-09-30T09:32:03.779935+0000","last_undegraded":"2023-09-30T09:32:03.779935+0000","last_fullsized":"2023-09-30T09:32:03.779935+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:01.743594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13631488,"num_objects":208,"num_object_clones":0,"num_object_copies":624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":208,"num_write_kb":13312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":681,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296574+0000","last_change":"2023-09-30T09:31:56.728868+0000","last_active":"2023-09-30T09:31:59.296574+0000","last_peered":"2023-09-30T09:31:59.296574+0000","last_clean":"2023-09-30T09:31:59.296574+0000","last_became_active":"2023-09-30T09:31:09.037968+0000","last_became_peered":"2023-09-30T09:31:09.037968+0000","last_unstale":"2023-09-30T09:31:59.296574+0000","last_undegraded":"2023-09-30T09:31:59.296574+0000","last_fullsized":"2023-09-30T09:31:59.296574+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:20.791826+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.521122+0000","last_change":"2023-09-30T09:20:23.580101+0000","last_active":"2023-09-30T09:31:59.521122+0000","last_peered":"2023-09-30T09:31:59.521122+0000","last_clean":"2023-09-30T09:31:59.521122+0000","last_became_active":"2023-09-30T09:20:23.579748+0000","last_became_peered":"2023-09-30T09:20:23.579748+0000","last_unstale":"2023-09-30T09:31:59.521122+0000","last_undegraded":"2023-09-30T09:31:59.521122+0000","last_fullsized":"2023-09-30T09:31:59.521122+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:57.995779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":168,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345767+0000","last_change":"2023-09-30T09:29:09.612070+0000","last_active":"2023-09-30T09:31:59.345767+0000","last_peered":"2023-09-30T09:31:59.345767+0000","last_clean":"2023-09-30T09:31:59.345767+0000","last_became_active":"2023-09-30T09:29:09.611725+0000","last_became_peered":"2023-09-30T09:29:09.611725+0000","last_unstale":"2023-09-30T09:31:59.345767+0000","last_undegraded":"2023-09-30T09:31:59.345767+0000","last_fullsized":"2023-09-30T09:31:59.345767+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:55.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"283'269","reported_seq":281,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.859545+0000","last_change":"2023-09-30T09:31:55.927639+0000","last_active":"2023-09-30T09:32:03.859545+0000","last_peered":"2023-09-30T09:32:03.859545+0000","last_clean":"2023-09-30T09:32:03.859545+0000","last_became_active":"2023-09-30T09:31:55.927308+0000","last_became_peered":"2023-09-30T09:31:55.927308+0000","last_unstale":"2023-09-30T09:32:03.859545+0000","last_undegraded":"2023-09-30T09:32:03.859545+0000","last_fullsized":"2023-09-30T09:32:03.859545+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":269,"log_dups_size":0,"ondisk_log_size":269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:51:52.415677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17629184,"num_objects":269,"num_object_clones":0,"num_object_copies":807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":269,"num_write_kb":17216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":729,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346176+0000","last_change":"2023-09-30T09:31:58.322133+0000","last_active":"2023-09-30T09:31:59.346176+0000","last_peered":"2023-09-30T09:31:59.346176+0000","last_clean":"2023-09-30T09:31:59.346176+0000","last_became_active":"2023-09-30T09:31:55.921237+0000","last_became_peered":"2023-09-30T09:31:55.921237+0000","last_unstale":"2023-09-30T09:31:59.346176+0000","last_undegraded":"2023-09-30T09:31:59.346176+0000","last_fullsized":"2023-09-30T09:31:59.346176+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:41:19.230724+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":252,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345875+0000","last_change":"2023-09-30T09:29:09.607621+0000","last_active":"2023-09-30T09:31:59.345875+0000","last_peered":"2023-09-30T09:31:59.345875+0000","last_clean":"2023-09-30T09:31:59.345875+0000","last_became_active":"2023-09-30T09:29:09.607245+0000","last_became_peered":"2023-09-30T09:29:09.607245+0000","last_unstale":"2023-09-30T09:31:59.345875+0000","last_undegraded":"2023-09-30T09:31:59.345875+0000","last_fullsized":"2023-09-30T09:31:59.345875+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:42:42.429421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":241,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346228+0000","last_change":"2023-09-30T09:20:23.575089+0000","last_active":"2023-09-30T09:31:59.346228+0000","last_peered":"2023-09-30T09:31:59.346228+0000","last_clean":"2023-09-30T09:31:59.346228+0000","last_became_active":"2023-09-30T09:20:23.574765+0000","last_became_peered":"2023-09-30T09:20:23.574765+0000","last_unstale":"2023-09-30T09:31:59.346228+0000","last_undegraded":"2023-09-30T09:31:59.346228+0000","last_fullsized":"2023-09-30T09:31:59.346228+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:23.224975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"283'238","reported_seq":250,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.859776+0000","last_change":"2023-09-30T09:31:55.922295+0000","last_active":"2023-09-30T09:32:03.859776+0000","last_peered":"2023-09-30T09:32:03.859776+0000","last_clean":"2023-09-30T09:32:03.859776+0000","last_became_active":"2023-09-30T09:31:55.922026+0000","last_became_peered":"2023-09-30T09:31:55.922026+0000","last_unstale":"2023-09-30T09:32:03.859776+0000","last_undegraded":"2023-09-30T09:32:03.859776+0000","last_fullsized":"2023-09-30T09:32:03.859776+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":238,"log_dups_size":0,"ondisk_log_size":238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:38:08.602806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15597568,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":238,"num_write_kb":15232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":510,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:32:00.381672+0000","last_change":"2023-09-30T09:32:00.381526+0000","last_active":"2023-09-30T09:32:00.381672+0000","last_peered":"2023-09-30T09:32:00.381672+0000","last_clean":"2023-09-30T09:32:00.381672+0000","last_became_active":"2023-09-30T09:31:09.035325+0000","last_became_peered":"2023-09-30T09:31:09.035325+0000","last_unstale":"2023-09-30T09:32:00.381672+0000","last_undegraded":"2023-09-30T09:32:00.381672+0000","last_fullsized":"2023-09-30T09:32:00.381672+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:34:48.440871+0000","scrub_duration":0.0015621879999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":250,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296756+0000","last_change":"2023-09-30T09:21:29.089363+0000","last_active":"2023-09-30T09:31:59.296756+0000","last_peered":"2023-09-30T09:31:59.296756+0000","last_clean":"2023-09-30T09:31:59.296756+0000","last_became_active":"2023-09-30T09:21:29.088930+0000","last_became_peered":"2023-09-30T09:21:29.088930+0000","last_unstale":"2023-09-30T09:31:59.296756+0000","last_undegraded":"2023-09-30T09:31:59.296756+0000","last_fullsized":"2023-09-30T09:31:59.296756+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:19.514982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":224,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345018+0000","last_change":"2023-09-30T09:21:29.092042+0000","last_active":"2023-09-30T09:31:59.345018+0000","last_peered":"2023-09-30T09:31:59.345018+0000","last_clean":"2023-09-30T09:31:59.345018+0000","last_became_active":"2023-09-30T09:21:29.091734+0000","last_became_peered":"2023-09-30T09:21:29.091734+0000","last_unstale":"2023-09-30T09:31:59.345018+0000","last_undegraded":"2023-09-30T09:31:59.345018+0000","last_fullsized":"2023-09-30T09:31:59.345018+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:26.782390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"283'202","reported_seq":217,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.766483+0000","last_change":"2023-09-30T09:31:55.916650+0000","last_active":"2023-09-30T09:32:03.766483+0000","last_peered":"2023-09-30T09:32:03.766483+0000","last_clean":"2023-09-30T09:32:03.766483+0000","last_became_active":"2023-09-30T09:31:55.916340+0000","last_became_peered":"2023-09-30T09:31:55.916340+0000","last_unstale":"2023-09-30T09:32:03.766483+0000","last_undegraded":"2023-09-30T09:32:03.766483+0000","last_fullsized":"2023-09-30T09:32:03.766483+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:10:59.242297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13238272,"num_objects":202,"num_object_clones":0,"num_object_copies":606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":202,"num_write_kb":12928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":557,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.521827+0000","last_change":"2023-09-30T09:31:56.428763+0000","last_active":"2023-09-30T09:31:59.521827+0000","last_peered":"2023-09-30T09:31:59.521827+0000","last_clean":"2023-09-30T09:31:59.521827+0000","last_became_active":"2023-09-30T09:31:09.046256+0000","last_became_peered":"2023-09-30T09:31:09.046256+0000","last_unstale":"2023-09-30T09:31:59.521827+0000","last_undegraded":"2023-09-30T09:31:59.521827+0000","last_fullsized":"2023-09-30T09:31:59.521827+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:13:35.942235+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":628,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.347619+0000","last_change":"2023-09-30T09:23:08.048438+0000","last_active":"2023-09-30T09:31:59.347619+0000","last_peered":"2023-09-30T09:31:59.347619+0000","last_clean":"2023-09-30T09:31:59.347619+0000","last_became_active":"2023-09-30T09:23:08.023806+0000","last_became_peered":"2023-09-30T09:23:08.023806+0000","last_unstale":"2023-09-30T09:31:59.347619+0000","last_undegraded":"2023-09-30T09:31:59.347619+0000","last_fullsized":"2023-09-30T09:31:59.347619+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:53:12.312828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":269,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300629+0000","last_change":"2023-09-30T09:29:09.611292+0000","last_active":"2023-09-30T09:31:59.300629+0000","last_peered":"2023-09-30T09:31:59.300629+0000","last_clean":"2023-09-30T09:31:59.300629+0000","last_became_active":"2023-09-30T09:29:09.611015+0000","last_became_peered":"2023-09-30T09:29:09.611015+0000","last_unstale":"2023-09-30T09:31:59.300629+0000","last_undegraded":"2023-09-30T09:31:59.300629+0000","last_fullsized":"2023-09-30T09:31:59.300629+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:48.124378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.344656+0000","last_change":"2023-09-30T09:26:51.324803+0000","last_active":"2023-09-30T09:31:59.344656+0000","last_peered":"2023-09-30T09:31:59.344656+0000","last_clean":"2023-09-30T09:31:59.344656+0000","last_became_active":"2023-09-30T09:26:51.324443+0000","last_became_peered":"2023-09-30T09:26:51.324443+0000","last_unstale":"2023-09-30T09:31:59.344656+0000","last_undegraded":"2023-09-30T09:31:59.344656+0000","last_fullsized":"2023-09-30T09:31:59.344656+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.178931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"283'244","reported_seq":256,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.865367+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:32:03.865367+0000","last_peered":"2023-09-30T09:32:03.865367+0000","last_clean":"2023-09-30T09:32:03.865367+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:32:03.865367+0000","last_undegraded":"2023-09-30T09:32:03.865367+0000","last_fullsized":"2023-09-30T09:32:03.865367+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":244,"log_dups_size":0,"ondisk_log_size":244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15990784,"num_objects":244,"num_object_clones":0,"num_object_copies":732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":244,"num_write_kb":15616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":725,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300109+0000","last_change":"2023-09-30T09:31:53.422514+0000","last_active":"2023-09-30T09:31:59.300109+0000","last_peered":"2023-09-30T09:31:59.300109+0000","last_clean":"2023-09-30T09:31:59.300109+0000","last_became_active":"2023-09-30T09:30:28.364065+0000","last_became_peered":"2023-09-30T09:30:28.364065+0000","last_unstale":"2023-09-30T09:31:59.300109+0000","last_undegraded":"2023-09-30T09:31:59.300109+0000","last_fullsized":"2023-09-30T09:31:59.300109+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:17:12.430898+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":432,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.344783+0000","last_change":"2023-09-30T09:23:23.254676+0000","last_active":"2023-09-30T09:31:59.344783+0000","last_peered":"2023-09-30T09:31:59.344783+0000","last_clean":"2023-09-30T09:31:59.344783+0000","last_became_active":"2023-09-30T09:23:23.251548+0000","last_became_peered":"2023-09-30T09:23:23.251548+0000","last_unstale":"2023-09-30T09:31:59.344783+0000","last_undegraded":"2023-09-30T09:31:59.344783+0000","last_fullsized":"2023-09-30T09:31:59.344783+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:11:17.088785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":262,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346256+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:31:59.346256+0000","last_peered":"2023-09-30T09:31:59.346256+0000","last_clean":"2023-09-30T09:31:59.346256+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:31:59.346256+0000","last_undegraded":"2023-09-30T09:31:59.346256+0000","last_fullsized":"2023-09-30T09:31:59.346256+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297415+0000","last_change":"2023-09-30T09:29:09.611507+0000","last_active":"2023-09-30T09:31:59.297415+0000","last_peered":"2023-09-30T09:31:59.297415+0000","last_clean":"2023-09-30T09:31:59.297415+0000","last_became_active":"2023-09-30T09:29:09.611120+0000","last_became_peered":"2023-09-30T09:29:09.611120+0000","last_unstale":"2023-09-30T09:31:59.297415+0000","last_undegraded":"2023-09-30T09:31:59.297415+0000","last_fullsized":"2023-09-30T09:31:59.297415+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:31.622647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"283'693","reported_seq":708,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:07.103024+0000","last_change":"2023-09-30T09:31:55.919211+0000","last_active":"2023-09-30T09:32:07.103024+0000","last_peered":"2023-09-30T09:32:07.103024+0000","last_clean":"2023-09-30T09:32:07.103024+0000","last_became_active":"2023-09-30T09:31:55.918891+0000","last_became_peered":"2023-09-30T09:31:55.918891+0000","last_unstale":"2023-09-30T09:32:07.103024+0000","last_undegraded":"2023-09-30T09:32:07.103024+0000","last_fullsized":"2023-09-30T09:32:07.103024+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":693,"log_dups_size":0,"ondisk_log_size":693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:44:45.070556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45416448,"num_objects":693,"num_object_clones":0,"num_object_copies":2079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":693,"num_write_kb":44352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":782,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.541519+0000","last_change":"2023-09-30T09:31:52.056893+0000","last_active":"2023-09-30T09:31:59.541519+0000","last_peered":"2023-09-30T09:31:59.541519+0000","last_clean":"2023-09-30T09:31:59.541519+0000","last_became_active":"2023-09-30T09:30:28.370995+0000","last_became_peered":"2023-09-30T09:30:28.370995+0000","last_unstale":"2023-09-30T09:31:59.541519+0000","last_undegraded":"2023-09-30T09:31:59.541519+0000","last_fullsized":"2023-09-30T09:31:59.541519+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:08:13.636726+0000","scrub_duration":0.003069217,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":457,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297197+0000","last_change":"2023-09-30T09:29:36.005128+0000","last_active":"2023-09-30T09:31:59.297197+0000","last_peered":"2023-09-30T09:31:59.297197+0000","last_clean":"2023-09-30T09:31:59.297197+0000","last_became_active":"2023-09-30T09:29:36.004563+0000","last_became_peered":"2023-09-30T09:29:36.004563+0000","last_unstale":"2023-09-30T09:31:59.297197+0000","last_undegraded":"2023-09-30T09:31:59.297197+0000","last_fullsized":"2023-09-30T09:31:59.297197+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:59.839484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":231,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345710+0000","last_change":"2023-09-30T09:29:09.607111+0000","last_active":"2023-09-30T09:31:59.345710+0000","last_peered":"2023-09-30T09:31:59.345710+0000","last_clean":"2023-09-30T09:31:59.345710+0000","last_became_active":"2023-09-30T09:29:09.606807+0000","last_became_peered":"2023-09-30T09:29:09.606807+0000","last_unstale":"2023-09-30T09:31:59.345710+0000","last_undegraded":"2023-09-30T09:31:59.345710+0000","last_fullsized":"2023-09-30T09:31:59.345710+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:49.380239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":279,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.344973+0000","last_change":"2023-09-30T09:26:51.324281+0000","last_active":"2023-09-30T09:31:59.344973+0000","last_peered":"2023-09-30T09:31:59.344973+0000","last_clean":"2023-09-30T09:31:59.344973+0000","last_became_active":"2023-09-30T09:26:51.323932+0000","last_became_peered":"2023-09-30T09:26:51.323932+0000","last_unstale":"2023-09-30T09:31:59.344973+0000","last_undegraded":"2023-09-30T09:31:59.344973+0000","last_fullsized":"2023-09-30T09:31:59.344973+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:27.823433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"283'389","reported_seq":404,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:04.574338+0000","last_change":"2023-09-30T09:31:55.921724+0000","last_active":"2023-09-30T09:32:04.574338+0000","last_peered":"2023-09-30T09:32:04.574338+0000","last_clean":"2023-09-30T09:32:04.574338+0000","last_became_active":"2023-09-30T09:31:55.921408+0000","last_became_peered":"2023-09-30T09:31:55.921408+0000","last_unstale":"2023-09-30T09:32:04.574338+0000","last_undegraded":"2023-09-30T09:32:04.574338+0000","last_fullsized":"2023-09-30T09:32:04.574338+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":389,"log_dups_size":0,"ondisk_log_size":389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:26:27.212199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25493504,"num_objects":389,"num_object_clones":0,"num_object_copies":1167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":389,"num_write_kb":24896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":744,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300320+0000","last_change":"2023-09-30T09:31:16.303084+0000","last_active":"2023-09-30T09:31:59.300320+0000","last_peered":"2023-09-30T09:31:59.300320+0000","last_clean":"2023-09-30T09:31:59.300320+0000","last_became_active":"2023-09-30T09:31:16.302581+0000","last_became_peered":"2023-09-30T09:31:16.302581+0000","last_unstale":"2023-09-30T09:31:59.300320+0000","last_undegraded":"2023-09-30T09:31:59.300320+0000","last_fullsized":"2023-09-30T09:31:59.300320+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:52:26.865953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":743,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.541389+0000","last_change":"2023-09-30T09:31:54.097604+0000","last_active":"2023-09-30T09:31:59.541389+0000","last_peered":"2023-09-30T09:31:59.541389+0000","last_clean":"2023-09-30T09:31:59.541389+0000","last_became_active":"2023-09-30T09:30:28.372087+0000","last_became_peered":"2023-09-30T09:30:28.372087+0000","last_unstale":"2023-09-30T09:31:59.541389+0000","last_undegraded":"2023-09-30T09:31:59.541389+0000","last_fullsized":"2023-09-30T09:31:59.541389+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:22:13.599174+0000","scrub_duration":0.0016120310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":200,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297131+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:31:59.297131+0000","last_peered":"2023-09-30T09:31:59.297131+0000","last_clean":"2023-09-30T09:31:59.297131+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:31:59.297131+0000","last_undegraded":"2023-09-30T09:31:59.297131+0000","last_fullsized":"2023-09-30T09:31:59.297131+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":225,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.521460+0000","last_change":"2023-09-30T09:12:09.760588+0000","last_active":"2023-09-30T09:31:59.521460+0000","last_peered":"2023-09-30T09:31:59.521460+0000","last_clean":"2023-09-30T09:31:59.521460+0000","last_became_active":"2023-09-30T09:12:09.760291+0000","last_became_peered":"2023-09-30T09:12:09.760291+0000","last_unstale":"2023-09-30T09:31:59.521460+0000","last_undegraded":"2023-09-30T09:31:59.521460+0000","last_fullsized":"2023-09-30T09:31:59.521460+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:22.696206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"283'302","reported_seq":314,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:04.234622+0000","last_change":"2023-09-30T09:31:55.933232+0000","last_active":"2023-09-30T09:32:04.234622+0000","last_peered":"2023-09-30T09:32:04.234622+0000","last_clean":"2023-09-30T09:32:04.234622+0000","last_became_active":"2023-09-30T09:31:55.932957+0000","last_became_peered":"2023-09-30T09:31:55.932957+0000","last_unstale":"2023-09-30T09:32:04.234622+0000","last_undegraded":"2023-09-30T09:32:04.234622+0000","last_fullsized":"2023-09-30T09:32:04.234622+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":302,"log_dups_size":0,"ondisk_log_size":302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:21:31.767047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19791872,"num_objects":302,"num_object_clones":0,"num_object_copies":906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":302,"num_write_kb":19328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":709,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346327+0000","last_change":"2023-09-30T09:31:54.412763+0000","last_active":"2023-09-30T09:31:59.346327+0000","last_peered":"2023-09-30T09:31:59.346327+0000","last_clean":"2023-09-30T09:31:59.346327+0000","last_became_active":"2023-09-30T09:30:28.369480+0000","last_became_peered":"2023-09-30T09:30:28.369480+0000","last_unstale":"2023-09-30T09:31:59.346327+0000","last_undegraded":"2023-09-30T09:31:59.346327+0000","last_fullsized":"2023-09-30T09:31:59.346327+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:32.875767+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":843,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.541305+0000","last_change":"2023-09-30T09:24:20.539818+0000","last_active":"2023-09-30T09:31:59.541305+0000","last_peered":"2023-09-30T09:31:59.541305+0000","last_clean":"2023-09-30T09:31:59.541305+0000","last_became_active":"2023-09-30T09:24:20.539443+0000","last_became_peered":"2023-09-30T09:24:20.539443+0000","last_unstale":"2023-09-30T09:31:59.541305+0000","last_undegraded":"2023-09-30T09:31:59.541305+0000","last_fullsized":"2023-09-30T09:31:59.541305+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:39.775820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":201,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.294604+0000","last_change":"2023-09-30T09:26:51.316523+0000","last_active":"2023-09-30T09:31:59.294604+0000","last_peered":"2023-09-30T09:31:59.294604+0000","last_clean":"2023-09-30T09:31:59.294604+0000","last_became_active":"2023-09-30T09:26:51.316171+0000","last_became_peered":"2023-09-30T09:26:51.316171+0000","last_unstale":"2023-09-30T09:31:59.294604+0000","last_undegraded":"2023-09-30T09:31:59.294604+0000","last_fullsized":"2023-09-30T09:31:59.294604+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:16.190372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":189,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.347088+0000","last_change":"2023-09-30T09:29:09.611376+0000","last_active":"2023-09-30T09:31:59.347088+0000","last_peered":"2023-09-30T09:31:59.347088+0000","last_clean":"2023-09-30T09:31:59.347088+0000","last_became_active":"2023-09-30T09:29:09.611090+0000","last_became_peered":"2023-09-30T09:29:09.611090+0000","last_unstale":"2023-09-30T09:31:59.347088+0000","last_undegraded":"2023-09-30T09:31:59.347088+0000","last_fullsized":"2023-09-30T09:31:59.347088+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:07:15.039397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"283'318","reported_seq":329,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:04.186133+0000","last_change":"2023-09-30T09:31:55.918185+0000","last_active":"2023-09-30T09:32:04.186133+0000","last_peered":"2023-09-30T09:32:04.186133+0000","last_clean":"2023-09-30T09:32:04.186133+0000","last_became_active":"2023-09-30T09:31:55.917905+0000","last_became_peered":"2023-09-30T09:31:55.917905+0000","last_unstale":"2023-09-30T09:32:04.186133+0000","last_undegraded":"2023-09-30T09:32:04.186133+0000","last_fullsized":"2023-09-30T09:32:04.186133+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":318,"log_dups_size":0,"ondisk_log_size":318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:28.915169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20840448,"num_objects":318,"num_object_clones":0,"num_object_copies":954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":318,"num_write_kb":20352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":557,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.521565+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:31:59.521565+0000","last_peered":"2023-09-30T09:31:59.521565+0000","last_clean":"2023-09-30T09:31:59.521565+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:31:59.521565+0000","last_undegraded":"2023-09-30T09:31:59.521565+0000","last_fullsized":"2023-09-30T09:31:59.521565+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":973,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.542125+0000","last_change":"2023-09-30T09:26:54.259826+0000","last_active":"2023-09-30T09:31:59.542125+0000","last_peered":"2023-09-30T09:31:59.542125+0000","last_clean":"2023-09-30T09:31:59.542125+0000","last_became_active":"2023-09-30T09:26:54.259011+0000","last_became_peered":"2023-09-30T09:26:54.259011+0000","last_unstale":"2023-09-30T09:31:59.542125+0000","last_undegraded":"2023-09-30T09:31:59.542125+0000","last_fullsized":"2023-09-30T09:31:59.542125+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:25:03.932355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":247,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.347796+0000","last_change":"2023-09-30T09:29:09.611808+0000","last_active":"2023-09-30T09:31:59.347796+0000","last_peered":"2023-09-30T09:31:59.347796+0000","last_clean":"2023-09-30T09:31:59.347796+0000","last_became_active":"2023-09-30T09:29:09.611484+0000","last_became_peered":"2023-09-30T09:29:09.611484+0000","last_unstale":"2023-09-30T09:31:59.347796+0000","last_undegraded":"2023-09-30T09:31:59.347796+0000","last_fullsized":"2023-09-30T09:31:59.347796+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:19.562461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":173,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.294346+0000","last_change":"2023-09-30T09:23:08.024454+0000","last_active":"2023-09-30T09:31:59.294346+0000","last_peered":"2023-09-30T09:31:59.294346+0000","last_clean":"2023-09-30T09:31:59.294346+0000","last_became_active":"2023-09-30T09:23:08.024130+0000","last_became_peered":"2023-09-30T09:23:08.024130+0000","last_unstale":"2023-09-30T09:31:59.294346+0000","last_undegraded":"2023-09-30T09:31:59.294346+0000","last_fullsized":"2023-09-30T09:31:59.294346+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:25.163177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"283'222","reported_seq":237,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.762692+0000","last_change":"2023-09-30T09:31:55.916432+0000","last_active":"2023-09-30T09:32:03.762692+0000","last_peered":"2023-09-30T09:32:03.762692+0000","last_clean":"2023-09-30T09:32:03.762692+0000","last_became_active":"2023-09-30T09:31:55.916097+0000","last_became_peered":"2023-09-30T09:31:55.916097+0000","last_unstale":"2023-09-30T09:32:03.762692+0000","last_undegraded":"2023-09-30T09:32:03.762692+0000","last_fullsized":"2023-09-30T09:32:03.762692+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":222,"log_dups_size":0,"ondisk_log_size":222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:27:52.188983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14548992,"num_objects":222,"num_object_clones":0,"num_object_copies":666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":222,"num_write_kb":14208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":735,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346213+0000","last_change":"2023-09-30T09:31:53.832809+0000","last_active":"2023-09-30T09:31:59.346213+0000","last_peered":"2023-09-30T09:31:59.346213+0000","last_clean":"2023-09-30T09:31:59.346213+0000","last_became_active":"2023-09-30T09:31:09.042735+0000","last_became_peered":"2023-09-30T09:31:09.042735+0000","last_unstale":"2023-09-30T09:31:59.346213+0000","last_undegraded":"2023-09-30T09:31:59.346213+0000","last_fullsized":"2023-09-30T09:31:59.346213+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:11:47.700087+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":269,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297642+0000","last_change":"2023-09-30T09:23:08.040696+0000","last_active":"2023-09-30T09:31:59.297642+0000","last_peered":"2023-09-30T09:31:59.297642+0000","last_clean":"2023-09-30T09:31:59.297642+0000","last_became_active":"2023-09-30T09:23:08.026654+0000","last_became_peered":"2023-09-30T09:23:08.026654+0000","last_unstale":"2023-09-30T09:31:59.297642+0000","last_undegraded":"2023-09-30T09:31:59.297642+0000","last_fullsized":"2023-09-30T09:31:59.297642+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:35.978656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1233,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.347149+0000","last_change":"2023-09-30T09:20:23.577882+0000","last_active":"2023-09-30T09:31:59.347149+0000","last_peered":"2023-09-30T09:31:59.347149+0000","last_clean":"2023-09-30T09:31:59.347149+0000","last_became_active":"2023-09-30T09:20:23.577329+0000","last_became_peered":"2023-09-30T09:20:23.577329+0000","last_unstale":"2023-09-30T09:31:59.347149+0000","last_undegraded":"2023-09-30T09:31:59.347149+0000","last_fullsized":"2023-09-30T09:31:59.347149+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:02.832070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":269,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296751+0000","last_change":"2023-09-30T09:26:51.317166+0000","last_active":"2023-09-30T09:31:59.296751+0000","last_peered":"2023-09-30T09:31:59.296751+0000","last_clean":"2023-09-30T09:31:59.296751+0000","last_became_active":"2023-09-30T09:26:51.316298+0000","last_became_peered":"2023-09-30T09:26:51.316298+0000","last_unstale":"2023-09-30T09:31:59.296751+0000","last_undegraded":"2023-09-30T09:31:59.296751+0000","last_fullsized":"2023-09-30T09:31:59.296751+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:36:28.893226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"283'239","reported_seq":254,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.783713+0000","last_change":"2023-09-30T09:31:55.915505+0000","last_active":"2023-09-30T09:32:03.783713+0000","last_peered":"2023-09-30T09:32:03.783713+0000","last_clean":"2023-09-30T09:32:03.783713+0000","last_became_active":"2023-09-30T09:31:55.915138+0000","last_became_peered":"2023-09-30T09:31:55.915138+0000","last_unstale":"2023-09-30T09:32:03.783713+0000","last_undegraded":"2023-09-30T09:32:03.783713+0000","last_fullsized":"2023-09-30T09:32:03.783713+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":239,"log_dups_size":0,"ondisk_log_size":239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:15.429908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15663104,"num_objects":239,"num_object_clones":0,"num_object_copies":717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":239,"num_write_kb":15296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":704,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-30T09:32:03.613939+0000","last_change":"2023-09-30T09:32:03.614022+0000","last_active":"2023-09-30T09:32:03.613939+0000","last_peered":"2023-09-30T09:32:03.613939+0000","last_clean":"2023-09-30T09:32:03.613939+0000","last_became_active":"2023-09-30T09:30:28.372210+0000","last_became_peered":"2023-09-30T09:30:28.372210+0000","last_unstale":"2023-09-30T09:32:03.613939+0000","last_undegraded":"2023-09-30T09:32:03.613939+0000","last_fullsized":"2023-09-30T09:32:03.613939+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:00.459062+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":560,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.522065+0000","last_change":"2023-09-30T09:29:09.643760+0000","last_active":"2023-09-30T09:31:59.522065+0000","last_peered":"2023-09-30T09:31:59.522065+0000","last_clean":"2023-09-30T09:31:59.522065+0000","last_became_active":"2023-09-30T09:29:09.620803+0000","last_became_peered":"2023-09-30T09:29:09.620803+0000","last_unstale":"2023-09-30T09:31:59.522065+0000","last_undegraded":"2023-09-30T09:31:59.522065+0000","last_fullsized":"2023-09-30T09:31:59.522065+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:54:49.883760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":161,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.347350+0000","last_change":"2023-09-30T09:26:51.327191+0000","last_active":"2023-09-30T09:31:59.347350+0000","last_peered":"2023-09-30T09:31:59.347350+0000","last_clean":"2023-09-30T09:31:59.347350+0000","last_became_active":"2023-09-30T09:26:51.317962+0000","last_became_peered":"2023-09-30T09:26:51.317962+0000","last_unstale":"2023-09-30T09:31:59.347350+0000","last_undegraded":"2023-09-30T09:31:59.347350+0000","last_fullsized":"2023-09-30T09:31:59.347350+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:57.609717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":224,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345891+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:31:59.345891+0000","last_peered":"2023-09-30T09:31:59.345891+0000","last_clean":"2023-09-30T09:31:59.345891+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:31:59.345891+0000","last_undegraded":"2023-09-30T09:31:59.345891+0000","last_fullsized":"2023-09-30T09:31:59.345891+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":659,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296757+0000","last_change":"2023-09-30T09:31:55.916431+0000","last_active":"2023-09-30T09:31:59.296757+0000","last_peered":"2023-09-30T09:31:59.296757+0000","last_clean":"2023-09-30T09:31:59.296757+0000","last_became_active":"2023-09-30T09:31:55.915855+0000","last_became_peered":"2023-09-30T09:31:55.915855+0000","last_unstale":"2023-09-30T09:31:59.296757+0000","last_undegraded":"2023-09-30T09:31:59.296757+0000","last_fullsized":"2023-09-30T09:31:59.296757+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:25.074276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:25.074276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":141,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346315+0000","last_change":"2023-09-30T09:26:51.324760+0000","last_active":"2023-09-30T09:31:59.346315+0000","last_peered":"2023-09-30T09:31:59.346315+0000","last_clean":"2023-09-30T09:31:59.346315+0000","last_became_active":"2023-09-30T09:26:51.324418+0000","last_became_peered":"2023-09-30T09:26:51.324418+0000","last_unstale":"2023-09-30T09:31:59.346315+0000","last_undegraded":"2023-09-30T09:31:59.346315+0000","last_fullsized":"2023-09-30T09:31:59.346315+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.809278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":195,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.347701+0000","last_change":"2023-09-30T09:26:51.319142+0000","last_active":"2023-09-30T09:31:59.347701+0000","last_peered":"2023-09-30T09:31:59.347701+0000","last_clean":"2023-09-30T09:31:59.347701+0000","last_became_active":"2023-09-30T09:26:51.318804+0000","last_became_peered":"2023-09-30T09:26:51.318804+0000","last_unstale":"2023-09-30T09:31:59.347701+0000","last_undegraded":"2023-09-30T09:31:59.347701+0000","last_fullsized":"2023-09-30T09:31:59.347701+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:16:10.293459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":444,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297519+0000","last_change":"2023-09-30T09:31:55.928041+0000","last_active":"2023-09-30T09:31:59.297519+0000","last_peered":"2023-09-30T09:31:59.297519+0000","last_clean":"2023-09-30T09:31:59.297519+0000","last_became_active":"2023-09-30T09:31:55.927642+0000","last_became_peered":"2023-09-30T09:31:55.927642+0000","last_unstale":"2023-09-30T09:31:59.297519+0000","last_undegraded":"2023-09-30T09:31:59.297519+0000","last_fullsized":"2023-09-30T09:31:59.297519+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:28:28.453679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:28:28.453679+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":244,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300423+0000","last_change":"2023-09-30T09:20:23.574350+0000","last_active":"2023-09-30T09:31:59.300423+0000","last_peered":"2023-09-30T09:31:59.300423+0000","last_clean":"2023-09-30T09:31:59.300423+0000","last_became_active":"2023-09-30T09:20:23.573987+0000","last_became_peered":"2023-09-30T09:20:23.573987+0000","last_unstale":"2023-09-30T09:31:59.300423+0000","last_undegraded":"2023-09-30T09:31:59.300423+0000","last_fullsized":"2023-09-30T09:31:59.300423+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:52:59.067951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":226,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297149+0000","last_change":"2023-09-30T09:29:09.609752+0000","last_active":"2023-09-30T09:31:59.297149+0000","last_peered":"2023-09-30T09:31:59.297149+0000","last_clean":"2023-09-30T09:31:59.297149+0000","last_became_active":"2023-09-30T09:29:09.609242+0000","last_became_peered":"2023-09-30T09:29:09.609242+0000","last_unstale":"2023-09-30T09:31:59.297149+0000","last_undegraded":"2023-09-30T09:31:59.297149+0000","last_fullsized":"2023-09-30T09:31:59.297149+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:31.366509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":227,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300483+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:31:59.300483+0000","last_peered":"2023-09-30T09:31:59.300483+0000","last_clean":"2023-09-30T09:31:59.300483+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:31:59.300483+0000","last_undegraded":"2023-09-30T09:31:59.300483+0000","last_fullsized":"2023-09-30T09:31:59.300483+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":257,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296978+0000","last_change":"2023-09-30T09:29:09.621218+0000","last_active":"2023-09-30T09:31:59.296978+0000","last_peered":"2023-09-30T09:31:59.296978+0000","last_clean":"2023-09-30T09:31:59.296978+0000","last_became_active":"2023-09-30T09:29:09.620972+0000","last_became_peered":"2023-09-30T09:29:09.620972+0000","last_unstale":"2023-09-30T09:31:59.296978+0000","last_undegraded":"2023-09-30T09:31:59.296978+0000","last_fullsized":"2023-09-30T09:31:59.296978+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:25.640288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":667,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:32:00.045851+0000","last_change":"2023-09-30T09:32:00.045773+0000","last_active":"2023-09-30T09:32:00.045851+0000","last_peered":"2023-09-30T09:32:00.045851+0000","last_clean":"2023-09-30T09:32:00.045851+0000","last_became_active":"2023-09-30T09:31:55.930054+0000","last_became_peered":"2023-09-30T09:31:55.930054+0000","last_unstale":"2023-09-30T09:32:00.045851+0000","last_undegraded":"2023-09-30T09:32:00.045851+0000","last_fullsized":"2023-09-30T09:32:00.045851+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:23:06.154566+0000","scrub_duration":0.0013966460000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":264,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.297923+0000","last_change":"2023-09-30T09:29:09.615385+0000","last_active":"2023-09-30T09:31:59.297923+0000","last_peered":"2023-09-30T09:31:59.297923+0000","last_clean":"2023-09-30T09:31:59.297923+0000","last_became_active":"2023-09-30T09:29:09.614479+0000","last_became_peered":"2023-09-30T09:29:09.614479+0000","last_unstale":"2023-09-30T09:31:59.297923+0000","last_undegraded":"2023-09-30T09:31:59.297923+0000","last_fullsized":"2023-09-30T09:31:59.297923+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:14.671222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":294,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.296867+0000","last_change":"2023-09-30T09:29:09.600457+0000","last_active":"2023-09-30T09:31:59.296867+0000","last_peered":"2023-09-30T09:31:59.296867+0000","last_clean":"2023-09-30T09:31:59.296867+0000","last_became_active":"2023-09-30T09:29:09.600080+0000","last_became_peered":"2023-09-30T09:29:09.600080+0000","last_unstale":"2023-09-30T09:31:59.296867+0000","last_undegraded":"2023-09-30T09:31:59.296867+0000","last_fullsized":"2023-09-30T09:31:59.296867+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:10:02.250349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":740,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.346611+0000","last_change":"2023-09-30T09:31:59.346611+0000","last_active":"2023-09-30T09:31:59.346611+0000","last_peered":"2023-09-30T09:31:59.346611+0000","last_clean":"2023-09-30T09:31:59.346611+0000","last_became_active":"2023-09-30T09:31:55.916135+0000","last_became_peered":"2023-09-30T09:31:55.916135+0000","last_unstale":"2023-09-30T09:31:59.346611+0000","last_undegraded":"2023-09-30T09:31:59.346611+0000","last_fullsized":"2023-09-30T09:31:59.346611+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:44:37.270612+0000","scrub_duration":0.004695744,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":166,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.300638+0000","last_change":"2023-09-30T09:26:51.337235+0000","last_active":"2023-09-30T09:31:59.300638+0000","last_peered":"2023-09-30T09:31:59.300638+0000","last_clean":"2023-09-30T09:31:59.300638+0000","last_became_active":"2023-09-30T09:26:51.324484+0000","last_became_peered":"2023-09-30T09:26:51.324484+0000","last_unstale":"2023-09-30T09:31:59.300638+0000","last_undegraded":"2023-09-30T09:31:59.300638+0000","last_fullsized":"2023-09-30T09:31:59.300638+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:43.684501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":241,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-30T09:31:59.345867+0000","last_change":"2023-09-30T09:20:23.577629+0000","last_active":"2023-09-30T09:31:59.345867+0000","last_peered":"2023-09-30T09:31:59.345867+0000","last_clean":"2023-09-30T09:31:59.345867+0000","last_became_active":"2023-09-30T09:20:23.577251+0000","last_became_peered":"2023-09-30T09:20:23.577251+0000","last_unstale":"2023-09-30T09:31:59.345867+0000","last_undegraded":"2023-09-30T09:31:59.345867+0000","last_fullsized":"2023-09-30T09:31:59.345867+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:20.201604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":363593728,"num_objects":5548,"num_object_clones":0,"num_object_copies":16644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5548,"num_write_kb":355072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1164247040,"data_stored":1164247040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5548,"ondisk_log_size":5548,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":1418,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16899,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":397806,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":68,"num_read_kb":126,"num_write":33,"num_write_kb":330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":703370,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234245,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":422720,"kb_used_data":201464,"kb_used_omap":0,"kb_used_meta":221248,"kb_avail":93322432,"statfs":{"total":95995035648,"available":95562170368,"internally_reserved":0,"allocated":206299136,"data_stored":205278607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":226557952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:31:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.343,"15min":1.0649999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.224},"max":{"1min":2.919,"5min":18.353000000000002,"15min":18.353000000000002},"last":0.495},{"interface":"front","average":{"1min":1.458,"5min":1.3120000000000001,"15min":0.996},"min":{"1min":0.35399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.5229999999999997,"5min":18.097999999999999,"15min":18.097999999999999},"last":0.33400000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:31:46 2023","interfaces":[{"interface":"back","average":{"1min":5.931,"5min":5.242,"15min":2.8500000000000001},"min":{"1min":0.35699999999999998,"5min":0.22,"15min":0.22},"max":{"1min":68.329999999999998,"5min":70.566999999999993,"15min":82.025999999999996},"last":0.54700000000000004},{"interface":"front","average":{"1min":5.4539999999999997,"5min":4.9500000000000002,"15min":2.7770000000000001},"min":{"1min":0.31900000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":68.266999999999996,"5min":70.626000000000005,"15min":82.012},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":2.0249999999999999,"15min":1.306},"min":{"1min":0.34499999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.6159999999999997,"5min":39.445,"15min":39.445},"last":0.504},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.9510000000000001,"15min":1.3400000000000001},"min":{"1min":0.38300000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.8479999999999999,"5min":40.988999999999997,"15min":40.988999999999997},"last":0.436}]},{"osd":3,"last update":"Sat Sep 30 09:31:46 2023","interfaces":[{"interface":"back","average":{"1min":11.862,"5min":5.3719999999999999,"15min":2.444},"min":{"1min":0.34499999999999997,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":155.31,"5min":155.31,"15min":155.31},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.6839999999999999,"5min":3.2810000000000001,"15min":1.6439999999999999},"min":{"1min":0.441,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.9429999999999996,"5min":77.048000000000002,"15min":77.048000000000002},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.2170000000000001,"15min":0.75},"min":{"1min":0.23899999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.8849999999999998,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0029999999999999,"15min":0.73499999999999999},"min":{"1min":0.25700000000000001,"5min":0.251,"15min":0.251},"max":{"1min":7.8890000000000002,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":0.373}]},{"osd":5,"last update":"Sat Sep 30 09:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.4139999999999999,"15min":0.87},"min":{"1min":0.27700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":6.6500000000000004,"5min":9.6790000000000003,"15min":9.6790000000000003},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.103,"5min":1.3660000000000001,"15min":0.88600000000000001},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.28599999999999998},"max":{"1min":3.79,"5min":11.022,"15min":11.022},"last":0.53800000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.2769999999999999,"15min":0.85499999999999998},"min":{"1min":0.36899999999999999,"5min":0.33300000000000002,"15min":0.33000000000000002},"max":{"1min":2.9670000000000001,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.1220000000000001,"15min":0.81899999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.915,"5min":7.915,"15min":7.915},"last":0.41899999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495882,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358260,"kb_used_data":132012,"kb_used_omap":0,"kb_used_meta":226240,"kb_avail":93386892,"statfs":{"total":95995035648,"available":95628177408,"internally_reserved":0,"allocated":135180288,"data_stored":134171376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":231669760},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:31:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":2.0129999999999999,"15min":1.27},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":6.1079999999999997,"5min":45.927999999999997,"15min":45.927999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.0939999999999999,"5min":2.585,"15min":1.482},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.23499999999999999},"max":{"1min":5.7649999999999997,"5min":45.316000000000003,"15min":45.316000000000003},"last":0.66800000000000004}]},{"osd":1,"last update":"Sat Sep 30 09:31:40 2023","interfaces":[{"interface":"back","average":{"1min":8.6289999999999996,"5min":4.3449999999999998,"15min":2.0430000000000001},"min":{"1min":0.28799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":76.433000000000007,"5min":76.433000000000007,"15min":76.433000000000007},"last":0.72999999999999998},{"interface":"front","average":{"1min":8.9809999999999999,"5min":4.718,"15min":2.2240000000000002},"min":{"1min":0.375,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":76.399000000000001,"5min":76.399000000000001,"15min":76.399000000000001},"last":0.73999999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:31:11 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.738,"15min":1.3540000000000001},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":2.5409999999999999,"5min":30.638999999999999,"15min":37.978000000000002},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.454,"15min":1.216},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.0810000000000004,"5min":29.759,"15min":37.759999999999998},"last":0.67900000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:31:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.2999999999999998,"15min":1.3600000000000001},"min":{"1min":0.33000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":31.353999999999999,"5min":31.353999999999999,"15min":31.353999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":3.6019999999999999,"5min":2.403,"15min":1.5049999999999999},"min":{"1min":0.40500000000000003,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":30.576000000000001,"5min":30.576000000000001,"15min":30.576000000000001},"last":0.76500000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:31:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6360000000000001,"5min":2.0329999999999999,"15min":1.403},"min":{"1min":0.30299999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.1059999999999999,"5min":49.286999999999999,"15min":73.656000000000006},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.0609999999999999,"5min":1.7010000000000001,"15min":1.2490000000000001},"min":{"1min":0.31900000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.7160000000000002,"5min":49.252000000000002,"15min":73.882999999999996},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:31:40 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":3.2789999999999999,"15min":1.591},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28499999999999998},"max":{"1min":10.103999999999999,"5min":76.613,"15min":76.613},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.8999999999999999,"5min":3.1699999999999999,"15min":1.5800000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":10.054,"5min":76.329999999999998,"15min":76.329999999999998},"last":0.52600000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:31:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":4.173,"15min":1.851},"min":{"1min":0.38700000000000001,"5min":0.34699999999999998,"15min":0.32100000000000001},"max":{"1min":8.0649999999999995,"5min":91.120999999999995,"15min":91.120999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.3220000000000001,"5min":4.0279999999999996,"15min":1.9059999999999999},"min":{"1min":0.39500000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":6.0490000000000004,"5min":91.733999999999995,"15min":91.733999999999995},"last":0.504}]}]},{"osd":1,"up_from":99,"seq":425201762447,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":797228,"kb_used_data":504036,"kb_used_omap":5,"kb_used_meta":293178,"kb_avail":92947924,"statfs":{"total":95995035648,"available":95178674176,"internally_reserved":0,"allocated":516132864,"data_stored":514555151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":300214454},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.3999999999999999,"15min":2.3570000000000002},"min":{"1min":0.23300000000000001,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":5.3710000000000004,"5min":88.045000000000002,"15min":150.93899999999999},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.415,"15min":2.4100000000000001},"min":{"1min":0.22500000000000001,"5min":0.216,"15min":0.216},"max":{"1min":3.8540000000000001,"5min":91.021000000000001,"15min":156.74199999999999},"last":0.51400000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:31:10 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":5.2649999999999997,"15min":2.823},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":3.2650000000000001,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.317},{"interface":"front","average":{"1min":1.0329999999999999,"5min":5.5270000000000001,"15min":2.839},"min":{"1min":0.28499999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.4199999999999999,"5min":144.62,"15min":144.62},"last":0.91000000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":4.242,"15min":1.9279999999999999},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.20200000000000001},"max":{"1min":5.9429999999999996,"5min":60.743000000000002,"15min":60.743000000000002},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.351,"5min":4.1699999999999999,"15min":1.8480000000000001},"min":{"1min":0.30599999999999999,"5min":0.29199999999999998,"15min":0.214},"max":{"1min":6.9619999999999997,"5min":61.201999999999998,"15min":61.201999999999998},"last":0.96499999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.361,"15min":0.98599999999999999},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.28299999999999997},"max":{"1min":3.8140000000000001,"5min":10.173,"15min":10.173},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.105,"5min":1.3360000000000001,"15min":0.99199999999999999},"min":{"1min":0.30399999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.5609999999999999,"5min":8.9299999999999997,"15min":8.9299999999999997},"last":1.02}]},{"osd":5,"last update":"Sat Sep 30 09:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":3.6509999999999998,"15min":1.732},"min":{"1min":0.315,"5min":0.308,"15min":0.30499999999999999},"max":{"1min":4.4669999999999996,"5min":195.512,"15min":195.512},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.228,"5min":1.365,"15min":1.02},"min":{"1min":0.40200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0060000000000002,"5min":10.128,"15min":10.128},"last":1.0509999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.5189999999999999,"15min":1.0720000000000001},"min":{"1min":0.438,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":15.791,"5min":15.791,"15min":15.791},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.2509999999999999,"5min":1.482,"15min":1.133},"min":{"1min":0.38700000000000001,"5min":0.375,"15min":0.313},"max":{"1min":4.4109999999999996,"5min":9.6660000000000004,"15min":15.864000000000001},"last":0.43099999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.7230000000000001,"15min":1.216},"min":{"1min":0.39300000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.6539999999999999,"5min":14.403,"15min":14.403},"last":0.98599999999999999},{"interface":"front","average":{"1min":1.9099999999999999,"5min":2.8500000000000001,"15min":1.655},"min":{"1min":0.42899999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.6519999999999992,"5min":122.56699999999999,"15min":122.56699999999999},"last":0.39600000000000002}]}]},{"osd":0,"up_from":94,"seq":403726925979,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347028,"kb_used_data":65676,"kb_used_omap":5,"kb_used_meta":281338,"kb_avail":93398124,"statfs":{"total":95995035648,"available":95639678976,"internally_reserved":0,"allocated":67252224,"data_stored":65581329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":288090365},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":3.4620000000000002,"15min":1.7110000000000001},"min":{"1min":0.23699999999999999,"5min":0.188,"15min":0.188},"max":{"1min":22.120999999999999,"5min":125.024,"15min":125.024},"last":1.7949999999999999},{"interface":"front","average":{"1min":1.915,"5min":3.2200000000000002,"15min":1.6619999999999999},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.192},"max":{"1min":22.058,"5min":124.901,"15min":124.901},"last":1.6839999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.9670000000000001,"15min":1.161},"min":{"1min":0.26400000000000001,"5min":0.23200000000000001,"15min":0.218},"max":{"1min":3.4260000000000002,"5min":55.823999999999998,"15min":55.823999999999998},"last":1.157},{"interface":"front","average":{"1min":0.94699999999999995,"5min":2.0950000000000002,"15min":1.232},"min":{"1min":0.27300000000000002,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":4.7249999999999996,"5min":55.790999999999997,"15min":55.790999999999997},"last":1.1240000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:31:46 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":2.5870000000000002,"15min":1.526},"min":{"1min":0.254,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":42.014000000000003,"5min":55.68,"15min":55.68},"last":1.552},{"interface":"front","average":{"1min":2.7810000000000001,"5min":2.649,"15min":1.462},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":42.380000000000003,"5min":55.244,"15min":55.244},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":2.379,"15min":1.3640000000000001},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":4.3490000000000002,"5min":121.486,"15min":121.486},"last":1.028},{"interface":"front","average":{"1min":1.228,"5min":2.843,"15min":1.611},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.31},"max":{"1min":3.6400000000000001,"5min":121.44499999999999,"15min":121.44499999999999},"last":1.048}]},{"osd":5,"last update":"Sat Sep 30 09:31:34 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":3.5379999999999998,"15min":1.9359999999999999},"min":{"1min":0.36199999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.1630000000000003,"5min":135.34899999999999,"15min":135.34899999999999},"last":1.8540000000000001},{"interface":"front","average":{"1min":1.1919999999999999,"5min":3.5350000000000001,"15min":1.9039999999999999},"min":{"1min":0.40899999999999997,"5min":0.35199999999999998,"15min":0.33700000000000002},"max":{"1min":5.7210000000000001,"5min":135.381,"15min":135.381},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:31:34 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.48,"15min":1.119},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.9180000000000001,"5min":13.811999999999999,"15min":13.811999999999999},"last":1.204},{"interface":"front","average":{"1min":1.617,"5min":1.3320000000000001,"15min":1.048},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":4.6849999999999996,"5min":6.407,"15min":7.556},"last":1.1870000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:31:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":5.298,"15min":2.4260000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":5.9199999999999999,"5min":150.96899999999999,"15min":150.96899999999999},"last":0.505},{"interface":"front","average":{"1min":1.786,"5min":5.0970000000000004,"15min":2.3079999999999998},"min":{"1min":0.39700000000000002,"5min":0.371,"15min":0.371},"max":{"1min":6.7889999999999997,"5min":150.328,"15min":150.328},"last":1.0820000000000001}]}]},{"osd":2,"up_from":0,"seq":446676598920,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435392,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":519016,"kb_used_data":134176,"kb_used_omap":2,"kb_used_meta":384829,"kb_avail":93226136,"statfs":{"total":95995035648,"available":95463563264,"internally_reserved":0,"allocated":137396224,"data_stored":135759493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":394065255},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.262,"15min":1.5620000000000001},"min":{"1min":0.20200000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":9.1539999999999999,"5min":10.856,"15min":130.946},"last":0.315},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.3049999999999999,"15min":1.5680000000000001},"min":{"1min":0.28100000000000003,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":4.694,"5min":12.361000000000001,"15min":130.96899999999999},"last":0.375}]},{"osd":1,"last update":"Sat Sep 30 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":2.9630000000000001,"15min":1.579},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.224},"max":{"1min":8.5,"5min":133.215,"15min":133.215},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.4259999999999999,"5min":3.069,"15min":1.5509999999999999},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.221},"max":{"1min":5.6859999999999999,"5min":133.16,"15min":133.16},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.7989999999999999,"15min":2.0009999999999999},"min":{"1min":0.32300000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5999999999999996,"5min":29.986000000000001,"15min":145.75899999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.9239999999999999,"15min":1.9159999999999999},"min":{"1min":0.35699999999999998,"5min":0.26200000000000001,"15min":0.22900000000000001},"max":{"1min":12.542,"5min":30.242999999999999,"15min":150.358},"last":0.44}]},{"osd":4,"last update":"Sat Sep 30 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":1.409,"15min":0.995},"min":{"1min":0.42599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.0199999999999996,"5min":15.960000000000001,"15min":15.960000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.454,"5min":1.425,"15min":1.0589999999999999},"min":{"1min":0.38800000000000001,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":4.5650000000000004,"5min":15.638,"15min":15.638},"last":0.504}]},{"osd":5,"last update":"Sat Sep 30 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.5860000000000001,"15min":1.1519999999999999},"min":{"1min":0.36499999999999999,"5min":0.33100000000000002,"15min":0.308},"max":{"1min":5.7240000000000002,"5min":9.0310000000000006,"15min":10.048999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.6279999999999999,"15min":1.2490000000000001},"min":{"1min":0.374,"5min":0.35299999999999998,"15min":0.33500000000000002},"max":{"1min":4.7380000000000004,"5min":11.231999999999999,"15min":12.785},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.4610000000000001,"15min":1.0569999999999999},"min":{"1min":0.40200000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":7.9900000000000002,"5min":10.308999999999999,"15min":10.348000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.625,"5min":1.6799999999999999,"15min":1.208},"min":{"1min":0.41599999999999998,"5min":0.38800000000000001,"15min":0.34799999999999998},"max":{"1min":5.8899999999999997,"5min":14.69,"15min":14.69},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":3.5419999999999998,"15min":1.887},"min":{"1min":0.39900000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2270000000000003,"5min":226.03999999999999,"15min":226.03999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.8779999999999999,"5min":3.5939999999999999,"15min":1.7629999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":11.696999999999999,"5min":226.06100000000001,"15min":226.06100000000001},"last":0.63200000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019155,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":242036,"kb_used_data":62060,"kb_used_omap":0,"kb_used_meta":179968,"kb_avail":93503116,"statfs":{"total":95995035648,"available":95747190784,"internally_reserved":0,"allocated":63549440,"data_stored":62540609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":184287232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:31:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":2.6930000000000001,"15min":2.6070000000000002},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":5.5599999999999996,"5min":45.884999999999998,"15min":128.79300000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.3240000000000001,"5min":2.411,"15min":2.4590000000000001},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.248},"max":{"1min":11.744,"5min":45.871000000000002,"15min":128.38800000000001},"last":0.66300000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:31:41 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.8479999999999999,"15min":1.6180000000000001},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.245},"max":{"1min":3.7290000000000001,"5min":93.244,"15min":93.244},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.1339999999999999,"5min":3.0369999999999999,"15min":1.667},"min":{"1min":0.253,"5min":0.253,"15min":0.23999999999999999},"max":{"1min":2.9860000000000002,"5min":93.465999999999994,"15min":93.465999999999994},"last":0.63800000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:31:19 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":3.2719999999999998,"15min":2.4820000000000002},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.29799999999999999},"max":{"1min":4.4340000000000002,"5min":77.620000000000005,"15min":140.601},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.5820000000000001,"5min":3.2120000000000002,"15min":2.456},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.484,"5min":77.638999999999996,"15min":140.54900000000001},"last":0.77600000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:31:41 2023","interfaces":[{"interface":"back","average":{"1min":5.0389999999999997,"5min":5.9260000000000002,"15min":2.8140000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":70.241,"5min":281.42200000000003,"15min":281.42200000000003},"last":0.67900000000000005},{"interface":"front","average":{"1min":4.5250000000000004,"5min":5.7149999999999999,"15min":2.782},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32600000000000001},"max":{"1min":70.433999999999997,"5min":281.44999999999999,"15min":281.44999999999999},"last":0.72399999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:31:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.8090000000000002,"15min":1.5609999999999999},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.252},"max":{"1min":11.808,"5min":134.30600000000001,"15min":134.30600000000001},"last":0.44},{"interface":"front","average":{"1min":1.4430000000000001,"5min":2.839,"15min":1.619},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":5.7999999999999998,"5min":134.35300000000001,"15min":134.35300000000001},"last":0.48099999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:31:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.109,"15min":0.76400000000000001},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":4.6829999999999998,"5min":6.8440000000000003,"15min":6.8440000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.98599999999999999,"15min":0.81999999999999995},"min":{"1min":0.312,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":4.5650000000000004,"5min":11.779999999999999,"15min":11.779999999999999},"last":0.70099999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:31:19 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":3.0680000000000001,"15min":1.722},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":11.444000000000001,"5min":136.27099999999999,"15min":136.27099999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.3600000000000001,"5min":2.8460000000000001,"15min":1.3859999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":5.5279999999999996,"5min":136.60400000000001,"15min":136.60400000000001},"last":0.51300000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757519,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":232040,"kb_used_data":36576,"kb_used_omap":0,"kb_used_meta":195456,"kb_avail":93513112,"statfs":{"total":95995035648,"available":95757426688,"internally_reserved":0,"allocated":37453824,"data_stored":36453183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":200146944},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.212,"15min":1.9079999999999999},"min":{"1min":0.224,"5min":0.22,"15min":0.22},"max":{"1min":2.9500000000000002,"5min":9.0299999999999994,"15min":148.79900000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.859,"15min":2.1869999999999998},"min":{"1min":0.44700000000000001,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":6.524,"5min":19.004000000000001,"15min":148.779},"last":0.433}]},{"osd":1,"last update":"Sat Sep 30 09:31:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":3.0110000000000001,"15min":2.4590000000000001},"min":{"1min":0.252,"5min":0.25,"15min":0.25},"max":{"1min":2.9900000000000002,"5min":114.20999999999999,"15min":166.708},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.71499999999999997,"5min":3.1240000000000001,"15min":2.419},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.6059999999999999,"5min":114.051,"15min":166.607},"last":0.372}]},{"osd":2,"last update":"Sat Sep 30 09:31:11 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":3.2570000000000001,"15min":2.6850000000000001},"min":{"1min":0.35299999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.6669999999999998,"5min":131.655,"15min":165.768},"last":0.313},{"interface":"front","average":{"1min":1.2130000000000001,"5min":2.8100000000000001,"15min":3.1429999999999998},"min":{"1min":0.35499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.0129999999999999,"5min":131.77600000000001,"15min":166.24000000000001},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":14.27,"5min":4.2939999999999996,"15min":2.2490000000000001},"min":{"1min":0.43099999999999999,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":226.34700000000001,"5min":226.34700000000001,"15min":226.34700000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":13.971,"5min":4.0540000000000003,"15min":2.085},"min":{"1min":0.34599999999999997,"5min":0.313,"15min":0.313},"max":{"1min":226.36000000000001,"5min":226.36000000000001,"15min":226.36000000000001},"last":0.5}]},{"osd":4,"last update":"Sat Sep 30 09:31:22 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.131,"15min":0.75900000000000001},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.222},"max":{"1min":7.4299999999999997,"5min":7.4299999999999997,"15min":8.5359999999999996},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.121,"15min":0.79600000000000004},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":2.9830000000000001,"5min":9.9640000000000004,"15min":9.9640000000000004},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.482,"15min":0.93500000000000005},"min":{"1min":0.33600000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.724,"5min":14.23,"15min":14.23},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.766,"5min":1.5409999999999999,"15min":1.075},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.826000000000001,"5min":18.960000000000001,"15min":18.960000000000001},"last":0.48299999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:31:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.212,"15min":1.008},"min":{"1min":0.317,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":2.9359999999999999,"5min":7.609,"15min":19.175999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.0569999999999999,"15min":0.88},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":2.306,"5min":7.4900000000000002,"15min":8.8810000000000002},"last":0.56299999999999994}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5962,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":80019456,"data_stored":80019456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":490209280,"data_stored":490209280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":160038912,"data_stored":160038912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":69926912,"data_stored":69926912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":60096512,"data_stored":60096512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":138870784,"data_stored":138870784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":165085184,"data_stored":165085184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:32:11.636 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:32:11.636 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-09-30T09:32:11.637 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:11.705 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:12.041 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:12.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:12.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:12.145 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:12.178 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:12.231 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:12.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:12.281 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:32:12.318 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:32:12.327 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:12.337 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:12.357 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:32:12.387 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:12.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:32:12.421 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 10 16 22578 22562 140.897 93 0.00691218 0.00709105 2023-09-30T09:32:12.444 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:32:12.489 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:32:12.528 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:32:12.571 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:32:12.572 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:32:12.572 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:32:12.572 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:32:12.572 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:32:12.572 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:32:12.572 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:32:12.572 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:32:12.573 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:32:12.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:12 smithi019 ceph-mon[216815]: pgmap v681: 119 pgs: 119 active+clean; 958 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-30T09:32:12.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1405650426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:12 smithi019 ceph-mon[224043]: pgmap v681: 119 pgs: 119 active+clean; 958 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-30T09:32:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1405650426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:12.593 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:12.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:12.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:12.599 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:12 smithi167 ceph-mon[192880]: pgmap v681: 119 pgs: 119 active+clean; 958 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-30T09:32:12.599 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1405650426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:12.654 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:32:12.700 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:12.751 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:12.877 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:12.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:12.989 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:13.044 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:13.139 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:13.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:13.247 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:13.272 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:13.280 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:13.343 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:13.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:13.421 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 11 16 24620 24604 139.69 127.625 0.00553003 0.00715317 2023-09-30T09:32:13.452 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:13.494 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:13.599 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:13.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:13.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:13.708 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:13.764 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:13.832 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:13.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:13.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:13.862 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:32:13.938 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:14.020 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:32:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:32:13] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2023-09-30T09:32:14.421 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 12 16 26554 26538 138.122 120.875 0.00515065 0.0072355 2023-09-30T09:32:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:14 smithi019 ceph-mon[224043]: pgmap v682: 119 pgs: 119 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-30T09:32:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:14 smithi019 ceph-mon[216815]: pgmap v682: 119 pgs: 119 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-30T09:32:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:14 smithi167 ceph-mon[192880]: pgmap v682: 119 pgs: 119 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-30T09:32:15.421 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 13 16 28370 28354 136.228 113.5 0.00860729 0.00733456 2023-09-30T09:32:15.422 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:15.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:15.527 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:15.603 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:15.677 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:15.704 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:15.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:15.813 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:15.881 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:15.884 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:32:15.884 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:32:15.884 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:29172 7.3 e284/279 7:c373ec76:::benchmark_data_smithi019_404601_object29171:head v 284'1817, mlcod=284'1815)", 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:15.880853+0000", 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.00029856300000000001, 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00032813299999999998, 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880853+0000", 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.885 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880853+0000", 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880853+0000", 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880918+0000", 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5277000000000005e-05 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880933+0000", 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5461e-05 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880940+0000", 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3999999999999997e-06 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880991+0000", 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0927999999999998e-05 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881004+0000", 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3166000000000001e-05 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:29175 7.b e284/279)", 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:15.881127+0000", 2023-09-30T09:32:15.889 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.4805999999999999e-05, 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000124368, 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881127+0000", 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881127+0000", 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881126+0000", 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:15.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881195+0000", 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8664999999999994e-05 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881204+0000", 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1439999999999992e-06 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881209+0000", 2023-09-30T09:32:15.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6210000000000003e-06 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881247+0000", 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8188000000000001e-05 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.893 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:29176 7.b e284/279)", 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:15.881218+0000", 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 4294967295.9999337, 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4623000000000005e-05, 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "queued for pg", 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:15.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881218+0000", 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881218+0000", 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881218+0000", 2023-09-30T09:32:15.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881263+0000", 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4709000000000002e-05 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881269+0000", 2023-09-30T09:32:15.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2600000000000002e-06 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.881273+0000", 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.749e-06 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.897 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:29157 7.7 7:ee1432d7:::benchmark_data_smithi019_404601_object29156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:15.870812+0000", 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.010339724, 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010575263999999999, 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 29157 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.898 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.870812+0000", 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.870812+0000", 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:15.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.870811+0000", 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.870836+0000", 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4927e-05 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.870844+0000", 2023-09-30T09:32:15.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5539999999999998e-06 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.870848+0000", 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0419999999999996e-06 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.870886+0000", 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7808000000000001e-05 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.871026+0000", 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013996200000000001 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.871156+0000", 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129892 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.873734+0000", 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0025774140000000001 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874472+0000", 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00073891400000000002 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:29162 7.7 7:eb7d3203:::benchmark_data_smithi019_404601_object29161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:15.871530+0000", 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.009622294, 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0099521809999999992, 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 29162 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.904 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.871530+0000", 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.871530+0000", 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.871529+0000", 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.871641+0000", 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011246799999999999 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:15.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.871657+0000", 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5451000000000002e-05 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.871663+0000", 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9660000000000001e-06 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.872444+0000", 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00078163600000000005 2023-09-30T09:32:15.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.872557+0000", 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112737 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.872682+0000", 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000125079 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.873833+0000", 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011505249999999999 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874688+0000", 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00085566600000000004 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:29170 7.7 7:eeef8b0a:::benchmark_data_smithi019_404601_object29169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:15.874701+0000", 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0064511880000000001, 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0068806800000000001, 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 29170 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.910 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874701+0000", 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874701+0000", 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874700+0000", 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874728+0000", 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8328999999999999e-05 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:15.912 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874736+0000", 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.678e-06 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874741+0000", 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4410000000000003e-06 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874787+0000", 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6047999999999997e-05 2023-09-30T09:32:15.913 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.874883+0000", 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5997999999999998e-05 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.875017+0000", 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013479900000000001 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.876454+0000", 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014362789999999999 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.877668+0000", 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001214098 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:29173 7.c 7:31e3267d:::benchmark_data_smithi019_404601_object29172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:15.878390+0000", 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.002761571, 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032967019999999999, 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 29173 2023-09-30T09:32:15.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878390+0000", 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878390+0000", 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878389+0000", 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878433+0000", 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3547999999999997e-05 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878445+0000", 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2592000000000001e-05 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878450+0000", 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6299999999999997e-06 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878492+0000", 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2267e-05 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878632+0000", 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013980000000000001 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.920 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.878787+0000", 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000154473 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880063+0000", 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012765700000000001 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:15.880084+0000", 2023-09-30T09:32:15.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0893000000000001e-05 2023-09-30T09:32:15.922 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.922 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:15.922 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.922 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:15.922 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:32:15.922 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 7 2023-09-30T09:32:15.922 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:32:16.024 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:16.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:16.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:16.137 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:16.184 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:16.272 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:16.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:16.375 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:16.422 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 14 16 30075 30059 134.107 106.562 0.00411126 0.00744978 2023-09-30T09:32:16.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:16 smithi019 ceph-mon[224043]: pgmap v683: 119 pgs: 119 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-30T09:32:16.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:16 smithi019 ceph-mon[216815]: pgmap v683: 119 pgs: 119 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-30T09:32:16.453 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:16 smithi167 ceph-mon[192880]: pgmap v683: 119 pgs: 119 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-30T09:32:16.580 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:16.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:16.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:16.633 INFO:teuthology.orchestra.run.smithi019.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-09-30T09:32:16.684 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:16.734 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:17.040 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:17.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:17.100 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:32:17.145 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:17.202 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:17.321 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:32:17.422 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 15 16 31647 31631 131.717 98.25 0.0074446 0.00758592 2023-09-30T09:32:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/863685625' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-30T09:32:17.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/863685625' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-30T09:32:17.665 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/863685625' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-30T09:32:18.422 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 16 16 33279 33263 129.86 102 0.022916 0.00769518 2023-09-30T09:32:18.619 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:18.624 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:18 smithi019 ceph-mon[216815]: pgmap v684: 119 pgs: 119 active+clean; 1.6 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-09-30T09:32:18.624 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:18.624 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:18.625 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:18 smithi019 ceph-mon[224043]: pgmap v684: 119 pgs: 119 active+clean; 1.6 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-09-30T09:32:18.625 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:18.625 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:18.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:18.726 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:32:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:18 smithi167 ceph-mon[192880]: pgmap v684: 119 pgs: 119 active+clean; 1.6 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-09-30T09:32:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:18.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:18.784 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:19.046 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:19.104 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:19.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:19.211 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:19.278 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:19.418 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:19.423 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 17 16 34454 34438 126.533 73.4375 0.010321 0.00789667 2023-09-30T09:32:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:19.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:19.530 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:19.586 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:19.773 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:19.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:19.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:19.879 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:19.942 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:20.081 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:32:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:32:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:32:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:32:20.082 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:32:20.326 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:20.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:20.388 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:32:20.423 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 18 16 36241 36225 125.708 111.688 0.00655606 0.00794924 2023-09-30T09:32:20.432 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:20 smithi167 ceph-mon[192880]: pgmap v685: 119 pgs: 119 active+clean; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-09-30T09:32:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1064000177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:20.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:20 smithi019 ceph-mon[224043]: pgmap v685: 119 pgs: 119 active+clean; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-09-30T09:32:20.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1064000177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:20.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:20 smithi019 ceph-mon[216815]: pgmap v685: 119 pgs: 119 active+clean; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-09-30T09:32:20.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1064000177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:20.753 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:21.151 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:32:21.369 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:21.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:21.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:21.423 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 19 16 37735 37719 124.007 93.375 0.012038 0.00805865 2023-09-30T09:32:21.477 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:21 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1064000177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:21 smithi167 ceph-mon[192880]: failed to encode map e285 with expected crc 2023-09-30T09:32:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:21 smithi167 ceph-mon[192880]: osdmap e285: 8 total, 8 up, 7 in 2023-09-30T09:32:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:21 smithi167 ceph-mon[192880]: failed to encode map e285 with expected crc 2023-09-30T09:32:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:21 smithi167 ceph-mon[192880]: failed to encode map e285 with expected crc 2023-09-30T09:32:21.693 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:21.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:21.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1064000177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[224043]: failed to encode map e285 with expected crc 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[224043]: osdmap e285: 8 total, 8 up, 7 in 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[224043]: failed to encode map e285 with expected crc 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[224043]: failed to encode map e285 with expected crc 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1064000177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[216815]: failed to encode map e285 with expected crc 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[216815]: osdmap e285: 8 total, 8 up, 7 in 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[216815]: failed to encode map e285 with expected crc 2023-09-30T09:32:21.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:21 smithi019 ceph-mon[216815]: failed to encode map e285 with expected crc 2023-09-30T09:32:22.299 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:22.425 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:32:22.424796+0000 min lat: 0.00127075 max lat: 0.26975 avg lat: 0.00819724 2023-09-30T09:32:22.425 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:32:22.425 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 20 16 39055 39039 121.925 82.5 0.00528842 0.00819724 2023-09-30T09:32:22.508 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:22.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:22.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:22.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:22 smithi167 ceph-mon[192880]: pgmap v687: 119 pgs: 119 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-30T09:32:22.570 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:22 smithi019 ceph-mon[224043]: pgmap v687: 119 pgs: 119 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-30T09:32:22.571 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:22 smithi019 ceph-mon[216815]: pgmap v687: 119 pgs: 119 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-30T09:32:22.619 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:22.650 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:22.700 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:22.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:22.783 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:32:22.804 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:22.825 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:32:22.880 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:32:22.899 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:22.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:32:22.971 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:32:22.980 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:22.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:22.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:23.009 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:32:23.042 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:32:23.080 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:32:23.081 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:32:23.081 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:32:23.081 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:32:23.081 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:32:23.082 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:32:23.082 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:32:23.082 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:32:23.082 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:32:23.086 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:23.147 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:23.346 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:23.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:23.385 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/873867848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:23.425 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 21 16 40356 40340 119.991 81.3125 0.00179922 0.00832153 2023-09-30T09:32:23.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:23.516 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:23.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/873867848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:23.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/873867848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:23.613 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:23.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:23.720 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:32:23.720 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:32:23.720 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:23.751 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:32:23.752 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:32:23.752 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-30T09:32:23.752 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:32:23.752 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:32:23.752 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-30T09:32:23.948 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:32:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:32:23] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.43.0" 2023-09-30T09:32:23.951 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:24.223 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:24.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:24.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:24.326 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:24.334 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:32:24.363 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:24.425 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 22 16 41412 41396 117.538 66 0.0111387 0.00850264 2023-09-30T09:32:24.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:24 smithi019 ceph-mon[224043]: pgmap v688: 119 pgs: 119 active+clean; 2.1 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-30T09:32:24.514 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:24 smithi019 ceph-mon[216815]: pgmap v688: 119 pgs: 119 active+clean; 2.1 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-30T09:32:24.690 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:24 smithi167 ceph-mon[192880]: pgmap v688: 119 pgs: 119 active+clean; 2.1 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-30T09:32:24.716 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:32:24.760 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:24.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:24.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:24.846 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:24.867 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:24.952 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:25.061 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:25.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:25.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:25.113 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:25.167 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:25.215 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:25.425 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 23 15 42926 42911 116.544 94.6875 0.00689797 0.00857534 2023-09-30T09:32:25.464 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:25.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:25.570 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:25.661 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:26.012 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:26.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:26.116 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:26.173 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:26.425 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 24 16 44103 44087 114.751 73.5 0.00484621 0.00867478 2023-09-30T09:32:26.426 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:26.427 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:26 smithi167 ceph-mon[192880]: pgmap v689: 119 pgs: 119 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-30T09:32:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:26.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:26.469 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:26.529 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:26.559 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:32:26.567 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:26.687 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:26.705 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:26 smithi019 ceph-mon[224043]: pgmap v689: 119 pgs: 119 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-30T09:32:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:26.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:26.709 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:26 smithi019 ceph-mon[216815]: pgmap v689: 119 pgs: 119 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-30T09:32:26.810 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:26.885 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:27.072 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:27.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:27.185 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:27.232 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:27.425 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 25 16 45307 45291 113.172 75.25 0.00852808 0.0088284 2023-09-30T09:32:27.632 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:27.675 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:32:27.675 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:32:27.675 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:32:27.908 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:27.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:27.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:28.015 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:28.061 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:28.241 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:28.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:28.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:28.348 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:28.389 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:28.417 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:32:28.427 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 26 16 46377 46361 111.382 66.875 0.0575002 0.00897119 2023-09-30T09:32:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:28 smithi167 ceph-mon[192880]: pgmap v690: 119 pgs: 119 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-30T09:32:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3343268455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:32:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:28 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:32:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:28 smithi019 ceph-mon[216815]: pgmap v690: 119 pgs: 119 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-30T09:32:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3343268455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:32:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:28 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:32:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:28 smithi019 ceph-mon[224043]: pgmap v690: 119 pgs: 119 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-30T09:32:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3343268455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:32:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:28 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:32:28.742 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:28.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:28.851 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:28.890 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:29.023 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:29.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:29.132 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:29.319 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:32:29.319 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:32:29.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 27 16 47813 47797 110.581 89.75 0.00838413 0.00903867 2023-09-30T09:32:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:32:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[224043]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[224043]: osdmap e286: 8 total, 8 up, 7 in 2023-09-30T09:32:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[224043]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[224043]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[224043]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.712 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[224043]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:29.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:32:29.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[216815]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[216815]: osdmap e286: 8 total, 8 up, 7 in 2023-09-30T09:32:29.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[216815]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[216815]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[216815]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:29 smithi019 ceph-mon[216815]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.721 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:29 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:29 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:32:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:29 smithi167 ceph-mon[192880]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:29 smithi167 ceph-mon[192880]: osdmap e286: 8 total, 8 up, 7 in 2023-09-30T09:32:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:29 smithi167 ceph-mon[192880]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:29 smithi167 ceph-mon[192880]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:29 smithi167 ceph-mon[192880]: failed to encode map e286 with expected crc 2023-09-30T09:32:29.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:29 smithi167 ceph-mon[192880]: failed to encode map e286 with expected crc 2023-09-30T09:32:30.347 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:30.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:30.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:30.397 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:32:31.186 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:31.187 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 28 16 49141 49125 109.596 83 0.0107945 0.00911991 2023-09-30T09:32:31.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:30 smithi167 ceph-mon[192880]: pgmap v692: 119 pgs: 119 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-30T09:32:31.215 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:30 smithi019 ceph-mon[216815]: pgmap v692: 119 pgs: 119 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-30T09:32:31.216 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:30 smithi019 ceph-mon[224043]: pgmap v692: 119 pgs: 119 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-30T09:32:31.270 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:31.365 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:31.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:31.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 29 16 50596 50580 108.953 90.9375 0.0192224 0.00917154 2023-09-30T09:32:31.454 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:32:31.472 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:31.490 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:32:31.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:31 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4049982419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:31.526 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:32:31.533 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:31.567 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:32:31.608 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:32:31.619 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:31.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:31.649 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:32:31.688 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:32:31.727 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:32:31.730 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:31.737 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:32:31.737 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:32:31.737 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:32:31.737 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:32:31.737 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:32:31.738 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:32:31.738 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:32:31.738 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:32:31.768 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:31 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4049982419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:31.769 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:31 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4049982419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:31.778 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:31.881 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:31.882 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:31.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:31.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:31.997 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:32.058 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:32.139 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:32.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:32.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:32.244 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:32.303 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:32.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 30 16 52049 52033 108.348 90.8125 0.0168167 0.0092248 2023-09-30T09:32:32.492 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:32.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:32.596 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:32.663 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:32 smithi167 ceph-mon[192880]: 1.1e scrub starts 2023-09-30T09:32:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:32 smithi167 ceph-mon[192880]: 1.1e scrub ok 2023-09-30T09:32:32.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:32 smithi167 ceph-mon[192880]: pgmap v693: 119 pgs: 119 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-30T09:32:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:32 smithi019 ceph-mon[216815]: 1.1e scrub starts 2023-09-30T09:32:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:32 smithi019 ceph-mon[216815]: 1.1e scrub ok 2023-09-30T09:32:32.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:32 smithi019 ceph-mon[216815]: pgmap v693: 119 pgs: 119 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-30T09:32:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:32 smithi019 ceph-mon[224043]: 1.1e scrub starts 2023-09-30T09:32:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:32 smithi019 ceph-mon[224043]: 1.1e scrub ok 2023-09-30T09:32:32.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:32 smithi019 ceph-mon[224043]: pgmap v693: 119 pgs: 119 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-30T09:32:33.172 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:33.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:33.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:33.217 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:32:33.278 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:32:33.308 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:33.354 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:32:33.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:33.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:32:33.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 31 16 53127 53111 107.027 67.375 0.0121447 0.00933801 2023-09-30T09:32:33.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:33 smithi019 ceph-mon[216815]: 1.c scrub starts 2023-09-30T09:32:33.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:33 smithi019 ceph-mon[216815]: 1.c scrub ok 2023-09-30T09:32:33.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:33.448 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:33 smithi019 ceph-mon[224043]: 1.c scrub starts 2023-09-30T09:32:33.448 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:33 smithi019 ceph-mon[224043]: 1.c scrub ok 2023-09-30T09:32:33.449 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:33.463 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:33.530 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:33.653 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:33.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:33.768 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:33.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:33 smithi167 ceph-mon[192880]: 1.c scrub starts 2023-09-30T09:32:33.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:33 smithi167 ceph-mon[192880]: 1.c scrub ok 2023-09-30T09:32:33.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:33.808 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:32:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:32:33] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2023-09-30T09:32:33.829 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:33.891 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:32:33.891 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:32:33.892 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-30T09:32:33.892 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-30T09:32:33.892 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.1 2023-09-30T09:32:33.947 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:33.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:34.055 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:34.085 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:32:33 smithi019 systemd[1]: Stopping Ceph osd.1 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:32:34.097 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:34.105 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:34.178 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:34.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:34.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:34.283 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:34.360 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:34.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 32 16 54368 54352 106.106 77.5625 0.00516833 0.00941899 2023-09-30T09:32:34.478 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:32:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:32:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:32:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:32:34.536 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:32:34.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:34.538 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:34.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:34.639 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:34.690 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:34 smithi019 ceph-mon[216815]: pgmap v694: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-09-30T09:32:34.691 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:34 smithi019 ceph-mon[224043]: pgmap v694: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-09-30T09:32:34.700 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:34.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:34 smithi167 ceph-mon[192880]: pgmap v694: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-09-30T09:32:34.861 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:34.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:34.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:34.974 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:35.062 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:32:34 smithi019 conmon[246226]: 2023-09-30T09:32:34.779+0000 7ffb67d84700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:32:35.062 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:32:34 smithi019 conmon[246226]: 2023-09-30T09:32:34.779+0000 7ffb67d84700 -1 osd.1 286 *** Got signal Terminated *** 2023-09-30T09:32:35.062 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:32:34 smithi019 conmon[246226]: 2023-09-30T09:32:34.779+0000 7ffb67d84700 -1 osd.1 286 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:32:35.069 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:35.157 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:35.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:35.190 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:32:35.269 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:35.364 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:35.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 33 16 55697 55681 105.408 83.0625 0.00938044 0.00948137 2023-09-30T09:32:35.511 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:32:35.629 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:35.630 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:35 smithi019 ceph-mon[224043]: osd.1 marked itself down and dead 2023-09-30T09:32:35.631 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:35 smithi019 ceph-mon[216815]: osd.1 marked itself down and dead 2023-09-30T09:32:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:35.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:35.733 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:35.760 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:35 smithi167 ceph-mon[192880]: osd.1 marked itself down and dead 2023-09-30T09:32:35.860 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:32:35.929 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:35.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:36.032 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:36.064 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:36.370 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:36.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:36.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:36.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 34 16 55782 55766 102.465 5.3125 0.00267782 0.00948586 2023-09-30T09:32:36.458 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:32:36 smithi019 bash[410025]: 5db49d11f877fec7503c3d8c7920fd6b72cd872dc53bce943268a1d1cc209175 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: pgmap v695: 119 pgs: 119 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: osdmap e287: 8 total, 7 up, 7 in 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.459 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:36 smithi167 ceph-mon[192880]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.474 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:36.515 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:36.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: pgmap v695: 119 pgs: 119 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-30T09:32:36.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:32:36.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: osdmap e287: 8 total, 7 up, 7 in 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[216815]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: pgmap v695: 119 pgs: 119 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-30T09:32:36.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: osdmap e287: 8 total, 7 up, 7 in 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: failed to encode map e287 with expected crc 2023-09-30T09:32:36.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:36 smithi019 ceph-mon[224043]: failed to encode map e287 with expected crc 2023-09-30T09:32:37.061 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:37.202 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:37.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:37.309 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:37.388 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:37.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 35 16 56748 56732 101.263 60.375 0.0114199 0.00987065 2023-09-30T09:32:37.508 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:37.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:37.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:37.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: osdmap e288: 8 total, 7 up, 7 in 2023-09-30T09:32:37.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:37 smithi167 ceph-mon[192880]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.616 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:37.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: osdmap e288: 8 total, 7 up, 7 in 2023-09-30T09:32:37.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[224043]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.689 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.689 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.690 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.690 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.690 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: osdmap e288: 8 total, 7 up, 7 in 2023-09-30T09:32:37.690 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.690 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.690 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.690 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:37 smithi019 ceph-mon[216815]: failed to encode map e288 with expected crc 2023-09-30T09:32:37.698 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:37.873 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:37.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:37.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:37.979 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:38.004 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35928 6.9 6:9537c3ec:::benchmark_data_smithi019_355417_object35927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.258963+0000", 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 238.741688915, 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0146549929999999, 2023-09-30T09:32:38.005 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35928 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.006 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.258963+0000", 2023-09-30T09:32:38.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.258963+0000", 2023-09-30T09:32:38.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.258961+0000", 2023-09-30T09:32:38.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:32:38.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.009 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.009 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259005+0000", 2023-09-30T09:32:38.009 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3911e-05 2023-09-30T09:32:38.009 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.009 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259021+0000", 2023-09-30T09:32:38.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6305000000000001e-05 2023-09-30T09:32:38.010 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.011 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259029+0000", 2023-09-30T09:32:38.011 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2870000000000004e-06 2023-09-30T09:32:38.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.012 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.303449+0000", 2023-09-30T09:32:38.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.044419783999999997 2023-09-30T09:32:38.012 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.012 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.012 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.303461+0000", 2023-09-30T09:32:38.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1722999999999999e-05 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.273536+0000", 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97007547500000002 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.013 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.273618+0000", 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2009999999999996e-05 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35938 6.3 6:c4d9c85e:::benchmark_data_smithi019_355417_object35937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.259041+0000", 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 238.74161154500001, 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0085380530000001, 2023-09-30T09:32:38.014 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.015 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:32:38.015 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.015 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.015 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.015 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35938 2023-09-30T09:32:38.015 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.015 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.015 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259041+0000", 2023-09-30T09:32:38.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.016 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.016 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259041+0000", 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259040+0000", 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259081+0000", 2023-09-30T09:32:38.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1119999999999999e-05 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259088+0000", 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8809999999999998e-06 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259093+0000", 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4730000000000002e-06 2023-09-30T09:32:38.018 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.304743+0000", 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.045650759999999999 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.304756+0000", 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2493999999999999e-05 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.020 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.267492+0000", 2023-09-30T09:32:38.020 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96273649500000003 2023-09-30T09:32:38.020 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.020 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.020 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.020 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.267579+0000", 2023-09-30T09:32:38.020 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6328000000000004e-05 2023-09-30T09:32:38.020 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.021 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.021 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.021 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.021 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.021 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35943 6.5 6:a25db399:::benchmark_data_smithi019_355417_object35942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:32:38.022 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.259098+0000", 2023-09-30T09:32:38.022 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 238.741554224, 2023-09-30T09:32:38.022 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.026377788, 2023-09-30T09:32:38.022 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.022 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:32:38.023 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.023 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.023 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.023 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35943 2023-09-30T09:32:38.023 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.023 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.024 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.024 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.024 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259098+0000", 2023-09-30T09:32:38.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.024 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.025 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.025 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259098+0000", 2023-09-30T09:32:38.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.025 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259098+0000", 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259137+0000", 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9613999999999997e-05 2023-09-30T09:32:38.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259143+0000", 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1229999999999997e-06 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259147+0000", 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.602e-06 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.027 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.301689+0000", 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.042542569000000002 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.301699+0000", 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9729999999999992e-06 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.277352+0000", 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97565302099999995 2023-09-30T09:32:38.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.278033+0000", 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00068037499999999999 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.285404+0000", 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0073710829999999996 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.285476+0000", 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1805999999999997e-05 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35946 6.19 6:9c9e6053:::benchmark_data_smithi019_355417_object35945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.260313+0000", 2023-09-30T09:32:38.030 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 238.740339229, 2023-09-30T09:32:38.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.031708208, 2023-09-30T09:32:38.031 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.031 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.031 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.031 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.031 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.031 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35946 2023-09-30T09:32:38.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260313+0000", 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260313+0000", 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260310+0000", 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999971 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260393+0000", 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3176999999999993e-05 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260415+0000", 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1444999999999999e-05 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260423+0000", 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4530000000000002e-06 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.295680+0000", 2023-09-30T09:32:38.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.035256919999999997 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.295687+0000", 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5200000000000003e-06 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.286058+0000", 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99037160300000004 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.286180+0000", 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000122154 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.286342+0000", 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016122 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.289202+0000", 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028603779999999998 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.289609+0000", 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00040729999999999998 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.291947+0000", 2023-09-30T09:32:38.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002337979 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.291981+0000", 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4224999999999998e-05 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.292021+0000", 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9678000000000002e-05 2023-09-30T09:32:38.038 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35947 6.18 6:1d4ebeb7:::benchmark_data_smithi019_355417_object35946:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.260434+0000", 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 238.74021857, 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.036275292, 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.039 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35947 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260434+0000", 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260434+0000", 2023-09-30T09:32:38.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260433+0000", 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260466+0000", 2023-09-30T09:32:38.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2790999999999998e-05 2023-09-30T09:32:38.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260476+0000", 2023-09-30T09:32:38.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0001e-05 2023-09-30T09:32:38.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260481+0000", 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7570000000000004e-06 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.310702+0000", 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.050221644000000003 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.043 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.310710+0000", 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5689999999999999e-06 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.289877+0000", 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97916740800000002 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.289987+0000", 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0001093 2023-09-30T09:32:38.044 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.290120+0000", 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000133151 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.293018+0000", 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028981200000000001 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.293401+0000", 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000383376 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.296635+0000", 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032337440000000002 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.046 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.296671+0000", 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6304999999999999e-05 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.296709+0000", 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7605000000000003e-05 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57773 6.12 6:4838cbd4:::benchmark_data_smithi019_355417_object57772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.504618+0000", 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 215.49603451199999, 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99969614299999998, 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57773 2023-09-30T09:32:38.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504618+0000", 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504618+0000", 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504617+0000", 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504670+0000", 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2459e-05 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504681+0000", 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1459e-05 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504686+0000", 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8239999999999999e-06 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504738+0000", 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1758999999999997e-05 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504752+0000", 2023-09-30T09:32:38.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3971000000000001e-05 2023-09-30T09:32:38.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.501144+0000", 2023-09-30T09:32:38.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99639237899999999 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.501265+0000", 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012081199999999999 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.501368+0000", 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103235 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504170+0000", 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028016859999999998 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504225+0000", 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5120000000000001e-05 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504249+0000", 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4437e-05 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504278+0000", 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9125999999999999e-05 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504314+0000", 2023-09-30T09:32:38.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5395000000000002e-05 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57838 6.12 6:48b85783:::benchmark_data_smithi019_355417_object57837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.559281+0000", 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 215.44137140300001, 2023-09-30T09:32:38.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94707381400000001, 2023-09-30T09:32:38.057 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.057 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.057 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.057 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.057 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.057 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57838 2023-09-30T09:32:38.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.057 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559281+0000", 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559281+0000", 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559279+0000", 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559386+0000", 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010671999999999999 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559405+0000", 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9114999999999999e-05 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559412+0000", 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6200000000000001e-06 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559471+0000", 2023-09-30T09:32:38.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.923e-05 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559492+0000", 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0428000000000002e-05 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.502496+0000", 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94300395100000001 2023-09-30T09:32:38.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.062 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.502618+0000", 2023-09-30T09:32:38.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012216399999999999 2023-09-30T09:32:38.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.502768+0000", 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000149837 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504395+0000", 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016269979999999999 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504437+0000", 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2043999999999999e-05 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506274+0000", 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018373 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506318+0000", 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3639e-05 2023-09-30T09:32:38.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506355+0000", 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7048000000000002e-05 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57893 6.12 6:49e4acbe:::benchmark_data_smithi019_355417_object57892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.598530+0000", 2023-09-30T09:32:38.065 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 215.40212258599999, 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.910345187, 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57893 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598530+0000", 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598530+0000", 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598528+0000", 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598584+0000", 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6011999999999998e-05 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598597+0000", 2023-09-30T09:32:38.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2286999999999999e-05 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598610+0000", 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3129000000000001e-05 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598681+0000", 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1112000000000006e-05 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598690+0000", 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1419999999999999e-06 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.503408+0000", 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90471810900000005 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.503531+0000", 2023-09-30T09:32:38.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012254900000000001 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.503661+0000", 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000130588 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.505271+0000", 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001609826 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506227+0000", 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00095588200000000002 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508808+0000", 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002581341 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508842+0000", 2023-09-30T09:32:38.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4025e-05 2023-09-30T09:32:38.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508875+0000", 2023-09-30T09:32:38.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2490000000000002e-05 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57910 6.12 6:4bac1af3:::benchmark_data_smithi019_355417_object57909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.607418+0000", 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 215.39323463599999, 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90162827800000001, 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.074 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57910 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607418+0000", 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.075 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607418+0000", 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607416+0000", 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607471+0000", 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4985999999999998e-05 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607497+0000", 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5318000000000001e-05 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607503+0000", 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6290000000000004e-06 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607578+0000", 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4721999999999994e-05 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607590+0000", 2023-09-30T09:32:38.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2534000000000001e-05 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504453+0000", 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.89686253000000005 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504539+0000", 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6562000000000002e-05 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504618+0000", 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8565000000000004e-05 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508765+0000", 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041465950000000003 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.080 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508774+0000", 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3190000000000001e-06 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508967+0000", 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019281699999999999 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509012+0000", 2023-09-30T09:32:38.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5592e-05 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509046+0000", 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3429000000000001e-05 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57916 6.12 6:4bf1e539:::benchmark_data_smithi019_355417_object57915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:32:38.082 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.607757+0000", 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 215.392895667, 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90142910499999995, 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57916 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.084 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607757+0000", 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607757+0000", 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607756+0000", 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607781+0000", 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4536999999999999e-05 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607787+0000", 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2450000000000003e-06 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607796+0000", 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8400000000000001e-06 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607846+0000", 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0528000000000001e-05 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607855+0000", 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3960000000000006e-06 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.505333+0000", 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.89747830500000003 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.505440+0000", 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010746799999999999 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.505560+0000", 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119641 2023-09-30T09:32:38.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508780+0000", 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032199139999999999 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508911+0000", 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013101899999999999 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509099+0000", 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018847800000000001 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509150+0000", 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0387999999999999e-05 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509186+0000", 2023-09-30T09:32:38.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5772000000000002e-05 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57940 6.12 6:4e8889e6:::benchmark_data_smithi019_355417_object57939:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.624369+0000", 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 215.37628339299999, 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.88492660599999995, 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.092 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57940 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624369+0000", 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.094 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624369+0000", 2023-09-30T09:32:38.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.094 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.094 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.094 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624368+0000", 2023-09-30T09:32:38.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624433+0000", 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4386000000000003e-05 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624442+0000", 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3090000000000001e-06 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624449+0000", 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5230000000000001e-06 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624513+0000", 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4858999999999997e-05 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.096 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624523+0000", 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6500000000000008e-06 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506403+0000", 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.881879682 2023-09-30T09:32:38.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506503+0000", 2023-09-30T09:32:38.098 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100552 2023-09-30T09:32:38.098 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506628+0000", 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012425000000000001 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508922+0000", 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002293986 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.099 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509226+0000", 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00030478500000000002 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509248+0000", 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2130999999999999e-05 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.100 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509274+0000", 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5259999999999999e-05 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509295+0000", 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1688999999999999e-05 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.101 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65378 6.3 6:c5bab36e:::benchmark_data_smithi019_355417_object65377:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.243566+0000", 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 198.75708633599999, 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0151177039999999, 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65378 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.102 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243566+0000", 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243566+0000", 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243565+0000", 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243608+0000", 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3086e-05 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243616+0000", 2023-09-30T09:32:38.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6139999999999992e-06 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243622+0000", 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4790000000000004e-06 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.248062+0000", 2023-09-30T09:32:38.105 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044400819999999997 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.248070+0000", 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4710000000000007e-06 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.255286+0000", 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0072160830000001 2023-09-30T09:32:38.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.255416+0000", 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012989100000000001 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.255522+0000", 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105604 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258502+0000", 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029799230000000002 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258529+0000", 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7623e-05 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258627+0000", 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7775000000000005e-05 2023-09-30T09:32:38.108 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258662+0000", 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4323999999999999e-05 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258684+0000", 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2010000000000001e-05 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.109 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65406 6.3 6:c0ab1d0f:::benchmark_data_smithi019_355417_object65405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.269356+0000", 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 198.731296235, 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99139678799999997, 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.110 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65406 2023-09-30T09:32:38.111 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.111 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.111 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.111 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269356+0000", 2023-09-30T09:32:38.111 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.111 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.111 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.112 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.112 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269356+0000", 2023-09-30T09:32:38.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.112 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.112 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.112 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269355+0000", 2023-09-30T09:32:38.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.113 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269455+0000", 2023-09-30T09:32:38.113 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9308999999999994e-05 2023-09-30T09:32:38.113 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269464+0000", 2023-09-30T09:32:38.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0289999999999994e-06 2023-09-30T09:32:38.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.114 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.114 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269469+0000", 2023-09-30T09:32:38.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6489999999999996e-06 2023-09-30T09:32:38.115 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.115 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.115 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.115 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269529+0000", 2023-09-30T09:32:38.115 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9685000000000002e-05 2023-09-30T09:32:38.116 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.116 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.116 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.116 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269575+0000", 2023-09-30T09:32:38.116 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6065e-05 2023-09-30T09:32:38.116 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.117 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.117 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.117 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.256853+0000", 2023-09-30T09:32:38.117 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98727761999999997 2023-09-30T09:32:38.117 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.117 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.118 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.118 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.256944+0000", 2023-09-30T09:32:38.118 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1009999999999998e-05 2023-09-30T09:32:38.118 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.118 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.118 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:38.119 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.257064+0000", 2023-09-30T09:32:38.119 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000120371 2023-09-30T09:32:38.119 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.119 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.119 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.119 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258738+0000", 2023-09-30T09:32:38.119 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001674009 2023-09-30T09:32:38.119 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.120 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.120 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.120 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258745+0000", 2023-09-30T09:32:38.120 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1600000000000001e-06 2023-09-30T09:32:38.120 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.121 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.121 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.121 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260668+0000", 2023-09-30T09:32:38.121 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019223910000000001 2023-09-30T09:32:38.121 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.121 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.122 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.122 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260716+0000", 2023-09-30T09:32:38.122 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8344000000000003e-05 2023-09-30T09:32:38.122 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.122 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.122 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.123 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260753+0000", 2023-09-30T09:32:38.123 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6743999999999999e-05 2023-09-30T09:32:38.123 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.123 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.123 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.123 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.124 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.124 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65419 6.3 6:c13ce430:::benchmark_data_smithi019_355417_object65418:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:32:38.124 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.282941+0000", 2023-09-30T09:32:38.124 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 198.71771086699999, 2023-09-30T09:32:38.124 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97912809700000003, 2023-09-30T09:32:38.124 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.124 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.124 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65419 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282941+0000", 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.125 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282941+0000", 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282941+0000", 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.126 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282987+0000", 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6019000000000001e-05 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282993+0000", 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4400000000000002e-06 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.127 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282997+0000", 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1010000000000002e-06 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.283149+0000", 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015114800000000001 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.128 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.283160+0000", 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1564e-05 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.257763+0000", 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97460299800000005 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.129 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.257886+0000", 2023-09-30T09:32:38.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012238 2023-09-30T09:32:38.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.130 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:38.130 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.257978+0000", 2023-09-30T09:32:38.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2826999999999997e-05 2023-09-30T09:32:38.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260561+0000", 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0025825790000000002 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260618+0000", 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6934000000000001e-05 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.261983+0000", 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013652079999999999 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262033+0000", 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9947000000000001e-05 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262069+0000", 2023-09-30T09:32:38.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6355e-05 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65425 6.3 6:c0030bff:::benchmark_data_smithi019_355417_object65424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.289950+0000", 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 198.71070180000001, 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97225219799999996, 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.133 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65425 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.289950+0000", 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.134 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.289950+0000", 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.289950+0000", 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290013+0000", 2023-09-30T09:32:38.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3658999999999995e-05 2023-09-30T09:32:38.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.136 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290024+0000", 2023-09-30T09:32:38.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0733999999999999e-05 2023-09-30T09:32:38.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290030+0000", 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6799999999999998e-06 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290088+0000", 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8155999999999997e-05 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290097+0000", 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3419999999999997e-06 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258766+0000", 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96866923900000002 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258867+0000", 2023-09-30T09:32:38.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100614 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258954+0000", 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7496000000000004e-05 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260766+0000", 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001811797 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260815+0000", 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8296e-05 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262121+0000", 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013066950000000001 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262168+0000", 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6592999999999999e-05 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262203+0000", 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4730999999999997e-05 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65449 6.3 6:c7b6375c:::benchmark_data_smithi019_355417_object65448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:32:38.141 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.309189+0000", 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 198.69146322899999, 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95312158499999999, 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65449 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.142 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309189+0000", 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309189+0000", 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.143 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309189+0000", 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309229+0000", 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0099999999999999e-05 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309237+0000", 2023-09-30T09:32:38.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3459999999999994e-06 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309241+0000", 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2950000000000004e-06 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309330+0000", 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9204999999999997e-05 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309340+0000", 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3700000000000001e-06 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.259580+0000", 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95023972499999998 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.259678+0000", 2023-09-30T09:32:38.147 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8717999999999999e-05 2023-09-30T09:32:38.147 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.147 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.147 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:38.147 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.259773+0000", 2023-09-30T09:32:38.147 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5106999999999996e-05 2023-09-30T09:32:38.147 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260874+0000", 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001101131 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260883+0000", 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0700000000000007e-06 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262260+0000", 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001377524 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262289+0000", 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9278999999999999e-05 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262311+0000", 2023-09-30T09:32:38.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1158e-05 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65485 6.3 6:c28c44bf:::benchmark_data_smithi019_355417_object65484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.342183+0000", 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 198.65846916300001, 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92270854099999999, 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.150 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65485 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342183+0000", 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.151 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342183+0000", 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342182+0000", 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342235+0000", 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3674999999999997e-05 2023-09-30T09:32:38.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342244+0000", 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3040000000000005e-06 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342248+0000", 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4320000000000001e-06 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342306+0000", 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7339000000000001e-05 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342317+0000", 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1049999999999999e-05 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.154 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260906+0000", 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.91858987800000003 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.261032+0000", 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000125473 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.261115+0000", 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3549999999999998e-05 2023-09-30T09:32:38.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264766+0000", 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036509120000000001 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264793+0000", 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6672000000000001e-05 2023-09-30T09:32:38.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264829+0000", 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6066999999999999e-05 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264866+0000", 2023-09-30T09:32:38.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6995000000000001e-05 2023-09-30T09:32:38.158 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.158 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.158 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.158 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264892+0000", 2023-09-30T09:32:38.158 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5449999999999999e-05 2023-09-30T09:32:38.158 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55766 7.5 7:aa9fdb5d:::benchmark_data_smithi019_404601_object55765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.516904469, 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0102513550000001, 2023-09-30T09:32:38.159 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55766 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.160 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483746+0000", 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.161 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483780+0000", 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4174999999999997e-05 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483797+0000", 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6626999999999999e-05 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.162 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483805+0000", 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3229999999999998e-06 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483862+0000", 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6236000000000002e-05 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.484005+0000", 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014357899999999999 2023-09-30T09:32:38.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.484145+0000", 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013967499999999999 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491724+0000", 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075794119999999998 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.509743+0000", 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.018018790999999999 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.509753+0000", 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7790000000000007e-06 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493928+0000", 2023-09-30T09:32:38.165 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98417540599999997 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493999+0000", 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0858999999999994e-05 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.166 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55755 7.9 7:9e422c00:::benchmark_data_smithi019_404601_object55754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.5141593680000001, 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0082494440000001, 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55755 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.168 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486491+0000", 2023-09-30T09:32:38.169 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:32:38.169 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.169 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.169 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.169 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486532+0000", 2023-09-30T09:32:38.169 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0222000000000002e-05 2023-09-30T09:32:38.169 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.169 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486545+0000", 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3288999999999999e-05 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486552+0000", 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6000000000000001e-06 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486609+0000", 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6268999999999999e-05 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486616+0000", 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0260000000000002e-06 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.495776+0000", 2023-09-30T09:32:38.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091603880000000002 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.495784+0000", 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3149999999999992e-06 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493615+0000", 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99783076999999998 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.494742+0000", 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001126993 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.173 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55773 7.9 7:9be0e4ac:::benchmark_data_smithi019_404601_object55772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.5134056060000001, 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0225480730000001, 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55773 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.174 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.175 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487246+0000", 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487313+0000", 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7360999999999994e-05 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487328+0000", 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5197999999999999e-05 2023-09-30T09:32:38.176 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487335+0000", 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.596e-06 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491649+0000", 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043143920000000002 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.177 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491656+0000", 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.092e-06 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.496453+0000", 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047967569999999996 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.496469+0000", 2023-09-30T09:32:38.178 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5926e-05 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.494766+0000", 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99829705800000001 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.495026+0000", 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00026024099999999998 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.179 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 6", 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.495185+0000", 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015893299999999999 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.500813+0000", 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0056275379999999996 2023-09-30T09:32:38.180 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509659+0000", 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088464240000000003 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509708+0000", 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8619000000000003e-05 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:32:38.181 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:32:38.182 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509795+0000", 2023-09-30T09:32:38.182 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6776999999999999e-05 2023-09-30T09:32:38.182 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.182 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.182 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.182 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:32:38.182 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:32:38.182 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:32:38.183 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:38.229 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:38.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:38.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:38.334 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:38.378 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:38.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 36 16 58203 58187 100.976 90.9375 0.00938153 0.00989844 2023-09-30T09:32:38.512 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:32:38.656 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:38.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:38 smithi019 ceph-mon[216815]: pgmap v698: 119 pgs: 22 stale+active+clean, 97 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-30T09:32:38.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3926892343' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:38.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:38 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:38.658 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:38 smithi019 ceph-mon[224043]: pgmap v698: 119 pgs: 22 stale+active+clean, 97 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-30T09:32:38.658 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3926892343' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:38.658 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:38 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:38.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:38.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:38.760 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:38.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:38 smithi167 ceph-mon[192880]: pgmap v698: 119 pgs: 22 stale+active+clean, 97 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-30T09:32:38.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3926892343' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:38.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:38 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:38.786 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:38.829 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:38.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:38.932 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:38.993 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:39.174 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:32:39.194 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:39.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:39.301 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:39.328 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:39.370 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:39.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:39.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:39.428 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 37 16 59236 59220 99.9917 64.5625 0.0128327 0.009996 2023-09-30T09:32:39.476 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:39.498 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:32:39.498 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:39.498 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: osdmap e289: 8 total, 7 up, 7 in 2023-09-30T09:32:39.498 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.498 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.498 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.498 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.498 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.499 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:39 smithi167 ceph-mon[192880]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.502 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:32:39.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:39.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: osdmap e289: 8 total, 7 up, 7 in 2023-09-30T09:32:39.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[216815]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.549 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:32:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:39.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: osdmap e289: 8 total, 7 up, 7 in 2023-09-30T09:32:39.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:39 smithi019 ceph-mon[224043]: failed to encode map e289 with expected crc 2023-09-30T09:32:39.920 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:39.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:39.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:40.024 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:40.069 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:40.160 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:40.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:40.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:40.275 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:40.327 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:40.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 38 16 60746 60730 99.8437 94.375 0.00805312 0.0100108 2023-09-30T09:32:40.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:40 smithi167 ceph-mon[192880]: pgmap v700: 119 pgs: 14 active+undersized+degraded, 35 active+undersized, 6 stale+active+clean, 64 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 1023 B/s rd, 96 MiB/s wr, 1.54k op/s; 29259/171843 objects degraded (17.027%) 2023-09-30T09:32:40.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:40 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 29259/171843 objects degraded (17.027%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:40.780 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:32:40.780 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:32:40.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:32:40.827 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:32:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:40 smithi019 ceph-mon[224043]: pgmap v700: 119 pgs: 14 active+undersized+degraded, 35 active+undersized, 6 stale+active+clean, 64 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 1023 B/s rd, 96 MiB/s wr, 1.54k op/s; 29259/171843 objects degraded (17.027%) 2023-09-30T09:32:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:40 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 29259/171843 objects degraded (17.027%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:40.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:40 smithi019 ceph-mon[216815]: pgmap v700: 119 pgs: 14 active+undersized+degraded, 35 active+undersized, 6 stale+active+clean, 64 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 1023 B/s rd, 96 MiB/s wr, 1.54k op/s; 29259/171843 objects degraded (17.027%) 2023-09-30T09:32:40.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:40 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 29259/171843 objects degraded (17.027%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:40.856 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:32:40.906 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:32:40.942 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:32:40.943 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:32:40.943 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-30T09:32:40.943 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:32:40.943 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:32:40.943 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:32:40.943 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:32:40.943 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:32:40.943 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:32:40.955 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:40.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:41.059 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:41.127 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:41.252 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:41.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:41.367 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:41.393 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:41.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 39 16 61875 61859 99.0929 70.5625 0.00200137 0.0100528 2023-09-30T09:32:41.437 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:41.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:41.543 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:41.552 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-30T09:32:41.596 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:41.700 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:32:41.701 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-30T09:32:41.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2752874535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:41.823 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:41.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:41.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2752874535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:41.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:41.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2752874535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:41.927 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:41.930 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:41.938 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:32:41.954 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:32:41.955 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:32:41.970 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:42.068 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:42.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:42.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:42.178 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:42.224 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:42.327 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:42.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:42.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:32:42.428892+0000 min lat: 0.00127075 max lat: 1.03693 avg lat: 0.0101022 2023-09-30T09:32:42.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:32:42.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 40 16 63364 63348 98.9419 93.0625 0.00905093 0.0101022 2023-09-30T09:32:42.438 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:42.504 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:42.630 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:42.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:42.741 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:42 smithi167 ceph-mon[192880]: pgmap v701: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 8.0 KiB/s rd, 55 MiB/s wr, 882 op/s; 36539/176121 objects degraded (20.747%); 26 KiB/s, 0 objects/s recovering 2023-09-30T09:32:42.804 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:42.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:42 smithi019 ceph-mon[216815]: pgmap v701: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 8.0 KiB/s rd, 55 MiB/s wr, 882 op/s; 36539/176121 objects degraded (20.747%); 26 KiB/s, 0 objects/s recovering 2023-09-30T09:32:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:42 smithi019 ceph-mon[224043]: pgmap v701: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 8.0 KiB/s rd, 55 MiB/s wr, 882 op/s; 36539/176121 objects degraded (20.747%); 26 KiB/s, 0 objects/s recovering 2023-09-30T09:32:42.862 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:42.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:42.970 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:42.995 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:43.100 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:43.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:43.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:43.210 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:43.290 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:43.393 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:43.401 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:43.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:43.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 41 16 64605 64589 98.4204 77.5625 0.00807638 0.0101554 2023-09-30T09:32:43.505 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:43.575 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:43.682 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:43.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:43.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:43.791 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:32:43.868 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:43.937 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:32:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:32:43] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2023-09-30T09:32:43.956 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:32:43.978 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:32:43.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:43.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:32:44.084 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:44.143 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:32:44.143 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:32:44.154 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:44.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 42 16 66021 66005 98.1841 88.5 0.00904936 0.0101802 2023-09-30T09:32:44.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:44 smithi167 ceph-mon[192880]: pgmap v702: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6.2 KiB/s rd, 42 MiB/s wr, 683 op/s; 36539/176121 objects degraded (20.747%); 20 KiB/s, 0 objects/s recovering 2023-09-30T09:32:44.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:44 smithi019 ceph-mon[224043]: pgmap v702: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6.2 KiB/s rd, 42 MiB/s wr, 683 op/s; 36539/176121 objects degraded (20.747%); 20 KiB/s, 0 objects/s recovering 2023-09-30T09:32:44.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:44 smithi019 ceph-mon[216815]: pgmap v702: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6.2 KiB/s rd, 42 MiB/s wr, 683 op/s; 36539/176121 objects degraded (20.747%); 20 KiB/s, 0 objects/s recovering 2023-09-30T09:32:45.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 43 16 67239 67223 97.6711 76.125 0.01476 0.010233 2023-09-30T09:32:45.567 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:45 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 40807/196425 objects degraded (20.775%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:45.757 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:45.759 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:32:45.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:45.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:45.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:45 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 40807/196425 objects degraded (20.775%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:45.869 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:45.953 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:46.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:45 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 40807/196425 objects degraded (20.775%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:46.156 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:46.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:46.265 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:46.268 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:46.329 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:46.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 44 16 68454 68438 97.1773 75.9375 0.0118639 0.0102861 2023-09-30T09:32:46.454 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:46.590 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:46.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:46.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:46.698 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:46.702 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-30T09:32:46.702 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:32:46.702 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:32:46.702 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-09-30T09:32:46.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-09-30T09:32:46.757 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:46.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:46 smithi019 ceph-mon[216815]: pgmap v703: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.5 KiB/s rd, 86 MiB/s wr, 1.38k op/s; 40807/196425 objects degraded (20.775%); 18 KiB/s, 0 objects/s recovering 2023-09-30T09:32:46.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:46 smithi019 ceph-mon[224043]: pgmap v703: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.5 KiB/s rd, 86 MiB/s wr, 1.38k op/s; 40807/196425 objects degraded (20.775%); 18 KiB/s, 0 objects/s recovering 2023-09-30T09:32:46.843 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:46.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:46.853 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:46 smithi167 ceph-mon[192880]: pgmap v703: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.5 KiB/s rd, 86 MiB/s wr, 1.38k op/s; 40807/196425 objects degraded (20.775%); 18 KiB/s, 0 objects/s recovering 2023-09-30T09:32:46.953 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:47.018 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:47.126 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:47.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:47.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:47.236 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:47.313 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:47.395 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:47.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:47.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:47.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 45 16 69779 69763 96.8579 82.8125 0.00219691 0.0103102 2023-09-30T09:32:47.502 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:47.563 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:47.595 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:32:47.661 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:47.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:47.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:47.673 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/617282379' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:47.773 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:47.827 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/617282379' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:47.827 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/617282379' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:32:47.829 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:48.221 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:48.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:48.330 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:48.369 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:48.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 46 16 71059 71043 96.4913 80 0.0122153 0.010359 2023-09-30T09:32:48.486 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:48.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:48.589 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:48.628 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: pgmap v704: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 4.8 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 40807/196425 objects degraded (20.775%); 16 KiB/s, 0 objects/s recovering 2023-09-30T09:32:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/617282379' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:32:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: osdmap e290: 8 total, 7 up, 7 in 2023-09-30T09:32:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:48.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: pgmap v704: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 4.8 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 40807/196425 objects degraded (20.775%); 16 KiB/s, 0 objects/s recovering 2023-09-30T09:32:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/617282379' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:32:48.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: osdmap e290: 8 total, 7 up, 7 in 2023-09-30T09:32:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: failed to encode map e290 with expected crc 2023-09-30T09:32:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: pgmap v704: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 4.8 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 40807/196425 objects degraded (20.775%); 16 KiB/s, 0 objects/s recovering 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/617282379' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: osdmap e290: 8 total, 7 up, 7 in 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: failed to encode map e290 with expected crc 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: failed to encode map e290 with expected crc 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: failed to encode map e290 with expected crc 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: failed to encode map e290 with expected crc 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: failed to encode map e290 with expected crc 2023-09-30T09:32:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:49.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:32:49.429 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 47 16 72200 72184 95.9556 71.3125 0.0107739 0.0104169 2023-09-30T09:32:49.759 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:49.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:49.865 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:49.906 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:49.986 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:50.102 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:50.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:50.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:50.135 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:32:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:32:50.211 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:32:50.216 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:50.247 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:32:50.278 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:50.279 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:32:50.318 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:32:50.348 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:32:50.348 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:32:50.348 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-30T09:32:50.348 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:32:50.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:32:50.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:32:50.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:32:50.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:32:50.349 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:32:50.374 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:50.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:50.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:50.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 48 16 73515 73499 95.6684 82.1875 0.0140678 0.0104487 2023-09-30T09:32:50.436 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:32:50.436 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:32:50.436 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:32:50.436 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:32:50.436 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:32:50.485 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:50.552 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:50.679 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:50.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:50.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:50.791 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:50.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:50 smithi019 ceph-mon[224043]: pgmap v706: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.1 KiB/s rd, 80 MiB/s wr, 1.29k op/s; 43861/210483 objects degraded (20.838%); 3.9 KiB/s, 0 objects/s recovering 2023-09-30T09:32:50.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1684283063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:50.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:50 smithi019 ceph-mon[216815]: pgmap v706: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.1 KiB/s rd, 80 MiB/s wr, 1.29k op/s; 43861/210483 objects degraded (20.838%); 3.9 KiB/s, 0 objects/s recovering 2023-09-30T09:32:50.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1684283063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:50.849 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:50 smithi167 ceph-mon[192880]: pgmap v706: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.1 KiB/s rd, 80 MiB/s wr, 1.29k op/s; 43861/210483 objects degraded (20.838%); 3.9 KiB/s, 0 objects/s recovering 2023-09-30T09:32:50.849 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1684283063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:32:50.852 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:50.969 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:50.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:51.079 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:32:51.126 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:51.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 49 16 74741 74725 95.2798 76.625 0.0298128 0.0104915 2023-09-30T09:32:51.602 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:32:51.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:51.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:32:51.654 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:32:51.709 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:52.153 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:51 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 44600/215148 objects degraded (20.730%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:52.153 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:51 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 44600/215148 objects degraded (20.730%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:52.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:51 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 44600/215148 objects degraded (20.730%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:52.195 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:52.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 50 16 75992 75976 94.9377 78.1875 0.00494046 0.0105215 2023-09-30T09:32:52.659 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:32:52.660 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:32:52.845 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:52.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:52.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:52 smithi019 ceph-mon[216815]: pgmap v707: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 44600/215148 objects degraded (20.730%) 2023-09-30T09:32:52.896 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:52 smithi019 ceph-mon[224043]: pgmap v707: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 44600/215148 objects degraded (20.730%) 2023-09-30T09:32:52.925 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:32:52.949 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:53.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:52 smithi167 ceph-mon[192880]: pgmap v707: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 44600/215148 objects degraded (20.730%) 2023-09-30T09:32:53.043 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:53.255 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:53.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:53.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:53.363 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:53.430 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:53.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 51 16 77091 77075 94.423 68.6875 0.0194323 0.0105861 2023-09-30T09:32:53.448 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:53.596 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:53.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:53.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:53.636 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:53.706 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:53.748 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:32:53.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:32:53.753 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:53.923 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:53.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:53.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:54.023 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:32:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:32:53] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2023-09-30T09:32:54.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:32:54.027 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:54.075 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:54.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 52 16 78199 78183 93.9389 69.25 0.0217272 0.0106398 2023-09-30T09:32:54.661 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:32:54.677 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.1 primary-affinity to 0 (802) 2023-09-30T09:32:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:54 smithi167 ceph-mon[192880]: pgmap v708: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 44600/215148 objects degraded (20.730%) 2023-09-30T09:32:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:54 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2864426724' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-30T09:32:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:54 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-30T09:32:55.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:54 smithi019 ceph-mon[216815]: pgmap v708: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 44600/215148 objects degraded (20.730%) 2023-09-30T09:32:55.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:54 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2864426724' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-30T09:32:55.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:54 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-30T09:32:55.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:54 smithi019 ceph-mon[224043]: pgmap v708: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 44600/215148 objects degraded (20.730%) 2023-09-30T09:32:55.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:54 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2864426724' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-30T09:32:55.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:54 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-30T09:32:55.420 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:55.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:55.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 53 16 79244 79228 93.3988 65.3125 0.0113972 0.0107026 2023-09-30T09:32:55.524 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:55.600 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:55.699 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:55.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:55.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:55.834 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:56.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:55 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-30T09:32:56.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:55 smithi167 ceph-mon[192880]: osdmap e291: 8 total, 7 up, 7 in 2023-09-30T09:32:56.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:55 smithi167 ceph-mon[192880]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:55 smithi167 ceph-mon[192880]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:55 smithi167 ceph-mon[192880]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:55 smithi167 ceph-mon[192880]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:55 smithi167 ceph-mon[192880]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:55 smithi167 ceph-mon[192880]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-30T09:32:56.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[224043]: osdmap e291: 8 total, 7 up, 7 in 2023-09-30T09:32:56.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[224043]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[224043]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[224043]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[224043]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[224043]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[224043]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[216815]: osdmap e291: 8 total, 7 up, 7 in 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[216815]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[216815]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[216815]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[216815]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[216815]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:55 smithi019 ceph-mon[216815]: failed to encode map e291 with expected crc 2023-09-30T09:32:56.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 54 16 80209 80193 92.7861 60.3125 0.0179455 0.0107725 2023-09-30T09:32:56.463 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:56.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:56.576 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:56.620 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:32:56.625 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:56.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:56 smithi019 ceph-mon[216815]: pgmap v710: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 48332/233202 objects degraded (20.725%) 2023-09-30T09:32:56.913 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:56.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:57.021 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:57.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:56 smithi167 ceph-mon[192880]: pgmap v710: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 48332/233202 objects degraded (20.725%) 2023-09-30T09:32:57.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:56 smithi019 ceph-mon[224043]: pgmap v710: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 48332/233202 objects degraded (20.725%) 2023-09-30T09:32:57.118 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:57.200 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:32:57.200 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:32:57.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:32:57.201 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:32:57.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.201 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80438 5.e 5:7ec295bd:::benchmark_data_smithi019_287194_object80437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:32:57.201 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:32:57.201 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 535.57436331999997, 2023-09-30T09:32:57.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5009281310000002, 2023-09-30T09:32:57.202 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.202 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:32:57.202 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.202 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.202 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.202 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80438 2023-09-30T09:32:57.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.202 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.203 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:32:57.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621658+0000", 2023-09-30T09:32:57.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621660+0000", 2023-09-30T09:32:57.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.423e-06 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621684+0000", 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4569000000000001e-05 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621694+0000", 2023-09-30T09:32:57.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5210000000000008e-06 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621700+0000", 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6710000000000002e-06 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.622254+0000", 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00055331 2023-09-30T09:32:57.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.622263+0000", 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8419999999999994e-06 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603497+0000", 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98123395899999999 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603507+0000", 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0285999999999999e-05 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.078297+0000", 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4747904549999999 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.084057+0000", 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0057597530000000003 2023-09-30T09:32:57.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122473+0000", 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.038415673999999997 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122586+0000", 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113668 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.209 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80444 5.5 5:a3c25906:::benchmark_data_smithi019_287194_object80443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 535.57431086999998, 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.335433417, 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.210 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80444 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621711+0000", 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3099999999999998e-07 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621733+0000", 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1899000000000002e-05 2023-09-30T09:32:57.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621740+0000", 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7290000000000003e-06 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621745+0000", 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3489999999999999e-06 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.213 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621802+0000", 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7148000000000002e-05 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621813+0000", 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1136e-05 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.602972+0000", 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98115875399999997 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.602983+0000", 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1294000000000001e-05 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.941598+0000", 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.338614129 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.941805+0000", 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020759399999999999 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.942011+0000", 2023-09-30T09:32:57.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020628199999999999 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949084+0000", 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0070728550000000003 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954359+0000", 2023-09-30T09:32:57.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0052750540000000004 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954690+0000", 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033033200000000002 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.218 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954747+0000", 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6943e-05 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957144+0000", 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023974880000000001 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.219 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80452 5.5 5:a7eeee83:::benchmark_data_smithi019_287194_object80451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 535.56352263199994, 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3299363909999999, 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80452 2023-09-30T09:32:57.220 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632499+0000", 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632500+0000", 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.576e-06 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632525+0000", 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4213999999999999e-05 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.223 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.223 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632543+0000", 2023-09-30T09:32:57.223 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8309e-05 2023-09-30T09:32:57.223 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.223 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.223 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.223 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632552+0000", 2023-09-30T09:32:57.224 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5860000000000005e-06 2023-09-30T09:32:57.224 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.224 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.224 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.224 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632615+0000", 2023-09-30T09:32:57.224 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3384000000000002e-05 2023-09-30T09:32:57.224 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.224 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.632628+0000", 2023-09-30T09:32:57.225 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2727e-05 2023-09-30T09:32:57.225 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.225 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603007+0000", 2023-09-30T09:32:57.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97037908100000003 2023-09-30T09:32:57.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.226 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.226 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603015+0000", 2023-09-30T09:32:57.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7049999999999999e-06 2023-09-30T09:32:57.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.227 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947003+0000", 2023-09-30T09:32:57.227 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.343988728 2023-09-30T09:32:57.227 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.227 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.227 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947114+0000", 2023-09-30T09:32:57.227 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000110746 2023-09-30T09:32:57.228 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.228 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.228 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:57.228 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.947240+0000", 2023-09-30T09:32:57.228 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000126136 2023-09-30T09:32:57.228 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.228 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.228 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.229 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954457+0000", 2023-09-30T09:32:57.229 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0072169809999999999 2023-09-30T09:32:57.229 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.229 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.229 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.229 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957159+0000", 2023-09-30T09:32:57.229 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00270178 2023-09-30T09:32:57.229 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.230 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.230 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.230 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.960528+0000", 2023-09-30T09:32:57.230 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033696120000000001 2023-09-30T09:32:57.230 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.230 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.230 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.230 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.960588+0000", 2023-09-30T09:32:57.231 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9361000000000003e-05 2023-09-30T09:32:57.231 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.231 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.231 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.231 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962435+0000", 2023-09-30T09:32:57.231 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001847465 2023-09-30T09:32:57.231 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.232 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.232 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.232 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.232 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.232 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80456 5.5 5:a29acd63:::benchmark_data_smithi019_287194_object80455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:32:57.232 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:32:57.232 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 535.55659989699996, 2023-09-30T09:32:57.233 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3250314799999998, 2023-09-30T09:32:57.233 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.233 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.233 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.233 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.233 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.233 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80456 2023-09-30T09:32:57.234 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.234 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.234 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.234 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.234 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:32:57.234 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.234 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.235 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.235 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.235 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639422+0000", 2023-09-30T09:32:57.235 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.235 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.235 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.235 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.236 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639424+0000", 2023-09-30T09:32:57.236 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0729999999999999e-06 2023-09-30T09:32:57.236 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.236 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.236 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.236 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639449+0000", 2023-09-30T09:32:57.236 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5324999999999999e-05 2023-09-30T09:32:57.236 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.237 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.237 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.237 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639467+0000", 2023-09-30T09:32:57.237 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7923000000000001e-05 2023-09-30T09:32:57.237 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.237 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.238 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.238 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639477+0000", 2023-09-30T09:32:57.238 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6809999999999993e-06 2023-09-30T09:32:57.238 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.238 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.238 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639552+0000", 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5191000000000006e-05 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.639566+0000", 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3654e-05 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.239 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.240 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.240 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603040+0000", 2023-09-30T09:32:57.240 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96347411699999996 2023-09-30T09:32:57.240 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.240 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.240 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.240 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603047+0000", 2023-09-30T09:32:57.241 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.126e-06 2023-09-30T09:32:57.241 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.241 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.241 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.241 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949179+0000", 2023-09-30T09:32:57.241 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3461320999999999 2023-09-30T09:32:57.241 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.241 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.242 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.242 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949348+0000", 2023-09-30T09:32:57.242 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016921899999999999 2023-09-30T09:32:57.242 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.242 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.242 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:32:57.243 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.949468+0000", 2023-09-30T09:32:57.243 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011944599999999999 2023-09-30T09:32:57.243 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.243 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.243 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.244 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.954573+0000", 2023-09-30T09:32:57.244 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005105371 2023-09-30T09:32:57.244 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.244 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.244 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.244 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.957167+0000", 2023-09-30T09:32:57.244 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025936269999999998 2023-09-30T09:32:57.245 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.245 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.245 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.245 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962500+0000", 2023-09-30T09:32:57.245 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0053332429999999997 2023-09-30T09:32:57.245 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.246 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.246 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.246 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.962562+0000", 2023-09-30T09:32:57.246 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2016999999999999e-05 2023-09-30T09:32:57.246 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.246 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.247 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.247 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.964453+0000", 2023-09-30T09:32:57.247 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018913669999999999 2023-09-30T09:32:57.247 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.247 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.247 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.247 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.247 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.248 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80465 5.e 5:72d43b9b:::benchmark_data_smithi019_287194_object80464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-30T09:32:57.248 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:32:57.248 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 535.53291930700004, 2023-09-30T09:32:57.248 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5017385250000004, 2023-09-30T09:32:57.248 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.248 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.248 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.248 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.249 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.249 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80465 2023-09-30T09:32:57.249 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.249 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.249 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.249 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.249 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:32:57.249 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.250 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.250 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.250 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.250 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663102+0000", 2023-09-30T09:32:57.250 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.250 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.250 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.251 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.251 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663104+0000", 2023-09-30T09:32:57.251 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9649999999999998e-06 2023-09-30T09:32:57.251 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.251 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.251 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.251 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663128+0000", 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3878000000000001e-05 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663145+0000", 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.649e-05 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.252 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663153+0000", 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5750000000000001e-06 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663245+0000", 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2249000000000004e-05 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.253 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.663259+0000", 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3422e-05 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603534+0000", 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94027541000000003 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.254 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.603543+0000", 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4120000000000001e-06 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122620+0000", 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5190771290000002 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.255 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122769+0000", 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014931999999999999 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.122900+0000", 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013077799999999999 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.131893+0000", 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089926810000000006 2023-09-30T09:32:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135017+0000", 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031246440000000002 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.162244+0000", 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027226509999999999 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.258 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.162314+0000", 2023-09-30T09:32:57.258 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0116000000000004e-05 2023-09-30T09:32:57.258 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.258 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.258 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.258 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.164841+0000", 2023-09-30T09:32:57.258 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002526946 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80467 5.e 5:7e7fcfbf:::benchmark_data_smithi019_287194_object80466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 533.45729901499999, 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4284252850000003, 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.259 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80467 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.260 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738723+0000", 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738724+0000", 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.823e-06 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.738752+0000", 2023-09-30T09:32:57.261 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7413e-05 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.740661+0000", 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001908795 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.740676+0000", 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5333999999999999e-05 2023-09-30T09:32:57.262 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125570+0000", 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3848938300000002 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125759+0000", 2023-09-30T09:32:57.263 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018891200000000001 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.125968+0000", 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020954400000000001 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.134957+0000", 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089886380000000002 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.264 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135084+0000", 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012721999999999999 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.164930+0000", 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.02984645 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.265 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.165000+0000", 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9052000000000002e-05 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167148+0000", 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021482739999999999 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.266 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80472 5.e 5:7c0728d2:::benchmark_data_smithi019_287194_object80471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 533.26149365699996, 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2349960290000004, 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80472 2023-09-30T09:32:57.267 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.268 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934528+0000", 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934529+0000", 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4750000000000001e-06 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934554+0000", 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4953999999999999e-05 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934572+0000", 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7317e-05 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.934580+0000", 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5080000000000005e-06 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.128799+0000", 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1942186069999998 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.128998+0000", 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000199447 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:32:57.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.129303+0000", 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00030457599999999998 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141779+0000", 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012476716000000001 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141802+0000", 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2903000000000001e-05 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167214+0000", 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.025411309 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.167261+0000", 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7747999999999998e-05 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169524+0000", 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002262469 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.274 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80493 5.e 5:7e14abc9:::benchmark_data_smithi019_287194_object80492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 533.15998276599998, 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1357754939999998, 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80493 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.275 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036039+0000", 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.276 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036040+0000", 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5269999999999999e-06 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036063+0000", 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3034999999999999e-05 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.277 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036076+0000", 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2979999999999999e-05 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.036085+0000", 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6340000000000007e-06 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.131929+0000", 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.095844402 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.132099+0000", 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016993999999999999 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.132318+0000", 2023-09-30T09:32:57.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021850799999999999 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141810+0000", 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0094919229999999993 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141869+0000", 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9252000000000001e-05 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169591+0000", 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027721632999999999 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.169673+0000", 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2330999999999995e-05 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171814+0000", 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002141329 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80495 5.e 5:72ad1293:::benchmark_data_smithi019_287194_object80494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 533.15643373, 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1346890580000002, 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.282 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80495 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.283 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039588+0000", 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039590+0000", 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0020000000000001e-06 2023-09-30T09:32:57.284 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039617+0000", 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6701000000000001e-05 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039628+0000", 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1557e-05 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.039635+0000", 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1829999999999997e-06 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135122+0000", 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0954865739999997 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135319+0000", 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019667600000000001 2023-09-30T09:32:57.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.135541+0000", 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022274 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141816+0000", 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062751109999999999 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151681+0000", 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0098647409999999998 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171874+0000", 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020192412999999999 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.171931+0000", 2023-09-30T09:32:57.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7716000000000001e-05 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174277+0000", 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002345644 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.289 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80498 5.e 5:7631c3ad:::benchmark_data_smithi019_287194_object80497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 533.14829092299999, 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1287919449999997, 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80498 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.290 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047731+0000", 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.291 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047732+0000", 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.832e-06 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047759+0000", 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6628999999999999e-05 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.292 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047779+0000", 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9468000000000001e-05 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.047789+0000", 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0864e-05 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138222+0000", 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0904321699999997 2023-09-30T09:32:57.293 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138406+0000", 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018471300000000001 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.138613+0000", 2023-09-30T09:32:57.294 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000206596 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.148131+0000", 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095180230000000005 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151779+0000", 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036479849999999999 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.295 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174332+0000", 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022552744 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.174400+0000", 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7886999999999996e-05 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176523+0000", 2023-09-30T09:32:57.296 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021230340000000002 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80506 5.e 5:7cef17fd:::benchmark_data_smithi019_287194_object80505:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 533.11210698599996, 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0963020300000004, 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.297 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80506 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.298 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083915+0000", 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083916+0000", 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9199999999999999e-07 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083934+0000", 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8879e-05 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.299 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083943+0000", 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0620000000000007e-06 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.083950+0000", 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1519999999999997e-06 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.300 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.141906+0000", 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0579564440000002 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.142127+0000", 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000220447 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.142337+0000", 2023-09-30T09:32:57.301 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000210867 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.148142+0000", 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058050389999999997 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.153209+0000", 2023-09-30T09:32:57.302 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050662160000000001 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176580+0000", 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.02337171 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.176646+0000", 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5775e-05 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.303 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.180217+0000", 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003570447 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80509 5.e 5:7a2f2184:::benchmark_data_smithi019_287194_object80508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:32:57.304 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 533.08944493700005, 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0801843260000004, 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80509 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.305 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106577+0000", 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.306 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106578+0000", 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6199999999999999e-06 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106608+0000", 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9841000000000001e-05 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106623+0000", 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4628e-05 2023-09-30T09:32:57.307 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:04.106632+0000", 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6560000000000004e-06 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145019+0000", 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0383863680000003 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.308 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145216+0000", 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000196942 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.145433+0000", 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000217436 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.151985+0000", 2023-09-30T09:32:57.309 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0065521779999999996 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.153262+0000", 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00127673 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.184424+0000", 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031162419 2023-09-30T09:32:57.310 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.184490+0000", 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5228999999999995e-05 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:10.186761+0000", 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022712790000000002 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.311 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:82845 5.3 5:c557c714:::benchmark_data_smithi019_287194_object67:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 508.00650139800001, 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.525600517, 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.312 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 82845 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.313 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189520+0000", 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189521+0000", 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9499999999999998e-07 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.314 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189523+0000", 2023-09-30T09:32:57.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1469999999999999e-06 2023-09-30T09:32:57.315 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.315 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.315 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.315 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189529+0000", 2023-09-30T09:32:57.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0719999999999996e-06 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.189534+0000", 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4090000000000002e-06 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.459994+0000", 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.270459909 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.316 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:29.466180+0000", 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061857609999999997 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.331107+0000", 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86492722099999997 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.339884+0000", 2023-09-30T09:32:57.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087769049999999998 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.642343+0000", 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.302458745 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.647436+0000", 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050929809999999999 2023-09-30T09:32:57.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.656592+0000", 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091567330000000002 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.695178+0000", 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.038585174999999999 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.697589+0000", 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024115120000000002 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.698724+0000", 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001134456 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.701606+0000", 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028820909999999998 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.715121+0000", 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013515005 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.321 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:83209 5.3 5:c8e07ed0:::benchmark_data_smithi019_287194_object431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 506.71129265399998, 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8400215319999997, 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 83209 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.322 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484729+0000", 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484730+0000", 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3769999999999999e-06 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484736+0000", 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2889999999999997e-06 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484750+0000", 2023-09-30T09:32:57.324 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4929999999999999e-05 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.484758+0000", 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1910000000000003e-06 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.512794+0000", 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028036607000000002 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:30.515536+0000", 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027418659999999999 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.902937+0000", 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3874005220000001 2023-09-30T09:32:57.326 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:31.909306+0000", 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063690129999999998 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.219767+0000", 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3104613839999999 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.229321+0000", 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095544229999999994 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.239410+0000", 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010088311000000001 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.268110+0000", 2023-09-30T09:32:57.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028700013 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.272145+0000", 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040355829999999997 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.298154+0000", 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026008963999999999 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.302424+0000", 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042697239999999999 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:33.324750+0000", 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022326334999999999 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:85292 5.3 5:c22e0b20:::benchmark_data_smithi019_287194_object2514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 491.09015436099997, 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0057864740000002, 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 85292 2023-09-30T09:32:57.331 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105867+0000", 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.332 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105869+0000", 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6419999999999999e-06 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105875+0000", 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6390000000000003e-06 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.333 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.334 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.334 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105892+0000", 2023-09-30T09:32:57.334 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6092000000000001e-05 2023-09-30T09:32:57.334 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.334 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.334 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.334 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.105902+0000", 2023-09-30T09:32:57.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0723000000000001e-05 2023-09-30T09:32:57.335 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.335 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.151432+0000", 2023-09-30T09:32:57.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.045529575000000003 2023-09-30T09:32:57.335 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.336 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:32:57.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:46.157840+0000", 2023-09-30T09:32:57.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0064082469999999997 2023-09-30T09:32:57.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.336 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:47.639845+0000", 2023-09-30T09:32:57.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4820053099999999 2023-09-30T09:32:57.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:32:57.337 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:47.646225+0000", 2023-09-30T09:32:57.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063791960000000002 2023-09-30T09:32:57.338 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.338 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.022311+0000", 2023-09-30T09:32:57.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.376086213 2023-09-30T09:32:57.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.027327+0000", 2023-09-30T09:32:57.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050160719999999999 2023-09-30T09:32:57.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:32:57.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.035994+0000", 2023-09-30T09:32:57.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086666720000000003 2023-09-30T09:32:57.340 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.078087+0000", 2023-09-30T09:32:57.341 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.042092989999999997 2023-09-30T09:32:57.341 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.341 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.341 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.341 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.079289+0000", 2023-09-30T09:32:57.341 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001202172 2023-09-30T09:32:57.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.342 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.342 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.342 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.081249+0000", 2023-09-30T09:32:57.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019603310000000001 2023-09-30T09:32:57.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.343 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.343 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.085070+0000", 2023-09-30T09:32:57.343 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038209870000000001 2023-09-30T09:32:57.343 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:49.111654+0000", 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026583612999999999 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.344 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87684 5.3 5:c18d78e7:::benchmark_data_smithi019_287194_object4906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 472.39961792899999, 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1651827400000001, 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87684 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796404+0000", 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796405+0000", 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.406e-06 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796411+0000", 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9139999999999996e-06 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796422+0000", 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1493e-05 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.796429+0000", 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.567e-06 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.070700+0000", 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.274271406 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.077044+0000", 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063433329999999996 2023-09-30T09:32:57.348 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.349 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.349 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.349 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.143755+0000", 2023-09-30T09:32:57.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.066710900000000004 2023-09-30T09:32:57.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.349 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.349 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:32:57.349 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.150381+0000", 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0066263609999999999 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.882648+0000", 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.732266774 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.888309+0000", 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005660867 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.897037+0000", 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087287739999999999 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.939441+0000", 2023-09-30T09:32:57.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.042403771999999999 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.941115+0000", 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001673316 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.943881+0000", 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027661420000000001 2023-09-30T09:32:57.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.947752+0000", 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038716940000000002 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.961586+0000", 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013834021 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.353 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87688 5.3 5:cc6b6e52:::benchmark_data_smithi019_287194_object4910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 472.39386487000002, 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1848223249999998, 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.354 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87688 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802157+0000", 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802159+0000", 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.322e-06 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.357 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802168+0000", 2023-09-30T09:32:57.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8470000000000002e-06 2023-09-30T09:32:57.357 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.357 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.357 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802199+0000", 2023-09-30T09:32:57.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1343000000000001e-05 2023-09-30T09:32:57.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.358 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.358 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:04.802212+0000", 2023-09-30T09:32:57.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2368000000000001e-05 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.077878+0000", 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27566626799999999 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.087824+0000", 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0099457680000000007 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.196502+0000", 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.10867827200000001 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.211705+0000", 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015202983 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.919095+0000", 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.707390357 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.924017+0000", 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049221409999999997 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.932712+0000", 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086950120000000002 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.942222+0000", 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0095098939999999996 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.963866+0000", 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.021643335999999999 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.966772+0000", 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029066550000000002 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.971211+0000", 2023-09-30T09:32:57.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0044391040000000001 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.986979+0000", 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015767654999999998 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.364 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:87754 5.3 5:cecacda3:::benchmark_data_smithi019_287194_object4976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 471.98381642800001, 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1694036080000001, 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 87754 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212205+0000", 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.366 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212206+0000", 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2869999999999999e-06 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212211+0000", 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6789999999999996e-06 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212224+0000", 2023-09-30T09:32:57.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2931999999999999e-05 2023-09-30T09:32:57.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.212233+0000", 2023-09-30T09:32:57.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0289999999999994e-06 2023-09-30T09:32:57.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.299820+0000", 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.087586586999999994 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:05.303276+0000", 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034559970000000001 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.858386+0000", 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5551105619999999 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:06.864253+0000", 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058669530000000003 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.276710+0000", 2023-09-30T09:32:57.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.412457187 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.287222+0000", 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010511855000000001 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.300806+0000", 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013584003000000001 2023-09-30T09:32:57.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.358020+0000", 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.057214108999999999 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.359098+0000", 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0010772889999999999 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.361377+0000", 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022794439999999998 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.367047+0000", 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0056699250000000001 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:25:08.381609+0000", 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.01456177 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145688 5.e 5:72fbdf89:::benchmark_data_smithi019_287194_object62910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 366.83133222700002, 2023-09-30T09:32:57.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.140027033, 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145688 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:32:57.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364691+0000", 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3039999999999999e-06 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364696+0000", 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5860000000000004e-06 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364709+0000", 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2938999999999999e-05 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364716+0000", 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7009999999999999e-06 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364773+0000", 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7463999999999997e-05 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364784+0000", 2023-09-30T09:32:57.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.077e-05 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305414+0000", 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94062986800000004 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305424+0000", 2023-09-30T09:32:57.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6339999999999996e-06 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.474904+0000", 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.169480576 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.475014+0000", 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011030400000000001 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4,5", 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.475093+0000", 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8994999999999998e-05 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478115+0000", 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030214360000000002 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.382 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.481419+0000", 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033041759999999998 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.487360+0000", 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0059407549999999998 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504511+0000", 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017150822 2023-09-30T09:32:57.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504581+0000", 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0104000000000005e-05 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504716+0000", 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000135599 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145692 5.e 5:778ecb27:::benchmark_data_smithi019_287194_object62914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 366.82488669200001, 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1338203870000001, 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145692 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.385 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:32:57.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371136+0000", 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4499999999999996e-07 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371141+0000", 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9710000000000003e-06 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371151+0000", 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0669000000000001e-05 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371157+0000", 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7219999999999996e-06 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371209+0000", 2023-09-30T09:32:57.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1801000000000001e-05 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371218+0000", 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7860000000000002e-06 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305986+0000", 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.934768932 2023-09-30T09:32:57.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305996+0000", 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7399999999999999e-06 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478142+0000", 2023-09-30T09:32:57.390 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172145298 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478283+0000", 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000141037 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4,5", 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478364+0000", 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1600999999999993e-05 2023-09-30T09:32:57.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.481503+0000", 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031385779999999999 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.487053+0000", 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055504949999999999 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.495859+0000", 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088061919999999991 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504770+0000", 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089107639999999998 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:32:57.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504831+0000", 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1209000000000003e-05 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504955+0000", 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012384700000000001 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.394 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:32:57.395 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:32:57.395 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:32:57.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:57.396 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:57.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:57.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 55 16 81324 81308 92.3661 69.6875 0.0250652 0.0108224 2023-09-30T09:32:57.497 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:57.561 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:57.675 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:57.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:32:57.796 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:32:57.882 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:57.990 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:32:58.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:58.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:32:58.103 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:32:58.134 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:58.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 56 16 82422 82406 91.9422 68.625 0.010311 0.010872 2023-09-30T09:32:58.438 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:32:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:58.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:32:58.542 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:32:58.602 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:58.658 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:32:58.696 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[224043]: pgmap v711: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s; 48332/233202 objects degraded (20.725%) 2023-09-30T09:32:58.696 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/547666739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:58.696 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 48332/233202 objects degraded (20.725%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:58.696 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:58.696 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:58.697 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[216815]: pgmap v711: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s; 48332/233202 objects degraded (20.725%) 2023-09-30T09:32:58.697 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/547666739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:58.697 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 48332/233202 objects degraded (20.725%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:58.697 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:58.697 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:58.713 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:32:59.020 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:32:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:58 smithi167 ceph-mon[192880]: pgmap v711: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s; 48332/233202 objects degraded (20.725%) 2023-09-30T09:32:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:58 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/547666739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:32:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 48332/233202 objects degraded (20.725%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:32:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:32:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:59.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:32:59.127 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:59.156 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:59.206 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:59.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:59.311 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:59.374 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:59.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 57 16 83402 83386 91.4037 61.25 0.0200006 0.010936 2023-09-30T09:32:59.462 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:59.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:59.571 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:32:59.632 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:59.677 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:32:59.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:59.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:32:59.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:59 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:59.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/547666739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:59.776 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:59 smithi167 ceph-mon[192880]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.776 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:59 smithi167 ceph-mon[192880]: osdmap e292: 8 total, 7 up, 7 in 2023-09-30T09:32:59.776 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:59 smithi167 ceph-mon[192880]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.776 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:59 smithi167 ceph-mon[192880]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.776 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:32:59 smithi167 ceph-mon[192880]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.783 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:32:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/547666739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[224043]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[224043]: osdmap e292: 8 total, 7 up, 7 in 2023-09-30T09:32:59.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[224043]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[224043]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[224043]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/547666739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[216815]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[216815]: osdmap e292: 8 total, 7 up, 7 in 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[216815]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[216815]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:32:59 smithi019 ceph-mon[216815]: failed to encode map e292 with expected crc 2023-09-30T09:32:59.863 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:32:59.967 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:32:59.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:32:59.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:00.075 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:00.133 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:00.226 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:00.234 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:00.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:00.341 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:00.418 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:00.430 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 58 16 84533 84517 91.0465 70.6875 0.0205113 0.0109787 2023-09-30T09:33:00.534 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:00.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:00.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:00.645 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:00.706 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:00.734 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:33:00.768 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:33:00.805 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:00.806 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:33:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:00.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:00.843 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:33:00.872 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:33:00.909 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:33:00.916 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:00.962 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:00 smithi167 ceph-mon[192880]: pgmap v713: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 50992/245256 objects degraded (20.791%) 2023-09-30T09:33:00.962 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1530469904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:00.965 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:00.969 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:33:00.969 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:33:00.969 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-30T09:33:00.969 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:33:00.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:33:00.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:33:00.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:33:00.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:33:00.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:33:00.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:00 smithi019 ceph-mon[224043]: pgmap v713: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 50992/245256 objects degraded (20.791%) 2023-09-30T09:33:00.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1530469904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:00.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:00 smithi019 ceph-mon[216815]: pgmap v713: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 50992/245256 objects degraded (20.791%) 2023-09-30T09:33:00.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1530469904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:01.071 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:01.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:01.180 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:01.233 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:01.421 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-30T09:33:01.421 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:33:01.421 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-30T09:33:01.421 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-30T09:33:01.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 59 16 85747 85731 90.7893 75.875 0.0088482 0.01101 2023-09-30T09:33:02.115 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:33:02.116 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:02.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:02.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:02.219 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:02.284 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:02.387 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:02.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:02.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:33:02.430882+0000 min lat: 0.00127075 max lat: 1.03693 avg lat: 0.0110515 2023-09-30T09:33:02.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:33:02.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 60 16 86870 86854 90.4459 70.1875 0.0161096 0.0110515 2023-09-30T09:33:02.501 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:02.527 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:02.567 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:02.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:02.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:02.672 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:02.720 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:02 smithi167 ceph-mon[192880]: pgmap v714: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 51700/249201 objects degraded (20.746%) 2023-09-30T09:33:03.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:02 smithi019 ceph-mon[224043]: pgmap v714: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 51700/249201 objects degraded (20.746%) 2023-09-30T09:33:03.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:02 smithi019 ceph-mon[216815]: pgmap v714: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 51700/249201 objects degraded (20.746%) 2023-09-30T09:33:03.102 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:03.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:03.118 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:33:03.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:03.120 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:33:03.220 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:03.310 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:03.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 61 16 87825 87809 89.9417 59.6875 0.00903914 0.0111136 2023-09-30T09:33:03.508 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:03.637 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:03.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:03.650 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:33:03.650 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:33:03.747 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:03.809 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:03.901 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:03.912 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:03.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:03.988 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:33:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:33:03] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-09-30T09:33:03.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:03.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 51700/249201 objects degraded (20.746%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:03.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:03.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 51700/249201 objects degraded (20.746%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:04.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 51700/249201 objects degraded (20.746%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:04.052 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:04.121 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:04.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:04.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:04.231 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:04.300 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:04.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 62 16 89004 88988 89.6795 73.6875 0.00823848 0.0111468 2023-09-30T09:33:05.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:04 smithi167 ceph-mon[192880]: pgmap v715: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 614 op/s; 51700/249201 objects degraded (20.746%) 2023-09-30T09:33:05.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:04 smithi019 ceph-mon[216815]: pgmap v715: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 614 op/s; 51700/249201 objects degraded (20.746%) 2023-09-30T09:33:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:04 smithi019 ceph-mon[224043]: pgmap v715: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 614 op/s; 51700/249201 objects degraded (20.746%) 2023-09-30T09:33:05.121 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:33:05.279 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:05.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:05.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:05.384 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:05.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 63 16 90238 90222 89.4801 77.125 0.0144195 0.0111711 2023-09-30T09:33:05.443 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:05.485 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:05.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:05.591 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:05.616 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:05.743 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:05.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:05.777 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:33:05.853 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:05.913 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:06.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 64 16 91252 91236 89.0722 63.375 0.00372049 0.0112184 2023-09-30T09:33:06.922 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:06.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:06.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:06 smithi167 ceph-mon[192880]: pgmap v716: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 55141/265728 objects degraded (20.751%) 2023-09-30T09:33:07.026 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:07.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:06 smithi019 ceph-mon[216815]: pgmap v716: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 55141/265728 objects degraded (20.751%) 2023-09-30T09:33:07.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:06 smithi019 ceph-mon[224043]: pgmap v716: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 55141/265728 objects degraded (20.751%) 2023-09-30T09:33:07.073 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:07.218 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:07.341 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:07.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:07.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:07.431 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 65 16 92337 92321 88.7451 67.8125 0.00765217 0.011264 2023-09-30T09:33:07.446 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:07.505 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:07.579 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:07.593 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:07.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:07.698 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:07.759 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:07.857 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:07.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:07.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:07.965 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:08.029 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:08.283 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:33:08.432 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 66 16 93429 93413 88.4335 68.25 0.011619 0.0113032 2023-09-30T09:33:08.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:08.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:08.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:08 smithi167 ceph-mon[192880]: pgmap v717: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 55141/265728 objects degraded (20.751%) 2023-09-30T09:33:08.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3215375858' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:08.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:08 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:08.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:33:08.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:33:08.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[216815]: pgmap v717: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 55141/265728 objects degraded (20.751%) 2023-09-30T09:33:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3215375858' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:33:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[224043]: pgmap v717: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 55141/265728 objects degraded (20.751%) 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3215375858' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:33:08.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:08 smithi019 conmon[216791]: 2023-09-30T09:33:08.703+0000 7faf52971700 -1 mon.a@0(leader).osd e293 definitely_dead 0 2023-09-30T09:33:09.432 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 67 16 94564 94548 88.1723 70.9375 0.00305121 0.0113233 2023-09-30T09:33:09.743 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:09.743 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:33:09.743 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: osdmap e293: 8 total, 7 up, 7 in 2023-09-30T09:33:09.743 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.743 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 55141/265728 objects degraded (20.751%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:09.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3092559643' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:33:09.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:09 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:33:09.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:09.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:33:09.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: osdmap e293: 8 total, 7 up, 7 in 2023-09-30T09:33:09.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 55141/265728 objects degraded (20.751%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:09.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3092559643' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:33:09.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:33:09.747 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:09.747 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:33:09.747 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: osdmap e293: 8 total, 7 up, 7 in 2023-09-30T09:33:09.747 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.747 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.748 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.748 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: failed to encode map e293 with expected crc 2023-09-30T09:33:09.748 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 55141/265728 objects degraded (20.751%), 20 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:09.748 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3092559643' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:33:09.748 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:09 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:33:09.951 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:09.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:09.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:10.034 INFO:teuthology.orchestra.run.smithi019.stderr:marked in osd.2. 2023-09-30T09:33:10.056 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:10.151 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:10.296 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:10.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:10.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:10.400 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:10.427 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:10.432 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 68 16 94579 94563 86.8897 0.9375 0.00326579 0.0113326 2023-09-30T09:33:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:10 smithi167 ceph-mon[192880]: failed to encode map e293 with expected crc 2023-09-30T09:33:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:10 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-30T09:33:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:10 smithi167 ceph-mon[192880]: osdmap e294: 8 total, 7 up, 8 in 2023-09-30T09:33:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:10 smithi167 ceph-mon[192880]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:10 smithi167 ceph-mon[192880]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:10 smithi167 ceph-mon[192880]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:10 smithi167 ceph-mon[192880]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:10 smithi167 ceph-mon[192880]: pgmap v720: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 57967/278451 objects degraded (20.818%) 2023-09-30T09:33:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[216815]: failed to encode map e293 with expected crc 2023-09-30T09:33:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-30T09:33:10.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[216815]: osdmap e294: 8 total, 7 up, 8 in 2023-09-30T09:33:10.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[216815]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[216815]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[216815]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[216815]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[216815]: pgmap v720: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 57967/278451 objects degraded (20.818%) 2023-09-30T09:33:10.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[224043]: failed to encode map e293 with expected crc 2023-09-30T09:33:10.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-30T09:33:10.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[224043]: osdmap e294: 8 total, 7 up, 8 in 2023-09-30T09:33:10.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[224043]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[224043]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[224043]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[224043]: failed to encode map e294 with expected crc 2023-09-30T09:33:10.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:10 smithi019 ceph-mon[224043]: pgmap v720: 119 pgs: 20 active+undersized+degraded, 47 active+undersized, 52 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 57967/278451 objects degraded (20.818%) 2023-09-30T09:33:10.735 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:10.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:10.838 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:10.870 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:10.918 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:10.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:10.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:11.023 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:11.052 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:11.097 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:11.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:11.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:11.202 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:11.229 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:11.271 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:11.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:11.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:11.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: failed to encode map e294 with expected crc 2023-09-30T09:33:11.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: osdmap e295: 8 total, 7 up, 8 in 2023-09-30T09:33:11.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.322 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: failed to encode map e294 with expected crc 2023-09-30T09:33:11.322 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.322 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: osdmap e295: 8 total, 7 up, 8 in 2023-09-30T09:33:11.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:11 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.376 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:11.403 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:11.432 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 69 16 94579 94563 85.6307 0 - 0.0113326 2023-09-30T09:33:11.452 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:11.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:11.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: failed to encode map e294 with expected crc 2023-09-30T09:33:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: osdmap e295: 8 total, 7 up, 8 in 2023-09-30T09:33:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:11 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:11.532 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-30T09:33:11.559 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:11.630 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:11.873 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:33:11.873 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:33:11.923 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:11.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:12.015 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:33:12.035 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:12.052 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:33:12.084 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:33:12.087 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:12.120 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:33:12.153 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:33:12.182 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:12.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:12.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:12.205 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:33:12.240 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:33:12.240 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:33:12.240 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-30T09:33:12.240 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:33:12.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:33:12.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:33:12.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:33:12.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:33:12.241 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:33:12.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:12.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e295 with expected crc 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: pgmap v723: 119 pgs: 10 unknown, 6 peering, 18 active+undersized+degraded, 40 active+undersized, 45 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46834/247014 objects degraded (18.960%) 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: 7.3 starting backfill to osd.2 from (0'0,0'0] MAX to 293'6020 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: osdmap e296: 8 total, 7 up, 8 in 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: 7.a starting backfill to osd.2 from (0'0,0'0] MAX to 293'6016 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: 7.8 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5959 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: 7.e starting backfill to osd.2 from (0'0,0'0] MAX to 293'5945 2023-09-30T09:33:12.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.256 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.256 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.256 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: 7.2 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5926 2023-09-30T09:33:12.256 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.256 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.256 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 293'5734 2023-09-30T09:33:12.256 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: 7.5 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5966 2023-09-30T09:33:12.256 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/878254568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:12.299 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:12.376 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:12.432 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 70 16 95500 95484 85.2297 28.7812 0.00657677 0.0117281 2023-09-30T09:33:12.451 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:12.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:12.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e295 with expected crc 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: pgmap v723: 119 pgs: 10 unknown, 6 peering, 18 active+undersized+degraded, 40 active+undersized, 45 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46834/247014 objects degraded (18.960%) 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: 7.3 starting backfill to osd.2 from (0'0,0'0] MAX to 293'6020 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: osdmap e296: 8 total, 7 up, 8 in 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: 7.a starting backfill to osd.2 from (0'0,0'0] MAX to 293'6016 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: 7.8 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5959 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: 7.e starting backfill to osd.2 from (0'0,0'0] MAX to 293'5945 2023-09-30T09:33:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: 7.2 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5926 2023-09-30T09:33:12.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 293'5734 2023-09-30T09:33:12.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: 7.5 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5966 2023-09-30T09:33:12.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/878254568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:12.555 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e295 with expected crc 2023-09-30T09:33:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: pgmap v723: 119 pgs: 10 unknown, 6 peering, 18 active+undersized+degraded, 40 active+undersized, 45 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46834/247014 objects degraded (18.960%) 2023-09-30T09:33:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: 7.3 starting backfill to osd.2 from (0'0,0'0] MAX to 293'6020 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: osdmap e296: 8 total, 7 up, 8 in 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: 7.a starting backfill to osd.2 from (0'0,0'0] MAX to 293'6016 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: 7.8 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5959 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: 7.e starting backfill to osd.2 from (0'0,0'0] MAX to 293'5945 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: 7.2 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5926 2023-09-30T09:33:12.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:12.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 293'5734 2023-09-30T09:33:12.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: 7.5 starting backfill to osd.2 from (0'0,0'0] MAX to 293'5966 2023-09-30T09:33:12.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/878254568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:12.605 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:12.715 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:13.370 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:33:13.371 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.378 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.378 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.378 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.378 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: osdmap e297: 8 total, 7 up, 8 in 2023-09-30T09:33:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:13.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:13.432 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 71 16 96419 96403 84.8383 57.4375 0.00974793 0.0117829 2023-09-30T09:33:13.481 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:13.509 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.509 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[216815]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: osdmap e297: 8 total, 7 up, 8 in 2023-09-30T09:33:13.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:13 smithi019 ceph-mon[224043]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e296 with expected crc 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: osdmap e297: 8 total, 7 up, 8 in 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:13 smithi167 ceph-mon[192880]: failed to encode map e297 with expected crc 2023-09-30T09:33:13.537 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:33:13.547 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:13.610 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:33:13.611 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:33:13.619 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:13.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:13.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:13.727 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:13.788 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:13.908 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:13.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:13.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:14.015 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:14.063 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:33:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:33:13] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-09-30T09:33:14.076 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:14.179 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:14.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:14.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:14.220 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:14.283 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:14.292 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:14.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 46834/247014 objects degraded (18.960%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:14.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: pgmap v725: 119 pgs: 10 unknown, 6 peering, 18 active+undersized+degraded, 40 active+undersized, 45 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46834/247014 objects degraded (18.960%) 2023-09-30T09:33:14.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: osdmap e298: 8 total, 7 up, 8 in 2023-09-30T09:33:14.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[216815]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 46834/247014 objects degraded (18.960%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: pgmap v725: 119 pgs: 10 unknown, 6 peering, 18 active+undersized+degraded, 40 active+undersized, 45 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46834/247014 objects degraded (18.960%) 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: osdmap e298: 8 total, 7 up, 8 in 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:14 smithi019 ceph-mon[224043]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.344 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:14.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 72 16 97443 97427 84.5489 64 0.00816602 0.0118229 2023-09-30T09:33:14.442 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:14.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 46834/247014 objects degraded (18.960%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: pgmap v725: 119 pgs: 10 unknown, 6 peering, 18 active+undersized+degraded, 40 active+undersized, 45 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46834/247014 objects degraded (18.960%) 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: osdmap e298: 8 total, 7 up, 8 in 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:14 smithi167 ceph-mon[192880]: failed to encode map e298 with expected crc 2023-09-30T09:33:14.555 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:14.606 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:14.689 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:14.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:14.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:14.801 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:14.860 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:14.946 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:14.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:14.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:15.055 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:15.125 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:15.181 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:15.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:15.295 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:15.321 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:15.365 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:15.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:15.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:15.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 73 16 98387 98371 84.1989 59 0.159689 0.0118679 2023-09-30T09:33:15.472 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:15.543 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:15.612 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:33:16.342 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:16.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:16.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:16.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 74 16 99515 99499 84.0137 70.5 0.00825371 0.0118979 2023-09-30T09:33:16.446 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:16.490 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:16.532 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0, 2] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-30T09:33:16.532 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:33:16.533 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-30T09:33:16.533 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-30T09:33:16.533 DEBUG:teuthology.orchestra.run.smithi019:osd.1> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-30T09:33:16.535 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl start ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.1 2023-09-30T09:33:16.558 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:16.564 INFO:journalctl@ceph.osd.1.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:33:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:16.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:16.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:16 smithi019 ceph-mon[216815]: pgmap v727: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 373 KiB/s rd, 184 MiB/s wr, 3.07k op/s; 60050/291129 objects degraded (20.627%); 36451/291129 objects misplaced (12.521%); 5.8 MiB/s, 1 keys/s, 101 objects/s recovering 2023-09-30T09:33:16.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:16 smithi019 ceph-mon[224043]: pgmap v727: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 373 KiB/s rd, 184 MiB/s wr, 3.07k op/s; 60050/291129 objects degraded (20.627%); 36451/291129 objects misplaced (12.521%); 5.8 MiB/s, 1 keys/s, 101 objects/s recovering 2023-09-30T09:33:16.666 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:16.741 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:16.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:16 smithi167 ceph-mon[192880]: pgmap v727: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 373 KiB/s rd, 184 MiB/s wr, 3.07k op/s; 60050/291129 objects degraded (20.627%); 36451/291129 objects misplaced (12.521%); 5.8 MiB/s, 1 keys/s, 101 objects/s recovering 2023-09-30T09:33:16.998 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:16 smithi019 systemd[1]: Starting Ceph osd.1 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:33:17.022 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:17.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:17.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:17.131 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:17.179 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:17.257 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:17.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:17.373 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:17.414 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:17.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 75 16 100464 100448 83.6843 59.3125 0.0131131 0.0119451 2023-09-30T09:33:17.485 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:17.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:17.530 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:17.596 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:17.739 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:17.827 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80421 5.a 5:536b7ad8:::benchmark_data_smithi019_287194_object80420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.575313+0000", 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 556.24836690799998, 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3514731449999999, 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:17.828 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80421 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575313+0000", 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575313+0000", 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575314+0000", 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1510000000000001e-06 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575338+0000", 2023-09-30T09:33:17.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3546000000000001e-05 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575355+0000", 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7292e-05 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.575363+0000", 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5979999999999999e-06 2023-09-30T09:33:17.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.582558+0000", 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0071956959999999997 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.582681+0000", 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000122588 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.582821+0000", 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013951 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.591425+0000", 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086042619999999997 2023-09-30T09:33:17.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.591442+0000", 2023-09-30T09:33:17.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7652e-05 2023-09-30T09:33:17.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620803+0000", 2023-09-30T09:33:17.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.029360377999999999 2023-09-30T09:33:17.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620815+0000", 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172e-05 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606293+0000", 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98547855500000003 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606306+0000", 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2862e-05 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.926699+0000", 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3203928170000001 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.926786+0000", 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7517999999999998e-05 2023-09-30T09:33:17.837 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80431 5.a 5:539a1d6a:::benchmark_data_smithi019_287194_object80430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.592490+0000", 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 556.23119018199998, 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3383307100000001, 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80431 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592490+0000", 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592490+0000", 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592490+0000", 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1199999999999998e-07 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592513+0000", 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2864999999999999e-05 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592520+0000", 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3910000000000003e-06 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592525+0000", 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0100000000000003e-06 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592804+0000", 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027989900000000001 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592953+0000", 2023-09-30T09:33:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014838199999999999 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.593107+0000", 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015438 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599129+0000", 2023-09-30T09:33:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0060221859999999997 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.604596+0000", 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0054663929999999999 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620852+0000", 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.01625619 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620861+0000", 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5620000000000004e-06 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606349+0000", 2023-09-30T09:33:17.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98548850600000004 2023-09-30T09:33:17.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606360+0000", 2023-09-30T09:33:17.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0954e-05 2023-09-30T09:33:17.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.930701+0000", 2023-09-30T09:33:17.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3243411780000001 2023-09-30T09:33:17.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.930821+0000", 2023-09-30T09:33:17.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011940200000000001 2023-09-30T09:33:17.848 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.848 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.849 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.849 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.849 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80432 5.a 5:5bdae413:::benchmark_data_smithi019_287194_object80431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:33:17.849 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.592531+0000", 2023-09-30T09:33:17.849 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 556.23114865499997, 2023-09-30T09:33:17.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.338405018, 2023-09-30T09:33:17.850 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.850 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:17.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.851 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80432 2023-09-30T09:33:17.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.851 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592531+0000", 2023-09-30T09:33:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592531+0000", 2023-09-30T09:33:17.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592532+0000", 2023-09-30T09:33:17.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.84e-07 2023-09-30T09:33:17.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592551+0000", 2023-09-30T09:33:17.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8927999999999999e-05 2023-09-30T09:33:17.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592557+0000", 2023-09-30T09:33:17.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0920000000000003e-06 2023-09-30T09:33:17.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.592562+0000", 2023-09-30T09:33:17.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8010000000000003e-06 2023-09-30T09:33:17.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.856 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.595900+0000", 2023-09-30T09:33:17.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033379730000000002 2023-09-30T09:33:17.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.596033+0000", 2023-09-30T09:33:17.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013348799999999999 2023-09-30T09:33:17.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:33:17.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.596167+0000", 2023-09-30T09:33:17.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013392100000000001 2023-09-30T09:33:17.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.604602+0000", 2023-09-30T09:33:17.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0084350959999999996 2023-09-30T09:33:17.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.607643+0000", 2023-09-30T09:33:17.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003040423 2023-09-30T09:33:17.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.860 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620894+0000", 2023-09-30T09:33:17.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013251639000000001 2023-09-30T09:33:17.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.861 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.620903+0000", 2023-09-30T09:33:17.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4479999999999994e-06 2023-09-30T09:33:17.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606397+0000", 2023-09-30T09:33:17.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98549406399999995 2023-09-30T09:33:17.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.606407+0000", 2023-09-30T09:33:17.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0677e-05 2023-09-30T09:33:17.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.930857+0000", 2023-09-30T09:33:17.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.324449378 2023-09-30T09:33:17.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.930936+0000", 2023-09-30T09:33:17.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9705999999999999e-05 2023-09-30T09:33:17.864 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.864 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.864 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.865 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:80436 5.d 5:bc14734e:::benchmark_data_smithi019_287194_object80435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-30T09:33:17.865 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:24:01.599675+0000", 2023-09-30T09:33:17.865 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 556.22400472699996, 2023-09-30T09:33:17.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3736680059999999, 2023-09-30T09:33:17.865 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.866 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:17.866 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.866 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.866 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.866 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 80436 2023-09-30T09:33:17.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.867 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599675+0000", 2023-09-30T09:33:17.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599675+0000", 2023-09-30T09:33:17.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599676+0000", 2023-09-30T09:33:17.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4099999999999999e-07 2023-09-30T09:33:17.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599700+0000", 2023-09-30T09:33:17.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4502e-05 2023-09-30T09:33:17.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599708+0000", 2023-09-30T09:33:17.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5730000000000002e-06 2023-09-30T09:33:17.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.599714+0000", 2023-09-30T09:33:17.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2149999999999998e-06 2023-09-30T09:33:17.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.601660+0000", 2023-09-30T09:33:17.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019455449999999999 2023-09-30T09:33:17.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.601801+0000", 2023-09-30T09:33:17.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000141008 2023-09-30T09:33:17.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.601953+0000", 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015205700000000001 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.610590+0000", 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086373960000000003 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.614958+0000", 2023-09-30T09:33:17.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043675220000000004 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621354+0000", 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063966839999999997 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:01.621364+0000", 2023-09-30T09:33:17.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8810000000000008e-06 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.607174+0000", 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98580998200000003 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:02.607192+0000", 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8068999999999999e-05 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.973241+0000", 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3660484550000001 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:24:03.973343+0000", 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000102576 2023-09-30T09:33:17.876 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119587 5.d 5:b6587ad4:::benchmark_data_smithi019_287194_object36809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.240740+0000", 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.58293994799999, 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0487021569999999, 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.877 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119587 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240740+0000", 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240740+0000", 2023-09-30T09:33:17.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240741+0000", 2023-09-30T09:33:17.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172e-06 2023-09-30T09:33:17.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240745+0000", 2023-09-30T09:33:17.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1949999999999996e-06 2023-09-30T09:33:17.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240753+0000", 2023-09-30T09:33:17.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4649999999999997e-06 2023-09-30T09:33:17.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240758+0000", 2023-09-30T09:33:17.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7060000000000003e-06 2023-09-30T09:33:17.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.882 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.882 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.306657+0000", 2023-09-30T09:33:17.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.065899345999999998 2023-09-30T09:33:17.882 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.882 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.882 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.306831+0000", 2023-09-30T09:33:17.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017438600000000001 2023-09-30T09:33:17.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.306966+0000", 2023-09-30T09:33:17.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013437099999999999 2023-09-30T09:33:17.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.315190+0000", 2023-09-30T09:33:17.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0082239329999999992 2023-09-30T09:33:17.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.315204+0000", 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4064e-05 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319784+0000", 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0045798760000000001 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.885 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.886 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319794+0000", 2023-09-30T09:33:17.886 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0005e-05 2023-09-30T09:33:17.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289378+0000", 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96958487100000001 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289442+0000", 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3767000000000001e-05 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.887 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119589 5.d 5:b4212818:::benchmark_data_smithi019_287194_object36811:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.240788+0000", 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.582892195, 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0491740350000001, 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119589 2023-09-30T09:33:17.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.889 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240788+0000", 2023-09-30T09:33:17.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240788+0000", 2023-09-30T09:33:17.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240788+0000", 2023-09-30T09:33:17.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4400000000000001e-07 2023-09-30T09:33:17.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.891 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.891 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.891 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240789+0000", 2023-09-30T09:33:17.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-06 2023-09-30T09:33:17.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.891 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.891 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.240794+0000", 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6020000000000002e-06 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.306657+0000", 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.06586322 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.893 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.310985+0000", 2023-09-30T09:33:17.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043277230000000003 2023-09-30T09:33:17.893 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.893 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.311112+0000", 2023-09-30T09:33:17.894 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012651399999999999 2023-09-30T09:33:17.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.894 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.894 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.894 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.311238+0000", 2023-09-30T09:33:17.894 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000126062 2023-09-30T09:33:17.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.894 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.320082+0000", 2023-09-30T09:33:17.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088445490000000002 2023-09-30T09:33:17.895 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.320091+0000", 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6920000000000008e-06 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289911+0000", 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96982005599999999 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289962+0000", 2023-09-30T09:33:17.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0992000000000002e-05 2023-09-30T09:33:17.897 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.897 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.897 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.898 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119593 5.d 5:bec5f115:::benchmark_data_smithi019_287194_object36815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-30T09:33:17.898 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.319460+0000", 2023-09-30T09:33:17.898 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.50422031400001, 2023-09-30T09:33:17.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98468280600000002, 2023-09-30T09:33:17.898 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.898 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:17.898 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.899 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.899 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.899 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119593 2023-09-30T09:33:17.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.899 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319460+0000", 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319460+0000", 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.900 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319461+0000", 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.212e-06 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.319465+0000", 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1450000000000001e-06 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.901 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.323330+0000", 2023-09-30T09:33:17.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038647619999999999 2023-09-30T09:33:17.902 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.323344+0000", 2023-09-30T09:33:17.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3753e-05 2023-09-30T09:33:17.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.323455+0000", 2023-09-30T09:33:17.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011108900000000001 2023-09-30T09:33:17.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.323466+0000", 2023-09-30T09:33:17.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1452e-05 2023-09-30T09:33:17.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.289993+0000", 2023-09-30T09:33:17.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96652676000000004 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.290096+0000", 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010355499999999999 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.290179+0000", 2023-09-30T09:33:17.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2276999999999998e-05 2023-09-30T09:33:17.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.906 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.906 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.303990+0000", 2023-09-30T09:33:17.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013811742 2023-09-30T09:33:17.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.907 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304056+0000", 2023-09-30T09:33:17.907 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5294000000000002e-05 2023-09-30T09:33:17.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304067+0000", 2023-09-30T09:33:17.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1251e-05 2023-09-30T09:33:17.908 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.908 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.908 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:17.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304103+0000", 2023-09-30T09:33:17.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.608e-05 2023-09-30T09:33:17.908 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304143+0000", 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9434000000000002e-05 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119597 5.d 5:b8437bdd:::benchmark_data_smithi019_287194_object36819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-30T09:33:17.909 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.324156+0000", 2023-09-30T09:33:17.910 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.49952439899999, 2023-09-30T09:33:17.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98024695500000003, 2023-09-30T09:33:17.910 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.910 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:17.910 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.910 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.910 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.910 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119597 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324156+0000", 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324156+0000", 2023-09-30T09:33:17.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324157+0000", 2023-09-30T09:33:17.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2109999999999999e-06 2023-09-30T09:33:17.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324162+0000", 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9960000000000001e-06 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324174+0000", 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.167e-05 2023-09-30T09:33:17.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324182+0000", 2023-09-30T09:33:17.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0590000000000003e-06 2023-09-30T09:33:17.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324312+0000", 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013004 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.324320+0000", 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0819999999999999e-06 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.293198+0000", 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96887842199999996 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.293327+0000", 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000128944 2023-09-30T09:33:17.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.917 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.917 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.917 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.293460+0000", 2023-09-30T09:33:17.917 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013269700000000001 2023-09-30T09:33:17.917 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.917 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.917 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.917 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304156+0000", 2023-09-30T09:33:17.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010696056000000001 2023-09-30T09:33:17.918 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.918 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.918 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304247+0000", 2023-09-30T09:33:17.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1582999999999994e-05 2023-09-30T09:33:17.918 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304311+0000", 2023-09-30T09:33:17.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3157999999999994e-05 2023-09-30T09:33:17.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:17.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304365+0000", 2023-09-30T09:33:17.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4490000000000002e-05 2023-09-30T09:33:17.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304403+0000", 2023-09-30T09:33:17.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7546999999999998e-05 2023-09-30T09:33:17.920 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.920 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.921 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.921 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119612 5.d 5:bd3bda5c:::benchmark_data_smithi019_287194_object36834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:33:17.921 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.384780+0000", 2023-09-30T09:33:17.921 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.43890009799998, 2023-09-30T09:33:17.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.91984285700000001, 2023-09-30T09:33:17.922 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.922 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:17.922 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.922 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.922 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.922 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119612 2023-09-30T09:33:17.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.923 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384780+0000", 2023-09-30T09:33:17.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384780+0000", 2023-09-30T09:33:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384781+0000", 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.043e-06 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384786+0000", 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.259e-06 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.925 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384802+0000", 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5756e-05 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384810+0000", 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1540000000000002e-06 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384940+0000", 2023-09-30T09:33:17.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000129352 2023-09-30T09:33:17.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.384949+0000", 2023-09-30T09:33:17.928 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7880000000000001e-06 2023-09-30T09:33:17.928 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.928 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.928 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.296569+0000", 2023-09-30T09:33:17.928 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.911620082 2023-09-30T09:33:17.928 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.929 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.929 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.296679+0000", 2023-09-30T09:33:17.929 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010973500000000001 2023-09-30T09:33:17.929 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.929 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.929 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.929 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.296787+0000", 2023-09-30T09:33:17.929 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000108106 2023-09-30T09:33:17.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304172+0000", 2023-09-30T09:33:17.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0073844059999999996 2023-09-30T09:33:17.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304482+0000", 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00031017600000000001 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304533+0000", 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1066999999999997e-05 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304584+0000", 2023-09-30T09:33:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0881999999999998e-05 2023-09-30T09:33:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304623+0000", 2023-09-30T09:33:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9051000000000003e-05 2023-09-30T09:33:17.932 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119631 5.d 5:b005cdcf:::benchmark_data_smithi019_287194_object36853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.406119+0000", 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.41756137900001, 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90989635000000002, 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.933 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119631 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406119+0000", 2023-09-30T09:33:17.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.935 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.935 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406119+0000", 2023-09-30T09:33:17.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406120+0000", 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.249e-06 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406125+0000", 2023-09-30T09:33:17.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7559999999999999e-06 2023-09-30T09:33:17.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.937 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.937 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406135+0000", 2023-09-30T09:33:17.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0463e-05 2023-09-30T09:33:17.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.937 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406142+0000", 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3300000000000001e-06 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406237+0000", 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4462999999999993e-05 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.939 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.939 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.406248+0000", 2023-09-30T09:33:17.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0614999999999999e-05 2023-09-30T09:33:17.939 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.939 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.939 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.939 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.300293+0000", 2023-09-30T09:33:17.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89404568500000003 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.300433+0000", 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013951 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.300571+0000", 2023-09-30T09:33:17.940 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000137923 2023-09-30T09:33:17.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304417+0000", 2023-09-30T09:33:17.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038467610000000002 2023-09-30T09:33:17.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304672+0000", 2023-09-30T09:33:17.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025463200000000002 2023-09-30T09:33:17.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.942 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.942 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.315933+0000", 2023-09-30T09:33:17.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011261353 2023-09-30T09:33:17.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:17.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.315976+0000", 2023-09-30T09:33:17.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2250999999999999e-05 2023-09-30T09:33:17.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.945 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.945 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.945 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316015+0000", 2023-09-30T09:33:17.945 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9359e-05 2023-09-30T09:33:17.945 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.945 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.945 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.946 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.946 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119646 5.d 5:b288c6e9:::benchmark_data_smithi019_287194_object36868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:33:17.946 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.427954+0000", 2023-09-30T09:33:17.946 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.39572575599999, 2023-09-30T09:33:17.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89682026999999997, 2023-09-30T09:33:17.946 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119646 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.947 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427954+0000", 2023-09-30T09:33:17.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427954+0000", 2023-09-30T09:33:17.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427956+0000", 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.217e-06 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427960+0000", 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6090000000000003e-06 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427976+0000", 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5390999999999999e-05 2023-09-30T09:33:17.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.427985+0000", 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2990000000000002e-06 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428087+0000", 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000102631 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428102+0000", 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4069e-05 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304713+0000", 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87661197499999999 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304820+0000", 2023-09-30T09:33:17.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010655 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.304934+0000", 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113917 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316036+0000", 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011102153 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316076+0000", 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0252e-05 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324680+0000", 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086032560000000001 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324736+0000", 2023-09-30T09:33:17.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5887999999999998e-05 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324775+0000", 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9063000000000002e-05 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119649 5.d 5:b97acc98:::benchmark_data_smithi019_287194_object36871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.428009+0000", 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.39567112700001, 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.896981166, 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119649 2023-09-30T09:33:17.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428009+0000", 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428009+0000", 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428009+0000", 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6499999999999999e-07 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428013+0000", 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0809999999999998e-06 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428020+0000", 2023-09-30T09:33:17.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3300000000000001e-06 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428026+0000", 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3099999999999997e-06 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428132+0000", 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010583500000000001 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.428141+0000", 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9949999999999994e-06 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.308041+0000", 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87989993799999999 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.308172+0000", 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013134999999999999 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.308274+0000", 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010208 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.962 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316086+0000", 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.007811162 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324840+0000", 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087541049999999999 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324896+0000", 2023-09-30T09:33:17.963 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5889e-05 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324954+0000", 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8155000000000001e-05 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.324990+0000", 2023-09-30T09:33:17.964 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6371000000000001e-05 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119659 5.d 5:b8082ea3:::benchmark_data_smithi019_287194_object36881:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.436320+0000", 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.38736033499998, 2023-09-30T09:33:17.965 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90387485099999998, 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119659 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.966 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436320+0000", 2023-09-30T09:33:17.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.967 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.967 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.967 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436320+0000", 2023-09-30T09:33:17.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436320+0000", 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4900000000000001e-07 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436323+0000", 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3410000000000001e-06 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436336+0000", 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3641e-05 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436342+0000", 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9510000000000003e-06 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436407+0000", 2023-09-30T09:33:17.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4412000000000006e-05 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.436416+0000", 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4970000000000007e-06 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316129+0000", 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87971309900000005 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.970 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316274+0000", 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014522499999999999 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.316389+0000", 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000114208 2023-09-30T09:33:17.971 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.325003+0000", 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086141789999999996 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.332748+0000", 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077455029999999999 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.972 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340107+0000", 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0073584699999999998 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340143+0000", 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6634999999999997e-05 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340195+0000", 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1240999999999999e-05 2023-09-30T09:33:17.973 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:119661 5.d 5:b77bcc20:::benchmark_data_smithi019_287194_object36883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:10.441103+0000", 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 427.382576653, 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89930644500000001, 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:17.974 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 119661 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441103+0000", 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.975 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.976 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441103+0000", 2023-09-30T09:33:17.976 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.976 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.976 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.976 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.976 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441105+0000", 2023-09-30T09:33:17.977 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4139999999999999e-06 2023-09-30T09:33:17.977 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.977 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.977 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.977 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441111+0000", 2023-09-30T09:33:17.978 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.049e-06 2023-09-30T09:33:17.978 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.978 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.978 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.978 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441125+0000", 2023-09-30T09:33:17.978 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4153e-05 2023-09-30T09:33:17.979 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.979 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.979 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.979 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441133+0000", 2023-09-30T09:33:17.979 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0450000000000001e-06 2023-09-30T09:33:17.979 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.980 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.980 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.980 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441245+0000", 2023-09-30T09:33:17.980 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000111558 2023-09-30T09:33:17.980 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.980 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.980 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.981 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:10.441256+0000", 2023-09-30T09:33:17.981 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1579999999999999e-05 2023-09-30T09:33:17.981 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.981 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.981 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.981 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.320272+0000", 2023-09-30T09:33:17.982 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87901538099999998 2023-09-30T09:33:17.982 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.982 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.982 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.982 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.320371+0000", 2023-09-30T09:33:17.982 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9270000000000003e-05 2023-09-30T09:33:17.982 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.983 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.983 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:33:17.983 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.320476+0000", 2023-09-30T09:33:17.983 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000104944 2023-09-30T09:33:17.983 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.983 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.984 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.984 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.336739+0000", 2023-09-30T09:33:17.984 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016263357999999999 2023-09-30T09:33:17.984 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.984 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.984 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.985 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340293+0000", 2023-09-30T09:33:17.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035540900000000002 2023-09-30T09:33:17.985 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.985 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.985 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.985 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340339+0000", 2023-09-30T09:33:17.986 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5862000000000002e-05 2023-09-30T09:33:17.986 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.986 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.986 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:17.986 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340377+0000", 2023-09-30T09:33:17.986 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8192999999999998e-05 2023-09-30T09:33:17.986 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.986 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:11.340410+0000", 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2548000000000001e-05 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145642 5.a 5:5d87f834:::benchmark_data_smithi019_287194_object62864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.286985+0000", 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 387.536695006, 2023-09-30T09:33:17.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6001666220000001, 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145642 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.988 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.286985+0000", 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.286985+0000", 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.286987+0000", 2023-09-30T09:33:17.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.657e-06 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.286993+0000", 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9259999999999997e-06 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287007+0000", 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3898e-05 2023-09-30T09:33:17.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287015+0000", 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2199999999999992e-06 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.289744+0000", 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027293949999999999 2023-09-30T09:33:17.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.289891+0000", 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014728899999999999 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.290029+0000", 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000137758 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:17.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.308124+0000", 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018094566999999999 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313660+0000", 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055359249999999997 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.324923+0000", 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011262853 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.324938+0000", 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5201999999999999e-05 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308729+0000", 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98379169399999999 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.994 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308743+0000", 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3352e-05 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.800834+0000", 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.492090793 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.800989+0000", 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015555500000000001 2023-09-30T09:33:17.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.887074+0000", 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.086085150999999999 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.887152+0000", 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7386999999999996e-05 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:17.996 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145643 5.a 5:52928c9d:::benchmark_data_smithi019_287194_object62865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.287030+0000", 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 387.53665038000003, 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6018917200000002, 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:17.997 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145643 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287030+0000", 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:17.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287030+0000", 2023-09-30T09:33:17.999 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:17.999 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:17.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:17.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287030+0000", 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3000000000000001e-07 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287033+0000", 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9610000000000001e-06 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:18.000 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287041+0000", 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3520000000000001e-06 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.287048+0000", 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9979999999999997e-06 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.294960+0000", 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0079122800000000007 2023-09-30T09:33:18.001 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.295082+0000", 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000122559 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.295203+0000", 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012053900000000001 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.002 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313664+0000", 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018460982000000001 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.315555+0000", 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018911620000000001 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.003 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325177+0000", 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0096223430000000002 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325203+0000", 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6016999999999998e-05 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308838+0000", 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98363487100000002 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308854+0000", 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6166000000000001e-05 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.802671+0000", 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4938168599999999 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:33:18.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.802866+0000", 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019441700000000001 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.888843+0000", 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.085977656999999999 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.888921+0000", 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8026000000000001e-05 2023-09-30T09:33:18.006 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145646 5.a 5:58abbb4b:::benchmark_data_smithi019_287194_object62868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.291309+0000", 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 387.53237141599999, 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6126824850000001, 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:18.007 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145646 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291309+0000", 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291309+0000", 2023-09-30T09:33:18.008 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291310+0000", 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3999999999999999e-06 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291315+0000", 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1950000000000002e-06 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.010 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:18.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291332+0000", 2023-09-30T09:33:18.010 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7005000000000001e-05 2023-09-30T09:33:18.010 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.010 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.010 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:18.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291341+0000", 2023-09-30T09:33:18.010 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1370000000000008e-06 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.303167+0000", 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011826066 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.303285+0000", 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000117348 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.011 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.303390+0000", 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000104719 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.313667+0000", 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010277873999999999 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.012 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.315592+0000", 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001924697 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325238+0000", 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0096459260000000008 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325246+0000", 2023-09-30T09:33:18.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1550000000000007e-06 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308908+0000", 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98366193300000004 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308919+0000", 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1124000000000001e-05 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.014 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.802923+0000", 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.494003881 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803085+0000", 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016144 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.015 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.903883+0000", 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.100798417 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.903991+0000", 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000108168 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.016 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145647 5.a 5:5b66dd4d:::benchmark_data_smithi019_287194_object62869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.291355+0000", 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 387.53232546800001, 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6146902910000001, 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145647 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.017 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:18.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:18.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291355+0000", 2023-09-30T09:33:18.018 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:18.018 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.019 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:18.019 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291355+0000", 2023-09-30T09:33:18.019 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:18.019 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.019 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.019 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:18.020 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291355+0000", 2023-09-30T09:33:18.020 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7599999999999997e-07 2023-09-30T09:33:18.020 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.020 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.020 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:18.020 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291358+0000", 2023-09-30T09:33:18.021 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7690000000000001e-06 2023-09-30T09:33:18.021 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.021 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.021 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:18.021 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291368+0000", 2023-09-30T09:33:18.021 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7799999999999995e-06 2023-09-30T09:33:18.022 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.022 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:18.022 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.291376+0000", 2023-09-30T09:33:18.022 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5210000000000002e-06 2023-09-30T09:33:18.022 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.308715+0000", 2023-09-30T09:33:18.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017338610000000001 2023-09-30T09:33:18.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.023 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:18.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.308876+0000", 2023-09-30T09:33:18.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016106900000000001 2023-09-30T09:33:18.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:33:18.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.309012+0000", 2023-09-30T09:33:18.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000136067 2023-09-30T09:33:18.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:18.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.315628+0000", 2023-09-30T09:33:18.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0066166080000000004 2023-09-30T09:33:18.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:18.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.315959+0000", 2023-09-30T09:33:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033090799999999998 2023-09-30T09:33:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325316+0000", 2023-09-30T09:33:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0093571009999999996 2023-09-30T09:33:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.325325+0000", 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3189999999999995e-06 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308955+0000", 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98363034999999999 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308980+0000", 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4943000000000002e-05 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803130+0000", 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4941497420000001 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:33:18.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803279+0000", 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000149476 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.905950+0000", 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.10267042 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.906045+0000", 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5131999999999997e-05 2023-09-30T09:33:18.029 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145680 5.a 5:5166d668:::benchmark_data_smithi019_287194_object62902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.355901+0000", 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 387.46777920400001, 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5249182000000001, 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:18.030 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145680 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355901+0000", 2023-09-30T09:33:18.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355901+0000", 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355903+0000", 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.655e-06 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355909+0000", 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.596e-06 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355921+0000", 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1505000000000001e-05 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355928+0000", 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9720000000000003e-06 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355981+0000", 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3405000000000001e-05 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.355989+0000", 2023-09-30T09:33:18.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6729999999999992e-06 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.309011+0000", 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95302198100000002 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.309020+0000", 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0340000000000002e-06 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803332+0000", 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4943121939999999 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803503+0000", 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017102799999999999 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.803685+0000", 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018205900000000001 2023-09-30T09:33:18.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.832506+0000", 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028821060999999999 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.833422+0000", 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00091556299999999999 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.879313+0000", 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.045890959000000002 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.879377+0000", 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4225000000000003e-05 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.880819+0000", 2023-09-30T09:33:18.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0014422899999999999 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145685 5.b 5:deae4cba:::benchmark_data_smithi019_287194_object62907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.359013+0000", 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 387.46466655199998, 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.420992553, 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:18.039 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145685 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359013+0000", 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359013+0000", 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359014+0000", 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2399999999999999e-07 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359016+0000", 2023-09-30T09:33:18.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.379e-06 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359022+0000", 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.626e-06 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359027+0000", 2023-09-30T09:33:18.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3639999999999998e-06 2023-09-30T09:33:18.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359084+0000", 2023-09-30T09:33:18.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7089000000000002e-05 2023-09-30T09:33:18.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.359091+0000", 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.516e-06 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308905+0000", 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94981431199999999 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.308916+0000", 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0776e-05 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.627874+0000", 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.318958147 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.627957+0000", 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2960999999999994e-05 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,7", 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.628052+0000", 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5212999999999998e-05 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.777472+0000", 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14941965300000001 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.778998+0000", 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001525649 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:18.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.779072+0000", 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4690999999999994e-05 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.779145+0000", 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2857999999999999e-05 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.780006+0000", 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00086089499999999998 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.048 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:18.049 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.049 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:18.049 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:18.049 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:33:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:18.050 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:18.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:18.153 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:18.228 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:18.383 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:18.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:18.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:18.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 76 16 101410 101394 83.3612 59.125 0.0111077 0.0119918 2023-09-30T09:33:18.455 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:33:18.496 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:18 smithi167 ceph-mon[192880]: pgmap v728: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 311 KiB/s rd, 153 MiB/s wr, 2.56k op/s; 60050/291129 objects degraded (20.627%); 36451/291129 objects misplaced (12.521%); 4.9 MiB/s, 1 keys/s, 84 objects/s recovering 2023-09-30T09:33:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1208576053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 60050/291129 objects degraded (20.627%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:18.559 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:18.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:18 smithi019 ceph-mon[224043]: pgmap v728: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 311 KiB/s rd, 153 MiB/s wr, 2.56k op/s; 60050/291129 objects degraded (20.627%); 36451/291129 objects misplaced (12.521%); 4.9 MiB/s, 1 keys/s, 84 objects/s recovering 2023-09-30T09:33:18.580 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:18.580 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1208576053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:18.580 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 60050/291129 objects degraded (20.627%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:18.580 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:18 smithi019 ceph-mon[216815]: pgmap v728: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 311 KiB/s rd, 153 MiB/s wr, 2.56k op/s; 60050/291129 objects degraded (20.627%); 36451/291129 objects misplaced (12.521%); 4.9 MiB/s, 1 keys/s, 84 objects/s recovering 2023-09-30T09:33:18.580 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:18.580 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1208576053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:18.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 60050/291129 objects degraded (20.627%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:18.653 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:18.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:18.759 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:18.784 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:18.825 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:18.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:18.931 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:18.992 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:19.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 77 16 102271 102255 82.9774 53.8125 0.00937695 0.0120465 2023-09-30T09:33:19.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:33:19.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1208576053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:33:19.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: osdmap e299: 8 total, 7 up, 8 in 2023-09-30T09:33:19.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[216815]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1208576053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: osdmap e299: 8 total, 7 up, 8 in 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:19 smithi019 ceph-mon[224043]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.759 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1208576053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: osdmap e299: 8 total, 7 up, 8 in 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:19 smithi167 ceph-mon[192880]: failed to encode map e299 with expected crc 2023-09-30T09:33:19.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:19.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:19.809 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:33:19.876 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:19.925 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:20.025 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:20.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:20.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:20.130 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:20.160 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:20.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 78 16 103242 103226 82.6916 60.6875 0.00720348 0.0120884 2023-09-30T09:33:20.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:20 smithi167 ceph-mon[192880]: pgmap v730: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 267 KiB/s rd, 167 MiB/s wr, 2.77k op/s; 62541/303045 objects degraded (20.638%); 38005/303045 objects misplaced (12.541%); 4.2 MiB/s, 1 keys/s, 72 objects/s recovering 2023-09-30T09:33:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:20 smithi019 ceph-mon[216815]: pgmap v730: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 267 KiB/s rd, 167 MiB/s wr, 2.77k op/s; 62541/303045 objects degraded (20.638%); 38005/303045 objects misplaced (12.541%); 4.2 MiB/s, 1 keys/s, 72 objects/s recovering 2023-09-30T09:33:20.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:20 smithi019 ceph-mon[224043]: pgmap v730: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 267 KiB/s rd, 167 MiB/s wr, 2.77k op/s; 62541/303045 objects degraded (20.638%); 38005/303045 objects misplaced (12.541%); 4.2 MiB/s, 1 keys/s, 72 objects/s recovering 2023-09-30T09:33:21.210 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:21.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:21.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:21.322 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:21.336 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:21 smithi019 bash[420675]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-30T09:33:21.336 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:21 smithi019 bash[420675]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-30T09:33:21.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:21 smithi019 bash[420675]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-30T09:33:21.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:21 smithi019 bash[420675]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-30T09:33:21.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:21 smithi019 bash[420675]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-30T09:33:21.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:21 smithi019 bash[420675]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-30T09:33:21.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:21 smithi019 bash[420675]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-30T09:33:21.337 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:21 smithi019 bash[420675]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-30T09:33:21.398 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:21.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 79 16 104070 104054 82.3 51.75 0.0103445 0.0121462 2023-09-30T09:33:21.987 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:21.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:22.095 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:22.148 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:22.232 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:22.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:22.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:22.339 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:22.413 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:22.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:33:22.433487+0000 min lat: 0.00127075 max lat: 2.24784 avg lat: 0.0121856 2023-09-30T09:33:22.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:33:22.433 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 80 16 105048 105032 82.0352 61.125 0.0073086 0.0121856 2023-09-30T09:33:22.507 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:22.537 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:22.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:22.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:22 smithi019 ceph-mon[216815]: pgmap v731: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 233 KiB/s rd, 153 MiB/s wr, 2.53k op/s; 62330/305751 objects degraded (20.386%); 38300/305751 objects misplaced (12.527%); 9.7 MiB/s, 1 keys/s, 160 objects/s recovering 2023-09-30T09:33:22.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:22 smithi019 ceph-mon[224043]: pgmap v731: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 233 KiB/s rd, 153 MiB/s wr, 2.53k op/s; 62330/305751 objects degraded (20.386%); 38300/305751 objects misplaced (12.527%); 9.7 MiB/s, 1 keys/s, 160 objects/s recovering 2023-09-30T09:33:22.644 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:22.689 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:33:22.710 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:22.721 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:33:22.765 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:33:22.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:22 smithi167 ceph-mon[192880]: pgmap v731: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 233 KiB/s rd, 153 MiB/s wr, 2.53k op/s; 62330/305751 objects degraded (20.386%); 38300/305751 objects misplaced (12.527%); 9.7 MiB/s, 1 keys/s, 160 objects/s recovering 2023-09-30T09:33:22.814 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:33:22.850 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:33:22.878 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:22.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:22.896 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:33:22.940 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:33:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:33:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-30T09:33:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:33:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:33:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:33:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:33:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:33:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:33:22.985 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:23.052 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:23.156 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:23.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:23.247 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:33:23.273 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:23.322 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:23.428 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:23.434 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 81 16 106045 106029 81.7917 62.3125 0.0109341 0.012221 2023-09-30T09:33:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:23.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:23.542 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:23.578 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4118931232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:23.578 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 62330/305751 objects degraded (20.386%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:23.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4118931232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:23.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 62330/305751 objects degraded (20.386%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:23.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4118931232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:23.589 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 62330/305751 objects degraded (20.386%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:23.597 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:23.667 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:23.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:23.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:23.775 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:23.828 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:33:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:33:23] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-09-30T09:33:23.831 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:24.434 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 82 16 106873 106857 81.4253 51.75 0.00877197 0.0122774 2023-09-30T09:33:24.459 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:33:24.460 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:24.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:24.561 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:33:24.561 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:33:24.572 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:24.701 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:24.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:24 smithi167 ceph-mon[192880]: pgmap v732: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 187 KiB/s rd, 123 MiB/s wr, 2.03k op/s; 62330/305751 objects degraded (20.386%); 38300/305751 objects misplaced (12.527%); 7.7 MiB/s, 1 keys/s, 128 objects/s recovering 2023-09-30T09:33:24.822 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:24.824 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:24 smithi019 ceph-mon[216815]: pgmap v732: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 187 KiB/s rd, 123 MiB/s wr, 2.03k op/s; 62330/305751 objects degraded (20.386%); 38300/305751 objects misplaced (12.527%); 7.7 MiB/s, 1 keys/s, 128 objects/s recovering 2023-09-30T09:33:24.824 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:24 smithi019 bash[421719]: d081a274ac32bbf428309a04f1773bd0329c82acad6506c5036e5697017c8b38 2023-09-30T09:33:24.824 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:24 smithi019 systemd[1]: Started Ceph osd.1 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:33:24.825 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:24 smithi019 conmon[421886]: 2023-09-30T09:33:24.456+0000 7fcfa9b2f640 -1 Falling back to public interface 2023-09-30T09:33:24.825 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:24 smithi019 ceph-mon[224043]: pgmap v732: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 187 KiB/s rd, 123 MiB/s wr, 2.03k op/s; 62330/305751 objects degraded (20.386%); 38300/305751 objects misplaced (12.527%); 7.7 MiB/s, 1 keys/s, 128 objects/s recovering 2023-09-30T09:33:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:24.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:24.926 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:24.998 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:25.086 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:25.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:25.194 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:25.264 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:25.342 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:25.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:25.436 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:25.436 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 83 16 107775 107759 81.1235 56.375 0.0155875 0.0123216 2023-09-30T09:33:25.447 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:25.504 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:25.522 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:25.586 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:25.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:25.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:25.693 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:25.754 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:25.837 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:25.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:25.951 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:26.033 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:26.146 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:33:26.147 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:33:26.194 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:26.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:26.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:26.310 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:26.373 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:26.434 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 84 16 108802 108786 80.9218 64.1875 0.00807806 0.0123531 2023-09-30T09:33:26.562 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:33:26.624 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:26 smithi019 ceph-mon[216815]: pgmap v733: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 64315/319722 objects degraded (20.116%); 40081/319722 objects misplaced (12.536%); 10 MiB/s, 166 objects/s recovering 2023-09-30T09:33:26.625 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:26 smithi019 ceph-mon[224043]: pgmap v733: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 64315/319722 objects degraded (20.116%); 40081/319722 objects misplaced (12.536%); 10 MiB/s, 166 objects/s recovering 2023-09-30T09:33:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:26 smithi167 ceph-mon[192880]: pgmap v733: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 64315/319722 objects degraded (20.116%); 40081/319722 objects misplaced (12.536%); 10 MiB/s, 166 objects/s recovering 2023-09-30T09:33:27.038 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:27.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:27.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:27.146 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:27.177 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:27.276 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:27.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:27.382 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:27.436 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 85 16 109666 109650 80.6036 54 0.0173989 0.0124016 2023-09-30T09:33:27.470 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:28.436 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 86 16 110481 110465 80.2581 50.9375 0.012073 0.0124549 2023-09-30T09:33:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:28 smithi019 ceph-mon[224043]: pgmap v734: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 64315/319722 objects degraded (20.116%); 40081/319722 objects misplaced (12.536%); 10 MiB/s, 166 objects/s recovering 2023-09-30T09:33:28.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 64315/319722 objects degraded (20.116%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:28.811 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:28.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:28 smithi019 ceph-mon[216815]: pgmap v734: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 64315/319722 objects degraded (20.116%); 40081/319722 objects misplaced (12.536%); 10 MiB/s, 166 objects/s recovering 2023-09-30T09:33:28.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 64315/319722 objects degraded (20.116%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:28.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:28.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:28.922 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:28.968 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:33:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:28 smithi167 ceph-mon[192880]: pgmap v734: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 64315/319722 objects degraded (20.116%); 40081/319722 objects misplaced (12.536%); 10 MiB/s, 166 objects/s recovering 2023-09-30T09:33:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 64315/319722 objects degraded (20.116%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:29.038 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:29.324 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:29.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:29.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:29.436 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 87 16 111296 111280 79.9209 50.9375 0.0217751 0.0125067 2023-09-30T09:33:29.443 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:29.512 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:29.593 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:29.746 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:29.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:29.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:29.872 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:29.911 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:29.974 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:29.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:30.078 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:30.123 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:30.124 INFO:teuthology.orchestra.run.smithi019.stderr:no valid command found; 10 closest matches: 2023-09-30T09:33:30.124 INFO:teuthology.orchestra.run.smithi019.stderr:0 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:1 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:2 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:abort 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:assert 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:bluefs debug_inject_read_zeros 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:bluefs files list 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:bluefs stats 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:bluestore allocator dump block 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:bluestore allocator fragmentation block 2023-09-30T09:33:30.125 INFO:teuthology.orchestra.run.smithi019.stderr:admin_socket: invalid command 2023-09-30T09:33:30.193 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:30.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:30.301 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:30.359 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:30.385 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:30 smithi019 conmon[421886]: 2023-09-30T09:33:30.242+0000 7fcfa9b2f640 -1 osd.1 286 log_to_monitors true 2023-09-30T09:33:30.437 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 88 16 112125 112109 79.6015 51.8125 0.00823034 0.012558 2023-09-30T09:33:30.472 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:30.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:30.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:30.579 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:30.594 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:33:30.663 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:33:30 smithi019 conmon[421886]: 2023-09-30T09:33:30.598+0000 7fcf96f93700 -1 osd.1 286 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:33:30.663 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:30 smithi019 ceph-mon[216815]: pgmap v735: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 66573/330714 objects degraded (20.130%); 41454/330714 objects misplaced (12.535%); 9.5 MiB/s, 152 objects/s recovering 2023-09-30T09:33:30.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/97243010' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:33:30.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:33:30.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:30 smithi019 ceph-mon[216815]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:33:30.664 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:30 smithi019 ceph-mon[224043]: pgmap v735: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 66573/330714 objects degraded (20.130%); 41454/330714 objects misplaced (12.535%); 9.5 MiB/s, 152 objects/s recovering 2023-09-30T09:33:30.664 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/97243010' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:33:30.664 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:33:30.664 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:30 smithi019 ceph-mon[224043]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:33:30.667 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:30 smithi167 ceph-mon[192880]: pgmap v735: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 66573/330714 objects degraded (20.130%); 41454/330714 objects misplaced (12.535%); 9.5 MiB/s, 152 objects/s recovering 2023-09-30T09:33:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/97243010' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:33:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:33:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:30 smithi167 ceph-mon[192880]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-30T09:33:30.942 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:31.437 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 89 16 113095 113079 79.3883 60.625 0.0152481 0.0125901 2023-09-30T09:33:31.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:31.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:33:31.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[216815]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:33:31.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[216815]: osdmap e300: 8 total, 7 up, 8 in 2023-09-30T09:33:31.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[216815]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:33:31.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[216815]: failed to encode map e300 with expected crc 2023-09-30T09:33:31.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[216815]: failed to encode map e300 with expected crc 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[216815]: failed to encode map e300 with expected crc 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[224043]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[224043]: osdmap e300: 8 total, 7 up, 8 in 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[224043]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[224043]: failed to encode map e300 with expected crc 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[224043]: failed to encode map e300 with expected crc 2023-09-30T09:33:31.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:31 smithi019 ceph-mon[224043]: failed to encode map e300 with expected crc 2023-09-30T09:33:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:31 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:33:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:31 smithi167 ceph-mon[192880]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-30T09:33:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:31 smithi167 ceph-mon[192880]: osdmap e300: 8 total, 7 up, 8 in 2023-09-30T09:33:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:31 smithi167 ceph-mon[192880]: from='osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:33:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:31 smithi167 ceph-mon[192880]: failed to encode map e300 with expected crc 2023-09-30T09:33:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:31 smithi167 ceph-mon[192880]: failed to encode map e300 with expected crc 2023-09-30T09:33:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:31 smithi167 ceph-mon[192880]: failed to encode map e300 with expected crc 2023-09-30T09:33:32.437 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 90 16 113275 113259 78.6313 11.25 0.0230413 0.0125943 2023-09-30T09:33:32.480 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:32.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:32.585 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:32.645 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:32.703 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:32.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:32.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:32.743 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: pgmap v737: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 66354/333183 objects degraded (19.915%); 41720/333183 objects misplaced (12.522%); 9.7 MiB/s, 155 objects/s recovering 2023-09-30T09:33:32.743 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377] boot 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: osdmap e301: 8 total, 8 up, 8 in 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.744 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[224043]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: pgmap v737: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 66354/333183 objects degraded (19.915%); 41720/333183 objects misplaced (12.522%); 9.7 MiB/s, 155 objects/s recovering 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377] boot 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: osdmap e301: 8 total, 8 up, 8 in 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:32 smithi019 ceph-mon[216815]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.808 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:32.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: pgmap v737: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 66354/333183 objects degraded (19.915%); 41720/333183 objects misplaced (12.522%); 9.7 MiB/s, 155 objects/s recovering 2023-09-30T09:33:32.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:33:32.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: osd.1 [v2:172.21.15.19:6810/2442769377,v1:172.21.15.19:6811/2442769377] boot 2023-09-30T09:33:32.832 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: osdmap e301: 8 total, 8 up, 8 in 2023-09-30T09:33:32.833 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-30T09:33:32.833 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.833 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.833 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.833 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.833 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.833 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.833 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:32 smithi167 ceph-mon[192880]: failed to encode map e301 with expected crc 2023-09-30T09:33:32.834 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:32.880 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:32.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:32.986 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:33.013 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:33.055 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:33.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:33.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:33.163 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:33.193 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:33:33.393 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-09-30T09:33:33.437 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 91 7 113275 113268 77.7736 0.5625 1.2837 0.0126957 2023-09-30T09:33:33.579 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:33.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:33.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:33.683 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:33.710 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:33.752 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:33.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:33.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:33:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:33:33] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.43.0" 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: osdmap e302: 8 total, 8 up, 8 in 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: 7.5 continuing backfill to osd.2 from (284'1900,300'7078] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 300'7078 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: 7.2 starting backfill to osd.2 from (284'1800,300'7138] MIN to 300'7141 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: 7.b starting backfill to osd.2 from (284'1800,300'6946] MIN to 300'6948 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: 7.e continuing backfill to osd.2 from (284'1800,300'7125] MIN to 300'7125 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 66354/333183 objects degraded (19.915%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:33.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: osdmap e302: 8 total, 8 up, 8 in 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: 7.5 continuing backfill to osd.2 from (284'1900,300'7078] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 300'7078 2023-09-30T09:33:33.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: 7.2 starting backfill to osd.2 from (284'1800,300'7138] MIN to 300'7141 2023-09-30T09:33:33.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: 7.b starting backfill to osd.2 from (284'1800,300'6946] MIN to 300'6948 2023-09-30T09:33:33.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: 7.e continuing backfill to osd.2 from (284'1800,300'7125] MIN to 300'7125 2023-09-30T09:33:33.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:33.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:33.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 66354/333183 objects degraded (19.915%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:33.856 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:33.881 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: osdmap e302: 8 total, 8 up, 8 in 2023-09-30T09:33:33.881 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.881 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: failed to encode map e302 with expected crc 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: 7.5 continuing backfill to osd.2 from (284'1900,300'7078] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 300'7078 2023-09-30T09:33:33.882 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: 7.2 starting backfill to osd.2 from (284'1800,300'7138] MIN to 300'7141 2023-09-30T09:33:33.883 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: 7.b starting backfill to osd.2 from (284'1800,300'6946] MIN to 300'6948 2023-09-30T09:33:33.883 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: 7.e continuing backfill to osd.2 from (284'1800,300'7125] MIN to 300'7125 2023-09-30T09:33:33.883 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:33:34.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:34.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 66354/333183 objects degraded (19.915%), 18 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:34.512 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total time run: 91.4503 2023-09-30T09:33:34.512 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total writes made: 113275 2023-09-30T09:33:34.512 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Write size: 65536 2023-09-30T09:33:34.512 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object size: 65536 2023-09-30T09:33:34.512 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Bandwidth (MB/sec): 77.4157 2023-09-30T09:33:34.512 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Bandwidth: 33.9498 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max bandwidth (MB/sec): 207.875 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min bandwidth (MB/sec): 0 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average IOPS: 1238 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev IOPS: 543.205 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max IOPS: 3326 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min IOPS: 0 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average Latency(s): 0.012834 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Latency(s): 0.0381551 2023-09-30T09:33:34.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max latency(s): 2.27325 2023-09-30T09:33:34.514 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min latency(s): 0.00127075 2023-09-30T09:33:34.514 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Cleaning up (deleting benchmark objects) 2023-09-30T09:33:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:34.515 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:34.515 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:34.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:34.616 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:34.753 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:33:34.834 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:34.926 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: pgmap v740: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 66354/333183 objects degraded (19.915%); 41720/333183 objects misplaced (12.522%); 6.9 MiB/s, 110 objects/s recovering 2023-09-30T09:33:34.957 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.957 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.957 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: osdmap e303: 8 total, 8 up, 8 in 2023-09-30T09:33:34.957 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.958 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.958 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.958 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.958 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.958 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.959 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[216815]: 7.a continuing backfill to osd.2 from (284'1900,300'7200] 7:511c6b92:::benchmark_data_smithi019_404601_object103871:head to 300'7200 2023-09-30T09:33:34.959 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: pgmap v740: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 66354/333183 objects degraded (19.915%); 41720/333183 objects misplaced (12.522%); 6.9 MiB/s, 110 objects/s recovering 2023-09-30T09:33:34.959 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.959 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: osdmap e303: 8 total, 8 up, 8 in 2023-09-30T09:33:34.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: failed to encode map e303 with expected crc 2023-09-30T09:33:34.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:34 smithi019 ceph-mon[224043]: 7.a continuing backfill to osd.2 from (284'1900,300'7200] 7:511c6b92:::benchmark_data_smithi019_404601_object103871:head to 300'7200 2023-09-30T09:33:34.970 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:34.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:34.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: pgmap v740: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 9 active+clean+remapped, 2 active+remapped+backfill_wait, 13 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 56 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 66354/333183 objects degraded (19.915%); 41720/333183 objects misplaced (12.522%); 6.9 MiB/s, 110 objects/s recovering 2023-09-30T09:33:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: failed to encode map e303 with expected crc 2023-09-30T09:33:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: failed to encode map e303 with expected crc 2023-09-30T09:33:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: osdmap e303: 8 total, 8 up, 8 in 2023-09-30T09:33:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: failed to encode map e303 with expected crc 2023-09-30T09:33:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: failed to encode map e303 with expected crc 2023-09-30T09:33:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: failed to encode map e303 with expected crc 2023-09-30T09:33:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: failed to encode map e303 with expected crc 2023-09-30T09:33:35.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: failed to encode map e303 with expected crc 2023-09-30T09:33:35.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: failed to encode map e303 with expected crc 2023-09-30T09:33:35.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:34 smithi167 ceph-mon[192880]: 7.a continuing backfill to osd.2 from (284'1900,300'7200] 7:511c6b92:::benchmark_data_smithi019_404601_object103871:head to 300'7200 2023-09-30T09:33:35.082 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:35.200 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:35.368 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:35.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:35.379 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:33:35.423 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:33:35.477 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:33:35.480 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:35.557 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:35.566 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:33:35.604 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:33:35.646 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:33:35.685 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:33:35.722 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:33:35.733 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:35.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:35.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:35.773 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:33:35.774 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:33:35.774 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:33:35.774 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:33:35.774 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:33:35.774 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:33:35.774 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:33:35.774 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:33:35.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1926219258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:35.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1926219258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:35.847 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:35.943 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:33:35.943 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:33:35.943 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:33:35.943 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:33:35.943 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:33:35.953 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:36.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1926219258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:36.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:36 smithi019 ceph-mon[216815]: pgmap v742: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s; 35889/339906 objects degraded (10.559%); 38688/339906 objects misplaced (11.382%); 13 MiB/s, 200 objects/s recovering 2023-09-30T09:33:36.962 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:36 smithi019 ceph-mon[224043]: pgmap v742: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s; 35889/339906 objects degraded (10.559%); 38688/339906 objects misplaced (11.382%); 13 MiB/s, 200 objects/s recovering 2023-09-30T09:33:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:36 smithi167 ceph-mon[192880]: pgmap v742: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s; 35889/339906 objects degraded (10.559%); 38688/339906 objects misplaced (11.382%); 13 MiB/s, 200 objects/s recovering 2023-09-30T09:33:37.204 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:37.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:37.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:37.312 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:37.390 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:37.486 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:37.515 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:33:37.558 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:37.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:37.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:37.687 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:37.732 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:37.812 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:37.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:37.920 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:37.954 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:38.022 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:38.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:38.128 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:38.173 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:38.239 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:38.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:38.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:38.347 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:38.379 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:38.394 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:33:38.443 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:38.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:38.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:38.550 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:38.590 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:38.650 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:38.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:38.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:38.765 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:38.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:38 smithi019 ceph-mon[216815]: pgmap v743: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s; 35889/339906 objects degraded (10.559%); 38688/339906 objects misplaced (11.382%); 10 MiB/s, 160 objects/s recovering 2023-09-30T09:33:38.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 35889/339906 objects degraded (10.559%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:38.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:38 smithi019 ceph-mon[224043]: pgmap v743: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s; 35889/339906 objects degraded (10.559%); 38688/339906 objects misplaced (11.382%); 10 MiB/s, 160 objects/s recovering 2023-09-30T09:33:38.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 35889/339906 objects degraded (10.559%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:38.855 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:38 smithi167 ceph-mon[192880]: pgmap v743: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s; 35889/339906 objects degraded (10.559%); 38688/339906 objects misplaced (11.382%); 10 MiB/s, 160 objects/s recovering 2023-09-30T09:33:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 35889/339906 objects degraded (10.559%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:39.149 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:39.152 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:39.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:39.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:39.255 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:33:39.267 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:39.403 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:39.419 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:39.743 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:39.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:39.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:39.860 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:40.005 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:40.631 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:40.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:40.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:40.739 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:40.764 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:40.806 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:40.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:40.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:40 smithi019 ceph-mon[216815]: pgmap v744: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 131 B/s rd, 18 MiB/s wr, 568 op/s; 35629/333363 objects degraded (10.688%); 37584/333363 objects misplaced (11.274%); 9.8 MiB/s, 157 objects/s recovering 2023-09-30T09:33:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:40 smithi019 ceph-mon[224043]: pgmap v744: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 131 B/s rd, 18 MiB/s wr, 568 op/s; 35629/333363 objects degraded (10.688%); 37584/333363 objects misplaced (11.274%); 9.8 MiB/s, 157 objects/s recovering 2023-09-30T09:33:40.913 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:41.005 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:41.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:40 smithi167 ceph-mon[192880]: pgmap v744: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 131 B/s rd, 18 MiB/s wr, 568 op/s; 35629/333363 objects degraded (10.688%); 37584/333363 objects misplaced (11.274%); 9.8 MiB/s, 157 objects/s recovering 2023-09-30T09:33:41.360 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:41.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:41.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:41.544 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:41.583 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:41.629 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:41.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:41.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:41.671 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:33:41.735 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:41.778 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:41.840 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:41.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:41.849 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1332520613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:41.950 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:42.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1332520613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:42.037 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1332520613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:42.045 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:42.161 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:42.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:42.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:42.268 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:42.313 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:42.790 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:42.817 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:42.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:42.919 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: pgmap v745: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 116 B/s rd, 16 MiB/s wr, 562 op/s; 35629/331812 objects degraded (10.738%); 37503/331812 objects misplaced (11.302%); 8.7 MiB/s, 139 objects/s recovering 2023-09-30T09:33:42.919 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:42.919 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1332520613' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:33:42.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: osdmap e304: 8 total, 8 up, 8 in 2023-09-30T09:33:42.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[224043]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.920 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: pgmap v745: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 116 B/s rd, 16 MiB/s wr, 562 op/s; 35629/331812 objects degraded (10.738%); 37503/331812 objects misplaced (11.302%); 8.7 MiB/s, 139 objects/s recovering 2023-09-30T09:33:42.920 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:42.921 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1332520613' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:33:42.921 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.921 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: osdmap e304: 8 total, 8 up, 8 in 2023-09-30T09:33:42.921 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.921 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.921 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.921 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:42 smithi019 ceph-mon[216815]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.930 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:42.970 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: pgmap v745: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 116 B/s rd, 16 MiB/s wr, 562 op/s; 35629/331812 objects degraded (10.738%); 37503/331812 objects misplaced (11.302%); 8.7 MiB/s, 139 objects/s recovering 2023-09-30T09:33:42.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:42.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1332520613' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:33:42.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: osdmap e304: 8 total, 8 up, 8 in 2023-09-30T09:33:42.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.971 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:42 smithi167 ceph-mon[192880]: failed to encode map e304 with expected crc 2023-09-30T09:33:42.974 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:43.040 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:43.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:43.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:43.149 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:43.307 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:43.334 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:33:43.334 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:33:43.334 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:33:43.334 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:33:43.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.335 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35934 6.16 6:6a0d4ce1:::benchmark_data_smithi019_355417_object35933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:33:43.335 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:33:43.335 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 304.06981989600001, 2023-09-30T09:33:43.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.022920797, 2023-09-30T09:33:43.335 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.335 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35934 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259898+0000", 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259900+0000", 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.573e-06 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.337 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259928+0000", 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8756999999999999e-05 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259941+0000", 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3066e-05 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259951+0000", 2023-09-30T09:33:43.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8050000000000001e-06 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311357+0000", 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.051405992999999997 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311380+0000", 2023-09-30T09:33:43.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.283e-05 2023-09-30T09:33:43.340 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.282746+0000", 2023-09-30T09:33:43.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97136601 2023-09-30T09:33:43.340 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.341 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.341 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.282819+0000", 2023-09-30T09:33:43.341 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2762999999999994e-05 2023-09-30T09:33:43.341 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.341 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.341 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.342 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.342 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35936 6.2 6:47063778:::benchmark_data_smithi019_355417_object35935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:33:43.342 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:33:43.342 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 304.06975191399999, 2023-09-30T09:33:43.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0116794739999999, 2023-09-30T09:33:43.342 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35936 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.343 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259966+0000", 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0800000000000005e-07 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259989+0000", 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2908000000000002e-05 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.259996+0000", 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8650000000000003e-06 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260001+0000", 2023-09-30T09:33:43.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2310000000000004e-06 2023-09-30T09:33:43.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.310543+0000", 2023-09-30T09:33:43.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.050541517000000001 2023-09-30T09:33:43.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.310552+0000", 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2320000000000007e-06 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271569+0000", 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96101677600000002 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271645+0000", 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6437e-05 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35941 6.6 6:62f338d5:::benchmark_data_smithi019_355417_object35940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 304.06970684499998, 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0179260409999999, 2023-09-30T09:33:43.348 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35941 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260011+0000", 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8299999999999998e-07 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260036+0000", 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4766999999999999e-05 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260043+0000", 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.387e-06 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.260047+0000", 2023-09-30T09:33:43.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9030000000000003e-06 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311366+0000", 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.051318726000000002 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.311377+0000", 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0621999999999999e-05 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268227+0000", 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95684991799999997 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268372+0000", 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014511 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:33:43.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.268552+0000", 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017976400000000001 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271663+0000", 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031116619999999998 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271672+0000", 2023-09-30T09:33:43.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3180000000000007e-06 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277851+0000", 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061798599999999997 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277902+0000", 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0611e-05 2023-09-30T09:33:43.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.277937+0000", 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5009999999999999e-05 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:35951 6.6 6:612781fc:::benchmark_data_smithi019_355417_object35950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:33:43.356 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 303.99841952000003, 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94995752200000005, 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 35951 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331298+0000", 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331301+0000", 2023-09-30T09:33:43.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9739999999999998e-06 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331348+0000", 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7113000000000001e-05 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331384+0000", 2023-09-30T09:33:43.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5253000000000003e-05 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331402+0000", 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8756000000000001e-05 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331491+0000", 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8213000000000004e-05 2023-09-30T09:33:43.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:39.331502+0000", 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1328999999999999e-05 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270400+0000", 2023-09-30T09:33:43.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.93889793200000005 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270562+0000", 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016258000000000001 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.270732+0000", 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016932199999999999 2023-09-30T09:33:43.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.271969+0000", 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00123719 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.274748+0000", 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002778689 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281163+0000", 2023-09-30T09:33:43.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0064157759999999998 2023-09-30T09:33:43.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281223+0000", 2023-09-30T09:33:43.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9302000000000002e-05 2023-09-30T09:33:43.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.365 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.365 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.365 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:28:40.281256+0000", 2023-09-30T09:33:43.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3093000000000003e-05 2023-09-30T09:33:43.365 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.366 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.366 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.366 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:99314 6.13 6:cb94d029:::benchmark_data_smithi019_355417_object7010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:33:43.366 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:29:59.583400+0000", 2023-09-30T09:33:43.366 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.746317731, 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94488777800000001, 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 99314 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583400+0000", 2023-09-30T09:33:43.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583400+0000", 2023-09-30T09:33:43.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583403+0000", 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.542e-06 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583414+0000", 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1259e-05 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583434+0000", 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0562999999999999e-05 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.583447+0000", 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2894999999999999e-05 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.624839+0000", 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.041391322000000001 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.626266+0000", 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00142702 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.811282+0000", 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.18501577399999999 2023-09-30T09:33:43.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:29:59.813318+0000", 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020367900000000001 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.396757+0000", 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.58343899600000004 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.398448+0000", 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001690951 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.401393+0000", 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029450679999999999 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.406617+0000", 2023-09-30T09:33:43.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0052237079999999997 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.407985+0000", 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001367769 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.523096+0000", 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.11511101999999999 2023-09-30T09:33:43.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.524223+0000", 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011267029999999999 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:00.528288+0000", 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040653979999999996 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.375 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:112907 6.13 6:cc36a634:::benchmark_data_smithi019_355417_object20603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:19.317462+0000", 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 204.01225542700001, 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.030684653, 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.376 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 112907 2023-09-30T09:33:43.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.377 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317462+0000", 2023-09-30T09:33:43.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317462+0000", 2023-09-30T09:33:43.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317464+0000", 2023-09-30T09:33:43.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2759999999999999e-06 2023-09-30T09:33:43.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.380 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.380 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317469+0000", 2023-09-30T09:33:43.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1490000000000002e-06 2023-09-30T09:33:43.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.380 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317482+0000", 2023-09-30T09:33:43.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2897e-05 2023-09-30T09:33:43.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.317490+0000", 2023-09-30T09:33:43.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7099999999999996e-06 2023-09-30T09:33:43.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.382 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.382 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.321704+0000", 2023-09-30T09:33:43.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042135260000000004 2023-09-30T09:33:43.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:33:43.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.322413+0000", 2023-09-30T09:33:43.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0007092 2023-09-30T09:33:43.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.704196+0000", 2023-09-30T09:33:43.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.38178283899999998 2023-09-30T09:33:43.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:33:43.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:19.706325+0000", 2023-09-30T09:33:43.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002128984 2023-09-30T09:33:43.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.285269+0000", 2023-09-30T09:33:43.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.57894403699999997 2023-09-30T09:33:43.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.287195+0000", 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001926072 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.290526+0000", 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033311479999999999 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.298483+0000", 2023-09-30T09:33:43.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0079569210000000005 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.316366+0000", 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017883202000000001 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.342631+0000", 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026264167000000001 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.343876+0000", 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012450569999999999 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:20.348147+0000", 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042714679999999996 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143205 6.13 6:ca286b4d:::benchmark_data_smithi019_355417_object50901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.317267+0000", 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 169.012451054, 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0055789479999999, 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.390 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143205 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317267+0000", 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317267+0000", 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317267+0000", 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5600000000000001e-07 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317270+0000", 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4930000000000001e-06 2023-09-30T09:33:43.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317278+0000", 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1939999999999998e-06 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317282+0000", 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.639e-06 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317574+0000", 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00029113799999999998 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.317687+0000", 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113625 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.318221+0000", 2023-09-30T09:33:43.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00053415699999999995 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.320841+0000", 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026193520000000001 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.324853+0000", 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040123479999999998 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.324865+0000", 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1793999999999999e-05 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316130+0000", 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99126465200000002 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:33:43.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316278+0000", 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000148326 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322784+0000", 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0065062449999999999 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322846+0000", 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1629000000000002e-05 2023-09-30T09:33:43.397 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143210 6.13 6:cd7c9817:::benchmark_data_smithi019_355417_object50906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.322320+0000", 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 169.00739784300001, 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0007609209999999, 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.398 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143210 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322320+0000", 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322320+0000", 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322321+0000", 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1999999999999998e-07 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322325+0000", 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2100000000000003e-06 2023-09-30T09:33:43.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322732+0000", 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000407291 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.322744+0000", 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.182e-05 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325773+0000", 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030286169999999999 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325782+0000", 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4979999999999995e-06 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.402 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316559+0000", 2023-09-30T09:33:43.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99077679100000005 2023-09-30T09:33:43.403 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.403 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316643+0000", 2023-09-30T09:33:43.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3658000000000004e-05 2023-09-30T09:33:43.403 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:33:43.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.316998+0000", 2023-09-30T09:33:43.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035565800000000002 2023-09-30T09:33:43.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322459+0000", 2023-09-30T09:33:43.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0054604650000000003 2023-09-30T09:33:43.405 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322886+0000", 2023-09-30T09:33:43.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00042728800000000001 2023-09-30T09:33:43.406 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.406 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.406 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.406 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322934+0000", 2023-09-30T09:33:43.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7895000000000001e-05 2023-09-30T09:33:43.406 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.407 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.407 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.407 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322969+0000", 2023-09-30T09:33:43.407 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5264e-05 2023-09-30T09:33:43.407 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.407 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.407 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.407 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323081+0000", 2023-09-30T09:33:43.408 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011154600000000001 2023-09-30T09:33:43.408 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.408 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.408 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.408 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.408 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.408 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143213 6.13 6:cf52ab7f:::benchmark_data_smithi019_355417_object50909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-09-30T09:33:43.409 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.325783+0000", 2023-09-30T09:33:43.409 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 169.00393532000001, 2023-09-30T09:33:43.409 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99744182599999998, 2023-09-30T09:33:43.409 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.409 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.409 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.410 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.410 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.410 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143213 2023-09-30T09:33:43.410 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.410 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.410 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325783+0000", 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325783+0000", 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325784+0000", 2023-09-30T09:33:43.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3209999999999999e-06 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325788+0000", 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.459e-06 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325798+0000", 2023-09-30T09:33:43.412 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6709999999999994e-06 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.325805+0000", 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0520000000000004e-06 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.326702+0000", 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00089691900000000004 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.326712+0000", 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0101e-05 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.318705+0000", 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99199274900000001 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.414 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.318821+0000", 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115704 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.319064+0000", 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000243847 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322593+0000", 2023-09-30T09:33:43.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003528659 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322913+0000", 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000319957 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323109+0000", 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019595999999999999 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323144+0000", 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4684000000000002e-05 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323224+0000", 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0742999999999996e-05 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.417 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143232 6.13 6:ce26f66a:::benchmark_data_smithi019_355417_object50928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.351507+0000", 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 168.97821093499999, 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97190067099999999, 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143232 2023-09-30T09:33:43.418 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351507+0000", 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351507+0000", 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.419 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351508+0000", 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.75e-07 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351510+0000", 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6230000000000001e-06 2023-09-30T09:33:43.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.421 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351522+0000", 2023-09-30T09:33:43.421 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2160999999999999e-05 2023-09-30T09:33:43.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.422 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351528+0000", 2023-09-30T09:33:43.422 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0889999999999996e-06 2023-09-30T09:33:43.422 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.422 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.422 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.422 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351658+0000", 2023-09-30T09:33:43.422 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000129782 2023-09-30T09:33:43.422 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.351667+0000", 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1439999999999992e-06 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.320634+0000", 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96896631099999997 2023-09-30T09:33:43.423 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.320745+0000", 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000111035 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.320999+0000", 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000253983 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.424 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.425 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.322763+0000", 2023-09-30T09:33:43.425 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001764568 2023-09-30T09:33:43.425 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323255+0000", 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000492122 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323302+0000", 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7076999999999997e-05 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323345+0000", 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2654000000000002e-05 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323408+0000", 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2546999999999996e-05 2023-09-30T09:33:43.427 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143245 6.13 6:c9599472:::benchmark_data_smithi019_355417_object50941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.361939+0000", 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 168.96777866900001, 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97745515400000005, 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.428 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143245 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361939+0000", 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.429 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361939+0000", 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361940+0000", 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5099999999999999e-07 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.430 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.431 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.431 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361943+0000", 2023-09-30T09:33:43.431 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1750000000000001e-06 2023-09-30T09:33:43.431 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.431 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.431 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.431 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361951+0000", 2023-09-30T09:33:43.431 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8939999999999993e-06 2023-09-30T09:33:43.432 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.432 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.432 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.432 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.361956+0000", 2023-09-30T09:33:43.432 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5840000000000002e-06 2023-09-30T09:33:43.432 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.432 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.432 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.433 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.362030+0000", 2023-09-30T09:33:43.433 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3905000000000005e-05 2023-09-30T09:33:43.433 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.433 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.433 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.434 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.362042+0000", 2023-09-30T09:33:43.434 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1392999999999999e-05 2023-09-30T09:33:43.434 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.434 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.434 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.434 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323462+0000", 2023-09-30T09:33:43.435 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96141999300000003 2023-09-30T09:33:43.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.435 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.435 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323546+0000", 2023-09-30T09:33:43.435 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4419000000000006e-05 2023-09-30T09:33:43.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.436 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.436 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:33:43.436 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.323803+0000", 2023-09-30T09:33:43.436 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025670000000000001 2023-09-30T09:33:43.436 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.437 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.437 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.437 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330037+0000", 2023-09-30T09:33:43.437 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062342020000000003 2023-09-30T09:33:43.437 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.437 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.438 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.438 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330102+0000", 2023-09-30T09:33:43.438 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4901999999999996e-05 2023-09-30T09:33:43.438 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.438 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.438 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.439 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339211+0000", 2023-09-30T09:33:43.439 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091090440000000002 2023-09-30T09:33:43.439 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.439 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.439 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.439 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339279+0000", 2023-09-30T09:33:43.440 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8442000000000006e-05 2023-09-30T09:33:43.440 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.440 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.440 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.440 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339394+0000", 2023-09-30T09:33:43.440 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011485 2023-09-30T09:33:43.440 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.441 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.441 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.441 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.441 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.441 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:143266 6.13 6:cd9d4c46:::benchmark_data_smithi019_355417_object50962:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-30T09:33:43.441 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:30:54.376173+0000", 2023-09-30T09:33:43.441 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 168.95354508299999, 2023-09-30T09:33:43.442 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96345502800000005, 2023-09-30T09:33:43.442 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.442 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.442 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.442 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.442 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.442 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 143266 2023-09-30T09:33:43.443 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.443 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.443 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.443 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.443 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376173+0000", 2023-09-30T09:33:43.443 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.444 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.444 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.444 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.444 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376173+0000", 2023-09-30T09:33:43.444 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.444 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.445 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.445 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.445 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376174+0000", 2023-09-30T09:33:43.445 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0559999999999999e-06 2023-09-30T09:33:43.445 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.445 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.446 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.446 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376178+0000", 2023-09-30T09:33:43.446 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.357e-06 2023-09-30T09:33:43.446 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.446 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.446 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.446 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376202+0000", 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3513e-05 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376209+0000", 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6660000000000008e-06 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376277+0000", 2023-09-30T09:33:43.447 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7251999999999999e-05 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:54.376289+0000", 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2873e-05 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.325866+0000", 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94957660499999996 2023-09-30T09:33:43.448 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.449 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.449 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.449 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.325992+0000", 2023-09-30T09:33:43.449 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012573700000000001 2023-09-30T09:33:43.449 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.449 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.449 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.326318+0000", 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00032582100000000001 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330166+0000", 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0038479159999999998 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.450 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.330176+0000", 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0284e-05 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339447+0000", 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0092713829999999994 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.451 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339506+0000", 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8841999999999997e-05 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:30:55.339628+0000", 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000121723 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.452 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.453 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.453 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.453 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35130.0:162579 6.2 6:42108c99:::benchmark_data_smithi019_355417_object70275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-30T09:33:43.453 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:31:15.355717+0000", 2023-09-30T09:33:43.453 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 147.97400135699999, 2023-09-30T09:33:43.453 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96398006300000005, 2023-09-30T09:33:43.454 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.454 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.454 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.454 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35130", 2023-09-30T09:33:43.454 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:43.455 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 162579 2023-09-30T09:33:43.455 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.455 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.455 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.455 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.455 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355717+0000", 2023-09-30T09:33:43.455 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.456 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.456 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.456 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.456 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355717+0000", 2023-09-30T09:33:43.456 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.457 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.457 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.457 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.457 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355718+0000", 2023-09-30T09:33:43.457 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6789999999999999e-06 2023-09-30T09:33:43.457 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.458 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.458 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.458 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355723+0000", 2023-09-30T09:33:43.458 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3510000000000004e-06 2023-09-30T09:33:43.458 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.459 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.459 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.459 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355739+0000", 2023-09-30T09:33:43.459 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6198e-05 2023-09-30T09:33:43.459 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.459 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.460 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.460 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355747+0000", 2023-09-30T09:33:43.460 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8299999999999996e-06 2023-09-30T09:33:43.460 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.460 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.461 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.461 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355829+0000", 2023-09-30T09:33:43.461 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2921000000000001e-05 2023-09-30T09:33:43.461 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.461 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.461 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.462 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:15.355842+0000", 2023-09-30T09:33:43.462 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2018000000000001e-05 2023-09-30T09:33:43.462 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.462 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.462 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.463 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.314498+0000", 2023-09-30T09:33:43.463 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95865608899999999 2023-09-30T09:33:43.463 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.463 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.463 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.463 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.314578+0000", 2023-09-30T09:33:43.463 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0556999999999995e-05 2023-09-30T09:33:43.464 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.464 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.464 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:33:43.464 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.314779+0000", 2023-09-30T09:33:43.464 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020123799999999999 2023-09-30T09:33:43.464 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.465 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.465 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.465 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.317791+0000", 2023-09-30T09:33:43.465 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030113269999999998 2023-09-30T09:33:43.465 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.465 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.465 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.466 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.317833+0000", 2023-09-30T09:33:43.466 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2048000000000001e-05 2023-09-30T09:33:43.466 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.466 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.466 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.466 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319642+0000", 2023-09-30T09:33:43.467 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00180915 2023-09-30T09:33:43.467 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.467 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.467 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.467 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319673+0000", 2023-09-30T09:33:43.468 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1548999999999999e-05 2023-09-30T09:33:43.468 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.468 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.468 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.468 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:31:16.319697+0000", 2023-09-30T09:33:43.468 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3108e-05 2023-09-30T09:33:43.469 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.469 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.469 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.469 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.469 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.469 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:55750 7.e 7:736da0d5:::benchmark_data_smithi019_404601_object55749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-30T09:33:43.470 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:32:35.464129+0000", 2023-09-30T09:33:43.470 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.865588720000005, 2023-09-30T09:33:43.470 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.029589514, 2023-09-30T09:33:43.470 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.470 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:43.471 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.471 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:33:43.471 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:33:43.471 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 55750 2023-09-30T09:33:43.471 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.471 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.472 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.472 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.472 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464129+0000", 2023-09-30T09:33:43.472 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.472 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.472 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.473 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.473 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464129+0000", 2023-09-30T09:33:43.473 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.473 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.473 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.473 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.474 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464130+0000", 2023-09-30T09:33:43.474 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8599999999999997e-07 2023-09-30T09:33:43.474 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.474 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.474 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.475 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464153+0000", 2023-09-30T09:33:43.475 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2727000000000001e-05 2023-09-30T09:33:43.475 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.475 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.475 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.475 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464160+0000", 2023-09-30T09:33:43.476 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3719999999999999e-06 2023-09-30T09:33:43.476 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.476 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.476 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.476 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464166+0000", 2023-09-30T09:33:43.476 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6679999999999998e-06 2023-09-30T09:33:43.477 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.477 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.477 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.477 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464244+0000", 2023-09-30T09:33:43.477 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7902000000000001e-05 2023-09-30T09:33:43.478 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.478 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.478 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.478 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464366+0000", 2023-09-30T09:33:43.478 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000122142 2023-09-30T09:33:43.479 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.479 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.479 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:43.479 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.464496+0000", 2023-09-30T09:33:43.479 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013052800000000001 2023-09-30T09:33:43.479 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.480 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.480 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.480 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.478930+0000", 2023-09-30T09:33:43.480 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014433293 2023-09-30T09:33:43.480 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.480 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.481 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.481 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.486135+0000", 2023-09-30T09:33:43.481 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0072057049999999997 2023-09-30T09:33:43.481 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.481 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.481 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.482 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.486149+0000", 2023-09-30T09:33:43.482 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3803e-05 2023-09-30T09:33:43.482 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.482 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.482 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.483 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.493637+0000", 2023-09-30T09:33:43.483 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0074882430000001 2023-09-30T09:33:43.483 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.483 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.483 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.483 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.493719+0000", 2023-09-30T09:33:43.484 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1244999999999999e-05 2023-09-30T09:33:43.484 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.484 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.484 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.484 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.484 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.485 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:55756 7.e 7:76bcace6:::benchmark_data_smithi019_404601_object55755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-30T09:33:43.485 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:32:35.468527+0000", 2023-09-30T09:33:43.485 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.861190679000003, 2023-09-30T09:33:43.485 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.025481224, 2023-09-30T09:33:43.485 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.486 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:43.486 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.486 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:33:43.486 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:33:43.486 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 55756 2023-09-30T09:33:43.486 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.487 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.487 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.487 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.487 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.468527+0000", 2023-09-30T09:33:43.487 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.487 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.488 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.488 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.488 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.468527+0000", 2023-09-30T09:33:43.488 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.488 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.488 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.488 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.489 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.468529+0000", 2023-09-30T09:33:43.489 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.573e-06 2023-09-30T09:33:43.489 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.489 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.489 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.489 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.468547+0000", 2023-09-30T09:33:43.489 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8352e-05 2023-09-30T09:33:43.489 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.490 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.490 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.490 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.468557+0000", 2023-09-30T09:33:43.490 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0261000000000001e-05 2023-09-30T09:33:43.490 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.490 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.490 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.491 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.468564+0000", 2023-09-30T09:33:43.491 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9589999999999998e-06 2023-09-30T09:33:43.491 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.491 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.491 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.491 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.469707+0000", 2023-09-30T09:33:43.491 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011423309999999999 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.469858+0000", 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015131699999999999 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.470013+0000", 2023-09-30T09:33:43.492 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000155239 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.478935+0000", 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089217619999999997 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.488214+0000", 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0092791669999999996 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.493 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.488224+0000", 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4490000000000005e-06 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.493952+0000", 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.005728671 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.494 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494008+0000", 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6143000000000001e-05 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:55760 7.e 7:7463d975:::benchmark_data_smithi019_404601_object55759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:32:35.469307+0000", 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.860410664, 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0247863699999999, 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.495 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 55760 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.469307+0000", 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.496 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.469307+0000", 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.469308+0000", 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1620000000000001e-06 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.497 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.469339+0000", 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.044e-05 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.469492+0000", 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015280200000000001 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.469499+0000", 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8099999999999998e-06 2023-09-30T09:33:43.498 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.472202+0000", 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027026239999999998 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.472351+0000", 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000148565 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.499 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:43.500 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.472495+0000", 2023-09-30T09:33:43.500 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014403699999999999 2023-09-30T09:33:43.500 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.500 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.500 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.500 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.488257+0000", 2023-09-30T09:33:43.500 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015762447999999998 2023-09-30T09:33:43.501 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.501 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.501 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.501 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.488264+0000", 2023-09-30T09:33:43.501 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8009999999999998e-06 2023-09-30T09:33:43.501 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.501 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.501 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494043+0000", 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0057793989999999 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494094+0000", 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0281999999999997e-05 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.502 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:55765 7.e 7:788f7208:::benchmark_data_smithi019_404601_object55764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:32:35.476097+0000", 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.853620728999999, 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0180823999999999, 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 55765 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.503 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.476097+0000", 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.476097+0000", 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.504 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.476099+0000", 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5379999999999999e-06 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.476138+0000", 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9490000000000003e-05 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.476151+0000", 2023-09-30T09:33:43.505 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2388999999999999e-05 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.476159+0000", 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1179999999999992e-06 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.480222+0000", 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040630309999999999 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.506 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.507 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.507 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.480352+0000", 2023-09-30T09:33:43.507 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012984300000000001 2023-09-30T09:33:43.507 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.507 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.480459+0000", 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000107206 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.488425+0000", 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0079663000000000008 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.508 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.488430+0000", 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3029999999999999e-06 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494124+0000", 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0056941159999999 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494180+0000", 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5065999999999998e-05 2023-09-30T09:33:43.509 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:55774 7.2 7:442c410d:::benchmark_data_smithi019_404601_object55773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:32:35.493497+0000", 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.836220628999996, 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.016929905, 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:33:43.510 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 55774 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.493497+0000", 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.493497+0000", 2023-09-30T09:33:43.511 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.493499+0000", 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.547e-06 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.493519+0000", 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0312000000000001e-05 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.512 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.493531+0000", 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2091e-05 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.493540+0000", 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3769999999999996e-06 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.493608+0000", 2023-09-30T09:33:43.513 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8077999999999994e-05 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.493619+0000", 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1786e-05 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.500260+0000", 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0066406400000001 2023-09-30T09:33:43.514 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.500372+0000", 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011216499999999999 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 7", 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.500486+0000", 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113303 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.515 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.504149+0000", 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036639369999999999 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.510285+0000", 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061355109999999997 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.510357+0000", 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2226999999999998e-05 2023-09-30T09:33:43.516 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.510427+0000", 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9931000000000005e-05 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:55777 7.1 7:89154f3b:::benchmark_data_smithi019_404601_object55776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:32:35.515235+0000", 2023-09-30T09:33:43.517 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.814482510000005, 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98782870499999997, 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 55777 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.518 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.515235+0000", 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.515235+0000", 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.515237+0000", 2023-09-30T09:33:43.519 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2190000000000001e-06 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.515256+0000", 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9848999999999999e-05 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.515269+0000", 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2601e-05 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.520 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.515277+0000", 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7449999999999995e-06 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.515331+0000", 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4060000000000001e-05 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.521 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.515339+0000", 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3299999999999999e-06 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.491280+0000", 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97594052099999995 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.522 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.491430+0000", 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015049200000000001 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6", 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.491534+0000", 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000103966 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494310+0000", 2023-09-30T09:33:43.523 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027754030000000001 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.502890+0000", 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0085799910000000004 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.502965+0000", 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4992000000000003e-05 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.524 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.503064+0000", 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9536000000000004e-05 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:55782 7.e 7:7c4b08af:::benchmark_data_smithi019_404601_object55781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:32:35.521014+0000", 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 67.808703537, 2023-09-30T09:33:43.525 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99139648499999999, 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 55782 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.521014+0000", 2023-09-30T09:33:43.526 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.521014+0000", 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.521015+0000", 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.054e-06 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.527 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.521033+0000", 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7989e-05 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.521046+0000", 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2238e-05 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.528 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.521052+0000", 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6889999999999998e-06 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.521103+0000", 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0385e-05 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:35.521108+0000", 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5539999999999997e-06 2023-09-30T09:33:43.529 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494279+0000", 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97317071799999999 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494401+0000", 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000122175 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.530 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 7", 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.494537+0000", 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013548 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.496549+0000", 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020127029999999998 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.512267+0000", 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.01571736 2023-09-30T09:33:43.531 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.512326+0000", 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9614000000000002e-05 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:32:36.512411+0000", 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4525999999999997e-05 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.532 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.533 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.533 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.533 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:118586 7.5 e304/301 7:a173e25b:::benchmark_data_smithi019_404601_object5308:head v 304'7397, mlcod=286'3496)", 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:33:43.548008+0000", 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0270028, 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027036072000000001, 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.587 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.548008+0000", 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.548008+0000", 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.548009+0000", 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0729999999999999e-06 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.548018+0000", 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6649999999999998e-06 2023-09-30T09:33:43.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.591 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.591 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.548035+0000", 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6223999999999999e-05 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.548044+0000", 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3680000000000008e-06 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.552313+0000", 2023-09-30T09:33:43.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042686920000000001 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.552345+0000", 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2217e-05 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:118579 7.0 e304/302 7:0b197c70:::benchmark_data_smithi019_404601_object5301:head v 304'7426, mlcod=286'3459)", 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:33:43.563960+0000", 2023-09-30T09:33:43.593 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.011050554000000001, 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011156294000000001, 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.563960+0000", 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.594 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.563960+0000", 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.563961+0000", 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.04e-06 2023-09-30T09:33:43.595 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.563969+0000", 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6769999999999995e-06 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.563977+0000", 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2549999999999997e-06 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.563984+0000", 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.131e-06 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.564040+0000", 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6048000000000003e-05 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.564063+0000", 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2646e-05 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:118595 7.5 e304/301 7:aae61295:::benchmark_data_smithi019_404601_object5317:head v 304'7398, mlcod=286'3496)", 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:33:43.568460+0000", 2023-09-30T09:33:43.600 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0065501609999999997, 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0067323620000000004, 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.568460+0000", 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.568460+0000", 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.568462+0000", 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.358e-06 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.568470+0000", 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3410000000000003e-06 2023-09-30T09:33:43.602 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.568488+0000", 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7927999999999998e-05 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.568498+0000", 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0030000000000001e-05 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.603 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570833+0000", 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023354510000000001 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570877+0000", 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3979e-05 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.604 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:118594 7.0 e304/302 7:0421957e:::benchmark_data_smithi019_404601_object5316:head v 304'7427, mlcod=286'3459)", 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:33:43.570704+0000", 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0043065559999999996, 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0045503699999999998, 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570704+0000", 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.605 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570704+0000", 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570705+0000", 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1349999999999999e-06 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570715+0000", 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9890000000000004e-06 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570724+0000", 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3239999999999992e-06 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570733+0000", 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.799e-06 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570829+0000", 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5727e-05 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.570859+0000", 2023-09-30T09:33:43.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0253999999999999e-05 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(7.b 304/301 [PushOp(7:de5e71e2:::benchmark_data_smithi019_404601_object65179:head, version: 289'3991, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:de5e71e2:::benchmark_data_smithi019_404601_object65179:head@289'3991, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:33:43.573943+0000", 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0010673989999999999, 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001399787, 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:33:43.609 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.573943+0000", 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.573943+0000", 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.573944+0000", 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.308e-06 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.573999+0000", 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4821000000000002e-05 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.574028+0000", 2023-09-30T09:33:43.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8419e-05 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.574048+0000", 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0712000000000001e-05 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.574173+0000", 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012430500000000001 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:33:43.612 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:33:43.574195+0000", 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2299000000000001e-05 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 5 2023-09-30T09:33:43.613 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:33:43.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 35629/331812 objects degraded (10.738%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:43.810 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 35629/331812 objects degraded (10.738%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:44.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 35629/331812 objects degraded (10.738%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:44.087 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:33:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:33:43] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.43.0" 2023-09-30T09:33:44.840 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:33:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:44 smithi167 ceph-mon[192880]: pgmap v747: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 B/s rd, 14 MiB/s wr, 504 op/s; 35629/331812 objects degraded (10.738%); 37503/331812 objects misplaced (11.302%); 7.8 MiB/s, 125 objects/s recovering 2023-09-30T09:33:45.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:44 smithi019 ceph-mon[224043]: pgmap v747: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 B/s rd, 14 MiB/s wr, 504 op/s; 35629/331812 objects degraded (10.738%); 37503/331812 objects misplaced (11.302%); 7.8 MiB/s, 125 objects/s recovering 2023-09-30T09:33:45.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:44 smithi019 ceph-mon[216815]: pgmap v747: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 B/s rd, 14 MiB/s wr, 504 op/s; 35629/331812 objects degraded (10.738%); 37503/331812 objects misplaced (11.302%); 7.8 MiB/s, 125 objects/s recovering 2023-09-30T09:33:45.166 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:45.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:45.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:45.254 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:33:45.269 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:45.358 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:45.441 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:45.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:45.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:45.548 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:45.631 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:45.795 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:33:46.134 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:46.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:46.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:46.246 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:46.346 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:46.472 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:46.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:46.578 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:46.644 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:46.662 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:46.700 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:46.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:46.712 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:46 smithi019 ceph-mon[216815]: pgmap v748: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 561 op/s; 35351/323052 objects degraded (10.943%); 36315/323052 objects misplaced (11.241%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:33:46.804 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:46.844 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:46.853 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:33:46.905 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:33:46.916 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:46.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:46.941 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:33:46.993 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:33:47.006 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:46 smithi019 ceph-mon[224043]: pgmap v748: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 561 op/s; 35351/323052 objects degraded (10.943%); 36315/323052 objects misplaced (11.241%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:33:47.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:46 smithi167 ceph-mon[192880]: pgmap v748: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 561 op/s; 35351/323052 objects degraded (10.943%); 36315/323052 objects misplaced (11.241%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:33:47.024 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:47.025 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:33:47.070 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:47.084 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:33:47.115 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:33:47.146 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:47.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:47.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:47.162 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:33:47.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:33:47.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:33:47.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:33:47.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:33:47.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:33:47.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:33:47.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:33:47.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:33:47.256 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:47.289 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:47.345 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:47.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:47.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:47.448 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:33:47.548 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:47.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2137237763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:47.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2137237763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2137237763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:48.951 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:33:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:48.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:33:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:48 smithi167 ceph-mon[192880]: pgmap v749: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 561 op/s; 35351/323052 objects degraded (10.943%); 36315/323052 objects misplaced (11.241%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:33:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 35351/323052 objects degraded (10.943%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:49.057 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:48 smithi019 ceph-mon[224043]: pgmap v749: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 561 op/s; 35351/323052 objects degraded (10.943%); 36315/323052 objects misplaced (11.241%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:33:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 35351/323052 objects degraded (10.943%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:49.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:48 smithi019 ceph-mon[216815]: pgmap v749: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 561 op/s; 35351/323052 objects degraded (10.943%); 36315/323052 objects misplaced (11.241%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:33:49.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:33:49.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 35351/323052 objects degraded (10.943%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:49.103 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:49.269 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:49.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:49.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:49.377 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:49.421 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:49.484 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:49.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:49.588 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:49.682 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:49.839 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:49.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:49.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:49.948 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:50.029 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:50.146 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:50.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:50.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:50.258 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:33:50.334 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:50.397 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:33:50.397 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:33:50.467 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:33:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:50.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:33:50.583 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:50.637 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:50.703 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:50.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:50.793 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:33:50.812 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:33:50.884 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:33:50.893 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:51.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:50 smithi167 ceph-mon[192880]: pgmap v750: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 35069/315330 objects degraded (11.121%); 35038/315330 objects misplaced (11.112%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:33:51.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3152206856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:51.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:50 smithi019 ceph-mon[216815]: pgmap v750: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 35069/315330 objects degraded (11.121%); 35038/315330 objects misplaced (11.112%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:33:51.030 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3152206856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:51.030 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:50 smithi019 ceph-mon[224043]: pgmap v750: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 35069/315330 objects degraded (11.121%); 35038/315330 objects misplaced (11.112%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:33:51.030 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3152206856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:33:51.063 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:33:51.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:51.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:33:51.171 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:51.189 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:33:51.189 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:33:51.190 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-30T09:33:51.190 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:33:51.190 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:33:51.190 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-30T09:33:51.214 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:51.303 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:51.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:51.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:51.409 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:51.438 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:51.583 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:33:51.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:51.783 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:33:51.989 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:51.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:51.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:51 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3152206856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:33:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:51 smithi167 ceph-mon[192880]: failed to encode map e305 with expected crc 2023-09-30T09:33:52.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:51 smithi167 ceph-mon[192880]: osdmap e305: 8 total, 8 up, 8 in 2023-09-30T09:33:52.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:51 smithi167 ceph-mon[192880]: failed to encode map e305 with expected crc 2023-09-30T09:33:52.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:33:52.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3152206856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:33:52.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[224043]: failed to encode map e305 with expected crc 2023-09-30T09:33:52.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[224043]: osdmap e305: 8 total, 8 up, 8 in 2023-09-30T09:33:52.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[224043]: failed to encode map e305 with expected crc 2023-09-30T09:33:52.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3152206856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:33:52.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[216815]: failed to encode map e305 with expected crc 2023-09-30T09:33:52.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[216815]: osdmap e305: 8 total, 8 up, 8 in 2023-09-30T09:33:52.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:51 smithi019 ceph-mon[216815]: failed to encode map e305 with expected crc 2023-09-30T09:33:52.098 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:52.141 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:52.216 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:52.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:52.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:52.318 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:52.319 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:52.413 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:52.518 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:52.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:52.624 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:52.687 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:52.828 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:52.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:52.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:52.940 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:52.965 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:52 smithi167 ceph-mon[192880]: pgmap v752: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 34971/313719 objects degraded (11.147%); 35038/313719 objects misplaced (11.169%); 3.4 MiB/s, 53 objects/s recovering 2023-09-30T09:33:52.967 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:52.984 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:52 smithi019 ceph-mon[216815]: pgmap v752: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 34971/313719 objects degraded (11.147%); 35038/313719 objects misplaced (11.169%); 3.4 MiB/s, 53 objects/s recovering 2023-09-30T09:33:52.984 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:52 smithi019 ceph-mon[224043]: pgmap v752: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 34971/313719 objects degraded (11.147%); 35038/313719 objects misplaced (11.169%); 3.4 MiB/s, 53 objects/s recovering 2023-09-30T09:33:53.013 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:53.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:53.115 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:53.222 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:53.345 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:53.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:53.408 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:33:53.449 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:33:53.479 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:53.557 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:33:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:53.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:33:53.661 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:33:53.747 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:53.939 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 34971/313719 objects degraded (11.147%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:53.998 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:33:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:54.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 34971/313719 objects degraded (11.147%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:54.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:33:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:33:53] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-09-30T09:33:54.018 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 34971/313719 objects degraded (11.147%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:54.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:33:54.120 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:33:54.138 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:33:54.216 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:54.464 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:33:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:54.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:33:54.570 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:54.661 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:33:54.661 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "" 2023-09-30T09:33:54.661 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:33:54.703 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:54.826 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:54 smithi019 ceph-mon[216815]: pgmap v753: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 34971/313719 objects degraded (11.147%); 35038/313719 objects misplaced (11.169%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:33:54.848 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:54.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:54.927 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:33:54.953 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:33:54.968 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:33:54.999 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:55.009 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:33:55.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:33:55.078 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:33:55.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:54 smithi019 ceph-mon[224043]: pgmap v753: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 34971/313719 objects degraded (11.147%); 35038/313719 objects misplaced (11.169%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:33:55.102 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:33:55.131 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:33:55.182 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:33:55.223 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:33:55.264 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:33:55.899 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:33:55.899 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:33:55.899 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:33:55.899 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:33:55.900 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:33:55.900 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:33:55.900 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:33:55.900 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:54 smithi167 ceph-mon[192880]: pgmap v753: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 34971/313719 objects degraded (11.147%); 35038/313719 objects misplaced (11.169%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:33:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:55.901 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:33:55.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:33:56.003 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:56.065 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:33:56.065 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:33:56.065 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:33:56.065 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:33:56.065 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:33:56.090 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4276670904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:56.198 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4276670904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:56.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4276670904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:33:56.243 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:57.098 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:57.105 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:56 smithi167 ceph-mon[192880]: pgmap v754: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 34649/305466 objects degraded (11.343%); 33983/305466 objects misplaced (11.125%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:33:57.106 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:33:57.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:57.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:57.110 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:56 smithi019 ceph-mon[224043]: pgmap v754: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 34649/305466 objects degraded (11.343%); 33983/305466 objects misplaced (11.125%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:33:57.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:56 smithi019 ceph-mon[216815]: pgmap v754: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 34649/305466 objects degraded (11.343%); 33983/305466 objects misplaced (11.125%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:33:57.209 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:33:57.302 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:57.400 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:33:57.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:57.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:33:57.513 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:33:57.596 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:57.619 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:33:57.739 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:33:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:57.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:33:57.855 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:57.914 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:57.989 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:57.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:57.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:58.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:57 smithi019 ceph-mon[224043]: 1.b scrub starts 2023-09-30T09:33:58.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:57 smithi019 ceph-mon[224043]: pgmap v755: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 34649/305466 objects degraded (11.343%); 33983/305466 objects misplaced (11.125%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:33:58.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:57 smithi019 ceph-mon[216815]: 1.b scrub starts 2023-09-30T09:33:58.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:57 smithi019 ceph-mon[216815]: pgmap v755: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 34649/305466 objects degraded (11.343%); 33983/305466 objects misplaced (11.125%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:33:58.093 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:33:58.152 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:57 smithi167 ceph-mon[192880]: 1.b scrub starts 2023-09-30T09:33:58.152 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:57 smithi167 ceph-mon[192880]: pgmap v755: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 34649/305466 objects degraded (11.343%); 33983/305466 objects misplaced (11.125%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:33:58.155 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:58.273 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:33:58.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:58.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:33:58.378 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:58.470 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:58.592 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:58.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:58.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:58.697 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:33:58.758 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:58.804 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:33:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:58.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:33:58.907 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:33:58.967 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:59.070 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:33:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:59.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:33:59.075 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 34649/305466 objects degraded (11.343%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:59.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:58 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4093054082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:59.076 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 34649/305466 objects degraded (11.343%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:59.076 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:58 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4093054082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:59.174 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:33:59.176 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:33:59.244 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:59.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 34649/305466 objects degraded (11.343%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:33:59.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:58 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4093054082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:33:59.388 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:33:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:59.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:33:59.412 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:33:59.500 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:59.535 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:59.657 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:59.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:59.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:59.762 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:33:59.813 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:33:59.892 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:33:59.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:33:59.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:33:59.917 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:33:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:33:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:33:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:33:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:135244 5.1 5:87738a27:::benchmark_data_smithi019_287194_object52466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-30T09:33:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:34.114532+0000", 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 445.79863777100002, 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21996616699999999, 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 135244 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114532+0000", 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114532+0000", 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114531+0000", 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114536+0000", 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0969999999999996e-06 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114546+0000", 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6520000000000001e-06 2023-09-30T09:33:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114551+0000", 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1529999999999996e-06 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114585+0000", 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4465999999999998e-05 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114665+0000", 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9425000000000006e-05 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.114738+0000", 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3571000000000004e-05 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.117493+0000", 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027542759999999999 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.118451+0000", 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00095841000000000003 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334392+0000", 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21594123600000001 2023-09-30T09:33:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334449+0000", 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.664e-05 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:34.334498+0000", 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9209999999999998e-05 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.44600.0:147190 5.1 5:8f06d857:::benchmark_data_smithi019_287194_object64412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:26:55.553920+0000", 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 424.359249358, 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.196767887, 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.44600", 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 147190 2023-09-30T09:33:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553920+0000", 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553920+0000", 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553920+0000", 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553925+0000", 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.361e-06 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553934+0000", 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7739999999999993e-06 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.553939+0000", 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4940000000000003e-06 2023-09-30T09:33:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.557550+0000", 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003610458 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.557685+0000", 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013476499999999999 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.557805+0000", 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012047199999999999 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.561618+0000", 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0038124930000000001 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.750555+0000", 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.18893771200000001 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.750596+0000", 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0507999999999999e-05 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.750646+0000", 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0364000000000001e-05 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:26:55.750688+0000", 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2097e-05 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71271 6.e 6:780eb166:::benchmark_data_smithi019_355417_object71270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 273.90679924599999, 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.228409374, 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:33:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71271 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006369+0000", 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:33:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006424+0000", 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5114000000000001e-05 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006439+0000", 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5241e-05 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006447+0000", 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6529999999999994e-06 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006492+0000", 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5022999999999999e-05 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006590+0000", 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7985000000000005e-05 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006695+0000", 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000104984 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.010237+0000", 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003542271 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.013024+0000", 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027868979999999999 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.231353+0000", 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.218328895 2023-09-30T09:33:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.231406+0000", 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3362000000000002e-05 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234780+0000", 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003373572 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71277 6.e 6:746a3c45:::benchmark_data_smithi019_355417_object71276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 273.90571749899999, 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.230281654, 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71277 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007451+0000", 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007503+0000", 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2033999999999997e-05 2023-09-30T09:33:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007515+0000", 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1418e-05 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007520+0000", 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.938e-06 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008550+0000", 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010299840000000001 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008653+0000", 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103688 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008752+0000", 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8879e-05 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.010372+0000", 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016195960000000001 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.013054+0000", 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026824940000000001 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234915+0000", 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22186042 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234978+0000", 2023-09-30T09:33:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3447000000000004e-05 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237734+0000", 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027555909999999999 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71282 6.e 6:707e64e4:::benchmark_data_smithi019_355417_object71281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:33:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 273.897505647, 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22500474600000001, 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71282 2023-09-30T09:33:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:33:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:33:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015663+0000", 2023-09-30T09:33:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:33:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015715+0000", 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1873999999999998e-05 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015725+0000", 2023-09-30T09:33:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0387000000000001e-05 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015730+0000", 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.814e-06 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015768+0000", 2023-09-30T09:33:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7673000000000003e-05 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015871+0000", 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103274 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015979+0000", 2023-09-30T09:33:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108563 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.018931+0000", 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029516809999999998 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020250+0000", 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013193289999999999 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237789+0000", 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21753858700000001 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237850+0000", 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0448999999999997e-05 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240669+0000", 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028193039999999999 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71283 6.e 6:7093bd40:::benchmark_data_smithi019_355417_object71282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 273.897165374, 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22746534299999999, 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:33:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71283 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:33:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016074+0000", 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0246999999999993e-05 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016082+0000", 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4610000000000002e-06 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016085+0000", 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5489999999999998e-06 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017714+0000", 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016290549999999999 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017801+0000", 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7076000000000005e-05 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017904+0000", 2023-09-30T09:33:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000102876 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020212+0000", 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023077570000000001 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.022334+0000", 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021222160000000001 2023-09-30T09:33:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240736+0000", 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.218401652 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240791+0000", 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.554e-05 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243470+0000", 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026784019999999999 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71284 6.e 6:7ce92b46:::benchmark_data_smithi019_355417_object71283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 273.89667699099999, 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22906620999999999, 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71284 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:33:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016492+0000", 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:33:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016557+0000", 2023-09-30T09:33:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4941e-05 2023-09-30T09:33:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016565+0000", 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3669999999999997e-06 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016570+0000", 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2390000000000004e-06 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.018977+0000", 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024078609999999999 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.019059+0000", 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1386999999999999e-05 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.019149+0000", 2023-09-30T09:33:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9938000000000005e-05 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020349+0000", 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012002530000000001 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.022546+0000", 2023-09-30T09:33:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021970819999999999 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243549+0000", 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22100276599999999 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243617+0000", 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8547999999999995e-05 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.245559+0000", 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019414879999999999 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:96725 6.e 6:743dbf6c:::benchmark_data_smithi019_355417_object4421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:33:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 243.429989345, 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.205057722, 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 96725 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483179+0000", 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483187+0000", 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3750000000000003e-06 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483203+0000", 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6215999999999999e-05 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483210+0000", 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0180000000000004e-06 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483254+0000", 2023-09-30T09:33:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3426000000000001e-05 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.485269+0000", 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002015644 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488763+0000", 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034939120000000001 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488819+0000", 2023-09-30T09:33:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6135999999999999e-05 2023-09-30T09:33:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488893+0000", 2023-09-30T09:33:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3734999999999998e-05 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.490570+0000", 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001677015 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.490591+0000", 2023-09-30T09:33:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0973000000000001e-05 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.686691+0000", 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.196100203 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.686739+0000", 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7395000000000003e-05 2023-09-30T09:33:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.688238+0000", 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001499301 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:98164 6.1e 6:7b1cf70c:::benchmark_data_smithi019_355417_object5860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:33:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 241.85262107899999, 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21486691799999999, 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 98164 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060547+0000", 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060554+0000", 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4069999999999998e-06 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060565+0000", 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1336e-05 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060571+0000", 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2219999999999999e-06 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.074775+0000", 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014203604999999999 2023-09-30T09:33:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.075990+0000", 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012149319999999999 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.092854+0000", 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016863878999999998 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.092955+0000", 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101431 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.093073+0000", 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000117923 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.097188+0000", 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041146919999999997 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.105705+0000", 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0085167469999999999 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.273965+0000", 2023-09-30T09:33:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16826055000000001 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.274028+0000", 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2502999999999994e-05 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.275416+0000", 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013878479999999999 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:98166 6.e 6:75ad8793:::benchmark_data_smithi019_355417_object5862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 241.85246756500001, 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.214711604, 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:33:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 98166 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:33:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060701+0000", 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060712+0000", 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0445000000000001e-05 2023-09-30T09:33:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060723+0000", 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1114999999999999e-05 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060730+0000", 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1130000000000003e-06 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.076100+0000", 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015369651999999999 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.076669+0000", 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00056970499999999999 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095225+0000", 2023-09-30T09:33:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.018555250999999998 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095303+0000", 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8380999999999994e-05 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095431+0000", 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012828399999999999 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.098950+0000", 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035189520000000001 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.105689+0000", 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0067384180000000004 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.273979+0000", 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.168290369 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.274032+0000", 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2456000000000001e-05 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:33:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.275414+0000", 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001382203 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:99939 6.1e 6:78215d60:::benchmark_data_smithi019_355417_object7635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 239.39497806700001, 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.201779021, 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 99939 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518190+0000", 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518198+0000", 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3209999999999999e-06 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518211+0000", 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3689000000000001e-05 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518219+0000", 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0320000000000003e-06 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518265+0000", 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5704999999999997e-05 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518919+0000", 2023-09-30T09:34:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00065372300000000004 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.707844+0000", 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.18892562099999999 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.707924+0000", 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0043999999999995e-05 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.708033+0000", 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108694 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.711930+0000", 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003896708 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.714999+0000", 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030689319999999999 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.718299+0000", 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033004290000000001 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.718348+0000", 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8959999999999999e-05 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.719971+0000", 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001622472 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:99940 6.e 6:70189e37:::benchmark_data_smithi019_355417_object7636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 239.393132694, 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20653424000000001, 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 99940 2023-09-30T09:34:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:34:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520036+0000", 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520043+0000", 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9550000000000003e-06 2023-09-30T09:34:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520056+0000", 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3206e-05 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520061+0000", 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.242e-06 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520098+0000", 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6636e-05 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520966+0000", 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00086822099999999997 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.013 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716327+0000", 2023-09-30T09:34:00.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.19536082900000001 2023-09-30T09:34:00.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.014 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.014 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.014 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716416+0000", 2023-09-30T09:34:00.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9463999999999996e-05 2023-09-30T09:34:00.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.014 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716529+0000", 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112768 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.721106+0000", 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004577378 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.015 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.723760+0000", 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026536900000000002 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.725227+0000", 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001467199 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.725281+0000", 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3758999999999998e-05 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.726571+0000", 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001290177 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.017 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:101192 6.e 6:74a2e37f:::benchmark_data_smithi019_355417_object8888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 237.56359861000001, 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.198305908, 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101192 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.018 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349570+0000", 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349577+0000", 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1400000000000002e-06 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.020 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349594+0000", 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7044000000000001e-05 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349601+0000", 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3339999999999996e-06 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349774+0000", 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017279799999999999 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.021 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.350391+0000", 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00061680600000000002 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534413+0000", 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.184022306 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.022 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534506+0000", 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3183000000000006e-05 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534635+0000", 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012914900000000001 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:00.023 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.537971+0000", 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033357170000000002 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.540045+0000", 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020742669999999999 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.024 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.546502+0000", 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006457043 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.546557+0000", 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4824999999999998e-05 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.547877+0000", 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013197580000000001 2023-09-30T09:34:00.025 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107544 6.e 6:73fbd947:::benchmark_data_smithi019_355417_object15240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 228.17962065099999, 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26128712500000001, 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.026 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107544 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733548+0000", 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733555+0000", 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6210000000000004e-06 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733572+0000", 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7017999999999999e-05 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733580+0000", 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1799999999999996e-06 2023-09-30T09:34:00.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733633+0000", 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2781999999999997e-05 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734357+0000", 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00072381799999999999 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739005+0000", 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046474890000000003 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739086+0000", 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1181999999999996e-05 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739168+0000", 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2346000000000001e-05 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745745+0000", 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0065768140000000003 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747298+0000", 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001552994 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.993625+0000", 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.24632679199999999 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.993663+0000", 2023-09-30T09:34:00.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8668000000000003e-05 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994836+0000", 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001172841 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.034 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107545 6.e 6:77969f61:::benchmark_data_smithi019_355417_object15241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 228.17842541799999, 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26112492199999998, 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:00.035 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107545 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:34:00.036 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734751+0000", 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9609999999999999e-06 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734761+0000", 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0096e-05 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734765+0000", 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5199999999999999e-06 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734813+0000", 2023-09-30T09:34:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7290000000000003e-05 2023-09-30T09:34:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:34:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735419+0000", 2023-09-30T09:34:00.039 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00060657400000000002 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742024+0000", 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0066044190000000003 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742129+0000", 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010515599999999999 2023-09-30T09:34:00.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742280+0000", 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015174899999999999 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747335+0000", 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050547270000000002 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747342+0000", 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4069999999999998e-06 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994874+0000", 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.24753244799999999 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994933+0000", 2023-09-30T09:34:00.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8959000000000003e-05 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995869+0000", 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00093624799999999998 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.043 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107547 6.e 6:7322664d:::benchmark_data_smithi019_355417_object15243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 228.17761695600001, 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26146208700000001, 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107547 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.044 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735552+0000", 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735558+0000", 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6540000000000004e-06 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.046 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735569+0000", 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1029000000000001e-05 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735575+0000", 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4320000000000004e-06 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735624+0000", 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9038e-05 2023-09-30T09:34:00.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.736267+0000", 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00064271499999999997 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745778+0000", 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0095116769999999996 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745840+0000", 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1790000000000003e-05 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745925+0000", 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5045999999999999e-05 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:00.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747718+0000", 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017924869999999999 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.748063+0000", 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00034562999999999998 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995923+0000", 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.247859211 2023-09-30T09:34:00.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995976+0000", 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3766e-05 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.997015+0000", 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001038436 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:114057 7.8 e303/295 7:116ea3d4:::benchmark_data_smithi019_404601_object779:head v 303'7213, mlcod=303'7211)", 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:33:35.902932+0000", 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 24.010237979999999, 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016763170000000001, 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.902932+0000", 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.902932+0000", 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.902930+0000", 2023-09-30T09:34:00.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.902947+0000", 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7626e-05 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.902989+0000", 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.142e-05 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.903006+0000", 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6833e-05 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.903137+0000", 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000131271 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.055 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.903213+0000", 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6441999999999998e-05 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.919652+0000", 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016438556999999999 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:35.919695+0000", 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2981000000000001e-05 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:120522 7.f e304/279 7:f5f13505:::benchmark_data_smithi019_404601_object7244:head v 304'7451, mlcod=304'7447)", 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:33:46.600409+0000", 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 13.312760588, 2023-09-30T09:34:00.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016402249000000001, 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.600409+0000", 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.600409+0000", 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.600408+0000", 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.600416+0000", 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4900000000000003e-06 2023-09-30T09:34:00.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.600427+0000", 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1246e-05 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.600433+0000", 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3049999999999998e-06 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.602620+0000", 2023-09-30T09:34:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021868090000000001 2023-09-30T09:34:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.602635+0000", 2023-09-30T09:34:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4718e-05 2023-09-30T09:34:00.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.616758+0000", 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014122878 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.616811+0000", 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3656999999999998e-05 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.062 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:120666 7.8 e304/295 7:1b3e6805:::benchmark_data_smithi019_404601_object7388:head v 304'7632, mlcod=304'7630)", 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:33:46.833857+0000", 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 13.079313190000001, 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.017603285999999999, 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833857+0000", 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833857+0000", 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833856+0000", 2023-09-30T09:34:00.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833865+0000", 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4949999999999997e-06 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833880+0000", 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5274e-05 2023-09-30T09:34:00.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833887+0000", 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4250000000000003e-06 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.836188+0000", 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023008460000000001 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.836216+0000", 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8377e-05 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.851424+0000", 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015207729 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.851460+0000", 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5960000000000001e-05 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:126534 7.8 e305/295 7:129da791:::benchmark_data_smithi019_404601_object13256:head v 305'8002, mlcod=305'8000)", 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:33:57.076697+0000", 2023-09-30T09:34:00.068 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.8364724830000001, 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014902047999999999, 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.076697+0000", 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.076697+0000", 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.076696+0000", 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.076709+0000", 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2680000000000001e-05 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.076747+0000", 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8587999999999997e-05 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.076756+0000", 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5019999999999992e-06 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.079291+0000", 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0025350289999999998 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.079318+0000", 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7103000000000001e-05 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.091546+0000", 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012227780000000001 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:00.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:57.091599+0000", 2023-09-30T09:34:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3526999999999998e-05 2023-09-30T09:34:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:00.074 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:00.074 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:34:00.075 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:34:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4093054082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[216815]: osdmap e306: 8 total, 8 up, 8 in 2023-09-30T09:34:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[216815]: pgmap v757: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 34350/298188 objects degraded (11.520%); 32747/298188 objects misplaced (10.982%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:34:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[216815]: failed to encode map e306 with expected crc 2023-09-30T09:34:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[216815]: 1.16 scrub starts 2023-09-30T09:34:00.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:34:00.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4093054082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:00.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[224043]: osdmap e306: 8 total, 8 up, 8 in 2023-09-30T09:34:00.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[224043]: pgmap v757: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 34350/298188 objects degraded (11.520%); 32747/298188 objects misplaced (10.982%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:34:00.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[224043]: failed to encode map e306 with expected crc 2023-09-30T09:34:00.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:33:59 smithi019 ceph-mon[224043]: 1.16 scrub starts 2023-09-30T09:34:00.098 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:59 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:34:00.098 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4093054082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:00.099 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:59 smithi167 ceph-mon[192880]: osdmap e306: 8 total, 8 up, 8 in 2023-09-30T09:34:00.099 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:59 smithi167 ceph-mon[192880]: pgmap v757: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 34350/298188 objects degraded (11.520%); 32747/298188 objects misplaced (10.982%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:34:00.099 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:59 smithi167 ceph-mon[192880]: failed to encode map e306 with expected crc 2023-09-30T09:34:00.099 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:33:59 smithi167 ceph-mon[192880]: 1.16 scrub starts 2023-09-30T09:34:00.102 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:00.194 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:00.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:00.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:00.297 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:00.374 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:00.750 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:00.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:00.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:00.877 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:00.903 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:34:00.903 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:34:00.904 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-30T09:34:00.904 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-30T09:34:00.904 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:00.951 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:01.004 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:00 smithi019 ceph-mon[216815]: failed to encode map e306 with expected crc 2023-09-30T09:34:01.004 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:00 smithi019 ceph-mon[216815]: 1.b scrub starts 2023-09-30T09:34:01.004 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:00 smithi019 ceph-mon[224043]: failed to encode map e306 with expected crc 2023-09-30T09:34:01.004 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:00 smithi019 ceph-mon[224043]: 1.b scrub starts 2023-09-30T09:34:01.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:34:01.101 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:01.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:01.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:01.225 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:00 smithi167 ceph-mon[192880]: failed to encode map e306 with expected crc 2023-09-30T09:34:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:00 smithi167 ceph-mon[192880]: 1.b scrub starts 2023-09-30T09:34:01.350 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:01.483 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:01.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:01.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:01.529 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:01.601 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:01.661 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:01.840 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:01.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:02.073 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:02.099 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:01 smithi167 ceph-mon[192880]: pgmap v758: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 34260/296511 objects degraded (11.554%); 32747/296511 objects misplaced (11.044%); 3.3 MiB/s, 53 objects/s recovering 2023-09-30T09:34:02.102 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:02.213 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:02.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:02.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:02.316 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:01 smithi019 ceph-mon[216815]: pgmap v758: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 34260/296511 objects degraded (11.554%); 32747/296511 objects misplaced (11.044%); 3.3 MiB/s, 53 objects/s recovering 2023-09-30T09:34:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:01 smithi019 ceph-mon[224043]: pgmap v758: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 34260/296511 objects degraded (11.554%); 32747/296511 objects misplaced (11.044%); 3.3 MiB/s, 53 objects/s recovering 2023-09-30T09:34:02.343 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:02.384 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:02.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:02.487 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:02.535 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:02.772 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:02.781 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:34:02.785 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:02.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:02.892 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:02.921 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:02.977 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:02.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:02.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:03.082 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:03.119 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:03.194 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:03.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:03.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:03.299 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:03.380 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:03.601 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:34:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:34:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:34:03.601 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:34:03.836 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:34:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:34:03] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-09-30T09:34:03.886 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:34:03.922 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:03.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:03.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:03.936 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:34:03.974 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:34:04.005 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:04.005 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:04.005 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:04 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 34260/296511 objects degraded (11.554%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:04.005 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:04 smithi167 ceph-mon[192880]: 1.b scrub starts 2023-09-30T09:34:04.005 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:04 smithi167 ceph-mon[192880]: pgmap v759: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 34260/296511 objects degraded (11.554%); 32747/296511 objects misplaced (11.044%); 3.3 MiB/s, 53 objects/s recovering 2023-09-30T09:34:04.005 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2027766987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:04.029 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:04.069 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:04.115 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:04.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:04.202 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:34:04.202 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:34:04.202 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:34:04.219 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:04.261 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:04.269 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:34:04.300 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:34:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 34260/296511 objects degraded (11.554%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[224043]: 1.b scrub starts 2023-09-30T09:34:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[224043]: pgmap v759: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 34260/296511 objects degraded (11.554%); 32747/296511 objects misplaced (11.044%); 3.3 MiB/s, 53 objects/s recovering 2023-09-30T09:34:04.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2027766987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:04.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:04.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:04.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 34260/296511 objects degraded (11.554%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:04.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[216815]: 1.b scrub starts 2023-09-30T09:34:04.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[216815]: pgmap v759: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 34260/296511 objects degraded (11.554%); 32747/296511 objects misplaced (11.044%); 3.3 MiB/s, 53 objects/s recovering 2023-09-30T09:34:04.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2027766987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:04.341 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:34:04.341 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:34:04.341 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:34:04.341 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:34:04.342 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:34:04.342 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:34:04.342 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:34:04.342 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:34:04.506 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:04.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:04.609 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:34:04.614 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:04.703 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:04.775 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:04.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:04.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:04.882 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:04.936 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:05.002 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:05.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:05.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:05.107 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:05.140 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:05.160 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:05.230 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:05.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:05.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:05.333 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:05.449 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:05.500 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:05.686 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:05.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:05.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:05.794 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:05.832 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:05.907 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:05.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:06.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:06.063 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:06.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:06 smithi019 conmon[216791]: 2023-09-30T09:34:06.295+0000 7faf52971700 -1 mon.a@0(leader).osd e306 definitely_dead 0 2023-09-30T09:34:06.374 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:06.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:06.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:06.488 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:06.503 INFO:teuthology.orchestra.run.smithi019.stderr:marked out osd.7. 2023-09-30T09:34:06.551 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:06 smithi019 ceph-mon[224043]: pgmap v760: 119 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 33940/288228 objects degraded (11.775%); 31695/288228 objects misplaced (10.997%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:34:06.551 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3088889172' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-30T09:34:06.551 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:06 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-30T09:34:06.551 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:06 smithi019 ceph-mon[224043]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-30T09:34:06.551 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:06 smithi019 ceph-mon[216815]: pgmap v760: 119 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 33940/288228 objects degraded (11.775%); 31695/288228 objects misplaced (10.997%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:34:06.551 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3088889172' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-30T09:34:06.551 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:06 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-30T09:34:06.552 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:06 smithi019 ceph-mon[216815]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-30T09:34:06.552 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:06.601 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:06.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:06.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:06.704 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:06.732 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:06 smithi167 ceph-mon[192880]: pgmap v760: 119 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 33940/288228 objects degraded (11.775%); 31695/288228 objects misplaced (10.997%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:34:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3088889172' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-30T09:34:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:06 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-30T09:34:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:06 smithi167 ceph-mon[192880]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-30T09:34:06.777 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:06.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:06.881 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:06.909 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:07.157 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:34:07.157 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:34:07.450 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:34:07.680 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:07.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:07 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-30T09:34:07.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:07 smithi019 ceph-mon[224043]: osdmap e307: 8 total, 8 up, 7 in 2023-09-30T09:34:07.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:07 smithi019 ceph-mon[224043]: failed to encode map e307 with expected crc 2023-09-30T09:34:07.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:07 smithi019 ceph-mon[224043]: osdmap e308: 8 total, 8 up, 7 in 2023-09-30T09:34:07.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:07 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-30T09:34:07.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:07 smithi019 ceph-mon[216815]: osdmap e307: 8 total, 8 up, 7 in 2023-09-30T09:34:07.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:07 smithi019 ceph-mon[216815]: failed to encode map e307 with expected crc 2023-09-30T09:34:07.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:07 smithi019 ceph-mon[216815]: osdmap e308: 8 total, 8 up, 7 in 2023-09-30T09:34:07.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:07.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:07 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-30T09:34:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:07 smithi167 ceph-mon[192880]: osdmap e307: 8 total, 8 up, 7 in 2023-09-30T09:34:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:07 smithi167 ceph-mon[192880]: failed to encode map e307 with expected crc 2023-09-30T09:34:07.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:07 smithi167 ceph-mon[192880]: osdmap e308: 8 total, 8 up, 7 in 2023-09-30T09:34:07.785 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:07.821 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:07.980 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:07.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:07.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:08.086 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:08.125 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:08.153 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:08.188 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:08.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:08.293 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:08.324 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:08.380 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:34:08.527 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:08.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:08.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: 1.b scrub starts 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: pgmap v762: 119 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 33940/288228 objects degraded (11.775%); 31695/288228 objects misplaced (10.997%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 33940/288228 objects degraded (11.775%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: osdmap e309: 8 total, 8 up, 7 in 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2806547645' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: 1.b scrub starts 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: pgmap v762: 119 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 33940/288228 objects degraded (11.775%); 31695/288228 objects misplaced (10.997%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 33940/288228 objects degraded (11.775%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: osdmap e309: 8 total, 8 up, 7 in 2023-09-30T09:34:08.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2806547645' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:08.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:08 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:08.635 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:08.704 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: 1.b scrub starts 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: pgmap v762: 119 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 33940/288228 objects degraded (11.775%); 31695/288228 objects misplaced (10.997%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: failed to encode map e308 with expected crc 2023-09-30T09:34:08.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:34:08.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 33940/288228 objects degraded (11.775%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:08.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: osdmap e309: 8 total, 8 up, 7 in 2023-09-30T09:34:08.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2806547645' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:08.706 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:08 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:08.707 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:08.845 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:08.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:08.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:08.960 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:08.961 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:09.074 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:09.220 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:09.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:09.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:09.329 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:09.370 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:09.449 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:09.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:09.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:09.485 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:34:09.556 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:09.584 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:09.732 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: 7.9 starting backfill to osd.2 from (0'0,0'0] MAX to 306'8137 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: 7.8 continuing backfill to osd.2 from (286'3000,306'8353] MIN to 306'8353 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 306'8267 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: 7.f starting backfill to osd.6 from (0'0,0'0] MAX to 306'8197 2023-09-30T09:34:09.733 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: 7.5 continuing backfill to osd.2 from (286'3000,306'8215] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 306'8215 2023-09-30T09:34:09.734 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:09.734 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:34:09.734 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:09 smithi167 ceph-mon[192880]: osdmap e310: 8 total, 8 up, 7 in 2023-09-30T09:34:09.780 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:09.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:09.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: 7.9 starting backfill to osd.2 from (0'0,0'0] MAX to 306'8137 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: 7.8 continuing backfill to osd.2 from (286'3000,306'8353] MIN to 306'8353 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 306'8267 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: 7.f starting backfill to osd.6 from (0'0,0'0] MAX to 306'8197 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: 7.5 continuing backfill to osd.2 from (286'3000,306'8215] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 306'8215 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:34:09.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[216815]: osdmap e310: 8 total, 8 up, 7 in 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: failed to encode map e309 with expected crc 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: 7.9 starting backfill to osd.2 from (0'0,0'0] MAX to 306'8137 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: 7.8 continuing backfill to osd.2 from (286'3000,306'8353] MIN to 306'8353 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 306'8267 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: 7.f starting backfill to osd.6 from (0'0,0'0] MAX to 306'8197 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: 7.5 continuing backfill to osd.2 from (286'3000,306'8215] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 306'8215 2023-09-30T09:34:09.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:09.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:34:09.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:09 smithi019 ceph-mon[224043]: osdmap e310: 8 total, 8 up, 7 in 2023-09-30T09:34:09.883 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:09.964 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:10.199 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:10.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:10.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:10.314 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:10.341 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:10.558 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: pgmap v765: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+clean+scrubbing, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 87 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 714 op/s; 33420/283923 objects degraded (11.771%); 50210/283923 objects misplaced (17.684%); 7.7 MiB/s, 116 objects/s recovering 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: pgmap v765: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+clean+scrubbing, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 87 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 714 op/s; 33420/283923 objects degraded (11.771%); 50210/283923 objects misplaced (17.684%); 7.7 MiB/s, 116 objects/s recovering 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[224043]: osdmap e311: 8 total, 8 up, 7 in 2023-09-30T09:34:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: pgmap v765: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+clean+scrubbing, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 87 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 714 op/s; 33420/283923 objects degraded (11.771%); 50210/283923 objects misplaced (17.684%); 7.7 MiB/s, 116 objects/s recovering 2023-09-30T09:34:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:10 smithi167 ceph-mon[192880]: osdmap e311: 8 total, 8 up, 7 in 2023-09-30T09:34:10.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: failed to encode map e310 with expected crc 2023-09-30T09:34:10.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:10 smithi019 ceph-mon[216815]: osdmap e311: 8 total, 8 up, 7 in 2023-09-30T09:34:10.962 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:10.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:10.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:11.066 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:11.154 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:11.289 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:11.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:11.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:11.404 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:11.548 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:11.634 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:34:11.670 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:34:11.670 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:34:11.674 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:34:11.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:11 smithi167 ceph-mon[192880]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:11 smithi167 ceph-mon[192880]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:11 smithi167 ceph-mon[192880]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:11 smithi167 ceph-mon[192880]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:11 smithi167 ceph-mon[192880]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:11 smithi167 ceph-mon[192880]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:11 smithi167 ceph-mon[192880]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.709 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:34:11.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:34:11.795 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:11.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:11.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[224043]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[224043]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[224043]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[224043]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[224043]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[224043]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[224043]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:11.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[216815]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[216815]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[216815]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[216815]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[216815]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[216815]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:11 smithi019 ceph-mon[216815]: failed to encode map e311 with expected crc 2023-09-30T09:34:11.815 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:34:11.863 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:34:11.901 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:11.901 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:34:11.945 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:34:11.975 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:34:11.976 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:34:11.976 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:34:11.976 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:34:11.976 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:34:11.976 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:34:11.977 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:34:11.977 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:34:12.020 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:12.251 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:12.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:12.359 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:12.442 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:12.815 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0, 2] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:34:12.816 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-30T09:34:12.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:12 smithi019 ceph-mon[216815]: pgmap v768: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 86 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 40 KiB/s rd, 105 KiB/s wr, 411 op/s; 40441/283662 objects degraded (14.257%); 50210/283662 objects misplaced (17.701%); 7.1 MiB/s, 118 objects/s recovering 2023-09-30T09:34:12.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2424556724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:12.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:12 smithi019 ceph-mon[224043]: pgmap v768: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 86 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 40 KiB/s rd, 105 KiB/s wr, 411 op/s; 40441/283662 objects degraded (14.257%); 50210/283662 objects misplaced (17.701%); 7.1 MiB/s, 118 objects/s recovering 2023-09-30T09:34:12.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2424556724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:12.876 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:34:12.916 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:12 smithi167 ceph-mon[192880]: pgmap v768: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 86 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 40 KiB/s rd, 105 KiB/s wr, 411 op/s; 40441/283662 objects degraded (14.257%); 50210/283662 objects misplaced (17.701%); 7.1 MiB/s, 118 objects/s recovering 2023-09-30T09:34:12.917 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2424556724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:12.936 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:34:13.363 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:13.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:13.483 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:13.602 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:13.669 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:13.723 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:13.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 40441/283662 objects degraded (14.257%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:13.732 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 40441/283662 objects degraded (14.257%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:13.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:13.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:13.834 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:13.901 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:13.902 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 40441/283662 objects degraded (14.257%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:14.044 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:14.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:14.057 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:34:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:34:13] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-09-30T09:34:14.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:14.075 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:14.137 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:14.158 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:14.205 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:14.485 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:14.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:14.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:14.592 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:14.633 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:14.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:14 smithi167 ceph-mon[192880]: pgmap v769: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 86 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 27 KiB/s rd, 71 KiB/s wr, 277 op/s; 40441/283662 objects degraded (14.257%); 50210/283662 objects misplaced (17.701%); 4.8 MiB/s, 79 objects/s recovering 2023-09-30T09:34:14.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:14 smithi019 ceph-mon[216815]: pgmap v769: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 86 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 27 KiB/s rd, 71 KiB/s wr, 277 op/s; 40441/283662 objects degraded (14.257%); 50210/283662 objects misplaced (17.701%); 4.8 MiB/s, 79 objects/s recovering 2023-09-30T09:34:14.779 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:14 smithi019 ceph-mon[224043]: pgmap v769: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 11 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 86 active+clean; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 27 KiB/s rd, 71 KiB/s wr, 277 op/s; 40441/283662 objects degraded (14.257%); 50210/283662 objects misplaced (17.701%); 4.8 MiB/s, 79 objects/s recovering 2023-09-30T09:34:15.500 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:15.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:15.508 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:34:15.604 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:15.689 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:16.455 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:34:16.973 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:34:16.974 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:34:16.974 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:34:16.975 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:16 smithi019 ceph-mon[216815]: pgmap v770: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 23 KiB/s rd, 61 KiB/s wr, 576 op/s; 40052/276600 objects degraded (14.480%); 48953/276600 objects misplaced (17.698%); 6.9 MiB/s, 113 objects/s recovering 2023-09-30T09:34:16.975 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:16 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:16.975 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:16 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:16.975 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:16 smithi167 ceph-mon[192880]: pgmap v770: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 23 KiB/s rd, 61 KiB/s wr, 576 op/s; 40052/276600 objects degraded (14.480%); 48953/276600 objects misplaced (17.698%); 6.9 MiB/s, 113 objects/s recovering 2023-09-30T09:34:16.976 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:16 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:16.980 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:16 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:16.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:16 smithi019 ceph-mon[224043]: pgmap v770: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 23 KiB/s rd, 61 KiB/s wr, 576 op/s; 40052/276600 objects degraded (14.480%); 48953/276600 objects misplaced (17.698%); 6.9 MiB/s, 113 objects/s recovering 2023-09-30T09:34:16.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:16 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:16.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:16 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:17.009 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:17.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:17.119 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:17.198 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:17.371 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:17.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:17.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:17.482 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:17.525 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:17.634 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:17.637 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:34:17.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:17.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:17.742 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:17.793 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:17.862 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:17.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:17.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:17.968 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:18.078 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:18.136 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:18.335 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:18.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:18.447 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:18.491 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:18.708 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:18.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:18.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:18.813 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:18.924 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:18 smithi019 ceph-mon[224043]: pgmap v771: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 308 op/s; 40052/276600 objects degraded (14.480%); 48953/276600 objects misplaced (17.698%); 2.0 MiB/s, 39 objects/s recovering 2023-09-30T09:34:18.924 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:18.924 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:18.924 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 40052/276600 objects degraded (14.480%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:18.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:18 smithi019 ceph-mon[216815]: pgmap v771: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 308 op/s; 40052/276600 objects degraded (14.480%); 48953/276600 objects misplaced (17.698%); 2.0 MiB/s, 39 objects/s recovering 2023-09-30T09:34:18.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:18.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:18.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 40052/276600 objects degraded (14.480%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:18.944 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:18 smithi167 ceph-mon[192880]: pgmap v771: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 308 op/s; 40052/276600 objects degraded (14.480%); 48953/276600 objects misplaced (17.698%); 2.0 MiB/s, 39 objects/s recovering 2023-09-30T09:34:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 40052/276600 objects degraded (14.480%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:19.645 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.1 to 1 (10000) 2023-09-30T09:34:19.646 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:34:19.960 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/510285331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:19.960 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4146380217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:34:20.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/510285331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:20.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4146380217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:34:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/510285331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:20.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4146380217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:34:20.178 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:34:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:34:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:34:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:34:20.179 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:34:20.648 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:20 smithi167 ceph-mon[192880]: pgmap v772: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 474 op/s; 39608/269874 objects degraded (14.676%); 47666/269874 objects misplaced (17.662%); 3.8 MiB/s, 67 objects/s recovering 2023-09-30T09:34:20.648 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:20 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:20.648 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/510285331' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[216815]: pgmap v772: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 474 op/s; 39608/269874 objects degraded (14.676%); 47666/269874 objects misplaced (17.662%); 3.8 MiB/s, 67 objects/s recovering 2023-09-30T09:34:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/510285331' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4146380217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:34:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[216815]: osdmap e312: 8 total, 8 up, 7 in 2023-09-30T09:34:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[216815]: failed to encode map e312 with expected crc 2023-09-30T09:34:20.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[216815]: failed to encode map e312 with expected crc 2023-09-30T09:34:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[224043]: pgmap v772: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 474 op/s; 39608/269874 objects degraded (14.676%); 47666/269874 objects misplaced (17.662%); 3.8 MiB/s, 67 objects/s recovering 2023-09-30T09:34:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/510285331' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4146380217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:34:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[224043]: osdmap e312: 8 total, 8 up, 7 in 2023-09-30T09:34:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[224043]: failed to encode map e312 with expected crc 2023-09-30T09:34:20.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:20 smithi019 ceph-mon[224043]: failed to encode map e312 with expected crc 2023-09-30T09:34:21.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4146380217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:34:21.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:20 smithi167 ceph-mon[192880]: osdmap e312: 8 total, 8 up, 7 in 2023-09-30T09:34:21.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:20 smithi167 ceph-mon[192880]: failed to encode map e312 with expected crc 2023-09-30T09:34:21.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:20 smithi167 ceph-mon[192880]: failed to encode map e312 with expected crc 2023-09-30T09:34:21.292 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:34:21.494 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:21.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:21.604 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:21.633 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:21.727 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:22.055 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:22.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:22.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:22.162 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:22.168 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-30T09:34:22.254 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:22.382 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:22.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:22.500 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:22.539 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:22.599 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:22.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:22.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:22.706 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:22.732 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:22.774 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:22.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:22.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:22.787 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:34:22.836 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:34:22.877 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:22.886 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:34:22.907 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:22 smithi167 ceph-mon[192880]: pgmap v774: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 39608/269412 objects degraded (14.702%); 47666/269412 objects misplaced (17.693%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:34:22.911 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:22.945 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:34:23.008 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:34:23.040 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:34:23.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:22 smithi019 ceph-mon[216815]: pgmap v774: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 39608/269412 objects degraded (14.702%); 47666/269412 objects misplaced (17.693%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:34:23.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:22 smithi019 ceph-mon[224043]: pgmap v774: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 39608/269412 objects degraded (14.702%); 47666/269412 objects misplaced (17.693%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:34:23.090 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:34:23.091 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:23.129 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:34:23.139 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:23.140 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:34:23.140 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:34:23.140 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:34:23.140 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:34:23.140 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:34:23.140 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:34:23.140 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:34:23.140 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:34:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:23.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:23.242 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:23.277 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:23.368 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:23.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:23.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:23.472 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:23.530 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:23.603 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:23.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:23.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:23.706 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:34:23.706 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:34:23.710 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:23.728 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1060039156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:23.728 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:23 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:23.728 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:23 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:23.728 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 39608/269412 objects degraded (14.702%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:23.755 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:23.918 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:34:23.942 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:34:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:34:23] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-09-30T09:34:23.943 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1060039156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:23.943 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:23 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:23.943 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:23 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:23.943 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 39608/269412 objects degraded (14.702%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:23.945 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1060039156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:23.945 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:23 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:23.945 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:23 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:23.945 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 39608/269412 objects degraded (14.702%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:24.406 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:24.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:24.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:24.523 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:24.558 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:24.669 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:24.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:24.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:24.692 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:24 smithi019 ceph-mon[224043]: pgmap v775: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 39608/269412 objects degraded (14.702%); 47666/269412 objects misplaced (17.693%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:34:24.692 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:34:24.692 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:34:24.692 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:24.694 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:24 smithi019 ceph-mon[216815]: pgmap v775: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 39608/269412 objects degraded (14.702%); 47666/269412 objects misplaced (17.693%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:34:24.694 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:34:24.694 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:34:24.694 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:24.730 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:34:24.774 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:24.808 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:24 smithi167 ceph-mon[192880]: pgmap v775: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 39608/269412 objects degraded (14.702%); 47666/269412 objects misplaced (17.693%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:34:24.808 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:34:24.809 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:34:24.809 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:24.811 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:24.893 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:24.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:24.995 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:25.034 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:25.099 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:25.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:25.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:25.206 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:25.217 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:25.301 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:25.385 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:25.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:25.487 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:25.515 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:25.568 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:25.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:25.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:25.684 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:25.774 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:26.019 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:26.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:26.029 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:26.130 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:26.159 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:26.206 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:26.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:26.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:26.308 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:26.347 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:26.559 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:26.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:26.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:26.663 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:26.730 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:34:26.782 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:26.886 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:26 smithi167 ceph-mon[192880]: pgmap v776: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 39176/262467 objects degraded (14.926%); 46376/262467 objects misplaced (17.669%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:34:27.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:26 smithi019 ceph-mon[216815]: pgmap v776: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 39176/262467 objects degraded (14.926%); 46376/262467 objects misplaced (17.669%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:34:27.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:26 smithi019 ceph-mon[224043]: pgmap v776: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 39176/262467 objects degraded (14.926%); 46376/262467 objects misplaced (17.669%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:34:27.117 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:34:27.117 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:34:27.501 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:27.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:27.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:27.610 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:27.643 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:27.687 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:27.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:27.770 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1514001586' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:34:27.771 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1514001586' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:34:27.790 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:27.809 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.3 to 1 (10000) 2023-09-30T09:34:27.827 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1514001586' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:34:28.037 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:28.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:28.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:28.130 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:34:28.141 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:28.225 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:28.318 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:28.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:28.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:28.425 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:28.462 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:28.554 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:28.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:28.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:28.663 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:28.663 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:28.697 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:28 smithi167 ceph-mon[192880]: pgmap v777: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 39176/262467 objects degraded (14.926%); 46376/262467 objects misplaced (17.669%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:34:28.697 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1514001586' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:34:28.697 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:28 smithi167 ceph-mon[192880]: osdmap e313: 8 total, 8 up, 7 in 2023-09-30T09:34:28.697 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:28 smithi167 ceph-mon[192880]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.697 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:28 smithi167 ceph-mon[192880]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.697 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:28 smithi167 ceph-mon[192880]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.697 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 39176/262467 objects degraded (14.926%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:28.709 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:28.810 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:28.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:28.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:28.913 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:28.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[216815]: pgmap v777: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 39176/262467 objects degraded (14.926%); 46376/262467 objects misplaced (17.669%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:34:28.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1514001586' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:34:28.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[216815]: osdmap e313: 8 total, 8 up, 7 in 2023-09-30T09:34:28.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[216815]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[216815]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[216815]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 39176/262467 objects degraded (14.926%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:28.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[224043]: pgmap v777: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 39176/262467 objects degraded (14.926%); 46376/262467 objects misplaced (17.669%); 4.5 MiB/s, 72 objects/s recovering 2023-09-30T09:34:28.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1514001586' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:34:28.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[224043]: osdmap e313: 8 total, 8 up, 7 in 2023-09-30T09:34:28.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[224043]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[224043]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[224043]: failed to encode map e313 with expected crc 2023-09-30T09:34:28.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 39176/262467 objects degraded (14.926%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:29.010 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-30T09:34:29.038 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:29.271 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:29.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:29.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:29.296 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:29.381 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:29.458 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:29.721 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:29.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:29.830 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:29.873 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:30.186 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:30.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:30.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:30.294 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:30.326 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:30.410 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:30.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:30.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:30.521 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:30.558 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:30.634 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:30.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:30.739 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:30.744 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:34:30.793 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:30.860 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:30.862 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:30 smithi167 ceph-mon[192880]: pgmap v779: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 38726/255513 objects degraded (15.156%); 44995/255513 objects misplaced (17.610%); 4.7 MiB/s, 75 objects/s recovering 2023-09-30T09:34:30.862 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3658478151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:34:30.863 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:34:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:30.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:30.871 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:34:30.871 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:34:30.964 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:31.071 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:30 smithi019 ceph-mon[216815]: pgmap v779: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 38726/255513 objects degraded (15.156%); 44995/255513 objects misplaced (17.610%); 4.7 MiB/s, 75 objects/s recovering 2023-09-30T09:34:31.071 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3658478151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:34:31.072 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:34:31.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:30 smithi019 ceph-mon[224043]: pgmap v779: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 38726/255513 objects degraded (15.156%); 44995/255513 objects misplaced (17.610%); 4.7 MiB/s, 75 objects/s recovering 2023-09-30T09:34:31.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3658478151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:34:31.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:34:31.083 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:31.890 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:34:32.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:32.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:34:32.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[224043]: osdmap e314: 8 total, 8 up, 7 in 2023-09-30T09:34:32.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[224043]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[224043]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[224043]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.008 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:32.008 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:34:32.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[216815]: osdmap e314: 8 total, 8 up, 7 in 2023-09-30T09:34:32.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[216815]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[216815]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:31 smithi019 ceph-mon[216815]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:31 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:34:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:31 smithi167 ceph-mon[192880]: osdmap e314: 8 total, 8 up, 7 in 2023-09-30T09:34:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:31 smithi167 ceph-mon[192880]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:31 smithi167 ceph-mon[192880]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:31 smithi167 ceph-mon[192880]: failed to encode map e314 with expected crc 2023-09-30T09:34:32.080 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:32.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:32.154 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:32.190 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:32.233 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:32.305 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:32.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:32.400 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:34:32.411 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:32.509 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:32.759 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:32.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:32.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:32.889 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:32.920 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:32 smithi167 ceph-mon[192880]: pgmap v781: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 38726/255009 objects degraded (15.186%); 44995/255009 objects misplaced (17.644%); 5.7 MiB/s, 91 objects/s recovering 2023-09-30T09:34:32.924 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:32.940 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:34:33.010 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:34:33.026 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:33.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:33.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:33.068 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:34:33.080 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:32 smithi019 ceph-mon[224043]: pgmap v781: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 38726/255009 objects degraded (15.186%); 44995/255009 objects misplaced (17.644%); 5.7 MiB/s, 91 objects/s recovering 2023-09-30T09:34:33.083 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:32 smithi019 ceph-mon[216815]: pgmap v781: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 38726/255009 objects degraded (15.186%); 44995/255009 objects misplaced (17.644%); 5.7 MiB/s, 91 objects/s recovering 2023-09-30T09:34:33.130 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:34:33.134 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:33.162 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:33.188 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:34:33.218 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:33.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:33.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:33.226 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:34:33.275 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:34:33.315 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:34:33.325 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:33.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:34:33.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:34:33.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:34:33.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:34:33.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:34:33.356 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:34:33.356 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:34:33.356 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:34:33.407 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:33.463 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:33.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:33.567 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:33.596 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:33.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:33 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1023990205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:33.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:33.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 38726/255009 objects degraded (15.186%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:33 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1023990205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 38726/255009 objects degraded (15.186%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:34.075 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:34:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:34:33] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-09-30T09:34:34.075 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:33 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1023990205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:34.075 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:34.075 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 38726/255009 objects degraded (15.186%), 17 pgs degraded (PG_DEGRADED) 2023-09-30T09:34:34.101 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:34.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:34.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:34.196 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:34:34.210 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:34.360 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:34.524 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:34:34.524 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:34:34.524 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35928 6.9 6:9537c3ec:::benchmark_data_smithi019_355417_object35927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.258963+0000", 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 355.26131761599999, 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0146549929999999, 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.525 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35928 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.258963+0000", 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.258963+0000", 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.258961+0000", 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259005+0000", 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3911e-05 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259021+0000", 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6305000000000001e-05 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259029+0000", 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2870000000000004e-06 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.303449+0000", 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.044419783999999997 2023-09-30T09:34:34.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.303461+0000", 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1722999999999999e-05 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.273536+0000", 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97007547500000002 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.529 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.273618+0000", 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2009999999999996e-05 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35938 6.3 6:c4d9c85e:::benchmark_data_smithi019_355417_object35937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.259041+0000", 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 355.261240246, 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0085380530000001, 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.530 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35938 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259041+0000", 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.531 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259041+0000", 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259040+0000", 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259081+0000", 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1119999999999999e-05 2023-09-30T09:34:34.532 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259088+0000", 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8809999999999998e-06 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259093+0000", 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4730000000000002e-06 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.533 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.304743+0000", 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.045650759999999999 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.304756+0000", 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2493999999999999e-05 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.267492+0000", 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96273649500000003 2023-09-30T09:34:34.534 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.267579+0000", 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6328000000000004e-05 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35943 6.5 6:a25db399:::benchmark_data_smithi019_355417_object35942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.259098+0000", 2023-09-30T09:34:34.535 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 355.26118292500001, 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.026377788, 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35943 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259098+0000", 2023-09-30T09:34:34.536 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259098+0000", 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259098+0000", 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.537 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259137+0000", 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9613999999999997e-05 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259143+0000", 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1229999999999997e-06 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.259147+0000", 2023-09-30T09:34:34.538 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.602e-06 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.301689+0000", 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.042542569000000002 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.301699+0000", 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9729999999999992e-06 2023-09-30T09:34:34.539 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.277352+0000", 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97565302099999995 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.278033+0000", 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00068037499999999999 2023-09-30T09:34:34.540 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.285404+0000", 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0073710829999999996 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.285476+0000", 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1805999999999997e-05 2023-09-30T09:34:34.541 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.542 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.542 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.542 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.542 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.542 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35946 6.19 6:9c9e6053:::benchmark_data_smithi019_355417_object35945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:34:34.542 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.260313+0000", 2023-09-30T09:34:34.543 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 355.25996793000002, 2023-09-30T09:34:34.543 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.031708208, 2023-09-30T09:34:34.543 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.543 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.543 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35946 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260313+0000", 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.544 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260313+0000", 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260310+0000", 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999971 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260393+0000", 2023-09-30T09:34:34.545 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3176999999999993e-05 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260415+0000", 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1444999999999999e-05 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260423+0000", 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4530000000000002e-06 2023-09-30T09:34:34.546 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.295680+0000", 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.035256919999999997 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.295687+0000", 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5200000000000003e-06 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.286058+0000", 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99037160300000004 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.286180+0000", 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000122154 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.286342+0000", 2023-09-30T09:34:34.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016122 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.289202+0000", 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028603779999999998 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.289609+0000", 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00040729999999999998 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.291947+0000", 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002337979 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.291981+0000", 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4224999999999998e-05 2023-09-30T09:34:34.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.292021+0000", 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9678000000000002e-05 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:35947 6.18 6:1d4ebeb7:::benchmark_data_smithi019_355417_object35946:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:28:39.260434+0000", 2023-09-30T09:34:34.551 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 355.25984727100001, 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.036275292, 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 35947 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.552 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260434+0000", 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260434+0000", 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260433+0000", 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.553 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260466+0000", 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2790999999999998e-05 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260476+0000", 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0001e-05 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.554 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.260481+0000", 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7570000000000004e-06 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.310702+0000", 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.050221644000000003 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:39.310710+0000", 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5689999999999999e-06 2023-09-30T09:34:34.555 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.289877+0000", 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97916740800000002 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.289987+0000", 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0001093 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.290120+0000", 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000133151 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.293018+0000", 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028981200000000001 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.293401+0000", 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000383376 2023-09-30T09:34:34.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.296635+0000", 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032337440000000002 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.296671+0000", 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6304999999999999e-05 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:28:40.296709+0000", 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7605000000000003e-05 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57773 6.12 6:4838cbd4:::benchmark_data_smithi019_355417_object57772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.504618+0000", 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 332.01566321299998, 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99969614299999998, 2023-09-30T09:34:34.559 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57773 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504618+0000", 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.560 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504618+0000", 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504617+0000", 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.561 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504670+0000", 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2459e-05 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504681+0000", 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1459e-05 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504686+0000", 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8239999999999999e-06 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504738+0000", 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1758999999999997e-05 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.504752+0000", 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3971000000000001e-05 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.501144+0000", 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99639237899999999 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.501265+0000", 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012081199999999999 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.501368+0000", 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103235 2023-09-30T09:34:34.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504170+0000", 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028016859999999998 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504225+0000", 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5120000000000001e-05 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.565 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.566 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504249+0000", 2023-09-30T09:34:34.566 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4437e-05 2023-09-30T09:34:34.566 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.566 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.566 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.566 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504278+0000", 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9125999999999999e-05 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504314+0000", 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5395000000000002e-05 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.567 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57838 6.12 6:48b85783:::benchmark_data_smithi019_355417_object57837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.559281+0000", 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 331.96100010399999, 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94707381400000001, 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.568 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57838 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559281+0000", 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559281+0000", 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.569 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559279+0000", 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559386+0000", 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010671999999999999 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559405+0000", 2023-09-30T09:34:34.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9114999999999999e-05 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559412+0000", 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6200000000000001e-06 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559471+0000", 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.923e-05 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.571 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.559492+0000", 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0428000000000002e-05 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.502496+0000", 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94300395100000001 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.502618+0000", 2023-09-30T09:34:34.572 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012216399999999999 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.502768+0000", 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000149837 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504395+0000", 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016269979999999999 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.573 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504437+0000", 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2043999999999999e-05 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506274+0000", 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018373 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.574 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506318+0000", 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3639e-05 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506355+0000", 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7048000000000002e-05 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57893 6.12 6:49e4acbe:::benchmark_data_smithi019_355417_object57892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:34:34.575 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.598530+0000", 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 331.92175128700001, 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.910345187, 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57893 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.576 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.577 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.577 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598530+0000", 2023-09-30T09:34:34.577 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.577 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.577 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.577 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.577 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598530+0000", 2023-09-30T09:34:34.577 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598528+0000", 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598584+0000", 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6011999999999998e-05 2023-09-30T09:34:34.578 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598597+0000", 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2286999999999999e-05 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598610+0000", 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3129000000000001e-05 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.579 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598681+0000", 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1112000000000006e-05 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.598690+0000", 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1419999999999999e-06 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.580 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.503408+0000", 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90471810900000005 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.503531+0000", 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012254900000000001 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.503661+0000", 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000130588 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.505271+0000", 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001609826 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506227+0000", 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00095588200000000002 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.582 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508808+0000", 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002581341 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508842+0000", 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4025e-05 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508875+0000", 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2490000000000002e-05 2023-09-30T09:34:34.583 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57910 6.12 6:4bac1af3:::benchmark_data_smithi019_355417_object57909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.607418+0000", 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 331.91286333699998, 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90162827800000001, 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.584 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57910 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607418+0000", 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607418+0000", 2023-09-30T09:34:34.585 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607416+0000", 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607471+0000", 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4985999999999998e-05 2023-09-30T09:34:34.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607497+0000", 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5318000000000001e-05 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607503+0000", 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6290000000000004e-06 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607578+0000", 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4721999999999994e-05 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607590+0000", 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2534000000000001e-05 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504453+0000", 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.89686253000000005 2023-09-30T09:34:34.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504539+0000", 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6562000000000002e-05 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.504618+0000", 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8565000000000004e-05 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508765+0000", 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041465950000000003 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508774+0000", 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3190000000000001e-06 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508967+0000", 2023-09-30T09:34:34.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019281699999999999 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509012+0000", 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5592e-05 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509046+0000", 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3429000000000001e-05 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.591 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57916 6.12 6:4bf1e539:::benchmark_data_smithi019_355417_object57915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.607757+0000", 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 331.91252436799999, 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90142910499999995, 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.592 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57916 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607757+0000", 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607757+0000", 2023-09-30T09:34:34.593 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607756+0000", 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607781+0000", 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4536999999999999e-05 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607787+0000", 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2450000000000003e-06 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607796+0000", 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8400000000000001e-06 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607846+0000", 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0528000000000001e-05 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.607855+0000", 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3960000000000006e-06 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.505333+0000", 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.89747830500000003 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.505440+0000", 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010746799999999999 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.505560+0000", 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119641 2023-09-30T09:34:34.597 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508780+0000", 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032199139999999999 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508911+0000", 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013101899999999999 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.598 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509099+0000", 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018847800000000001 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509150+0000", 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0387999999999999e-05 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509186+0000", 2023-09-30T09:34:34.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5772000000000002e-05 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:57940 6.12 6:4e8889e6:::benchmark_data_smithi019_355417_object57939:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:02.624369+0000", 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 331.89591209399998, 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.88492660599999995, 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.600 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 57940 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624369+0000", 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624369+0000", 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624368+0000", 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624433+0000", 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4386000000000003e-05 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.602 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624442+0000", 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3090000000000001e-06 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624449+0000", 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5230000000000001e-06 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624513+0000", 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4858999999999997e-05 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:02.624523+0000", 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6500000000000008e-06 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506403+0000", 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.881879682 2023-09-30T09:34:34.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506503+0000", 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100552 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.506628+0000", 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012425000000000001 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.508922+0000", 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002293986 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509226+0000", 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00030478500000000002 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509248+0000", 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2130999999999999e-05 2023-09-30T09:34:34.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509274+0000", 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5259999999999999e-05 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:03.509295+0000", 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1688999999999999e-05 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.607 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.608 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.608 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.608 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65378 6.3 6:c5bab36e:::benchmark_data_smithi019_355417_object65377:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-09-30T09:34:34.608 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.243566+0000", 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 315.27671503699997, 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0151177039999999, 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65378 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243566+0000", 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243566+0000", 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243565+0000", 2023-09-30T09:34:34.610 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243608+0000", 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3086e-05 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243616+0000", 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6139999999999992e-06 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.243622+0000", 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4790000000000004e-06 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.248062+0000", 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044400819999999997 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.248070+0000", 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4710000000000007e-06 2023-09-30T09:34:34.612 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.255286+0000", 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0072160830000001 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.255416+0000", 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012989100000000001 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.613 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.255522+0000", 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105604 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258502+0000", 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029799230000000002 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258529+0000", 2023-09-30T09:34:34.614 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7623e-05 2023-09-30T09:34:34.615 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.615 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.615 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.615 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258627+0000", 2023-09-30T09:34:34.617 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7775000000000005e-05 2023-09-30T09:34:34.617 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.617 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.617 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.617 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258662+0000", 2023-09-30T09:34:34.617 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4323999999999999e-05 2023-09-30T09:34:34.617 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258684+0000", 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2010000000000001e-05 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65406 6.3 6:c0ab1d0f:::benchmark_data_smithi019_355417_object65405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:34:34.618 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.269356+0000", 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 315.25092493599999, 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99139678799999997, 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65406 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.619 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269356+0000", 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269356+0000", 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269355+0000", 2023-09-30T09:34:34.620 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269455+0000", 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9308999999999994e-05 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269464+0000", 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0289999999999994e-06 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.621 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269469+0000", 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6489999999999996e-06 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269529+0000", 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9685000000000002e-05 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.622 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.269575+0000", 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6065e-05 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.256853+0000", 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98727761999999997 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.256944+0000", 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1009999999999998e-05 2023-09-30T09:34:34.623 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.257064+0000", 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000120371 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258738+0000", 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001674009 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.624 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258745+0000", 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1600000000000001e-06 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260668+0000", 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019223910000000001 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260716+0000", 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8344000000000003e-05 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.625 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260753+0000", 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6743999999999999e-05 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65419 6.3 6:c13ce430:::benchmark_data_smithi019_355417_object65418:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.282941+0000", 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 315.23733956799998, 2023-09-30T09:34:34.626 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97912809700000003, 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65419 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282941+0000", 2023-09-30T09:34:34.627 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282941+0000", 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282941+0000", 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.628 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282987+0000", 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6019000000000001e-05 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282993+0000", 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4400000000000002e-06 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.282997+0000", 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1010000000000002e-06 2023-09-30T09:34:34.629 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.283149+0000", 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015114800000000001 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.283160+0000", 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1564e-05 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.630 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.257763+0000", 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97460299800000005 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.257886+0000", 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012238 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.257978+0000", 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2826999999999997e-05 2023-09-30T09:34:34.631 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260561+0000", 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0025825790000000002 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260618+0000", 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6934000000000001e-05 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.632 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.261983+0000", 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013652079999999999 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262033+0000", 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9947000000000001e-05 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262069+0000", 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6355e-05 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.633 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65425 6.3 6:c0030bff:::benchmark_data_smithi019_355417_object65424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.289950+0000", 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 315.23033050100003, 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97225219799999996, 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.634 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65425 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.289950+0000", 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.289950+0000", 2023-09-30T09:34:34.635 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.289950+0000", 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290013+0000", 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3658999999999995e-05 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.636 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290024+0000", 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0733999999999999e-05 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290030+0000", 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6799999999999998e-06 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.637 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290088+0000", 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8155999999999997e-05 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.290097+0000", 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3419999999999997e-06 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258766+0000", 2023-09-30T09:34:34.638 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96866923900000002 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258867+0000", 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100614 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.258954+0000", 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7496000000000004e-05 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.639 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260766+0000", 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001811797 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260815+0000", 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8296e-05 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.640 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262121+0000", 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013066950000000001 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262168+0000", 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6592999999999999e-05 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.641 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262203+0000", 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4730999999999997e-05 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65449 6.3 6:c7b6375c:::benchmark_data_smithi019_355417_object65448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.309189+0000", 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 315.21109193000001, 2023-09-30T09:34:34.642 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95312158499999999, 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65449 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309189+0000", 2023-09-30T09:34:34.643 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309189+0000", 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309189+0000", 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.644 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309229+0000", 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0099999999999999e-05 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309237+0000", 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3459999999999994e-06 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.645 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309241+0000", 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2950000000000004e-06 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309330+0000", 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9204999999999997e-05 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.309340+0000", 2023-09-30T09:34:34.646 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3700000000000001e-06 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.259580+0000", 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95023972499999998 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.259678+0000", 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8717999999999999e-05 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.647 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.259773+0000", 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5106999999999996e-05 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260874+0000", 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001101131 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260883+0000", 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0700000000000007e-06 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262260+0000", 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001377524 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262289+0000", 2023-09-30T09:34:34.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9278999999999999e-05 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.262311+0000", 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1158e-05 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:65485 6.3 6:c28c44bf:::benchmark_data_smithi019_355417_object65484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-30T09:34:34.650 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:19.342183+0000", 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 315.17809786399999, 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92270854099999999, 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 65485 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.651 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342183+0000", 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342183+0000", 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342182+0000", 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:34.652 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342235+0000", 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3674999999999997e-05 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342244+0000", 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3040000000000005e-06 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.653 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342248+0000", 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4320000000000001e-06 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342306+0000", 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7339000000000001e-05 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.654 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:19.342317+0000", 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1049999999999999e-05 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.260906+0000", 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.91858987800000003 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.655 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.261032+0000", 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000125473 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.261115+0000", 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3549999999999998e-05 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264766+0000", 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036509120000000001 2023-09-30T09:34:34.656 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264793+0000", 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6672000000000001e-05 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264829+0000", 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6066999999999999e-05 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.657 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264866+0000", 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6995000000000001e-05 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:20.264892+0000", 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5449999999999999e-05 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55766 7.5 7:aa9fdb5d:::benchmark_data_smithi019_404601_object55765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 119.03653317, 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0102513550000001, 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55766 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.659 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483746+0000", 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483780+0000", 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4174999999999997e-05 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483797+0000", 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6626999999999999e-05 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483805+0000", 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3229999999999998e-06 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483862+0000", 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6236000000000002e-05 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.662 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.484005+0000", 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014357899999999999 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.484145+0000", 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013967499999999999 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491724+0000", 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075794119999999998 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.663 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.509743+0000", 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.018018790999999999 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.509753+0000", 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7790000000000007e-06 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.664 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493928+0000", 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98417540599999997 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493999+0000", 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0858999999999994e-05 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.665 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55755 7.9 7:9e422c00:::benchmark_data_smithi019_404601_object55754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 119.033788069, 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0082494440000001, 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55755 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.666 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486491+0000", 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486532+0000", 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0222000000000002e-05 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486545+0000", 2023-09-30T09:34:34.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3288999999999999e-05 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486552+0000", 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6000000000000001e-06 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486609+0000", 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6268999999999999e-05 2023-09-30T09:34:34.669 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486616+0000", 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0260000000000002e-06 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.495776+0000", 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091603880000000002 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.670 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.495784+0000", 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3149999999999992e-06 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493615+0000", 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99783076999999998 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.671 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.494742+0000", 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001126993 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55773 7.9 7:9be0e4ac:::benchmark_data_smithi019_404601_object55772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 119.03303430699999, 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0225480730000001, 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:34:34.672 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55773 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487246+0000", 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:34:34.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487313+0000", 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7360999999999994e-05 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487328+0000", 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5197999999999999e-05 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487335+0000", 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.596e-06 2023-09-30T09:34:34.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491649+0000", 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043143920000000002 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491656+0000", 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.092e-06 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.496453+0000", 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047967569999999996 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.496469+0000", 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5926e-05 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.494766+0000", 2023-09-30T09:34:34.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99829705800000001 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.495026+0000", 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00026024099999999998 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 6", 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.495185+0000", 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015893299999999999 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.500813+0000", 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0056275379999999996 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509659+0000", 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088464240000000003 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:34:34.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509708+0000", 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8619000000000003e-05 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509795+0000", 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6776999999999999e-05 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:34:34.680 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:34:34.842 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:34.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:34.959 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:34.991 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:34 smithi167 ceph-mon[192880]: pgmap v782: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 38726/255009 objects degraded (15.186%); 44995/255009 objects misplaced (17.644%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:34:34.994 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:35.059 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:35.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:35.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:35.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:34 smithi019 ceph-mon[224043]: pgmap v782: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 38726/255009 objects degraded (15.186%); 44995/255009 objects misplaced (17.644%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:34:35.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:34 smithi019 ceph-mon[216815]: pgmap v782: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 38726/255009 objects degraded (15.186%); 44995/255009 objects misplaced (17.644%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:34:35.163 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:35.238 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:35.287 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:35.520 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:35.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:35.529 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:34:35.626 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:35.842 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:36.161 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:36.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:36.271 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:36.309 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:36.367 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:36.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:36.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:36.473 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:36.535 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:36.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:36 smithi019 ceph-mon[216815]: pgmap v783: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 38237/247776 objects degraded (15.432%); 43630/247776 objects misplaced (17.609%); 5.9 MiB/s, 94 objects/s recovering 2023-09-30T09:34:36.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:36 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3601003194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:34:36.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:36 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:34:36.846 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.4 to 1 (10000) 2023-09-30T09:34:36.952 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:34:37.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:36 smithi019 ceph-mon[224043]: pgmap v783: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 38237/247776 objects degraded (15.432%); 43630/247776 objects misplaced (17.609%); 5.9 MiB/s, 94 objects/s recovering 2023-09-30T09:34:37.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:36 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3601003194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:34:37.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:36 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:34:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:36 smithi167 ceph-mon[192880]: pgmap v783: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 38237/247776 objects degraded (15.432%); 43630/247776 objects misplaced (17.609%); 5.9 MiB/s, 94 objects/s recovering 2023-09-30T09:34:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:36 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3601003194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:34:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:36 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:34:37.305 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:37.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:37.409 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:37.436 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:38.040 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-30T09:34:38.040 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:38.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:38.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:34:38.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: osdmap e315: 8 total, 8 up, 7 in 2023-09-30T09:34:38.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[224043]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.181 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:38.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:34:38.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: osdmap e315: 8 total, 8 up, 7 in 2023-09-30T09:34:38.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:37 smithi019 ceph-mon[216815]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.190 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:38.213 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:34:38.213 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: osdmap e315: 8 total, 8 up, 7 in 2023-09-30T09:34:38.213 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.213 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.213 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.213 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.213 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:37 smithi167 ceph-mon[192880]: failed to encode map e315 with expected crc 2023-09-30T09:34:38.215 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:38.266 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:38.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:38.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:38.371 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:38.398 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:38.406 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:38.444 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:38.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:38.550 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:38.584 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:38.687 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:38.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:38.791 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:38.820 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:38.850 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:34:38.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:38.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:38.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:38.985 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:39.013 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:39.059 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:39.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: pgmap v785: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 38237/247776 objects degraded (15.432%); 43630/247776 objects misplaced (17.609%); 3.1 MiB/s, 48 objects/s recovering 2023-09-30T09:34:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: osdmap e316: 8 total, 8 up, 7 in 2023-09-30T09:34:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: 7.9 starting backfill to osd.4 from (0'0,0'0] MAX to 314'8943 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 38237/247776 objects degraded (15.432%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1864074301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: pgmap v785: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 38237/247776 objects degraded (15.432%); 43630/247776 objects misplaced (17.609%); 3.1 MiB/s, 48 objects/s recovering 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: osdmap e316: 8 total, 8 up, 7 in 2023-09-30T09:34:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: 7.9 starting backfill to osd.4 from (0'0,0'0] MAX to 314'8943 2023-09-30T09:34:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 38237/247776 objects degraded (15.432%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1864074301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:38 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:39.169 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:39.202 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: pgmap v785: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 38237/247776 objects degraded (15.432%); 43630/247776 objects misplaced (17.609%); 3.1 MiB/s, 48 objects/s recovering 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: osdmap e316: 8 total, 8 up, 7 in 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: failed to encode map e316 with expected crc 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: 7.9 starting backfill to osd.4 from (0'0,0'0] MAX to 314'8943 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 38237/247776 objects degraded (15.432%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1864074301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:39.203 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:38 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:34:39.207 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:39.301 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:39.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:39.405 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:39.441 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:39.518 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:39.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:39.625 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:39.657 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:39.731 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:39.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: osdmap e317: 8 total, 8 up, 7 in 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: failed to encode map e317 with expected crc 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: failed to encode map e317 with expected crc 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: failed to encode map e317 with expected crc 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: failed to encode map e317 with expected crc 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: 7.1 starting backfill to osd.4 from (0'0,0'0] MAX to 314'9010 2023-09-30T09:34:39.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: failed to encode map e317 with expected crc 2023-09-30T09:34:39.839 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: 7.4 starting backfill to osd.4 from (0'0,0'0] MAX to 314'9137 2023-09-30T09:34:39.839 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: failed to encode map e317 with expected crc 2023-09-30T09:34:39.839 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:39 smithi167 ceph-mon[192880]: failed to encode map e317 with expected crc 2023-09-30T09:34:39.839 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:39.890 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:39.963 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:39.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:40.077 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:40.115 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:40.173 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:40.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:40.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:40.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:34:40.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:40.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: osdmap e317: 8 total, 8 up, 7 in 2023-09-30T09:34:40.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: 7.1 starting backfill to osd.4 from (0'0,0'0] MAX to 314'9010 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: 7.4 starting backfill to osd.4 from (0'0,0'0] MAX to 314'9137 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[216815]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:34:40.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: osdmap e317: 8 total, 8 up, 7 in 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: 7.1 starting backfill to osd.4 from (0'0,0'0] MAX to 314'9010 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: 7.4 starting backfill to osd.4 from (0'0,0'0] MAX to 314'9137 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:39 smithi019 ceph-mon[224043]: failed to encode map e317 with expected crc 2023-09-30T09:34:40.280 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:40.322 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:40.384 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:40.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:40.490 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:40.608 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:40.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: pgmap v788: 119 pgs: 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 37533/243585 objects degraded (15.409%); 51901/243585 objects misplaced (21.307%); 11 MiB/s, 175 objects/s recovering 2023-09-30T09:34:40.870 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:34:40.871 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: osdmap e318: 8 total, 8 up, 7 in 2023-09-30T09:34:40.871 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: failed to encode map e318 with expected crc 2023-09-30T09:34:40.871 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: failed to encode map e318 with expected crc 2023-09-30T09:34:40.871 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: failed to encode map e318 with expected crc 2023-09-30T09:34:40.871 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: failed to encode map e318 with expected crc 2023-09-30T09:34:40.871 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: failed to encode map e318 with expected crc 2023-09-30T09:34:40.871 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.151 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:41.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: pgmap v788: 119 pgs: 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 37533/243585 objects degraded (15.409%); 51901/243585 objects misplaced (21.307%); 11 MiB/s, 175 objects/s recovering 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: osdmap e318: 8 total, 8 up, 7 in 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[224043]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:41.156 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:40 smithi019 ceph-mon[216815]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.257 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:41.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: pgmap v788: 119 pgs: 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 37533/243585 objects degraded (15.409%); 51901/243585 objects misplaced (21.307%); 11 MiB/s, 175 objects/s recovering 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: osdmap e318: 8 total, 8 up, 7 in 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:40 smithi167 ceph-mon[192880]: failed to encode map e318 with expected crc 2023-09-30T09:34:41.284 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:41.354 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:41.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:41.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:41.464 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:41.508 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:41.570 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:41.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:41.678 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:41.762 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:41.894 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[224043]: osdmap e319: 8 total, 8 up, 7 in 2023-09-30T09:34:41.894 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[224043]: failed to encode map e319 with expected crc 2023-09-30T09:34:41.894 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[224043]: failed to encode map e319 with expected crc 2023-09-30T09:34:41.894 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[224043]: failed to encode map e319 with expected crc 2023-09-30T09:34:41.894 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[224043]: failed to encode map e319 with expected crc 2023-09-30T09:34:41.894 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[224043]: failed to encode map e319 with expected crc 2023-09-30T09:34:41.895 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[224043]: failed to encode map e319 with expected crc 2023-09-30T09:34:41.966 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:41.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:41.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:41.971 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:42.072 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:42.119 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:34:42.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[216815]: osdmap e319: 8 total, 8 up, 7 in 2023-09-30T09:34:42.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[216815]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.148 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[216815]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.148 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[216815]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.148 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[216815]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.148 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[216815]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.148 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:41 smithi019 ceph-mon[216815]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.170 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:42.193 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:34:42.244 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:34:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:41 smithi167 ceph-mon[192880]: osdmap e319: 8 total, 8 up, 7 in 2023-09-30T09:34:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:41 smithi167 ceph-mon[192880]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:41 smithi167 ceph-mon[192880]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:41 smithi167 ceph-mon[192880]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:41 smithi167 ceph-mon[192880]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:41 smithi167 ceph-mon[192880]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:41 smithi167 ceph-mon[192880]: failed to encode map e319 with expected crc 2023-09-30T09:34:42.293 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:34:42.344 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:34:42.390 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:34:42.436 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:34:42.486 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:34:42.489 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:42.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:42.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:42.514 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:34:42.514 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:34:42.514 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:34:42.514 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:34:42.514 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:34:42.515 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:34:42.515 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:34:42.515 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:34:42.526 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:34:42.526 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:34:42.608 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:42.644 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:42.703 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:42.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:42.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:42.810 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:42.890 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:42.904 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:42 smithi167 ceph-mon[192880]: pgmap v791: 119 pgs: 1 peering, 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 94 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 37533/243336 objects degraded (15.424%); 51901/243336 objects misplaced (21.329%); 10 MiB/s, 165 objects/s recovering 2023-09-30T09:34:42.904 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4012805595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:42 smithi019 ceph-mon[216815]: pgmap v791: 119 pgs: 1 peering, 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 94 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 37533/243336 objects degraded (15.424%); 51901/243336 objects misplaced (21.329%); 10 MiB/s, 165 objects/s recovering 2023-09-30T09:34:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4012805595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:42 smithi019 ceph-mon[224043]: pgmap v791: 119 pgs: 1 peering, 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 94 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 37533/243336 objects degraded (15.424%); 51901/243336 objects misplaced (21.329%); 10 MiB/s, 165 objects/s recovering 2023-09-30T09:34:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4012805595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:43.600 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:43.605 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:34:43.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:43.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:43.717 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:43.793 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:43.840 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:34:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:34:43] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-09-30T09:34:44.231 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:44.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:44.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:44.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 37533/243336 objects degraded (15.424%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:44.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:43 smithi019 ceph-mon[224043]: pgmap v792: 119 pgs: 1 peering, 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 94 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 37533/243336 objects degraded (15.424%); 51901/243336 objects misplaced (21.329%); 7.4 MiB/s, 118 objects/s recovering 2023-09-30T09:34:44.241 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 37533/243336 objects degraded (15.424%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:44.241 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:43 smithi019 ceph-mon[216815]: pgmap v792: 119 pgs: 1 peering, 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 94 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 37533/243336 objects degraded (15.424%); 51901/243336 objects misplaced (21.329%); 7.4 MiB/s, 118 objects/s recovering 2023-09-30T09:34:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 37533/243336 objects degraded (15.424%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:43 smithi167 ceph-mon[192880]: pgmap v792: 119 pgs: 1 peering, 2 activating+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 94 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 37533/243336 objects degraded (15.424%); 51901/243336 objects misplaced (21.329%); 7.4 MiB/s, 118 objects/s recovering 2023-09-30T09:34:44.341 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:44.367 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:44.410 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:44.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:44.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:44.517 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:44.558 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:44.614 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:44.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:44.721 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:44.748 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:44.797 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:44.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:44.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:44.905 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:44.967 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:45.177 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:45.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:45.235 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:34:45.294 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:45.426 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:45.547 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:45.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:45.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:45.654 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:45.743 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:45.783 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:45.892 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:45.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:46.012 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:46.056 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:46.118 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:46.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:46.223 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:46.257 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:46.314 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:46.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:46.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:46.422 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:46.468 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:46.517 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:46 smithi019 ceph-mon[216815]: pgmap v793: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 36966/235494 objects degraded (15.697%); 50102/235494 objects misplaced (21.275%); 11 MiB/s, 171 objects/s recovering 2023-09-30T09:34:46.517 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2882684121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:34:46.524 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.5 to 1 (10000) 2023-09-30T09:34:46.526 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:46.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:46.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:46.596 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:46.636 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:46.678 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:46 smithi167 ceph-mon[192880]: pgmap v793: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 36966/235494 objects degraded (15.697%); 50102/235494 objects misplaced (21.275%); 11 MiB/s, 171 objects/s recovering 2023-09-30T09:34:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2882684121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:34:46.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:46 smithi019 ceph-mon[224043]: pgmap v793: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 36966/235494 objects degraded (15.697%); 50102/235494 objects misplaced (21.275%); 11 MiB/s, 171 objects/s recovering 2023-09-30T09:34:46.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2882684121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:34:46.830 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:46.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:46.934 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:46.960 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:47.005 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:47.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:47.110 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:47.138 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:47.173 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2882684121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: osdmap e320: 8 total, 8 up, 7 in 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:34:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[224043]: osdmap e321: 8 total, 8 up, 7 in 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2882684121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: osdmap e320: 8 total, 8 up, 7 in 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: failed to encode map e320 with expected crc 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:34:47.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:47 smithi019 ceph-mon[216815]: osdmap e321: 8 total, 8 up, 7 in 2023-09-30T09:34:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2882684121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:34:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: osdmap e320: 8 total, 8 up, 7 in 2023-09-30T09:34:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: failed to encode map e320 with expected crc 2023-09-30T09:34:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: failed to encode map e320 with expected crc 2023-09-30T09:34:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: failed to encode map e320 with expected crc 2023-09-30T09:34:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: failed to encode map e320 with expected crc 2023-09-30T09:34:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: failed to encode map e320 with expected crc 2023-09-30T09:34:48.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:34:48.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:47 smithi167 ceph-mon[192880]: osdmap e321: 8 total, 8 up, 7 in 2023-09-30T09:34:48.238 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:48.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:48.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:48.325 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-30T09:34:48.342 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:48.368 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:48.428 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:48.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:48.544 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:48.579 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:48.661 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: pgmap v795: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 36966/235494 objects degraded (15.697%); 50102/235494 objects misplaced (21.275%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 319'9288 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:48.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:48.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 36966/235494 objects degraded (15.697%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:48.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: osdmap e322: 8 total, 8 up, 7 in 2023-09-30T09:34:48.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: 7.e continuing backfill to osd.2 from (290'4806,319'9328] MIN to 319'9328 2023-09-30T09:34:48.666 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:48 smithi167 ceph-mon[192880]: 7.e starting backfill to osd.5 from (0'0,0'0] MAX to 319'9328 2023-09-30T09:34:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:48.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:48.768 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:48.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: pgmap v795: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 36966/235494 objects degraded (15.697%); 50102/235494 objects misplaced (21.275%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:34:48.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 319'9288 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 36966/235494 objects degraded (15.697%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: osdmap e322: 8 total, 8 up, 7 in 2023-09-30T09:34:48.788 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: 7.e continuing backfill to osd.2 from (290'4806,319'9328] MIN to 319'9328 2023-09-30T09:34:48.789 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[224043]: 7.e starting backfill to osd.5 from (0'0,0'0] MAX to 319'9328 2023-09-30T09:34:48.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: pgmap v795: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 36966/235494 objects degraded (15.697%); 50102/235494 objects misplaced (21.275%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:34:48.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 319'9288 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: failed to encode map e321 with expected crc 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 36966/235494 objects degraded (15.697%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: osdmap e322: 8 total, 8 up, 7 in 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: 7.e continuing backfill to osd.2 from (290'4806,319'9328] MIN to 319'9328 2023-09-30T09:34:48.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:48 smithi019 ceph-mon[216815]: 7.e starting backfill to osd.5 from (0'0,0'0] MAX to 319'9328 2023-09-30T09:34:48.804 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:48.869 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:48.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:48.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:48.906 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:48.978 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:49.011 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:49.083 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:49.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:49.189 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:49.229 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:49.307 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:49.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:49.419 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:49.459 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:49.517 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:49.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:49.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:49.626 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: 7.b starting backfill to osd.5 from (0'0,0'0] MAX to 319'9167 2023-09-30T09:34:49.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:34:49.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:49 smithi167 ceph-mon[192880]: osdmap e323: 8 total, 8 up, 7 in 2023-09-30T09:34:49.667 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:49.754 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:49.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:49.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:49.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: 7.b starting backfill to osd.5 from (0'0,0'0] MAX to 319'9167 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[216815]: osdmap e323: 8 total, 8 up, 7 in 2023-09-30T09:34:49.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: failed to encode map e322 with expected crc 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: 7.b starting backfill to osd.5 from (0'0,0'0] MAX to 319'9167 2023-09-30T09:34:49.842 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:34:49.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:49 smithi019 ceph-mon[224043]: osdmap e323: 8 total, 8 up, 7 in 2023-09-30T09:34:49.861 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:49.923 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:49.973 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:49.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:50.077 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:34:50.106 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:50.157 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:34:50.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:50.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:34:50.261 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:50.324 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:50.414 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:50.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:50.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:50.472 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:34:50.523 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:50.596 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:50.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: pgmap v799: 119 pgs: 2 peering, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 35678/231030 objects degraded (15.443%); 50172/231030 objects misplaced (21.717%); 9.1 MiB/s, 156 objects/s recovering 2023-09-30T09:34:50.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/365921611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:50.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:50.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:50.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:34:50.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:50 smithi167 ceph-mon[192880]: osdmap e324: 8 total, 8 up, 7 in 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: pgmap v799: 119 pgs: 2 peering, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 35678/231030 objects degraded (15.443%); 50172/231030 objects misplaced (21.717%); 9.1 MiB/s, 156 objects/s recovering 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/365921611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:34:50.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[224043]: osdmap e324: 8 total, 8 up, 7 in 2023-09-30T09:34:50.813 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.813 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.813 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: failed to encode map e323 with expected crc 2023-09-30T09:34:50.813 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: pgmap v799: 119 pgs: 2 peering, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 35678/231030 objects degraded (15.443%); 50172/231030 objects misplaced (21.717%); 9.1 MiB/s, 156 objects/s recovering 2023-09-30T09:34:50.813 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/365921611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:50.813 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:34:50.813 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:34:50.814 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:34:50.814 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:50 smithi019 ceph-mon[216815]: osdmap e324: 8 total, 8 up, 7 in 2023-09-30T09:34:51.062 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:51.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:51.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:51.170 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:51.220 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:51.273 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:51.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:51.379 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:51.408 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:51.505 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:51.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:51.615 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:51.649 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:51.723 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:51.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:51.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:51 smithi167 ceph-mon[192880]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:51 smithi167 ceph-mon[192880]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:51 smithi167 ceph-mon[192880]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:51 smithi167 ceph-mon[192880]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:51 smithi167 ceph-mon[192880]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[216815]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[216815]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[216815]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[216815]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[216815]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[224043]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[224043]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[224043]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[224043]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:51 smithi019 ceph-mon[224043]: failed to encode map e324 with expected crc 2023-09-30T09:34:51.846 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:51.920 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:52.134 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:52.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:52.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:52.247 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:52.307 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:52.355 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:52.569 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:52 smithi019 ceph-mon[216815]: pgmap v801: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 28665/230775 objects degraded (12.421%); 54887/230775 objects misplaced (23.784%); 11 MiB/s, 175 objects/s recovering 2023-09-30T09:34:52.771 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:34:52.771 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:34:52.771 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:52.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:52.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:52 smithi019 ceph-mon[224043]: pgmap v801: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 28665/230775 objects degraded (12.421%); 54887/230775 objects misplaced (23.784%); 11 MiB/s, 175 objects/s recovering 2023-09-30T09:34:52.883 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:52.943 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:53.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:52 smithi167 ceph-mon[192880]: pgmap v801: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 28665/230775 objects degraded (12.421%); 54887/230775 objects misplaced (23.784%); 11 MiB/s, 175 objects/s recovering 2023-09-30T09:34:53.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28665/230775 objects degraded (12.421%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:53.841 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:34:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:34:53] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2023-09-30T09:34:53.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28665/230775 objects degraded (12.421%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:53.996 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:34:54.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28665/230775 objects degraded (12.421%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:54.459 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:54.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:54.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:54.568 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:34:54.673 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:54.723 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:54 smithi019 ceph-mon[216815]: pgmap v802: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 28665/230775 objects degraded (12.421%); 54887/230775 objects misplaced (23.784%); 7.5 MiB/s, 118 objects/s recovering 2023-09-30T09:34:54.724 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:54 smithi019 ceph-mon[224043]: pgmap v802: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 28665/230775 objects degraded (12.421%); 54887/230775 objects misplaced (23.784%); 7.5 MiB/s, 118 objects/s recovering 2023-09-30T09:34:54.740 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:34:54.807 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:34:54.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:54.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:34:54.924 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:54.959 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:54 smithi167 ceph-mon[192880]: pgmap v802: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 99 active+clean; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 28665/230775 objects degraded (12.421%); 54887/230775 objects misplaced (23.784%); 7.5 MiB/s, 118 objects/s recovering 2023-09-30T09:34:54.963 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:55.017 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:55.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:55.123 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:55.210 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:55.557 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:55.563 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:34:55.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:55.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:55.603 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:34:55.667 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:34:55.695 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:55.742 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:34:55.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:55.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:34:55.796 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:34:55.796 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:34:55.801 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:34:55.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3311716109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:55.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3311716109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:55.841 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:34:55.848 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:55.883 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3311716109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:34:55.888 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:55.896 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:34:55.951 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:34:55.959 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:34:55.959 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:34:55.959 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:34:55.960 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:34:55.960 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:34:55.960 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:34:55.960 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:34:55.960 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:34:55.975 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:55.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:55.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:56.087 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:34:56.213 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:56.614 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:56 smithi019 ceph-mon[216815]: pgmap v803: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6.0 KiB/s rd, 16 KiB/s wr, 612 op/s; 28560/222663 objects degraded (12.827%); 52480/222663 objects misplaced (23.569%); 9.6 MiB/s, 152 objects/s recovering 2023-09-30T09:34:56.979 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:56 smithi019 ceph-mon[224043]: pgmap v803: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6.0 KiB/s rd, 16 KiB/s wr, 612 op/s; 28560/222663 objects degraded (12.827%); 52480/222663 objects misplaced (23.569%); 9.6 MiB/s, 152 objects/s recovering 2023-09-30T09:34:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:56.980 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:34:56.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:34:56.980 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:57.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:56 smithi167 ceph-mon[192880]: pgmap v803: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6.0 KiB/s rd, 16 KiB/s wr, 612 op/s; 28560/222663 objects degraded (12.827%); 52480/222663 objects misplaced (23.569%); 9.6 MiB/s, 152 objects/s recovering 2023-09-30T09:34:57.185 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:34:57.224 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:57.307 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:34:57.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:57.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:34:57.412 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:57.454 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:57.517 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:57.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:57.625 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:57.712 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:34:57.712 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:59.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:58 smithi167 ceph-mon[192880]: pgmap v804: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 351 op/s; 28560/222663 objects degraded (12.827%); 52480/222663 objects misplaced (23.569%); 3.9 MiB/s, 55 objects/s recovering 2023-09-30T09:34:59.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28560/222663 objects degraded (12.827%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:59.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:34:58 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3658771878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:34:59.081 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:34:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:59.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:34:59.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:58 smithi019 ceph-mon[224043]: pgmap v804: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 351 op/s; 28560/222663 objects degraded (12.827%); 52480/222663 objects misplaced (23.569%); 3.9 MiB/s, 55 objects/s recovering 2023-09-30T09:34:59.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28560/222663 objects degraded (12.827%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:59.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:34:58 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3658771878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:34:59.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:58 smithi019 ceph-mon[216815]: pgmap v804: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 351 op/s; 28560/222663 objects degraded (12.827%); 52480/222663 objects misplaced (23.569%); 3.9 MiB/s, 55 objects/s recovering 2023-09-30T09:34:59.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28560/222663 objects degraded (12.827%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:34:59.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:34:58 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3658771878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:34:59.188 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:34:59.233 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:59.270 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:34:59.293 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:34:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:59.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:34:59.402 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:59.438 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.6 to 1 (10000) 2023-09-30T09:34:59.447 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:59.508 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:59.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:59.614 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:34:59.662 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:34:59.741 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:34:59.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:34:59.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:34:59.848 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:34:59.939 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:00.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3658771878' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:35:00.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:00 smithi167 ceph-mon[192880]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:00 smithi167 ceph-mon[192880]: osdmap e325: 8 total, 8 up, 7 in 2023-09-30T09:35:00.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:00 smithi167 ceph-mon[192880]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:00 smithi167 ceph-mon[192880]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:00 smithi167 ceph-mon[192880]: pgmap v806: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 600 op/s; 28454/214644 objects degraded (13.256%); 50184/214644 objects misplaced (23.380%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:35:00.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:00 smithi167 ceph-mon[192880]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3658771878' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:35:00.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[224043]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[224043]: osdmap e325: 8 total, 8 up, 7 in 2023-09-30T09:35:00.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[224043]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[224043]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[224043]: pgmap v806: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 600 op/s; 28454/214644 objects degraded (13.256%); 50184/214644 objects misplaced (23.380%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:35:00.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[224043]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3658771878' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:35:00.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[216815]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[216815]: osdmap e325: 8 total, 8 up, 7 in 2023-09-30T09:35:00.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[216815]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[216815]: failed to encode map e325 with expected crc 2023-09-30T09:35:00.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[216815]: pgmap v806: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 600 op/s; 28454/214644 objects degraded (13.256%); 50184/214644 objects misplaced (23.380%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:35:00.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:00 smithi019 ceph-mon[216815]: failed to encode map e325 with expected crc 2023-09-30T09:35:01.082 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:01.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:01.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:01.124 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-30T09:35:01.194 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:01.329 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:02.202 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:02.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:02.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:02.319 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:02.418 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:02.449 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:02.462 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:02.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:02 smithi019 ceph-mon[224043]: pgmap v807: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 557 op/s; 28496/214146 objects degraded (13.307%); 50022/214146 objects misplaced (23.359%); 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:35:02.531 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:02 smithi019 ceph-mon[216815]: pgmap v807: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 557 op/s; 28496/214146 objects degraded (13.307%); 50022/214146 objects misplaced (23.359%); 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:35:02.531 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:02 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2504481586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:02.531 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:02 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:02.582 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:35:02.734 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:02.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:02.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:02 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2504481586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:02.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:02 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:02.850 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:02.893 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:02 smithi167 ceph-mon[192880]: pgmap v807: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 557 op/s; 28496/214146 objects degraded (13.307%); 50022/214146 objects misplaced (23.359%); 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:35:02.893 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:02 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2504481586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:02.893 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:02 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:02.897 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:02.980 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:02.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:03.088 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:03.196 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:03.811 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:35:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:35:03] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2023-09-30T09:35:03.811 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: osdmap e326: 8 total, 8 up, 7 in 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: failed to encode map e326 with expected crc 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: failed to encode map e326 with expected crc 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: failed to encode map e326 with expected crc 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28496/214146 objects degraded (13.307%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:03.812 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:03.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:35:03.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: osdmap e326: 8 total, 8 up, 7 in 2023-09-30T09:35:03.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: failed to encode map e326 with expected crc 2023-09-30T09:35:03.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: failed to encode map e326 with expected crc 2023-09-30T09:35:03.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: failed to encode map e326 with expected crc 2023-09-30T09:35:03.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:03.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:03.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28496/214146 objects degraded (13.307%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:35:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: osdmap e326: 8 total, 8 up, 7 in 2023-09-30T09:35:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: failed to encode map e326 with expected crc 2023-09-30T09:35:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: failed to encode map e326 with expected crc 2023-09-30T09:35:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: failed to encode map e326 with expected crc 2023-09-30T09:35:04.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:04.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:04.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28496/214146 objects degraded (13.307%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:04.126 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:35:04.126 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:35:04.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:128358 7.d e306/302 7:b44ac402:::benchmark_data_smithi019_404601_object15080:head v 306'7966, mlcod=286'3458)", 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:00.013599+0000", 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 64.021952260999996, 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.29377689000000001, 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.127 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013599+0000", 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013599+0000", 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013600+0000", 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.08e-06 2023-09-30T09:35:04.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013611+0000", 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1062e-05 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013633+0000", 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1801e-05 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013642+0000", 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8449999999999992e-06 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.284819+0000", 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27117700500000003 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.284845+0000", 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6444e-05 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.307320+0000", 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022475095 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.131 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.307376+0000", 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5557999999999999e-05 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:128360 7.b e306/301 7:d422a517:::benchmark_data_smithi019_404601_object15082:head v 306'7873, mlcod=291'4872)", 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:00.013667+0000", 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 64.021884431000004, 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.28383565399999999, 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.132 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013667+0000", 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013667+0000", 2023-09-30T09:35:04.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013667+0000", 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0199999999999998e-07 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013671+0000", 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1570000000000002e-06 2023-09-30T09:35:04.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013677+0000", 2023-09-30T09:35:04.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2480000000000001e-06 2023-09-30T09:35:04.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013682+0000", 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8500000000000002e-06 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.284779+0000", 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27109628299999999 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.284817+0000", 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7972000000000002e-05 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.297458+0000", 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012641584000000001 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.297502+0000", 2023-09-30T09:35:04.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4258000000000002e-05 2023-09-30T09:35:04.138 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.138 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.138 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.138 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:128357 7.a e306/302 7:503abca5:::benchmark_data_smithi019_404601_object15079:head v 306'8145, mlcod=286'3542)", 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:00.013842+0000", 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 64.021709079000004, 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.28302493899999998, 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013842+0000", 2023-09-30T09:35:04.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013842+0000", 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013843+0000", 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3900000000000004e-07 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013849+0000", 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.438e-06 2023-09-30T09:35:04.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.285216+0000", 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27136657600000003 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.285223+0000", 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9009999999999997e-06 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.285444+0000", 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022070100000000001 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.285472+0000", 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8716000000000001e-05 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.296810+0000", 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011337774 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.296867+0000", 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6993999999999997e-05 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.144 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:128356 7.5 e306/301 7:a01fc4fa:::benchmark_data_smithi019_404601_object15078:head v 306'8001, mlcod=286'3496)", 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:00.015151+0000", 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 64.020399918999999, 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.292259722, 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.015151+0000", 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.015151+0000", 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.015152+0000", 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7700000000000004e-07 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.015162+0000", 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0353000000000001e-05 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.287271+0000", 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.272108774 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.287287+0000", 2023-09-30T09:35:04.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6019999999999999e-05 2023-09-30T09:35:04.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.288039+0000", 2023-09-30T09:35:04.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0007517 2023-09-30T09:35:04.150 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.150 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.150 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.288074+0000", 2023-09-30T09:35:04.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5580999999999997e-05 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.307381+0000", 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.019306092 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.307411+0000", 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0425000000000001e-05 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.151 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145240 7.0 e314/302 7:058eeb6c:::benchmark_data_smithi019_404601_object31962:head v 314'9029, mlcod=290'4806)", 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.506969+0000", 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.528582356000001, 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.223555431, 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.506969+0000", 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.506969+0000", 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.506970+0000", 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4799999999999997e-07 2023-09-30T09:35:04.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.506975+0000", 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4210000000000003e-06 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.506983+0000", 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6529999999999994e-06 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.506991+0000", 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2150000000000001e-06 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.507878+0000", 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00088737899999999999 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.507897+0000", 2023-09-30T09:35:04.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8627999999999999e-05 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.730480+0000", 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22258310100000001 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.730524+0000", 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4085999999999998e-05 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.156 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145241 7.1 e314/302 7:8e04c091:::benchmark_data_smithi019_404601_object31963:head v 314'9005, mlcod=286'3525)", 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.509677+0000", 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.525874022, 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21276693399999999, 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.157 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509677+0000", 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509677+0000", 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.158 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509678+0000", 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.083e-06 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509687+0000", 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3550000000000005e-06 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509699+0000", 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2099999999999999e-05 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509708+0000", 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1400000000000006e-06 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509879+0000", 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000171655 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509904+0000", 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4145999999999999e-05 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.722410+0000", 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21250633799999999 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.722444+0000", 2023-09-30T09:35:04.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4116999999999999e-05 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(7.2 314/301 [PushOp(7:488d3d82:::benchmark_data_smithi019_404601_object69684:head, version: 289'4385, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:488d3d82:::benchmark_data_smithi019_404601_object69684:head@289'4385, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.509733+0000", 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.525817777, 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.224720905, 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.162 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:35:04.163 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.163 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.163 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.163 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509733+0000", 2023-09-30T09:35:04.163 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.163 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.163 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.163 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509733+0000", 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509734+0000", 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4900000000000001e-07 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509764+0000", 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0636000000000003e-05 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509781+0000", 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6806000000000001e-05 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509796+0000", 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4793e-05 2023-09-30T09:35:04.165 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.732377+0000", 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22258154799999999 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.732390+0000", 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2508e-05 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.166 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.734454+0000", 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002064265 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145250 7.9 e314/308 7:96a8486b:::benchmark_data_smithi019_404601_object31972:head v 314'8936, mlcod=286'3382)", 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.509976+0000", 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.525574863999999, 2023-09-30T09:35:04.167 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21804420299999999, 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509976+0000", 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.168 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509976+0000", 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509977+0000", 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0599999999999999e-07 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509987+0000", 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0039e-05 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509998+0000", 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0842000000000001e-05 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.510005+0000", 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.396e-06 2023-09-30T09:35:04.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.514783+0000", 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0047772300000000004 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.514807+0000", 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4343999999999999e-05 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.171 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.727976+0000", 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21316924600000001 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.728020+0000", 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4299999999999999e-05 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.172 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145246 7.a e314/302 7:5dc6241f:::benchmark_data_smithi019_404601_object31968:head v 314'9220, mlcod=286'3542)", 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.513320+0000", 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.522230722, 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.226737997, 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513320+0000", 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513320+0000", 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513321+0000", 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3099999999999996e-07 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.174 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513329+0000", 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2130000000000002e-06 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513339+0000", 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0776e-05 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.175 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513348+0000", 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1950000000000003e-06 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513429+0000", 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1637999999999993e-05 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513447+0000", 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7969e-05 2023-09-30T09:35:04.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.740015+0000", 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.226567501 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.740058+0000", 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3773999999999998e-05 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.177 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.178 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.178 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.178 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145245 7.2 e314/301 7:40511ea0:::benchmark_data_smithi019_404601_object31967:head v 314'9169, mlcod=288'3633)", 2023-09-30T09:35:04.178 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.513364+0000", 2023-09-30T09:35:04.178 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.522187009, 2023-09-30T09:35:04.178 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.225725501, 2023-09-30T09:35:04.178 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.178 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513364+0000", 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513364+0000", 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.179 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513364+0000", 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4700000000000002e-07 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513368+0000", 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8820000000000003e-06 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.180 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513373+0000", 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6700000000000002e-06 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.513376+0000", 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.472e-06 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.514225+0000", 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00084865899999999998 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.181 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.514240+0000", 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4942e-05 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.739040+0000", 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22479961600000001 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.739090+0000", 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9913000000000001e-05 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145253 7.9 e314/308 7:91f792be:::benchmark_data_smithi019_404601_object31975:head v 314'8937, mlcod=286'3382)", 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.515383+0000", 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.520167711999999, 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21482290300000001, 2023-09-30T09:35:04.183 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.515383+0000", 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.515383+0000", 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.515385+0000", 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1689999999999999e-06 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.515399+0000", 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4069e-05 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.515410+0000", 2023-09-30T09:35:04.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1772e-05 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.515418+0000", 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7170000000000008e-06 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.722509+0000", 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20709048199999999 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.722530+0000", 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1288e-05 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.730164+0000", 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0076337310000000004 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.730206+0000", 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2675000000000001e-05 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145247 7.6 e314/308 7:6ffe3934:::benchmark_data_smithi019_404601_object31969:head v 314'9003, mlcod=284'1160)", 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.517093+0000", 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.518458197000001, 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25551791000000001, 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.188 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517093+0000", 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517093+0000", 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.189 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517094+0000", 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.296e-06 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517103+0000", 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6549999999999993e-06 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719169+0000", 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20206605999999999 2023-09-30T09:35:04.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719181+0000", 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2208000000000001e-05 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719584+0000", 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000402354 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.191 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719615+0000", 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1711999999999997e-05 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.772565+0000", 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.052949323 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.772611+0000", 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6301999999999998e-05 2023-09-30T09:35:04.192 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145255 7.d e314/302 7:b15ce7f6:::benchmark_data_smithi019_404601_object31977:head v 314'9043, mlcod=286'3458)", 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.520411+0000", 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 27.515140015, 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.209314375, 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.520411+0000", 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.520411+0000", 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.520412+0000", 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.111e-06 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.520424+0000", 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.169e-05 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719200+0000", 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.19877589300000001 2023-09-30T09:35:04.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719207+0000", 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5730000000000002e-06 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719507+0000", 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00029977299999999998 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719525+0000", 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7793000000000001e-05 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.729676+0000", 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010151380999999999 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.729725+0000", 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9160999999999999e-05 2023-09-30T09:35:04.197 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155902 7.d e325/302 7:b6a3d450:::benchmark_data_smithi019_404601_object42624:head v 325'9721, mlcod=286'3458)", 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.307314+0000", 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 3.7282373409999998, 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.23823868200000001, 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.198 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307314+0000", 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307314+0000", 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.199 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307314+0000", 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6399999999999998e-07 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307318+0000", 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6219999999999998e-06 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307323+0000", 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6289999999999998e-06 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.200 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307328+0000", 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4399999999999998e-06 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307385+0000", 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6990999999999997e-05 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307402+0000", 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7577000000000001e-05 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317476+0000", 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010073268999999999 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545552+0000", 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22807689 2023-09-30T09:35:04.202 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155897 7.2 e325/301 7:45a09b7b:::benchmark_data_smithi019_404601_object42619:head v 325'9840, mlcod=288'3633)", 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.309357+0000", 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 3.7261941859999999, 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25470845600000003, 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309357+0000", 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309357+0000", 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309358+0000", 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9800000000000002e-07 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309365+0000", 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9029999999999998e-06 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309376+0000", 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1236000000000001e-05 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309386+0000", 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0273e-05 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309471+0000", 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4431999999999994e-05 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.206 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309497+0000", 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6186e-05 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564016+0000", 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25451902799999998 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564065+0000", 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9400000000000001e-05 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155904 7.1 e325/316 7:8f01ff57:::benchmark_data_smithi019_404601_object42626:head v 325'9702, mlcod=0'0)", 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.313701+0000", 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 3.721850479, 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.24660228100000001, 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.208 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313701+0000", 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313701+0000", 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313702+0000", 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.850000000000001e-07 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313709+0000", 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8199999999999997e-06 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313720+0000", 2023-09-30T09:35:04.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0589e-05 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313729+0000", 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5590000000000006e-06 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545499+0000", 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.231770059 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545542+0000", 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3189999999999998e-05 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.560235+0000", 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014692935000000001 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.560303+0000", 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8144000000000004e-05 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155910 7.2 e325/301 7:4c9226c2:::benchmark_data_smithi019_404601_object42632:head v 325'9841, mlcod=288'3633)", 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.313754+0000", 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 3.721797504, 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25034391900000003, 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.213 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313754+0000", 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313754+0000", 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313754+0000", 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7700000000000001e-07 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313758+0000", 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7890000000000001e-06 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313764+0000", 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.083e-06 2023-09-30T09:35:04.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313770+0000", 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3459999999999999e-06 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545530+0000", 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.23175974899999999 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545556+0000", 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5942e-05 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564071+0000", 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018515485000000002 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564098+0000", 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6248e-05 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.217 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155898 7.a e325/302 7:5ec05665:::benchmark_data_smithi019_404601_object42620:head v 325'9895, mlcod=286'3542)", 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.313782+0000", 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 3.7217690179999998, 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25034780899999998, 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.218 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313782+0000", 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313782+0000", 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313782+0000", 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6400000000000001e-07 2023-09-30T09:35:04.219 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313789+0000", 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9569999999999996e-06 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545496+0000", 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.23170665600000001 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.220 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545512+0000", 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6263e-05 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545610+0000", 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7787000000000004e-05 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545636+0000", 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6353e-05 2023-09-30T09:35:04.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564104+0000", 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018467378999999999 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564130+0000", 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6250000000000001e-05 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.222 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155909 7.9 e325/315 7:991cfd1d:::benchmark_data_smithi019_404601_object42631:head v 325'9572, mlcod=0'0)", 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.314429+0000", 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 3.7211216509999998, 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.245911667, 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.314429+0000", 2023-09-30T09:35:04.223 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.224 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.224 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.224 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.224 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.314429+0000", 2023-09-30T09:35:04.224 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.224 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.224 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.314430+0000", 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2799999999999995e-07 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.314442+0000", 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1884e-05 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545563+0000", 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.23112110999999999 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545570+0000", 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9890000000000003e-06 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.547758+0000", 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021876830000000002 2023-09-30T09:35:04.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.547783+0000", 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5232000000000002e-05 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.560308+0000", 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012524709 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.560341+0000", 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3331999999999999e-05 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155905 7.5 e325/308 7:a9bd4049:::benchmark_data_smithi019_404601_object42627:head v 325'9726, mlcod=289'4080)", 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.317398+0000", 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 3.7181531329999999, 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.256421277, 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:04.228 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317398+0000", 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317398+0000", 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.229 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317399+0000", 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0890000000000001e-06 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317407+0000", 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8529999999999992e-06 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545519+0000", 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22811198999999999 2023-09-30T09:35:04.230 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545529+0000", 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0433e-05 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.553328+0000", 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077986449999999999 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:04.231 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.553358+0000", 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0039000000000002e-05 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.573785+0000", 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020426889 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.573819+0000", 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4338999999999998e-05 2023-09-30T09:35:04.232 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.233 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.233 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.233 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:04.233 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:04.233 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:35:04.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:04 smithi019 ceph-mon[224043]: pgmap v809: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28496/214146 objects degraded (13.307%); 50022/214146 objects misplaced (23.359%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:35:04.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:04 smithi019 ceph-mon[216815]: pgmap v809: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28496/214146 objects degraded (13.307%); 50022/214146 objects misplaced (23.359%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:35:05.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:04 smithi167 ceph-mon[192880]: pgmap v809: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28496/214146 objects degraded (13.307%); 50022/214146 objects misplaced (23.359%); 2.5 MiB/s, 39 objects/s recovering 2023-09-30T09:35:05.543 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:05.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:05.652 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:05.699 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:35:05.788 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:06.222 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:06.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:06.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:06.332 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:06.362 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:06.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:06 smithi019 ceph-mon[216815]: pgmap v810: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 28436/205776 objects degraded (13.819%); 47551/205776 objects misplaced (23.108%); 5.0 MiB/s, 80 objects/s recovering 2023-09-30T09:35:06.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:06 smithi019 ceph-mon[224043]: pgmap v810: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 28436/205776 objects degraded (13.819%); 47551/205776 objects misplaced (23.108%); 5.0 MiB/s, 80 objects/s recovering 2023-09-30T09:35:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:06 smithi167 ceph-mon[192880]: pgmap v810: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 28436/205776 objects degraded (13.819%); 47551/205776 objects misplaced (23.108%); 5.0 MiB/s, 80 objects/s recovering 2023-09-30T09:35:07.338 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:07.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:07.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:07.449 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:07.588 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:35:07.632 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:07.915 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:07.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:08.028 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:08.062 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:08.121 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:08.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:08.229 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:08.394 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:08.572 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:08.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:08.589 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:08.686 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:08.722 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:35:08 smithi167 conmon[172457]: logger=cleanup t=2023-09-30T09:35:08.372001639Z level=info msg="Completed cleanup jobs" duration=64.925013ms 2023-09-30T09:35:08.722 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:08 smithi167 ceph-mon[192880]: pgmap v811: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 28436/205776 objects degraded (13.819%); 47551/205776 objects misplaced (23.108%); 3.5 MiB/s, 55 objects/s recovering 2023-09-30T09:35:08.722 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28436/205776 objects degraded (13.819%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:08.727 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:08.785 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:08.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:08.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:08.827 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:35:08.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:08 smithi019 ceph-mon[224043]: pgmap v811: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 28436/205776 objects degraded (13.819%); 47551/205776 objects misplaced (23.108%); 3.5 MiB/s, 55 objects/s recovering 2023-09-30T09:35:08.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28436/205776 objects degraded (13.819%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:08.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:08 smithi019 ceph-mon[216815]: pgmap v811: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 28436/205776 objects degraded (13.819%); 47551/205776 objects misplaced (23.108%); 3.5 MiB/s, 55 objects/s recovering 2023-09-30T09:35:08.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28436/205776 objects degraded (13.819%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:08.883 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:35:08.893 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:08.922 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:08.930 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:35:08.977 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:35:09.005 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:09.032 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:35:09.072 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:35:09.099 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:35:09.134 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:35:09.141 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:35:09.141 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:35:09.141 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:35:09.141 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:35:09.141 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:35:09.141 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:35:09.141 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:35:09.141 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:35:09.203 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:09.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:09.326 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:09.421 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:09.657 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.0 to 1 (10000) 2023-09-30T09:35:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3305608091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:10.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2697770190' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:35:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3305608091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2697770190' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:35:10.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3305608091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2697770190' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:35:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:10 smithi167 ceph-mon[192880]: pgmap v812: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 28319/198399 objects degraded (14.274%); 45337/198399 objects misplaced (22.851%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:35:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2697770190' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:35:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:10 smithi167 ceph-mon[192880]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:10 smithi167 ceph-mon[192880]: osdmap e327: 8 total, 8 up, 7 in 2023-09-30T09:35:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:10 smithi167 ceph-mon[192880]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:10 smithi167 ceph-mon[192880]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[216815]: pgmap v812: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 28319/198399 objects degraded (14.274%); 45337/198399 objects misplaced (22.851%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:35:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2697770190' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:35:11.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[216815]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[216815]: osdmap e327: 8 total, 8 up, 7 in 2023-09-30T09:35:11.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[216815]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[216815]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[224043]: pgmap v812: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 103 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 28319/198399 objects degraded (14.274%); 45337/198399 objects misplaced (22.851%); 4.2 MiB/s, 67 objects/s recovering 2023-09-30T09:35:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2697770190' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:35:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[224043]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[224043]: osdmap e327: 8 total, 8 up, 7 in 2023-09-30T09:35:11.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[224043]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:10 smithi019 ceph-mon[224043]: failed to encode map e327 with expected crc 2023-09-30T09:35:11.545 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:11.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:11.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:11.659 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:11.712 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:11.801 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:11.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:11.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: osdmap e328: 8 total, 8 up, 7 in 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.e starting backfill to osd.0 from (0'0,0'0] MAX to 326'10071 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.5 starting backfill to osd.0 from (0'0,0'0] MAX to 326'10058 2023-09-30T09:35:11.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.f starting backfill to osd.0 from (0'0,0'0] MAX to 326'10005 2023-09-30T09:35:11.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.4 starting backfill to osd.0 from (0'0,0'0] MAX to 326'10077 2023-09-30T09:35:11.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.9 starting backfill to osd.0 from (0'0,0'0] MAX to 326'9888 2023-09-30T09:35:11.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.5 continuing backfill to osd.2 from (286'3000,326'10058] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 326'10058 2023-09-30T09:35:11.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.e continuing backfill to osd.2 from (292'5506,326'10071] MIN to 326'10071 2023-09-30T09:35:11.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.e continuing backfill to osd.5 from (292'5506,326'10071] MIN to 326'10071 2023-09-30T09:35:11.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.4 continuing backfill to osd.4 from (292'5233,326'10077] MIN to 326'10077 2023-09-30T09:35:11.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[224043]: 7.9 continuing backfill to osd.4 from (292'5076,326'9888] MIN to 326'9888 2023-09-30T09:35:11.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: osdmap e328: 8 total, 8 up, 7 in 2023-09-30T09:35:11.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.e starting backfill to osd.0 from (0'0,0'0] MAX to 326'10071 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.5 starting backfill to osd.0 from (0'0,0'0] MAX to 326'10058 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.f starting backfill to osd.0 from (0'0,0'0] MAX to 326'10005 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.4 starting backfill to osd.0 from (0'0,0'0] MAX to 326'10077 2023-09-30T09:35:11.842 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.9 starting backfill to osd.0 from (0'0,0'0] MAX to 326'9888 2023-09-30T09:35:11.843 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.5 continuing backfill to osd.2 from (286'3000,326'10058] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 326'10058 2023-09-30T09:35:11.843 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.e continuing backfill to osd.2 from (292'5506,326'10071] MIN to 326'10071 2023-09-30T09:35:11.843 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.e continuing backfill to osd.5 from (292'5506,326'10071] MIN to 326'10071 2023-09-30T09:35:11.843 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.4 continuing backfill to osd.4 from (292'5233,326'10077] MIN to 326'10077 2023-09-30T09:35:11.843 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:11 smithi019 ceph-mon[216815]: 7.9 continuing backfill to osd.4 from (292'5076,326'9888] MIN to 326'9888 2023-09-30T09:35:11.907 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:11.951 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.951 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.951 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.951 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: osdmap e328: 8 total, 8 up, 7 in 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.e starting backfill to osd.0 from (0'0,0'0] MAX to 326'10071 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: failed to encode map e328 with expected crc 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.5 starting backfill to osd.0 from (0'0,0'0] MAX to 326'10058 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.f starting backfill to osd.0 from (0'0,0'0] MAX to 326'10005 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.4 starting backfill to osd.0 from (0'0,0'0] MAX to 326'10077 2023-09-30T09:35:11.952 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.9 starting backfill to osd.0 from (0'0,0'0] MAX to 326'9888 2023-09-30T09:35:11.953 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.5 continuing backfill to osd.2 from (286'3000,326'10058] 7:a7815b78:::benchmark_data_smithi019_404601_object53537:head to 326'10058 2023-09-30T09:35:11.953 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.e continuing backfill to osd.2 from (292'5506,326'10071] MIN to 326'10071 2023-09-30T09:35:11.953 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.e continuing backfill to osd.5 from (292'5506,326'10071] MIN to 326'10071 2023-09-30T09:35:11.953 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.4 continuing backfill to osd.4 from (292'5233,326'10077] MIN to 326'10077 2023-09-30T09:35:11.953 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:11 smithi167 ceph-mon[192880]: 7.9 continuing backfill to osd.4 from (292'5076,326'9888] MIN to 326'9888 2023-09-30T09:35:11.954 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:12.019 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:12.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:12.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:12.123 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:35:12.173 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:12.255 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:35:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:12.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:35:12.368 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:12.537 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:12.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: pgmap v815: 119 pgs: 3 remapped+peering, 4 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 96 active+clean; 4.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 28346/197913 objects degraded (14.322%); 45174/197913 objects misplaced (22.825%); 5.4 MiB/s, 88 objects/s recovering 2023-09-30T09:35:12.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: osdmap e329: 8 total, 8 up, 7 in 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[216815]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: pgmap v815: 119 pgs: 3 remapped+peering, 4 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 96 active+clean; 4.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 28346/197913 objects degraded (14.322%); 45174/197913 objects misplaced (22.825%); 5.4 MiB/s, 88 objects/s recovering 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-30T09:35:12.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: osdmap e329: 8 total, 8 up, 7 in 2023-09-30T09:35:12.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: failed to encode map e329 with expected crc 2023-09-30T09:35:12.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:12 smithi019 ceph-mon[224043]: failed to encode map e329 with expected crc 2023-09-30T09:35:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: pgmap v815: 119 pgs: 3 remapped+peering, 4 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 96 active+clean; 4.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 28346/197913 objects degraded (14.322%); 45174/197913 objects misplaced (22.825%); 5.4 MiB/s, 88 objects/s recovering 2023-09-30T09:35:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-30T09:35:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: osdmap e329: 8 total, 8 up, 7 in 2023-09-30T09:35:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: failed to encode map e329 with expected crc 2023-09-30T09:35:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: failed to encode map e329 with expected crc 2023-09-30T09:35:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: failed to encode map e329 with expected crc 2023-09-30T09:35:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: failed to encode map e329 with expected crc 2023-09-30T09:35:13.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: failed to encode map e329 with expected crc 2023-09-30T09:35:13.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:12 smithi167 ceph-mon[192880]: failed to encode map e329 with expected crc 2023-09-30T09:35:13.597 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:13.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:13.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[224043]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[224043]: osdmap e330: 8 total, 8 up, 7 in 2023-09-30T09:35:13.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[224043]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[224043]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[224043]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.694 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[224043]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.694 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28346/197913 objects degraded (14.322%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:13.694 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[216815]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.694 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[216815]: osdmap e330: 8 total, 8 up, 7 in 2023-09-30T09:35:13.694 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[216815]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[216815]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[216815]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[216815]: failed to encode map e330 with expected crc 2023-09-30T09:35:13.695 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28346/197913 objects degraded (14.322%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:13.713 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:13.823 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:13 smithi167 ceph-mon[192880]: failed to encode map e330 with expected crc 2023-09-30T09:35:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:13 smithi167 ceph-mon[192880]: osdmap e330: 8 total, 8 up, 7 in 2023-09-30T09:35:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:13 smithi167 ceph-mon[192880]: failed to encode map e330 with expected crc 2023-09-30T09:35:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:13 smithi167 ceph-mon[192880]: failed to encode map e330 with expected crc 2023-09-30T09:35:14.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:13 smithi167 ceph-mon[192880]: failed to encode map e330 with expected crc 2023-09-30T09:35:14.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:13 smithi167 ceph-mon[192880]: failed to encode map e330 with expected crc 2023-09-30T09:35:14.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28346/197913 objects degraded (14.322%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:14.067 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:35:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:35:13] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2023-09-30T09:35:14.457 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-30T09:35:14.478 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:14.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:14.610 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:14.653 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:14.712 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:14.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[216815]: pgmap v818: 119 pgs: 3 remapped+peering, 4 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 96 active+clean; 4.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 28346/197913 objects degraded (14.322%); 45174/197913 objects misplaced (22.825%); 2.1 MiB/s, 36 objects/s recovering 2023-09-30T09:35:14.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[216815]: osdmap e331: 8 total, 8 up, 7 in 2023-09-30T09:35:14.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[216815]: failed to encode map e331 with expected crc 2023-09-30T09:35:14.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[216815]: failed to encode map e331 with expected crc 2023-09-30T09:35:14.714 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[216815]: failed to encode map e331 with expected crc 2023-09-30T09:35:14.714 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[216815]: failed to encode map e331 with expected crc 2023-09-30T09:35:14.714 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[216815]: failed to encode map e331 with expected crc 2023-09-30T09:35:14.714 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[216815]: failed to encode map e331 with expected crc 2023-09-30T09:35:14.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:14.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:14.769 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:35:14.818 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:14.921 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:14 smithi167 ceph-mon[192880]: pgmap v818: 119 pgs: 3 remapped+peering, 4 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 96 active+clean; 4.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 28346/197913 objects degraded (14.322%); 45174/197913 objects misplaced (22.825%); 2.1 MiB/s, 36 objects/s recovering 2023-09-30T09:35:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:14 smithi167 ceph-mon[192880]: osdmap e331: 8 total, 8 up, 7 in 2023-09-30T09:35:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:14 smithi167 ceph-mon[192880]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:14 smithi167 ceph-mon[192880]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:14 smithi167 ceph-mon[192880]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:14 smithi167 ceph-mon[192880]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:14 smithi167 ceph-mon[192880]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:14 smithi167 ceph-mon[192880]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[224043]: pgmap v818: 119 pgs: 3 remapped+peering, 4 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 96 active+clean; 4.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 28346/197913 objects degraded (14.322%); 45174/197913 objects misplaced (22.825%); 2.1 MiB/s, 36 objects/s recovering 2023-09-30T09:35:15.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[224043]: osdmap e331: 8 total, 8 up, 7 in 2023-09-30T09:35:15.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[224043]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[224043]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[224043]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[224043]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[224043]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:14 smithi019 ceph-mon[224043]: failed to encode map e331 with expected crc 2023-09-30T09:35:15.218 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:15.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:15.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:15.330 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:15.359 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:15.380 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:35:15.380 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:35:15.408 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:15.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:15.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:15.513 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:15.558 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:15.623 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:15.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:15.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:15.729 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:15.776 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:15.850 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:15.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:15.957 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:15.995 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:16.052 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:16.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:16.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:16.162 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:16.262 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:16.327 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:16.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:16.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:16.431 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:16.461 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:16.526 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:16.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:16.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:16.632 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:35:16.679 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:16.739 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:16 smithi167 ceph-mon[192880]: pgmap v820: 119 pgs: 1 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 407 KiB/s rd, 274 KiB/s wr, 669 op/s; 28940/191736 objects degraded (15.094%); 59623/191736 objects misplaced (31.096%); 4.4 MiB/s, 77 objects/s recovering 2023-09-30T09:35:16.739 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/636435196' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:35:16.753 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:35:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:16.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:35:16.773 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:35:16.862 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:16.905 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:16.973 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:16.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:16.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:17.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:16 smithi019 ceph-mon[216815]: pgmap v820: 119 pgs: 1 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 407 KiB/s rd, 274 KiB/s wr, 669 op/s; 28940/191736 objects degraded (15.094%); 59623/191736 objects misplaced (31.096%); 4.4 MiB/s, 77 objects/s recovering 2023-09-30T09:35:17.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/636435196' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:35:17.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:16 smithi019 ceph-mon[224043]: pgmap v820: 119 pgs: 1 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 407 KiB/s rd, 274 KiB/s wr, 669 op/s; 28940/191736 objects degraded (15.094%); 59623/191736 objects misplaced (31.096%); 4.4 MiB/s, 77 objects/s recovering 2023-09-30T09:35:17.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/636435196' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:35:17.079 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:17.186 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:17.722 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:17.769 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:35:17.926 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:17.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:17.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:17 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/636435196' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:35:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:17 smithi167 ceph-mon[192880]: osdmap e332: 8 total, 8 up, 7 in 2023-09-30T09:35:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:17 smithi167 ceph-mon[192880]: failed to encode map e332 with expected crc 2023-09-30T09:35:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:17 smithi167 ceph-mon[192880]: failed to encode map e332 with expected crc 2023-09-30T09:35:18.044 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:35:18.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:18.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/636435196' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:35:18.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[224043]: osdmap e332: 8 total, 8 up, 7 in 2023-09-30T09:35:18.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[224043]: failed to encode map e332 with expected crc 2023-09-30T09:35:18.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[224043]: failed to encode map e332 with expected crc 2023-09-30T09:35:18.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:18.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/636435196' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:35:18.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[216815]: osdmap e332: 8 total, 8 up, 7 in 2023-09-30T09:35:18.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[216815]: failed to encode map e332 with expected crc 2023-09-30T09:35:18.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:17 smithi019 ceph-mon[216815]: failed to encode map e332 with expected crc 2023-09-30T09:35:18.076 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:18.112 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:18.137 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:35:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:18.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:35:18.243 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:35:18.284 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:18.347 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:35:18.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:18.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:35:18.452 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:35:18.491 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:18.548 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:35:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:18.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:35:18.654 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:18.763 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:18.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[224043]: failed to encode map e332 with expected crc 2023-09-30T09:35:18.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[224043]: pgmap v822: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28954/191736 objects degraded (15.101%); 59623/191736 objects misplaced (31.096%); 3.6 MiB/s, 59 objects/s recovering 2023-09-30T09:35:18.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:35:18.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:35:18.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:18.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28954/191736 objects degraded (15.101%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:18.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[216815]: failed to encode map e332 with expected crc 2023-09-30T09:35:18.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[216815]: pgmap v822: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28954/191736 objects degraded (15.101%); 59623/191736 objects misplaced (31.096%); 3.6 MiB/s, 59 objects/s recovering 2023-09-30T09:35:18.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:35:18.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:35:18.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:18.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28954/191736 objects degraded (15.101%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:18 smithi167 ceph-mon[192880]: failed to encode map e332 with expected crc 2023-09-30T09:35:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:18 smithi167 ceph-mon[192880]: pgmap v822: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28954/191736 objects degraded (15.101%); 59623/191736 objects misplaced (31.096%); 3.6 MiB/s, 59 objects/s recovering 2023-09-30T09:35:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:18 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:35:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:18 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:35:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28954/191736 objects degraded (15.101%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:20.095 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:35:20.440 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:20.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:20.565 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:20.639 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:20.863 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:20.931 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:20.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:20.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:21.021 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:20 smithi019 ceph-mon[216815]: pgmap v823: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 28598/184461 objects degraded (15.504%); 57098/184461 objects misplaced (30.954%); 5.1 MiB/s, 82 objects/s recovering 2023-09-30T09:35:21.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:20 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:35:21.022 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:20 smithi019 ceph-mon[224043]: pgmap v823: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 28598/184461 objects degraded (15.504%); 57098/184461 objects misplaced (30.954%); 5.1 MiB/s, 82 objects/s recovering 2023-09-30T09:35:21.022 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:20 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:35:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:20 smithi167 ceph-mon[192880]: pgmap v823: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 28598/184461 objects degraded (15.504%); 57098/184461 objects misplaced (30.954%); 5.1 MiB/s, 82 objects/s recovering 2023-09-30T09:35:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:20 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:35:21.052 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:21.170 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:21.352 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:21.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:21.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:21.466 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:21.490 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:21.605 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:21.785 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:21.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:21.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:21.899 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:21.933 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:22.002 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:22.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:22.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:22.116 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:22.151 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:22.166 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:35:22.211 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:35:22.222 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:22.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:22.270 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:35:22.316 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:35:22.328 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:22.361 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:22.370 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:35:22.409 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:35:22.431 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:22.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:22.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:22.452 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:35:22.489 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:35:22.493 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:35:22.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:35:22.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:35:22.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:35:22.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:35:22.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:35:22.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:35:22.494 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:35:22.538 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:22.626 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:22.811 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.2 to 1 (10000) 2023-09-30T09:35:23.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[224043]: pgmap v824: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 28598/184038 objects degraded (15.539%); 56963/184038 objects misplaced (30.952%); 5.4 MiB/s, 87 objects/s recovering 2023-09-30T09:35:23.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:35:23.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1185668965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:23.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3562469624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:35:23.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:35:23.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[216815]: pgmap v824: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 28598/184038 objects degraded (15.539%); 56963/184038 objects misplaced (30.952%); 5.4 MiB/s, 87 objects/s recovering 2023-09-30T09:35:23.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:35:23.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1185668965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:23.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3562469624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:35:23.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:22 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:35:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:22 smithi167 ceph-mon[192880]: pgmap v824: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 28598/184038 objects degraded (15.539%); 56963/184038 objects misplaced (30.952%); 5.4 MiB/s, 87 objects/s recovering 2023-09-30T09:35:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:22 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:35:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1185668965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3562469624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:35:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:22 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:35:23.314 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:23.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:23.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:23.419 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:23.514 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:24.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: osdmap e333: 8 total, 8 up, 7 in 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28598/184038 objects degraded (15.539%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:24.070 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[216815]: 1.14 scrub starts 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:35:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:35:23] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: osdmap e333: 8 total, 8 up, 7 in 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.071 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28598/184038 objects degraded (15.539%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:24.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:23 smithi019 ceph-mon[224043]: 1.14 scrub starts 2023-09-30T09:35:24.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: osdmap e333: 8 total, 8 up, 7 in 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: failed to encode map e333 with expected crc 2023-09-30T09:35:24.277 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28598/184038 objects degraded (15.539%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:24.278 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:23 smithi167 ceph-mon[192880]: 1.14 scrub starts 2023-09-30T09:35:24.481 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:24.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:24.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:24.794 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:24.825 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:24.827 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: pgmap v826: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 320 op/s; 28598/184038 objects degraded (15.539%); 56963/184038 objects misplaced (30.952%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:35:24.827 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: osdmap e334: 8 total, 8 up, 7 in 2023-09-30T09:35:24.827 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: failed to encode map e334 with expected crc 2023-09-30T09:35:24.827 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: failed to encode map e334 with expected crc 2023-09-30T09:35:24.827 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: failed to encode map e334 with expected crc 2023-09-30T09:35:24.874 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:24.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:24.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:24.978 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:25.072 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:25.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:35:25.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:35:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: pgmap v826: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 320 op/s; 28598/184038 objects degraded (15.539%); 56963/184038 objects misplaced (30.952%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:35:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: osdmap e334: 8 total, 8 up, 7 in 2023-09-30T09:35:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:35:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:35:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: pgmap v826: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 320 op/s; 28598/184038 objects degraded (15.539%); 56963/184038 objects misplaced (30.952%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:35:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: osdmap e334: 8 total, 8 up, 7 in 2023-09-30T09:35:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: failed to encode map e334 with expected crc 2023-09-30T09:35:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:35:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:35:25.378 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:25.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:25.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:25.485 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:25.538 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:25.624 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:25.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:25.732 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:25.788 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:26.052 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-30T09:35:26.052 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:35:26.294 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:26.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:26.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:26.418 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:26.553 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:26.793 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:26.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:26.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:26.903 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:26.953 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:27.047 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:27.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:27.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:27.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:26 smithi019 ceph-mon[216815]: pgmap v828: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 28096/176445 objects degraded (15.923%); 54428/176445 objects misplaced (30.847%); 5.8 MiB/s, 93 objects/s recovering 2023-09-30T09:35:27.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:26 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:35:27.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:26 smithi019 ceph-mon[224043]: pgmap v828: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 28096/176445 objects degraded (15.923%); 54428/176445 objects misplaced (30.847%); 5.8 MiB/s, 93 objects/s recovering 2023-09-30T09:35:27.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:26 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:35:27.154 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:27.200 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:26 smithi167 ceph-mon[192880]: pgmap v828: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 28096/176445 objects degraded (15.923%); 54428/176445 objects misplaced (30.847%); 5.8 MiB/s, 93 objects/s recovering 2023-09-30T09:35:27.200 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:26 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:35:27.208 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:27.306 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:35:27.306 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:35:27.306 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:35:27.306 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:35:27.306 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:35:27.313 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:27.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:27.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:27.424 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:27.432 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:27.451 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:27.518 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:27.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:27.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:27.626 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:27.747 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:27.894 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:35:28.048 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:35:28.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1695549547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:28.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1695549547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:28.194 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:28.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1695549547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:35:28.311 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:28.433 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: pgmap v829: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 28096/176445 objects degraded (15.923%); 54428/176445 objects misplaced (30.847%); 4.1 MiB/s, 66 objects/s recovering 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1695549547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: osdmap e335: 8 total, 8 up, 7 in 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28096/176445 objects degraded (15.923%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:29.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:28 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:35:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: pgmap v829: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 28096/176445 objects degraded (15.923%); 54428/176445 objects misplaced (30.847%); 4.1 MiB/s, 66 objects/s recovering 2023-09-30T09:35:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:35:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1695549547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:35:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: osdmap e335: 8 total, 8 up, 7 in 2023-09-30T09:35:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28096/176445 objects degraded (15.923%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: pgmap v829: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 28096/176445 objects degraded (15.923%); 54428/176445 objects misplaced (30.847%); 4.1 MiB/s, 66 objects/s recovering 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1695549547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: osdmap e335: 8 total, 8 up, 7 in 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: failed to encode map e335 with expected crc 2023-09-30T09:35:29.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28096/176445 objects degraded (15.923%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:29.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:28 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:35:29.928 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:35:29.967 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:29.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:29.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:30.090 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:30.131 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:30.201 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:30.206 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:29 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:35:30.206 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:29 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:30.206 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:29 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:30.206 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:29 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:35:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:30.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:30.308 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:29 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:35:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:29 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:29 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:29 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:35:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:29 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:35:30.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:30.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:30.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:29 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:35:30.426 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:30.838 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:30.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:30.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:30.957 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:30.992 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:31.042 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:31.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:31.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:31.150 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:31.177 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:30 smithi167 ceph-mon[192880]: pgmap v831: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 27411/170316 objects degraded (16.094%); 52477/170316 objects misplaced (30.812%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:35:31.182 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:31.250 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:31.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:31.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:30 smithi019 ceph-mon[216815]: pgmap v831: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 27411/170316 objects degraded (16.094%); 52477/170316 objects misplaced (30.812%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:35:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:30 smithi019 ceph-mon[224043]: pgmap v831: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 27411/170316 objects degraded (16.094%); 52477/170316 objects misplaced (30.812%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:35:31.356 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:31.470 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:32.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:31 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:35:32.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:31 smithi167 ceph-mon[192880]: pgmap v832: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 27419/169818 objects degraded (16.146%); 52320/169818 objects misplaced (30.809%); 8.4 MiB/s, 134 objects/s recovering 2023-09-30T09:35:32.281 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:32.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:32.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:32.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:31 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:35:32.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:31 smithi019 ceph-mon[224043]: pgmap v832: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 27419/169818 objects degraded (16.146%); 52320/169818 objects misplaced (30.809%); 8.4 MiB/s, 134 objects/s recovering 2023-09-30T09:35:32.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:31 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:35:32.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:31 smithi019 ceph-mon[216815]: pgmap v832: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 27419/169818 objects degraded (16.146%); 52320/169818 objects misplaced (30.809%); 8.4 MiB/s, 134 objects/s recovering 2023-09-30T09:35:32.390 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:32.393 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:35:32.425 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:32.485 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:35:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:32.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:35:32.569 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:32.592 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:32.653 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:32.940 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:32.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:33.031 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:35:33.053 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:33.085 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:33.142 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:33.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:33.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:33.235 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:32 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:35:33.235 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:32 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:35:33.248 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:32 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:35:33.397 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:33.676 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:33.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:33.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:33.743 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:35:33.760 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:35:33.793 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:33.831 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:33.918 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:33.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:34.024 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 27419/169818 objects degraded (16.146%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[224043]: pgmap v833: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 27419/169818 objects degraded (16.146%); 52320/169818 objects misplaced (30.809%); 7.0 MiB/s, 111 objects/s recovering 2023-09-30T09:35:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[224043]: from='client.35634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:35:34.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:34.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:34.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 27419/169818 objects degraded (16.146%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:34.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[216815]: pgmap v833: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 27419/169818 objects degraded (16.146%); 52320/169818 objects misplaced (30.809%); 7.0 MiB/s, 111 objects/s recovering 2023-09-30T09:35:34.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:33 smithi019 ceph-mon[216815]: from='client.35634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:35:34.088 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:35:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:35:33] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.43.0" 2023-09-30T09:35:34.109 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 27419/169818 objects degraded (16.146%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:33 smithi167 ceph-mon[192880]: pgmap v833: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 27419/169818 objects degraded (16.146%); 52320/169818 objects misplaced (30.809%); 7.0 MiB/s, 111 objects/s recovering 2023-09-30T09:35:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:33 smithi167 ceph-mon[192880]: from='client.35634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:35:35.276 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:34 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:35:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:34 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:35:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:34 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:35:36.023 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:36 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:35:36.023 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:36 smithi019 ceph-mon[224043]: pgmap v834: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 26494/163785 objects degraded (16.176%); 50493/163785 objects misplaced (30.829%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:35:36.058 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:36.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:36.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:36.166 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:36 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:35:36.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:36 smithi019 ceph-mon[216815]: pgmap v834: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 26494/163785 objects degraded (16.176%); 50493/163785 objects misplaced (30.829%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:35:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:36 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:35:36.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:36 smithi167 ceph-mon[192880]: pgmap v834: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 26494/163785 objects degraded (16.176%); 50493/163785 objects misplaced (30.829%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:35:36.865 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:36.911 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":833,"stamp":"2023-09-30T09:35:33.465979+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3712760676,"num_objects":56606,"num_object_clones":0,"num_object_copies":169818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27419,"num_objects_misplaced":52320,"num_objects_unfound":0,"num_objects_dirty":56606,"num_whiteouts":0,"num_read":6114,"num_read_kb":12467,"num_write":170963,"num_write_kb":7263166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12749,"num_bytes_recovered":830382552,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86061,"ondisk_log_size":86061,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":397,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9608532,"kb_used_data":7434132,"kb_used_omap":23,"kb_used_meta":2174248,"kb_avail":646607532,"statfs":{"total":671965249536,"available":662126112768,"internally_reserved":0,"allocated":7612551168,"data_stored":7601372157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2226430211},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,2,11,1,5,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-310640640,"num_objects":-4740,"num_object_clones":0,"num_object_copies":-14220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1179,"num_objects_misplaced":-4643,"num_objects_unfound":0,"num_objects_dirty":-4740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4740,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1074,"num_bytes_recovered":70255146,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.639565"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":799,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842721+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:35:23.842721+0000","last_peered":"2023-09-30T09:35:23.842721+0000","last_clean":"2023-09-30T09:35:23.842721+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:35:23.842721+0000","last_undegraded":"2023-09-30T09:35:23.842721+0000","last_fullsized":"2023-09-30T09:35:23.842721+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":353,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847228+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:35:23.847228+0000","last_peered":"2023-09-30T09:35:23.847228+0000","last_clean":"2023-09-30T09:35:23.847228+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:35:23.847228+0000","last_undegraded":"2023-09-30T09:35:23.847228+0000","last_fullsized":"2023-09-30T09:35:23.847228+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":315,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833647+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:35:23.833647+0000","last_peered":"2023-09-30T09:35:23.833647+0000","last_clean":"2023-09-30T09:35:23.833647+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:35:23.833647+0000","last_undegraded":"2023-09-30T09:35:23.833647+0000","last_fullsized":"2023-09-30T09:35:23.833647+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":779,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842127+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:35:23.842127+0000","last_peered":"2023-09-30T09:35:23.842127+0000","last_clean":"2023-09-30T09:35:23.842127+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:35:23.842127+0000","last_undegraded":"2023-09-30T09:35:23.842127+0000","last_fullsized":"2023-09-30T09:35:23.842127+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.832727+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:35:23.832727+0000","last_peered":"2023-09-30T09:35:23.832727+0000","last_clean":"2023-09-30T09:35:23.832727+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:35:23.832727+0000","last_undegraded":"2023-09-30T09:35:23.832727+0000","last_fullsized":"2023-09-30T09:35:23.832727+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":239,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.831689+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:35:23.831689+0000","last_peered":"2023-09-30T09:35:23.831689+0000","last_clean":"2023-09-30T09:35:23.831689+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:35:23.831689+0000","last_undegraded":"2023-09-30T09:35:23.831689+0000","last_fullsized":"2023-09-30T09:35:23.831689+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":548,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847136+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:35:23.847136+0000","last_peered":"2023-09-30T09:35:23.847136+0000","last_clean":"2023-09-30T09:35:23.847136+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:35:23.847136+0000","last_undegraded":"2023-09-30T09:35:23.847136+0000","last_fullsized":"2023-09-30T09:35:23.847136+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":381,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.830001+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:35:23.830001+0000","last_peered":"2023-09-30T09:35:23.830001+0000","last_clean":"2023-09-30T09:35:23.830001+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:35:23.830001+0000","last_undegraded":"2023-09-30T09:35:23.830001+0000","last_fullsized":"2023-09-30T09:35:23.830001+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":311,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.837016+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:35:23.837016+0000","last_peered":"2023-09-30T09:35:23.837016+0000","last_clean":"2023-09-30T09:35:23.837016+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:35:23.837016+0000","last_undegraded":"2023-09-30T09:35:23.837016+0000","last_fullsized":"2023-09-30T09:35:23.837016+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":758,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833466+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:35:23.833466+0000","last_peered":"2023-09-30T09:35:23.833466+0000","last_clean":"2023-09-30T09:35:23.833466+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:35:23.833466+0000","last_undegraded":"2023-09-30T09:35:23.833466+0000","last_fullsized":"2023-09-30T09:35:23.833466+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":230,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.829588+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:35:23.829588+0000","last_peered":"2023-09-30T09:35:23.829588+0000","last_clean":"2023-09-30T09:35:23.829588+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:35:23.829588+0000","last_undegraded":"2023-09-30T09:35:23.829588+0000","last_fullsized":"2023-09-30T09:35:23.829588+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":316,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:24.249782+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:35:24.249782+0000","last_peered":"2023-09-30T09:35:24.249782+0000","last_clean":"2023-09-30T09:35:24.249782+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:35:24.249782+0000","last_undegraded":"2023-09-30T09:35:24.249782+0000","last_fullsized":"2023-09-30T09:35:24.249782+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":498,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.877954+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:35:27.877954+0000","last_peered":"2023-09-30T09:35:27.877954+0000","last_clean":"2023-09-30T09:35:27.877954+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:35:27.877954+0000","last_undegraded":"2023-09-30T09:35:27.877954+0000","last_fullsized":"2023-09-30T09:35:27.877954+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841644+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:35:23.841644+0000","last_peered":"2023-09-30T09:35:23.841644+0000","last_clean":"2023-09-30T09:35:23.841644+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:35:23.841644+0000","last_undegraded":"2023-09-30T09:35:23.841644+0000","last_fullsized":"2023-09-30T09:35:23.841644+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":262,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.891032+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:35:27.891032+0000","last_peered":"2023-09-30T09:35:27.891032+0000","last_clean":"2023-09-30T09:35:27.891032+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:35:27.891032+0000","last_undegraded":"2023-09-30T09:35:27.891032+0000","last_fullsized":"2023-09-30T09:35:27.891032+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":501,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840662+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:35:23.840662+0000","last_peered":"2023-09-30T09:35:23.840662+0000","last_clean":"2023-09-30T09:35:23.840662+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:35:23.840662+0000","last_undegraded":"2023-09-30T09:35:23.840662+0000","last_fullsized":"2023-09-30T09:35:23.840662+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":56,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833723+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:35:23.833723+0000","last_peered":"2023-09-30T09:35:23.833723+0000","last_clean":"2023-09-30T09:35:23.833723+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:35:23.833723+0000","last_undegraded":"2023-09-30T09:35:23.833723+0000","last_fullsized":"2023-09-30T09:35:23.833723+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":248,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.836572+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:35:23.836572+0000","last_peered":"2023-09-30T09:35:23.836572+0000","last_clean":"2023-09-30T09:35:23.836572+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:35:23.836572+0000","last_undegraded":"2023-09-30T09:35:23.836572+0000","last_fullsized":"2023-09-30T09:35:23.836572+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":494,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835413+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:35:23.835413+0000","last_peered":"2023-09-30T09:35:23.835413+0000","last_clean":"2023-09-30T09:35:23.835413+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:35:23.835413+0000","last_undegraded":"2023-09-30T09:35:23.835413+0000","last_fullsized":"2023-09-30T09:35:23.835413+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":319,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848910+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:35:23.848910+0000","last_peered":"2023-09-30T09:35:23.848910+0000","last_clean":"2023-09-30T09:35:23.848910+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:35:23.848910+0000","last_undegraded":"2023-09-30T09:35:23.848910+0000","last_fullsized":"2023-09-30T09:35:23.848910+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":330,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847450+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:35:23.847450+0000","last_peered":"2023-09-30T09:35:23.847450+0000","last_clean":"2023-09-30T09:35:23.847450+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:35:23.847450+0000","last_undegraded":"2023-09-30T09:35:23.847450+0000","last_fullsized":"2023-09-30T09:35:23.847450+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":612,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.829170+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:35:23.829170+0000","last_peered":"2023-09-30T09:35:23.829170+0000","last_clean":"2023-09-30T09:35:23.829170+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:35:23.829170+0000","last_undegraded":"2023-09-30T09:35:23.829170+0000","last_fullsized":"2023-09-30T09:35:23.829170+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":57,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847331+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:35:23.847331+0000","last_peered":"2023-09-30T09:35:23.847331+0000","last_clean":"2023-09-30T09:35:23.847331+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:35:23.847331+0000","last_undegraded":"2023-09-30T09:35:23.847331+0000","last_fullsized":"2023-09-30T09:35:23.847331+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":346,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.883365+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:35:27.883365+0000","last_peered":"2023-09-30T09:35:27.883365+0000","last_clean":"2023-09-30T09:35:27.883365+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:35:27.883365+0000","last_undegraded":"2023-09-30T09:35:27.883365+0000","last_fullsized":"2023-09-30T09:35:27.883365+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":681,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842879+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:35:23.842879+0000","last_peered":"2023-09-30T09:35:23.842879+0000","last_clean":"2023-09-30T09:35:23.842879+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:35:23.842879+0000","last_undegraded":"2023-09-30T09:35:23.842879+0000","last_fullsized":"2023-09-30T09:35:23.842879+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":238,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.838816+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:35:23.838816+0000","last_peered":"2023-09-30T09:35:23.838816+0000","last_clean":"2023-09-30T09:35:23.838816+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:35:23.838816+0000","last_undegraded":"2023-09-30T09:35:23.838816+0000","last_fullsized":"2023-09-30T09:35:23.838816+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":24,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841282+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:35:23.841282+0000","last_peered":"2023-09-30T09:35:23.841282+0000","last_clean":"2023-09-30T09:35:23.841282+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:35:23.841282+0000","last_undegraded":"2023-09-30T09:35:23.841282+0000","last_fullsized":"2023-09-30T09:35:23.841282+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":773,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.830784+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:35:23.830784+0000","last_peered":"2023-09-30T09:35:23.830784+0000","last_clean":"2023-09-30T09:35:23.830784+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:35:23.830784+0000","last_undegraded":"2023-09-30T09:35:23.830784+0000","last_fullsized":"2023-09-30T09:35:23.830784+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":40,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848161+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:35:23.848161+0000","last_peered":"2023-09-30T09:35:23.848161+0000","last_clean":"2023-09-30T09:35:23.848161+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:35:23.848161+0000","last_undegraded":"2023-09-30T09:35:23.848161+0000","last_fullsized":"2023-09-30T09:35:23.848161+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":238,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835272+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:35:23.835272+0000","last_peered":"2023-09-30T09:35:23.835272+0000","last_clean":"2023-09-30T09:35:23.835272+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:35:23.835272+0000","last_undegraded":"2023-09-30T09:35:23.835272+0000","last_fullsized":"2023-09-30T09:35:23.835272+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":800,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840933+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:35:23.840933+0000","last_peered":"2023-09-30T09:35:23.840933+0000","last_clean":"2023-09-30T09:35:23.840933+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:35:23.840933+0000","last_undegraded":"2023-09-30T09:35:23.840933+0000","last_fullsized":"2023-09-30T09:35:23.840933+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":494,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834041+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:35:23.834041+0000","last_peered":"2023-09-30T09:35:23.834041+0000","last_clean":"2023-09-30T09:35:23.834041+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:35:23.834041+0000","last_undegraded":"2023-09-30T09:35:23.834041+0000","last_fullsized":"2023-09-30T09:35:23.834041+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":343,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835402+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:35:23.835402+0000","last_peered":"2023-09-30T09:35:23.835402+0000","last_clean":"2023-09-30T09:35:23.835402+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:35:23.835402+0000","last_undegraded":"2023-09-30T09:35:23.835402+0000","last_fullsized":"2023-09-30T09:35:23.835402+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"335'10401","reported_seq":14377,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:28.788342+0000","last_change":"2023-09-30T09:34:48.517696+0000","last_active":"2023-09-30T09:35:28.788342+0000","last_peered":"2023-09-30T09:35:28.788342+0000","last_clean":"2023-09-30T09:32:35.457744+0000","last_became_active":"2023-09-30T09:34:48.499533+0000","last_became_peered":"2023-09-30T09:34:48.499533+0000","last_unstale":"2023-09-30T09:35:28.788342+0000","last_undegraded":"2023-09-30T09:34:48.469688+0000","last_fullsized":"2023-09-30T09:35:28.788342+0000","mapping_epoch":321,"log_start":"292'5621","ondisk_log_start":"292'5621","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4780,"log_dups_size":0,"ondisk_log_size":4780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:05:42.164341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229179392,"num_objects":3497,"num_object_clones":0,"num_object_copies":10491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1608,"num_objects_misplaced":3497,"num_objects_unfound":0,"num_objects_dirty":3497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10401,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1934,"num_bytes_recovered":126746624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[6,1,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"1,4,6","objects":1889},{"shards":"4,6","objects":1608}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":790,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:28.705369+0000","last_change":"2023-09-30T09:35:28.705369+0000","last_active":"2023-09-30T09:35:28.705369+0000","last_peered":"2023-09-30T09:35:28.705369+0000","last_clean":"2023-09-30T09:35:28.705369+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:35:28.705369+0000","last_undegraded":"2023-09-30T09:35:28.705369+0000","last_fullsized":"2023-09-30T09:35:28.705369+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":69,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848317+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:35:23.848317+0000","last_peered":"2023-09-30T09:35:23.848317+0000","last_clean":"2023-09-30T09:35:23.848317+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:35:23.848317+0000","last_undegraded":"2023-09-30T09:35:23.848317+0000","last_fullsized":"2023-09-30T09:35:23.848317+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":24,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842632+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:35:23.842632+0000","last_peered":"2023-09-30T09:35:23.842632+0000","last_clean":"2023-09-30T09:35:23.842632+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:35:23.842632+0000","last_undegraded":"2023-09-30T09:35:23.842632+0000","last_fullsized":"2023-09-30T09:35:23.842632+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"335'10810","reported_seq":12066,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:29.230137+0000","last_change":"2023-09-30T09:35:10.766663+0000","last_active":"2023-09-30T09:35:29.230137+0000","last_peered":"2023-09-30T09:35:29.230137+0000","last_clean":"2023-09-30T09:32:35.454324+0000","last_became_active":"2023-09-30T09:33:33.826036+0000","last_became_peered":"2023-09-30T09:33:33.826036+0000","last_unstale":"2023-09-30T09:35:29.230137+0000","last_undegraded":"2023-09-30T09:33:33.732722+0000","last_fullsized":"2023-09-30T09:35:29.230137+0000","mapping_epoch":302,"log_start":"298'6299","ondisk_log_start":"298'6299","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4511,"log_dups_size":0,"ondisk_log_size":4511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:42.597216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235274240,"num_objects":3590,"num_object_clones":0,"num_object_copies":10770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10810,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":582,"num_bytes_recovered":38141952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,5,1],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":258},{"shards":"3,5","objects":3332}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":784,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834320+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:35:23.834320+0000","last_peered":"2023-09-30T09:35:23.834320+0000","last_clean":"2023-09-30T09:35:23.834320+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:35:23.834320+0000","last_undegraded":"2023-09-30T09:35:23.834320+0000","last_fullsized":"2023-09-30T09:35:23.834320+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":47,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847002+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:35:23.847002+0000","last_peered":"2023-09-30T09:35:23.847002+0000","last_clean":"2023-09-30T09:35:23.847002+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:35:23.847002+0000","last_undegraded":"2023-09-30T09:35:23.847002+0000","last_fullsized":"2023-09-30T09:35:23.847002+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":327,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848092+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:35:23.848092+0000","last_peered":"2023-09-30T09:35:23.848092+0000","last_clean":"2023-09-30T09:35:23.848092+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:35:23.848092+0000","last_undegraded":"2023-09-30T09:35:23.848092+0000","last_fullsized":"2023-09-30T09:35:23.848092+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"335'10431","reported_seq":10665,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:29.025179+0000","last_change":"2023-09-30T09:35:10.879711+0000","last_active":"2023-09-30T09:35:29.025179+0000","last_peered":"2023-09-30T09:35:29.025179+0000","last_clean":"2023-09-30T09:32:35.467048+0000","last_became_active":"2023-09-30T09:35:10.845361+0000","last_became_peered":"2023-09-30T09:35:10.845361+0000","last_unstale":"2023-09-30T09:35:29.025179+0000","last_undegraded":"2023-09-30T09:35:10.791338+0000","last_fullsized":"2023-09-30T09:35:29.025179+0000","mapping_epoch":327,"log_start":"292'5576","ondisk_log_start":"292'5576","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4855,"log_dups_size":0,"ondisk_log_size":4855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:41.089234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232325120,"num_objects":3545,"num_object_clones":0,"num_object_copies":10635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3545,"num_objects_misplaced":7090,"num_objects_unfound":0,"num_objects_dirty":3545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10431,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":3997696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[6,1,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":3545}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":618,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.836939+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:35:23.836939+0000","last_peered":"2023-09-30T09:35:23.836939+0000","last_clean":"2023-09-30T09:35:23.836939+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:35:23.836939+0000","last_undegraded":"2023-09-30T09:35:23.836939+0000","last_fullsized":"2023-09-30T09:35:23.836939+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":28,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841215+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:35:23.841215+0000","last_peered":"2023-09-30T09:35:23.841215+0000","last_clean":"2023-09-30T09:35:23.841215+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:35:23.841215+0000","last_undegraded":"2023-09-30T09:35:23.841215+0000","last_fullsized":"2023-09-30T09:35:23.841215+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":314,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.836435+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:35:23.836435+0000","last_peered":"2023-09-30T09:35:23.836435+0000","last_clean":"2023-09-30T09:35:23.836435+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:35:23.836435+0000","last_undegraded":"2023-09-30T09:35:23.836435+0000","last_fullsized":"2023-09-30T09:35:23.836435+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"335'10733","reported_seq":10810,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:35:29.204312+0000","last_change":"2023-09-30T09:34:08.527633+0000","last_active":"2023-09-30T09:35:29.204312+0000","last_peered":"2023-09-30T09:35:29.204312+0000","last_clean":"2023-09-30T09:33:09.318417+0000","last_became_active":"2023-09-30T09:34:08.510746+0000","last_became_peered":"2023-09-30T09:34:08.510746+0000","last_unstale":"2023-09-30T09:35:29.204312+0000","last_undegraded":"2023-09-30T09:35:29.204312+0000","last_fullsized":"2023-09-30T09:35:29.204312+0000","mapping_epoch":308,"log_start":"298'6176","ondisk_log_start":"298'6176","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4557,"log_dups_size":0,"ondisk_log_size":4557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:07.374969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233635840,"num_objects":3565,"num_object_clones":0,"num_object_copies":10695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3565,"num_objects_unfound":0,"num_objects_dirty":3565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10733,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":3565}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":464,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:25.763024+0000","last_change":"2023-09-30T09:35:25.762995+0000","last_active":"2023-09-30T09:35:25.763024+0000","last_peered":"2023-09-30T09:35:25.763024+0000","last_clean":"2023-09-30T09:35:25.763024+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:35:25.763024+0000","last_undegraded":"2023-09-30T09:35:25.763024+0000","last_fullsized":"2023-09-30T09:35:25.763024+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":294,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841827+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:35:23.841827+0000","last_peered":"2023-09-30T09:35:23.841827+0000","last_clean":"2023-09-30T09:35:23.841827+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:35:23.841827+0000","last_undegraded":"2023-09-30T09:35:23.841827+0000","last_fullsized":"2023-09-30T09:35:23.841827+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":240,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839270+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:35:23.839270+0000","last_peered":"2023-09-30T09:35:23.839270+0000","last_clean":"2023-09-30T09:35:23.839270+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:35:23.839270+0000","last_undegraded":"2023-09-30T09:35:23.839270+0000","last_fullsized":"2023-09-30T09:35:23.839270+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"335'10586","reported_seq":10675,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:35:29.188870+0000","last_change":"2023-09-30T09:35:10.779076+0000","last_active":"2023-09-30T09:35:29.188870+0000","last_peered":"2023-09-30T09:35:29.188870+0000","last_clean":"2023-09-30T09:34:06.422096+0000","last_became_active":"2023-09-30T09:35:10.747716+0000","last_became_peered":"2023-09-30T09:35:10.747716+0000","last_unstale":"2023-09-30T09:35:29.188870+0000","last_undegraded":"2023-09-30T09:35:29.188870+0000","last_fullsized":"2023-09-30T09:35:29.188870+0000","mapping_epoch":327,"log_start":"298'6020","ondisk_log_start":"298'6020","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4566,"log_dups_size":0,"ondisk_log_size":4566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:09.191024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226754560,"num_objects":3460,"num_object_clones":0,"num_object_copies":10380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3460,"num_objects_unfound":0,"num_objects_dirty":3460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10586,"num_write_kb":449472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":3460}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.832454+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:35:23.832454+0000","last_peered":"2023-09-30T09:35:23.832454+0000","last_clean":"2023-09-30T09:35:23.832454+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:35:23.832454+0000","last_undegraded":"2023-09-30T09:35:23.832454+0000","last_fullsized":"2023-09-30T09:35:23.832454+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":362,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833922+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:35:23.833922+0000","last_peered":"2023-09-30T09:35:23.833922+0000","last_clean":"2023-09-30T09:35:23.833922+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:35:23.833922+0000","last_undegraded":"2023-09-30T09:35:23.833922+0000","last_fullsized":"2023-09-30T09:35:23.833922+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":293,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848983+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:35:23.848983+0000","last_peered":"2023-09-30T09:35:23.848983+0000","last_clean":"2023-09-30T09:35:23.848983+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:35:23.848983+0000","last_undegraded":"2023-09-30T09:35:23.848983+0000","last_fullsized":"2023-09-30T09:35:23.848983+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"335'10661","reported_seq":10882,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:29.230255+0000","last_change":"2023-09-30T09:35:10.871065+0000","last_active":"2023-09-30T09:35:29.230255+0000","last_peered":"2023-09-30T09:35:29.230255+0000","last_clean":"2023-09-30T09:32:35.449714+0000","last_became_active":"2023-09-30T09:35:10.838732+0000","last_became_peered":"2023-09-30T09:35:10.838732+0000","last_unstale":"2023-09-30T09:35:29.230255+0000","last_undegraded":"2023-09-30T09:35:10.818868+0000","last_fullsized":"2023-09-30T09:35:29.230255+0000","mapping_epoch":327,"log_start":"298'6106","ondisk_log_start":"298'6106","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4555,"log_dups_size":0,"ondisk_log_size":4555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:44.318381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235339776,"num_objects":3591,"num_object_clones":0,"num_object_copies":10773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3589,"num_objects_misplaced":7184,"num_objects_unfound":0,"num_objects_dirty":3591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10661,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":3866624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[3,7,1],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"1,3,7","objects":2},{"shards":"3,7","objects":3589}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":756,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835385+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:35:23.835385+0000","last_peered":"2023-09-30T09:35:23.835385+0000","last_clean":"2023-09-30T09:35:23.835385+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:35:23.835385+0000","last_undegraded":"2023-09-30T09:35:23.835385+0000","last_fullsized":"2023-09-30T09:35:23.835385+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":312,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834334+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:35:23.834334+0000","last_peered":"2023-09-30T09:35:23.834334+0000","last_clean":"2023-09-30T09:35:23.834334+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:35:23.834334+0000","last_undegraded":"2023-09-30T09:35:23.834334+0000","last_fullsized":"2023-09-30T09:35:23.834334+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":29,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847481+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:35:23.847481+0000","last_peered":"2023-09-30T09:35:23.847481+0000","last_clean":"2023-09-30T09:35:23.847481+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:35:23.847481+0000","last_undegraded":"2023-09-30T09:35:23.847481+0000","last_fullsized":"2023-09-30T09:35:23.847481+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"335'10534","reported_seq":13382,"reported_epoch":335,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:29.021775+0000","last_change":"2023-09-30T09:35:10.766317+0000","last_active":"2023-09-30T09:35:29.021775+0000","last_peered":"2023-09-30T09:35:29.021775+0000","last_clean":"2023-09-30T09:32:35.473843+0000","last_became_active":"2023-09-30T09:33:33.842621+0000","last_became_peered":"2023-09-30T09:33:33.842621+0000","last_unstale":"2023-09-30T09:35:29.021775+0000","last_undegraded":"2023-09-30T09:33:33.676280+0000","last_fullsized":"2023-09-30T09:33:33.596470+0000","mapping_epoch":302,"log_start":"292'5721","ondisk_log_start":"292'5721","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4813,"log_dups_size":0,"ondisk_log_size":4813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:19:47.448341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229965848,"num_objects":3510,"num_object_clones":0,"num_object_copies":10530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2181,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3510,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10534,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1381,"num_bytes_recovered":90505216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":1329}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":770,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839084+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:35:23.839084+0000","last_peered":"2023-09-30T09:35:23.839084+0000","last_clean":"2023-09-30T09:35:23.839084+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:35:23.839084+0000","last_undegraded":"2023-09-30T09:35:23.839084+0000","last_fullsized":"2023-09-30T09:35:23.839084+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":331,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847962+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:35:23.847962+0000","last_peered":"2023-09-30T09:35:23.847962+0000","last_clean":"2023-09-30T09:35:23.847962+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:35:23.847962+0000","last_undegraded":"2023-09-30T09:35:23.847962+0000","last_fullsized":"2023-09-30T09:35:23.847962+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":310,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848663+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:35:23.848663+0000","last_peered":"2023-09-30T09:35:23.848663+0000","last_clean":"2023-09-30T09:35:23.848663+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:35:23.848663+0000","last_undegraded":"2023-09-30T09:35:23.848663+0000","last_fullsized":"2023-09-30T09:35:23.848663+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"335'10632","reported_seq":10713,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:35:28.883799+0000","last_change":"2023-09-30T09:34:47.630136+0000","last_active":"2023-09-30T09:35:28.883799+0000","last_peered":"2023-09-30T09:35:28.883799+0000","last_clean":"2023-09-30T09:34:06.422409+0000","last_became_active":"2023-09-30T09:34:08.626702+0000","last_became_peered":"2023-09-30T09:34:08.626702+0000","last_unstale":"2023-09-30T09:35:28.883799+0000","last_undegraded":"2023-09-30T09:35:28.883799+0000","last_fullsized":"2023-09-30T09:35:28.883799+0000","mapping_epoch":308,"log_start":"292'5806","ondisk_log_start":"292'5806","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4826,"log_dups_size":0,"ondisk_log_size":4826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:53.753813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232259584,"num_objects":3544,"num_object_clones":0,"num_object_copies":10632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3544,"num_objects_unfound":0,"num_objects_dirty":3544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10632,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3544}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":478,"reported_epoch":335,"state":"active+clean+scrubbing","last_fresh":"2023-09-30T09:35:28.678924+0000","last_change":"2023-09-30T09:35:28.678924+0000","last_active":"2023-09-30T09:35:28.678924+0000","last_peered":"2023-09-30T09:35:28.678924+0000","last_clean":"2023-09-30T09:35:28.678924+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:35:28.678924+0000","last_undegraded":"2023-09-30T09:35:28.678924+0000","last_fullsized":"2023-09-30T09:35:28.678924+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":325,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.829843+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:35:23.829843+0000","last_peered":"2023-09-30T09:35:23.829843+0000","last_clean":"2023-09-30T09:35:23.829843+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:35:23.829843+0000","last_undegraded":"2023-09-30T09:35:23.829843+0000","last_fullsized":"2023-09-30T09:35:23.829843+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":102,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835254+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:35:23.835254+0000","last_peered":"2023-09-30T09:35:23.835254+0000","last_clean":"2023-09-30T09:35:23.835254+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:35:23.835254+0000","last_undegraded":"2023-09-30T09:35:23.835254+0000","last_fullsized":"2023-09-30T09:35:23.835254+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"335'10570","reported_seq":10765,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:29.230201+0000","last_change":"2023-09-30T09:34:39.166112+0000","last_active":"2023-09-30T09:35:29.230201+0000","last_peered":"2023-09-30T09:35:29.230201+0000","last_clean":"2023-09-30T09:32:35.465008+0000","last_became_active":"2023-09-30T09:34:39.150257+0000","last_became_peered":"2023-09-30T09:34:39.150257+0000","last_unstale":"2023-09-30T09:35:29.230201+0000","last_undegraded":"2023-09-30T09:34:38.934740+0000","last_fullsized":"2023-09-30T09:35:29.230201+0000","mapping_epoch":316,"log_start":"298'6078","ondisk_log_start":"298'6078","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4492,"log_dups_size":0,"ondisk_log_size":4492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:50.805799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223870976,"num_objects":3416,"num_object_clones":0,"num_object_copies":10248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3415,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":3416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10570,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":3473408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6,1],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1},{"shards":"3,6","objects":3415}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":629,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.837097+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:35:23.837097+0000","last_peered":"2023-09-30T09:35:23.837097+0000","last_clean":"2023-09-30T09:35:23.837097+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:35:23.837097+0000","last_undegraded":"2023-09-30T09:35:23.837097+0000","last_fullsized":"2023-09-30T09:35:23.837097+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":526,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839730+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:35:23.839730+0000","last_peered":"2023-09-30T09:35:23.839730+0000","last_clean":"2023-09-30T09:35:23.839730+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:35:23.839730+0000","last_undegraded":"2023-09-30T09:35:23.839730+0000","last_fullsized":"2023-09-30T09:35:23.839730+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":331,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841491+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:35:23.841491+0000","last_peered":"2023-09-30T09:35:23.841491+0000","last_clean":"2023-09-30T09:35:23.841491+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:35:23.841491+0000","last_undegraded":"2023-09-30T09:35:23.841491+0000","last_fullsized":"2023-09-30T09:35:23.841491+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840670+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:35:23.840670+0000","last_peered":"2023-09-30T09:35:23.840670+0000","last_clean":"2023-09-30T09:35:23.840670+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:35:23.840670+0000","last_undegraded":"2023-09-30T09:35:23.840670+0000","last_fullsized":"2023-09-30T09:35:23.840670+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"335'10702","reported_seq":10765,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:29.026861+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:35:29.026861+0000","last_peered":"2023-09-30T09:35:29.026861+0000","last_clean":"2023-09-30T09:35:29.026861+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:35:29.026861+0000","last_undegraded":"2023-09-30T09:35:29.026861+0000","last_fullsized":"2023-09-30T09:35:29.026861+0000","mapping_epoch":279,"log_start":"292'5857","ondisk_log_start":"292'5857","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4845,"log_dups_size":0,"ondisk_log_size":4845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232914944,"num_objects":3554,"num_object_clones":0,"num_object_copies":10662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10702,"num_write_kb":456192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":797,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840585+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:35:23.840585+0000","last_peered":"2023-09-30T09:35:23.840585+0000","last_clean":"2023-09-30T09:35:23.840585+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:35:23.840585+0000","last_undegraded":"2023-09-30T09:35:23.840585+0000","last_fullsized":"2023-09-30T09:35:23.840585+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":326,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834286+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:35:23.834286+0000","last_peered":"2023-09-30T09:35:23.834286+0000","last_clean":"2023-09-30T09:35:23.834286+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:35:23.834286+0000","last_undegraded":"2023-09-30T09:35:23.834286+0000","last_fullsized":"2023-09-30T09:35:23.834286+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":314,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847526+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:35:23.847526+0000","last_peered":"2023-09-30T09:35:23.847526+0000","last_clean":"2023-09-30T09:35:23.847526+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:35:23.847526+0000","last_undegraded":"2023-09-30T09:35:23.847526+0000","last_fullsized":"2023-09-30T09:35:23.847526+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":52,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848589+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:35:23.848589+0000","last_peered":"2023-09-30T09:35:23.848589+0000","last_clean":"2023-09-30T09:35:23.848589+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:35:23.848589+0000","last_undegraded":"2023-09-30T09:35:23.848589+0000","last_fullsized":"2023-09-30T09:35:23.848589+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"335'10686","reported_seq":10890,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:29.024322+0000","last_change":"2023-09-30T09:35:10.808625+0000","last_active":"2023-09-30T09:35:29.024322+0000","last_peered":"2023-09-30T09:35:29.024322+0000","last_clean":"2023-09-30T09:32:35.460821+0000","last_became_active":"2023-09-30T09:35:10.778683+0000","last_became_peered":"2023-09-30T09:35:10.778683+0000","last_unstale":"2023-09-30T09:35:29.024322+0000","last_undegraded":"2023-09-30T09:35:10.766189+0000","last_fullsized":"2023-09-30T09:35:29.024322+0000","mapping_epoch":327,"log_start":"293'5833","ondisk_log_start":"293'5833","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4853,"log_dups_size":0,"ondisk_log_size":4853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:45.905717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240254976,"num_objects":3666,"num_object_clones":0,"num_object_copies":10998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3665,"num_objects_misplaced":7332,"num_objects_unfound":0,"num_objects_dirty":3666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10686,"num_write_kb":459264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":3342336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[6,1,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"1,5,6","objects":1},{"shards":"5,6","objects":3665}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":694,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.838890+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:35:23.838890+0000","last_peered":"2023-09-30T09:35:23.838890+0000","last_clean":"2023-09-30T09:35:23.838890+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:35:23.838890+0000","last_undegraded":"2023-09-30T09:35:23.838890+0000","last_fullsized":"2023-09-30T09:35:23.838890+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":513,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835690+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:35:23.835690+0000","last_peered":"2023-09-30T09:35:23.835690+0000","last_clean":"2023-09-30T09:35:23.835690+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:35:23.835690+0000","last_undegraded":"2023-09-30T09:35:23.835690+0000","last_fullsized":"2023-09-30T09:35:23.835690+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":60,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835579+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:35:23.835579+0000","last_peered":"2023-09-30T09:35:23.835579+0000","last_clean":"2023-09-30T09:35:23.835579+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:35:23.835579+0000","last_undegraded":"2023-09-30T09:35:23.835579+0000","last_fullsized":"2023-09-30T09:35:23.835579+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":254,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840835+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:35:23.840835+0000","last_peered":"2023-09-30T09:35:23.840835+0000","last_clean":"2023-09-30T09:35:23.840835+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:35:23.840835+0000","last_undegraded":"2023-09-30T09:35:23.840835+0000","last_fullsized":"2023-09-30T09:35:23.840835+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"335'10545","reported_seq":12967,"reported_epoch":335,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:29.953097+0000","last_change":"2023-09-30T09:35:23.879286+0000","last_active":"2023-09-30T09:35:29.953097+0000","last_peered":"2023-09-30T09:35:29.953097+0000","last_clean":"2023-09-30T09:34:06.430942+0000","last_became_active":"2023-09-30T09:34:47.755052+0000","last_became_peered":"2023-09-30T09:34:47.755052+0000","last_unstale":"2023-09-30T09:35:29.953097+0000","last_undegraded":"2023-09-30T09:34:47.557422+0000","last_fullsized":"2023-09-30T09:34:47.514378+0000","mapping_epoch":320,"log_start":"290'4350","ondisk_log_start":"290'4350","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":6195,"log_dups_size":0,"ondisk_log_size":6195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:44:56.113242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228130816,"num_objects":3481,"num_object_clones":0,"num_object_copies":10443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":141,"num_objects_misplaced":3479,"num_objects_unfound":0,"num_objects_dirty":3481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10545,"num_write_kb":448832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1170,"num_bytes_recovered":76349440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-139},{"shards":"0,3,7","objects":3479}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":907,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842219+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:35:23.842219+0000","last_peered":"2023-09-30T09:35:23.842219+0000","last_clean":"2023-09-30T09:35:23.842219+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:35:23.842219+0000","last_undegraded":"2023-09-30T09:35:23.842219+0000","last_fullsized":"2023-09-30T09:35:23.842219+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":599,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.838645+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:35:23.838645+0000","last_peered":"2023-09-30T09:35:23.838645+0000","last_clean":"2023-09-30T09:35:23.838645+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:35:23.838645+0000","last_undegraded":"2023-09-30T09:35:23.838645+0000","last_fullsized":"2023-09-30T09:35:23.838645+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":253,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842053+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:35:23.842053+0000","last_peered":"2023-09-30T09:35:23.842053+0000","last_clean":"2023-09-30T09:35:23.842053+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:35:23.842053+0000","last_undegraded":"2023-09-30T09:35:23.842053+0000","last_fullsized":"2023-09-30T09:35:23.842053+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":287,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835692+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:35:23.835692+0000","last_peered":"2023-09-30T09:35:23.835692+0000","last_clean":"2023-09-30T09:35:23.835692+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:35:23.835692+0000","last_undegraded":"2023-09-30T09:35:23.835692+0000","last_fullsized":"2023-09-30T09:35:23.835692+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"335'10631","reported_seq":18883,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:29.341564+0000","last_change":"2023-09-30T09:35:10.837720+0000","last_active":"2023-09-30T09:35:29.341564+0000","last_peered":"2023-09-30T09:35:29.341564+0000","last_clean":"2023-09-30T09:32:35.481650+0000","last_became_active":"2023-09-30T09:35:10.788829+0000","last_became_peered":"2023-09-30T09:35:10.788829+0000","last_unstale":"2023-09-30T09:35:29.341564+0000","last_undegraded":"2023-09-30T09:35:10.757584+0000","last_fullsized":"2023-09-30T09:35:29.341564+0000","mapping_epoch":327,"log_start":"286'3000","ondisk_log_start":"286'3000","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":7631,"log_dups_size":0,"ondisk_log_size":7631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:05.384343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231145472,"num_objects":3527,"num_object_clones":0,"num_object_copies":10581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1837,"num_objects_misplaced":6123,"num_objects_unfound":0,"num_objects_dirty":3527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10631,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4068,"num_bytes_recovered":266600448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[7,1,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":931},{"shards":"2,6,7","objects":1240},{"shards":"6,7","objects":1356}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":480,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835513+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:35:23.835513+0000","last_peered":"2023-09-30T09:35:23.835513+0000","last_clean":"2023-09-30T09:35:23.835513+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:35:23.835513+0000","last_undegraded":"2023-09-30T09:35:23.835513+0000","last_fullsized":"2023-09-30T09:35:23.835513+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":710,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.838721+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:35:23.838721+0000","last_peered":"2023-09-30T09:35:23.838721+0000","last_clean":"2023-09-30T09:35:23.838721+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:35:23.838721+0000","last_undegraded":"2023-09-30T09:35:23.838721+0000","last_fullsized":"2023-09-30T09:35:23.838721+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":262,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.829085+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:35:23.829085+0000","last_peered":"2023-09-30T09:35:23.829085+0000","last_clean":"2023-09-30T09:35:23.829085+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:35:23.829085+0000","last_undegraded":"2023-09-30T09:35:23.829085+0000","last_fullsized":"2023-09-30T09:35:23.829085+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.831437+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:35:23.831437+0000","last_peered":"2023-09-30T09:35:23.831437+0000","last_clean":"2023-09-30T09:35:23.831437+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:35:23.831437+0000","last_undegraded":"2023-09-30T09:35:23.831437+0000","last_fullsized":"2023-09-30T09:35:23.831437+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"335'10595","reported_seq":15037,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:29.276117+0000","last_change":"2023-09-30T09:34:47.553318+0000","last_active":"2023-09-30T09:35:29.276117+0000","last_peered":"2023-09-30T09:35:29.276117+0000","last_clean":"2023-09-30T09:32:35.469075+0000","last_became_active":"2023-09-30T09:33:33.792569+0000","last_became_peered":"2023-09-30T09:33:33.792569+0000","last_unstale":"2023-09-30T09:35:29.276117+0000","last_undegraded":"2023-09-30T09:33:33.610726+0000","last_fullsized":"2023-09-30T09:33:33.587755+0000","mapping_epoch":302,"log_start":"291'5045","ondisk_log_start":"291'5045","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":5550,"log_dups_size":0,"ondisk_log_size":5550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:16.121133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233373696,"num_objects":3561,"num_object_clones":0,"num_object_copies":10683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10595,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2178,"num_bytes_recovered":142737408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2121}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":609,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834735+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:35:23.834735+0000","last_peered":"2023-09-30T09:35:23.834735+0000","last_clean":"2023-09-30T09:35:23.834735+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:35:23.834735+0000","last_undegraded":"2023-09-30T09:35:23.834735+0000","last_fullsized":"2023-09-30T09:35:23.834735+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":843,"reported_epoch":334,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:23.849441+0000","last_change":"2023-09-30T09:35:11.685841+0000","last_active":"2023-09-30T09:35:23.849441+0000","last_peered":"2023-09-30T09:35:23.849441+0000","last_clean":"2023-09-30T09:26:47.308941+0000","last_became_active":"2023-09-30T09:35:11.685274+0000","last_became_peered":"2023-09-30T09:35:11.685274+0000","last_unstale":"2023-09-30T09:35:23.849441+0000","last_undegraded":"2023-09-30T09:35:11.661789+0000","last_fullsized":"2023-09-30T09:35:11.661213+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:38.516737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[1,4],"avail_no_missing":["1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":60,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848432+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:35:23.848432+0000","last_peered":"2023-09-30T09:35:23.848432+0000","last_clean":"2023-09-30T09:35:23.848432+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:35:23.848432+0000","last_undegraded":"2023-09-30T09:35:23.848432+0000","last_fullsized":"2023-09-30T09:35:23.848432+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":234,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839411+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:35:23.839411+0000","last_peered":"2023-09-30T09:35:23.839411+0000","last_clean":"2023-09-30T09:35:23.839411+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:35:23.839411+0000","last_undegraded":"2023-09-30T09:35:23.839411+0000","last_fullsized":"2023-09-30T09:35:23.839411+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"335'10721","reported_seq":10886,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:35:29.170259+0000","last_change":"2023-09-30T09:33:32.864041+0000","last_active":"2023-09-30T09:35:29.170259+0000","last_peered":"2023-09-30T09:35:29.170259+0000","last_clean":"2023-09-30T09:33:09.329441+0000","last_became_active":"2023-09-30T09:33:11.506402+0000","last_became_peered":"2023-09-30T09:33:11.506402+0000","last_unstale":"2023-09-30T09:35:29.170259+0000","last_undegraded":"2023-09-30T09:35:29.170259+0000","last_fullsized":"2023-09-30T09:35:29.170259+0000","mapping_epoch":295,"log_start":"298'6198","ondisk_log_start":"298'6198","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4523,"log_dups_size":0,"ondisk_log_size":4523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:51:06.485777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230883328,"num_objects":3523,"num_object_clones":0,"num_object_copies":10569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3498,"num_objects_unfound":0,"num_objects_dirty":3523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10721,"num_write_kb":455808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":3523}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":807,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847716+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:35:23.847716+0000","last_peered":"2023-09-30T09:35:23.847716+0000","last_clean":"2023-09-30T09:35:23.847716+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:35:23.847716+0000","last_undegraded":"2023-09-30T09:35:23.847716+0000","last_fullsized":"2023-09-30T09:35:23.847716+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":335,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835333+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:35:23.835333+0000","last_peered":"2023-09-30T09:35:23.835333+0000","last_clean":"2023-09-30T09:35:23.835333+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:35:23.835333+0000","last_undegraded":"2023-09-30T09:35:23.835333+0000","last_fullsized":"2023-09-30T09:35:23.835333+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1101,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834573+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:35:23.834573+0000","last_peered":"2023-09-30T09:35:23.834573+0000","last_clean":"2023-09-30T09:35:23.834573+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:35:23.834573+0000","last_undegraded":"2023-09-30T09:35:23.834573+0000","last_fullsized":"2023-09-30T09:35:23.834573+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":352,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.906364+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:35:27.906364+0000","last_peered":"2023-09-30T09:35:27.906364+0000","last_clean":"2023-09-30T09:35:27.906364+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:35:27.906364+0000","last_undegraded":"2023-09-30T09:35:27.906364+0000","last_fullsized":"2023-09-30T09:35:27.906364+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"335'10737","reported_seq":12720,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:29.140890+0000","last_change":"2023-09-30T09:34:37.846918+0000","last_active":"2023-09-30T09:35:29.140890+0000","last_peered":"2023-09-30T09:35:29.140890+0000","last_clean":"2023-09-30T09:32:35.469896+0000","last_became_active":"2023-09-30T09:33:32.854592+0000","last_became_peered":"2023-09-30T09:33:32.854592+0000","last_unstale":"2023-09-30T09:35:29.140890+0000","last_undegraded":"2023-09-30T09:33:32.787590+0000","last_fullsized":"2023-09-30T09:33:32.601623+0000","mapping_epoch":301,"log_start":"298'6250","ondisk_log_start":"298'6250","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:02.368619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232456192,"num_objects":3547,"num_object_clones":0,"num_object_copies":10641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2653,"num_objects_misplaced":3547,"num_objects_unfound":0,"num_objects_dirty":3547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10737,"num_write_kb":457088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":946,"num_bytes_recovered":61997056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":894}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":800,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834705+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:35:23.834705+0000","last_peered":"2023-09-30T09:35:23.834705+0000","last_clean":"2023-09-30T09:35:23.834705+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:35:23.834705+0000","last_undegraded":"2023-09-30T09:35:23.834705+0000","last_fullsized":"2023-09-30T09:35:23.834705+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":660,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.837245+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:35:23.837245+0000","last_peered":"2023-09-30T09:35:23.837245+0000","last_clean":"2023-09-30T09:35:23.837245+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:35:23.837245+0000","last_undegraded":"2023-09-30T09:35:23.837245+0000","last_fullsized":"2023-09-30T09:35:23.837245+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":22,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.831528+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:35:23.831528+0000","last_peered":"2023-09-30T09:35:23.831528+0000","last_clean":"2023-09-30T09:35:23.831528+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:35:23.831528+0000","last_undegraded":"2023-09-30T09:35:23.831528+0000","last_fullsized":"2023-09-30T09:35:23.831528+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":276,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848029+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:35:23.848029+0000","last_peered":"2023-09-30T09:35:23.848029+0000","last_clean":"2023-09-30T09:35:23.848029+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:35:23.848029+0000","last_undegraded":"2023-09-30T09:35:23.848029+0000","last_fullsized":"2023-09-30T09:35:23.848029+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":748,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.832370+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:35:23.832370+0000","last_peered":"2023-09-30T09:35:23.832370+0000","last_clean":"2023-09-30T09:35:23.832370+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:35:23.832370+0000","last_undegraded":"2023-09-30T09:35:23.832370+0000","last_fullsized":"2023-09-30T09:35:23.832370+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":18,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839344+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:35:23.839344+0000","last_peered":"2023-09-30T09:35:23.839344+0000","last_clean":"2023-09-30T09:35:23.839344+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:35:23.839344+0000","last_undegraded":"2023-09-30T09:35:23.839344+0000","last_fullsized":"2023-09-30T09:35:23.839344+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:25.342082+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:35:25.342082+0000","last_peered":"2023-09-30T09:35:25.342082+0000","last_clean":"2023-09-30T09:35:25.342082+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:35:25.342082+0000","last_undegraded":"2023-09-30T09:35:25.342082+0000","last_fullsized":"2023-09-30T09:35:25.342082+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":519,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.858975+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:35:23.858975+0000","last_peered":"2023-09-30T09:35:23.858975+0000","last_clean":"2023-09-30T09:35:23.858975+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:35:23.858975+0000","last_undegraded":"2023-09-30T09:35:23.858975+0000","last_fullsized":"2023-09-30T09:35:23.858975+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":313,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841765+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:35:23.841765+0000","last_peered":"2023-09-30T09:35:23.841765+0000","last_clean":"2023-09-30T09:35:23.841765+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:35:23.841765+0000","last_undegraded":"2023-09-30T09:35:23.841765+0000","last_fullsized":"2023-09-30T09:35:23.841765+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":29,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.846976+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:35:23.846976+0000","last_peered":"2023-09-30T09:35:23.846976+0000","last_clean":"2023-09-30T09:35:23.846976+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:35:23.846976+0000","last_undegraded":"2023-09-30T09:35:23.846976+0000","last_fullsized":"2023-09-30T09:35:23.846976+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":279,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841418+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:35:23.841418+0000","last_peered":"2023-09-30T09:35:23.841418+0000","last_clean":"2023-09-30T09:35:23.841418+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:35:23.841418+0000","last_undegraded":"2023-09-30T09:35:23.841418+0000","last_fullsized":"2023-09-30T09:35:23.841418+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":29,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847308+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:35:23.847308+0000","last_peered":"2023-09-30T09:35:23.847308+0000","last_clean":"2023-09-30T09:35:23.847308+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:35:23.847308+0000","last_undegraded":"2023-09-30T09:35:23.847308+0000","last_fullsized":"2023-09-30T09:35:23.847308+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":463,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.837663+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:35:23.837663+0000","last_peered":"2023-09-30T09:35:23.837663+0000","last_clean":"2023-09-30T09:35:23.837663+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:35:23.837663+0000","last_undegraded":"2023-09-30T09:35:23.837663+0000","last_fullsized":"2023-09-30T09:35:23.837663+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":331,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.891275+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:35:27.891275+0000","last_peered":"2023-09-30T09:35:27.891275+0000","last_clean":"2023-09-30T09:35:27.891275+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:35:27.891275+0000","last_undegraded":"2023-09-30T09:35:27.891275+0000","last_fullsized":"2023-09-30T09:35:27.891275+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":355,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.832579+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:35:23.832579+0000","last_peered":"2023-09-30T09:35:23.832579+0000","last_clean":"2023-09-30T09:35:23.832579+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:35:23.832579+0000","last_undegraded":"2023-09-30T09:35:23.832579+0000","last_fullsized":"2023-09-30T09:35:23.832579+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":783,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833541+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:35:23.833541+0000","last_peered":"2023-09-30T09:35:23.833541+0000","last_clean":"2023-09-30T09:35:23.833541+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:35:23.833541+0000","last_undegraded":"2023-09-30T09:35:23.833541+0000","last_fullsized":"2023-09-30T09:35:23.833541+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":38,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.872212+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:35:23.872212+0000","last_peered":"2023-09-30T09:35:23.872212+0000","last_clean":"2023-09-30T09:35:23.872212+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:35:23.872212+0000","last_undegraded":"2023-09-30T09:35:23.872212+0000","last_fullsized":"2023-09-30T09:35:23.872212+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":310,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.846837+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:35:23.846837+0000","last_peered":"2023-09-30T09:35:23.846837+0000","last_clean":"2023-09-30T09:35:23.846837+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:35:23.846837+0000","last_undegraded":"2023-09-30T09:35:23.846837+0000","last_fullsized":"2023-09-30T09:35:23.846837+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3707764760,"num_objects":56577,"num_object_clones":0,"num_object_copies":169731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27405,"num_objects_misplaced":52320,"num_objects_unfound":0,"num_objects_dirty":56577,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169975,"num_write_kb":7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12530,"num_bytes_recovered":820576256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9541595136,"data_stored":9541582920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80039,"ondisk_log_size":80039,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234286,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495923,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2419452,"kb_used_data":2045300,"kb_used_omap":0,"kb_used_meta":374080,"kb_avail":91325700,"statfs":{"total":95995035648,"available":93517516800,"internally_reserved":0,"allocated":2094387200,"data_stored":2093223099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":383057920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,9,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:34:48 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.0680000000000001,"15min":1.387},"min":{"1min":0.23699999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.968,"5min":19.882000000000001,"15min":45.927999999999997},"last":1.704},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.724,"15min":1.7070000000000001},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.23499999999999999},"max":{"1min":5.8730000000000002,"5min":20.373000000000001,"15min":45.316000000000003},"last":9.0939999999999994}]},{"osd":1,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.6039999999999992,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":1.7390000000000001},{"interface":"front","average":{"1min":2.0630000000000002,"5min":2.0630000000000002,"15min":2.0630000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":8.5869999999999997,"5min":8.5869999999999997,"15min":8.5869999999999997},"last":0.92500000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":1.738,"15min":1.7290000000000001},"min":{"1min":0.32800000000000001,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":8.0850000000000009,"5min":22.158000000000001,"15min":37.978000000000002},"last":0.82699999999999996},{"interface":"front","average":{"1min":2.6360000000000001,"5min":2.0289999999999999,"15min":1.641},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.706,"5min":22.356999999999999,"15min":37.759999999999998},"last":1.8140000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":3.1960000000000002,"15min":1.921},"min":{"1min":0.34699999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1920000000000002,"5min":80.436999999999998,"15min":80.436999999999998},"last":1.77},{"interface":"front","average":{"1min":2.827,"5min":3.6720000000000002,"15min":2.1970000000000001},"min":{"1min":0.39000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":13.340999999999999,"5min":80.039000000000001,"15min":80.039000000000001},"last":4.7169999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:34:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.496,"15min":1.554},"min":{"1min":0.31,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.2890000000000001,"5min":8.1059999999999999,"15min":73.656000000000006},"last":0.872},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.2330000000000001,"15min":1.3520000000000001},"min":{"1min":0.30099999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.8200000000000001,"5min":7.7160000000000002,"15min":73.882999999999996},"last":1.8380000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":3.0289999999999999,"15min":1.7809999999999999},"min":{"1min":0.33200000000000002,"5min":0.29099999999999998,"15min":0.28499999999999998},"max":{"1min":5.4880000000000004,"5min":76.613,"15min":76.613},"last":1.5149999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":2.9039999999999999,"15min":1.7250000000000001},"min":{"1min":0.36199999999999999,"5min":0.31,"15min":0.31},"max":{"1min":1.246,"5min":76.329999999999998,"15min":76.329999999999998},"last":1.7909999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:35:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.712,"15min":2.1320000000000001},"min":{"1min":0.38,"5min":0.377,"15min":0.32100000000000001},"max":{"1min":8.7400000000000002,"5min":8.7400000000000002,"15min":91.120999999999995},"last":2.669},{"interface":"front","average":{"1min":1.306,"5min":1.448,"15min":2.0710000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.9489999999999998,"5min":9.9489999999999998,"15min":91.733999999999995},"last":1.863}]}]},{"osd":1,"up_from":301,"seq":1292785156121,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591400,"kb_used_data":463136,"kb_used_omap":9,"kb_used_meta":128246,"kb_avail":93153752,"statfs":{"total":95995035648,"available":95389442048,"internally_reserved":0,"allocated":474251264,"data_stored":472356897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":131323922},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":2.5419999999999998,"15min":2.5419999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.016999999999999,"5min":10.016999999999999,"15min":10.016999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.024,"5min":3.024,"15min":3.024},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":10.061,"5min":10.061,"15min":10.061},"last":0.437}]},{"osd":2,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":2.7959999999999998,"15min":2.7959999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.095000000000001,"5min":10.095000000000001,"15min":10.095000000000001},"last":3.5529999999999999},{"interface":"front","average":{"1min":2.706,"5min":2.706,"15min":2.706},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.8260000000000005,"5min":9.8260000000000005,"15min":9.8260000000000005},"last":0.67800000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":9.4469999999999992,"5min":9.4469999999999992,"15min":9.4469999999999992},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":77.959000000000003,"5min":77.959000000000003,"15min":77.959000000000003},"last":4.4729999999999999},{"interface":"front","average":{"1min":10.177,"5min":10.177,"15min":10.177},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":77.828999999999994,"5min":77.828999999999994,"15min":77.828999999999994},"last":9.3930000000000007}]},{"osd":4,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":9.8770000000000007,"5min":9.8770000000000007,"15min":9.8770000000000007},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.6319999999999997,"5min":4.6319999999999997,"15min":4.6319999999999997},"last":0.45600000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.109,"15min":2.109},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":9.9860000000000007,"5min":9.9860000000000007,"15min":9.9860000000000007},"last":3.915},{"interface":"front","average":{"1min":1.601,"5min":1.601,"15min":1.601},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":5.7130000000000001,"5min":5.7130000000000001,"15min":5.7130000000000001},"last":0.497}]},{"osd":6,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":17.097999999999999,"5min":17.097999999999999,"15min":17.097999999999999},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":209.221,"5min":209.221,"15min":209.221},"last":0.752},{"interface":"front","average":{"1min":17.015999999999998,"5min":17.015999999999998,"15min":17.015999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":208.43600000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":3.879}]},{"osd":7,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":12.445,"5min":12.445,"15min":12.445},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":165.59399999999999,"5min":165.59399999999999,"15min":165.59399999999999},"last":3.5960000000000001},{"interface":"front","average":{"1min":12.679,"5min":12.679,"15min":12.679},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":165.833,"5min":165.833,"15min":165.833},"last":3.625}]}]},{"osd":0,"up_from":94,"seq":403726926020,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1011324,"kb_used_data":684144,"kb_used_omap":5,"kb_used_meta":327162,"kb_avail":92733828,"statfs":{"total":95995035648,"available":94959439872,"internally_reserved":0,"allocated":700563456,"data_stored":698539232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":335014141},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":2.9980000000000002,"15min":2.9980000000000002},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":17.966000000000001,"5min":17.966000000000001,"15min":17.966000000000001},"last":0.22700000000000001},{"interface":"front","average":{"1min":9.4250000000000007,"5min":9.4250000000000007,"15min":9.4250000000000007},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":138.92400000000001,"5min":138.92400000000001,"15min":138.92400000000001},"last":0.378}]},{"osd":2,"last update":"Sat Sep 30 09:35:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7639999999999998,"5min":1.756,"15min":1.5720000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.6769999999999996,"5min":11.218999999999999,"15min":55.823999999999998},"last":0.41499999999999998},{"interface":"front","average":{"1min":5.6159999999999997,"5min":2.0830000000000002,"15min":1.748},"min":{"1min":0.29299999999999998,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":40.128999999999998,"5min":40.128999999999998,"15min":55.790999999999997},"last":0.84699999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:34:53 2023","interfaces":[{"interface":"back","average":{"1min":26.254999999999999,"5min":6.7469999999999999,"15min":3.419},"min":{"1min":0.35599999999999998,"5min":0.221,"15min":0.221},"max":{"1min":218.06999999999999,"5min":218.06999999999999,"15min":218.06999999999999},"last":9.1720000000000006},{"interface":"front","average":{"1min":18.782,"5min":5.2729999999999997,"15min":2.8730000000000002},"min":{"1min":0.46100000000000002,"5min":0.245,"15min":0.245},"max":{"1min":218.63200000000001,"5min":218.63200000000001,"15min":218.63200000000001},"last":5.0309999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:34:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.0580000000000001,"15min":1.4530000000000001},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":4.4370000000000003,"5min":7.2830000000000004,"15min":121.486},"last":0.47199999999999998},{"interface":"front","average":{"1min":9.1699999999999999,"5min":2.8980000000000001,"15min":2.242},"min":{"1min":0.40300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":136.262,"5min":136.262,"15min":136.262},"last":0.35399999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:34:45 2023","interfaces":[{"interface":"back","average":{"1min":9.3550000000000004,"5min":5.2199999999999998,"15min":2.6360000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":136.34,"5min":136.34,"15min":136.34},"last":9.0459999999999994},{"interface":"front","average":{"1min":1.9019999999999999,"5min":3.609,"15min":2.0830000000000002},"min":{"1min":0.41799999999999998,"5min":0.39900000000000002,"15min":0.33700000000000002},"max":{"1min":8.7430000000000003,"5min":135.381,"15min":135.381},"last":0.69799999999999995}]},{"osd":6,"last update":"Sat Sep 30 09:34:45 2023","interfaces":[{"interface":"back","average":{"1min":22.914000000000001,"5min":6.7789999999999999,"15min":3.0179999999999998},"min":{"1min":0.39100000000000001,"5min":0.36199999999999999,"15min":0.33300000000000002},"max":{"1min":214.785,"5min":214.785,"15min":214.785},"last":0.66700000000000004},{"interface":"front","average":{"1min":23.077000000000002,"5min":6.7469999999999999,"15min":2.9580000000000002},"min":{"1min":0.40600000000000003,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":214.93899999999999,"5min":214.93899999999999,"15min":214.93899999999999},"last":0.443}]},{"osd":7,"last update":"Sat Sep 30 09:34:45 2023","interfaces":[{"interface":"back","average":{"1min":7.4210000000000003,"5min":5.0599999999999996,"15min":2.9569999999999999},"min":{"1min":0.432,"5min":0.373,"15min":0.373},"max":{"1min":97.382000000000005,"5min":150.96899999999999,"15min":150.96899999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":6.8760000000000003,"5min":4.8369999999999997,"15min":2.7789999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":97.015000000000001,"5min":150.328,"15min":150.328},"last":0.51000000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598961,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":423072,"kb_used_data":130712,"kb_used_omap":5,"kb_used_meta":292346,"kb_avail":93322080,"statfs":{"total":95995035648,"available":95561809920,"internally_reserved":0,"allocated":133849088,"data_stored":132024437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":299362957},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":2.9449999999999998,"15min":1.9179999999999999},"min":{"1min":0.22800000000000001,"5min":0.187,"15min":0.187},"max":{"1min":77.379000000000005,"5min":77.379000000000005,"15min":77.379000000000005},"last":5.9180000000000001},{"interface":"front","average":{"1min":6.2549999999999999,"5min":2.9660000000000002,"15min":1.8140000000000001},"min":{"1min":0.25600000000000001,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":76.796999999999997,"5min":77.066999999999993,"15min":77.066999999999993},"last":3.2010000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:34:32 2023","interfaces":[{"interface":"back","average":{"1min":3.5720000000000001,"5min":3.5720000000000001,"15min":3.5720000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":14.707000000000001,"5min":14.707000000000001,"15min":14.707000000000001},"last":6.0999999999999996},{"interface":"front","average":{"1min":3.0569999999999999,"5min":3.0569999999999999,"15min":3.0569999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.179,"5min":10.179,"15min":10.179},"last":5.8559999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.4169999999999998,"5min":3.883,"15min":2.085},"min":{"1min":0.35199999999999998,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":77.370000000000005,"5min":78.915999999999997,"15min":78.915999999999997},"last":3.2349999999999999},{"interface":"front","average":{"1min":8.0079999999999991,"5min":3.9199999999999999,"15min":2.2669999999999999},"min":{"1min":0.30199999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":80.834000000000003,"5min":80.834000000000003,"15min":80.834000000000003},"last":5.9470000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.8029999999999999,"15min":1.25},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":5.524,"5min":35.972000000000001,"15min":35.972000000000001},"last":3.2599999999999998},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.7470000000000001,"15min":1.4810000000000001},"min":{"1min":0.34799999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":5.8789999999999996,"5min":36.518000000000001,"15min":36.518000000000001},"last":5.8789999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.732,"15min":1.9410000000000001},"min":{"1min":0.377,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":12.651999999999999,"5min":12.651999999999999,"15min":155.477},"last":6.0190000000000001},{"interface":"front","average":{"1min":1.794,"5min":1.4119999999999999,"15min":1.8660000000000001},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":4.4560000000000004,"5min":6.048,"15min":155.46000000000001},"last":3.286}]},{"osd":6,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":6.7009999999999996,"15min":3.1899999999999999},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.9729999999999999,"5min":154.417,"15min":154.417},"last":5.9729999999999999},{"interface":"front","average":{"1min":3.1589999999999998,"5min":6.9749999999999996,"15min":3.1960000000000002},"min":{"1min":0.47399999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":12.710000000000001,"5min":154.04400000000001,"15min":154.04400000000001},"last":6.0490000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":3.665,"15min":3},"min":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8399999999999999,"5min":123.294,"15min":213.404},"last":5.8399999999999999},{"interface":"front","average":{"1min":2.234,"5min":3.9769999999999999,"15min":3.0219999999999998},"min":{"1min":0.438,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":12.686999999999999,"5min":123.16,"15min":213.27199999999999},"last":6.0750000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435433,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3072108,"kb_used_data":2507360,"kb_used_omap":2,"kb_used_meta":564733,"kb_avail":90673044,"statfs":{"total":95995035648,"available":92849197056,"internally_reserved":0,"allocated":2567536640,"data_stored":2565581236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":578286951},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,5,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":3.1619999999999999,"15min":2.395},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":17.241,"5min":102.57299999999999,"15min":130.946},"last":2.3969999999999998},{"interface":"front","average":{"1min":4.3620000000000001,"5min":3.516,"15min":2.476},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":26.687000000000001,"5min":102.745,"15min":130.96899999999999},"last":2.1970000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:34:31 2023","interfaces":[{"interface":"back","average":{"1min":21.266999999999999,"5min":21.266999999999999,"15min":21.266999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":273.99599999999998,"5min":273.99599999999998,"15min":273.99599999999998},"last":5.6529999999999996},{"interface":"front","average":{"1min":19.021999999999998,"5min":19.021999999999998,"15min":19.021999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":274.21600000000001,"5min":274.21600000000001,"15min":274.21600000000001},"last":13.000999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":7.2629999999999999,"5min":2.851,"15min":2.6669999999999998},"min":{"1min":0.378,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":76.974000000000004,"5min":76.974000000000004,"15min":145.75899999999999},"last":2.7509999999999999},{"interface":"front","average":{"1min":6.0830000000000002,"5min":2.5299999999999998,"15min":2.496},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":77.593999999999994,"5min":77.593999999999994,"15min":150.358},"last":0.375}]},{"osd":4,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.788,"15min":1.2589999999999999},"min":{"1min":0.32400000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":17.300999999999998,"5min":17.300999999999998,"15min":17.300999999999998},"last":0.54900000000000004},{"interface":"front","average":{"1min":3.9079999999999999,"5min":1.7569999999999999,"15min":1.3640000000000001},"min":{"1min":0.378,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":10.731,"5min":10.731,"15min":15.638},"last":2.2999999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":3.7839999999999998,"5min":2.4350000000000001,"15min":1.6399999999999999},"min":{"1min":0.52200000000000002,"5min":0.33100000000000002,"15min":0.308},"max":{"1min":10.77,"5min":37.485999999999997,"15min":37.485999999999997},"last":2.3380000000000001},{"interface":"front","average":{"1min":3.8730000000000002,"5min":2.7120000000000002,"15min":1.79},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.33500000000000002},"max":{"1min":24.654,"5min":37.448,"15min":37.448},"last":2.7970000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":1.905,"15min":1.3400000000000001},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":17.344999999999999,"5min":17.344999999999999,"15min":17.344999999999999},"last":0.504},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.0259999999999998,"15min":1.5469999999999999},"min":{"1min":0.49399999999999999,"5min":0.38900000000000001,"15min":0.34799999999999998},"max":{"1min":10.818,"5min":10.818,"15min":14.69},"last":2.3610000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":4.3120000000000003,"5min":2.4529999999999998,"15min":2.3809999999999998},"min":{"1min":0.504,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":24.687000000000001,"5min":26.469000000000001,"15min":226.03999999999999},"last":2.8330000000000002},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.0640000000000001,"15min":2.1110000000000002},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":17.379000000000001,"5min":26.428000000000001,"15min":226.06100000000001},"last":0.45400000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019196,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1145716,"kb_used_data":904876,"kb_used_omap":0,"kb_used_meta":240832,"kb_avail":92599436,"statfs":{"total":95995035648,"available":94821822464,"internally_reserved":0,"allocated":926593024,"data_stored":925436618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":246611968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:34:31 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":1.1930000000000001,"15min":2.6760000000000002},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.20599999999999999},"max":{"1min":2.8660000000000001,"5min":5.5599999999999996,"15min":128.79300000000001},"last":1.4490000000000001},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.357,"15min":2.6080000000000001},"min":{"1min":0.311,"5min":0.25,"15min":0.248},"max":{"1min":9.1460000000000008,"5min":11.744,"15min":128.38800000000001},"last":2.7090000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:34:31 2023","interfaces":[{"interface":"back","average":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":9.5039999999999996},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":160.69999999999999,"5min":160.69999999999999,"15min":160.69999999999999},"last":2.4079999999999999},{"interface":"front","average":{"1min":9.5649999999999995,"5min":9.5649999999999995,"15min":9.5649999999999995},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":160.50700000000001,"5min":160.50700000000001,"15min":160.50700000000001},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:35:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":1.8200000000000001,"15min":2.8410000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.096,"5min":19.391999999999999,"15min":140.601},"last":0.63200000000000001},{"interface":"front","average":{"1min":2.089,"5min":1.865,"15min":2.8130000000000002},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.309},"max":{"1min":11.057,"5min":19.422999999999998,"15min":140.54900000000001},"last":0.63900000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:34:44 2023","interfaces":[{"interface":"back","average":{"1min":5.1029999999999998,"5min":3.1320000000000001,"15min":3.2770000000000001},"min":{"1min":0.315,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":52.762,"5min":70.241,"15min":281.42200000000003},"last":3.3399999999999999},{"interface":"front","average":{"1min":4.7389999999999999,"5min":2.9169999999999998,"15min":3.218},"min":{"1min":0.32800000000000001,"5min":0.312,"15min":0.312},"max":{"1min":52.790999999999997,"5min":70.433999999999997,"15min":281.44999999999999},"last":3.403}]},{"osd":5,"last update":"Sat Sep 30 09:34:31 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":2.8500000000000001,"15min":1.641},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":1.4950000000000001,"5min":134.30600000000001,"15min":134.30600000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.0089999999999999,"5min":2.8500000000000001,"15min":1.746},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":6.3929999999999998,"5min":134.35300000000001,"15min":134.35300000000001},"last":0.47699999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":3.081,"15min":1.514},"min":{"1min":0.30499999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.4470000000000001,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.95699999999999996,"5min":2.9609999999999999,"15min":1.5469999999999999},"min":{"1min":0.30199999999999999,"5min":0.249,"15min":0.249},"max":{"1min":7.8079999999999998,"5min":202.42400000000001,"15min":202.42400000000001},"last":0.499}]},{"osd":7,"last update":"Sat Sep 30 09:35:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":4.6449999999999996,"15min":2.9969999999999999},"min":{"1min":0.35099999999999998,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":7.5220000000000002,"5min":230.90100000000001,"15min":230.90100000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.67500000000000004,"5min":4.2539999999999996,"15min":2.5510000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":3.7360000000000002,"5min":229.97,"15min":229.97},"last":0.67300000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757560,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":945460,"kb_used_data":698604,"kb_used_omap":0,"kb_used_meta":246848,"kb_avail":92799692,"statfs":{"total":95995035648,"available":95026884608,"internally_reserved":0,"allocated":715370496,"data_stored":714210638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":252772352},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:34:42 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":1.3899999999999999,"15min":2.1339999999999999},"min":{"1min":0.253,"5min":0.224,"15min":0.22},"max":{"1min":9.1140000000000008,"5min":9.2680000000000007,"15min":148.79900000000001},"last":2.1749999999999998},{"interface":"front","average":{"1min":1.77,"5min":1.456,"15min":2.3879999999999999},"min":{"1min":0.28199999999999997,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":5.7160000000000002,"5min":6.524,"15min":148.779},"last":2.9319999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:34:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.6520000000000001,"5min":5.6520000000000001,"15min":5.6520000000000001},"last":5.101},{"interface":"front","average":{"1min":3.1619999999999999,"5min":3.1619999999999999,"15min":3.1619999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.603999999999999,"5min":10.603999999999999,"15min":10.603999999999999},"last":2.2519999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:35:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2679999999999998,"5min":1.8089999999999999,"15min":3.0049999999999999},"min":{"1min":0.42799999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":30.613,"5min":30.613,"15min":165.768},"last":2.9129999999999998},{"interface":"front","average":{"1min":3.484,"5min":1.6910000000000001,"15min":3.4540000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":36.488,"5min":36.488,"15min":166.24000000000001},"last":4.3920000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:34:51 2023","interfaces":[{"interface":"back","average":{"1min":3.7850000000000001,"5min":7.5519999999999996,"15min":3.5590000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":35.737000000000002,"5min":226.34700000000001,"15min":226.34700000000001},"last":1.621},{"interface":"front","average":{"1min":4.3319999999999999,"5min":7.5780000000000003,"15min":3.4220000000000002},"min":{"1min":0.39100000000000001,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":35.689,"5min":226.36000000000001,"15min":226.36000000000001},"last":0.48599999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:34:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.081,"15min":0.85199999999999998},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":5.7160000000000002,"5min":7.4299999999999997,"15min":8.5359999999999996},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.877,"5min":1.0700000000000001,"15min":0.90500000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.7029999999999998,"5min":4.2389999999999999,"15min":9.9640000000000004},"last":2.2080000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:35:02 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":3.2669999999999999,"15min":1.716},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":5.7539999999999996,"5min":155.602,"15min":155.602},"last":2.633},{"interface":"front","average":{"1min":0.92900000000000005,"5min":3.2749999999999999,"15min":1.841},"min":{"1min":0.36499999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.8620000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":2.8759999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.766,"15min":1.633},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":2.8260000000000001,"5min":133.40700000000001,"15min":133.40700000000001},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.3939999999999999,"5min":2.8260000000000001,"15min":1.5629999999999999},"min":{"1min":0.35599999999999998,"5min":0.313,"15min":0.29999999999999999},"max":{"1min":5.6829999999999998,"5min":132.797,"15min":132.797},"last":0.44500000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":687804416,"data_stored":687800344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":461180928,"data_stored":461176856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":128647168,"data_stored":128647168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2545090560,"data_stored":2545090560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":921698304,"data_stored":921698304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":709033984,"data_stored":709033984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2083917824,"data_stored":2083913752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2004221952,"data_stored":2004221952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:35:36.919 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:35:37.219 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:37.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:37.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:37.325 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:37.612 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:37.635 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:35:37.635 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145688 5.e 5:72fbdf89:::benchmark_data_smithi019_287194_object62910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 527.26369726899998, 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.140027033, 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.636 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145688 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:35:37.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364689+0000", 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.638 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364691+0000", 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3039999999999999e-06 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364696+0000", 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5860000000000004e-06 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364709+0000", 2023-09-30T09:35:37.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2938999999999999e-05 2023-09-30T09:35:37.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364716+0000", 2023-09-30T09:35:37.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7009999999999999e-06 2023-09-30T09:35:37.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364773+0000", 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7463999999999997e-05 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.364784+0000", 2023-09-30T09:35:37.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.077e-05 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305414+0000", 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94062986800000004 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305424+0000", 2023-09-30T09:35:37.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6339999999999996e-06 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.474904+0000", 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.169480576 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.475014+0000", 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011030400000000001 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4,5", 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.475093+0000", 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8994999999999998e-05 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478115+0000", 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030214360000000002 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.481419+0000", 2023-09-30T09:35:37.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033041759999999998 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.487360+0000", 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0059407549999999998 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504511+0000", 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017150822 2023-09-30T09:35:37.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504581+0000", 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0104000000000005e-05 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504716+0000", 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000135599 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.646 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.44600.0:145692 5.e 5:778ecb27:::benchmark_data_smithi019_287194_object62914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 527.25725173399996, 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1338203870000001, 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.44600", 2023-09-30T09:35:37.647 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/3328809644", 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145692 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371135+0000", 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371136+0000", 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4499999999999996e-07 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371141+0000", 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9710000000000003e-06 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371151+0000", 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0669000000000001e-05 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371157+0000", 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7219999999999996e-06 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371209+0000", 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1801000000000001e-05 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:50.371218+0000", 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7860000000000002e-06 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305986+0000", 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.934768932 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:51.305996+0000", 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7399999999999999e-06 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478142+0000", 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172145298 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478283+0000", 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000141037 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,4,5", 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.478364+0000", 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1600999999999993e-05 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.481503+0000", 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031385779999999999 2023-09-30T09:35:37.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.487053+0000", 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055504949999999999 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.495859+0000", 2023-09-30T09:35:37.654 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088061919999999991 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504770+0000", 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0089107639999999998 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504831+0000", 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1209000000000003e-05 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:26:52.504955+0000", 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012384700000000001 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:137040 7.6 7:679b4560:::benchmark_data_smithi019_404601_object23762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:19.548961+0000", 2023-09-30T09:35:37.656 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 78.079425874999998, 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14513814899999999, 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 137040 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.548961+0000", 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.548961+0000", 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.548962+0000", 2023-09-30T09:35:37.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.198e-06 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.548968+0000", 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1020000000000002e-06 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.548980+0000", 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2009e-05 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.548988+0000", 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2900000000000002e-06 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.555415+0000", 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0064265320000000004 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.555616+0000", 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020122599999999999 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.555786+0000", 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000170025 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.560572+0000", 2023-09-30T09:35:37.661 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0047862290000000004 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.569778+0000", 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0092054930000000004 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.689054+0000", 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.119276295 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.689121+0000", 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6826e-05 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:19.694099+0000", 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049779239999999999 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:138048 7.6 7:6a43176f:::benchmark_data_smithi019_404601_object24770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:21.559406+0000", 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 76.068980491000005, 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14075759199999999, 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 138048 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.664 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559406+0000", 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559406+0000", 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559407+0000", 2023-09-30T09:35:37.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.237e-06 2023-09-30T09:35:37.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559413+0000", 2023-09-30T09:35:37.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2129999999999999e-06 2023-09-30T09:35:37.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559426+0000", 2023-09-30T09:35:37.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3356000000000001e-05 2023-09-30T09:35:37.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559434+0000", 2023-09-30T09:35:37.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1210000000000007e-06 2023-09-30T09:35:37.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559502+0000", 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7787999999999995e-05 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559620+0000", 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0001177 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.669 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.559738+0000", 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000118071 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.564372+0000", 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0046348029999999998 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.670 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.671 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.578439+0000", 2023-09-30T09:35:37.671 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014066703 2023-09-30T09:35:37.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.671 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.671 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.694978+0000", 2023-09-30T09:35:37.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.116539014 2023-09-30T09:35:37.672 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.672 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.695061+0000", 2023-09-30T09:35:37.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2909000000000002e-05 2023-09-30T09:35:37.672 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:21.700164+0000", 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0051026769999999999 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:144152 7.6 7:69c63477:::benchmark_data_smithi019_404601_object30874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-09-30T09:35:37.673 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:34.342714+0000", 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 63.285672173000002, 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15453692599999999, 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 144152 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.674 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.342714+0000", 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.342714+0000", 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.342716+0000", 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6360000000000001e-06 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.342723+0000", 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5119999999999997e-06 2023-09-30T09:35:37.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.342737+0000", 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4630999999999999e-05 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.342746+0000", 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4349999999999997e-06 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.350635+0000", 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0078897510000000004 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.350813+0000", 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000178111 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.678 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.350919+0000", 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010544600000000001 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.366697+0000", 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015778057000000002 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.366717+0000", 2023-09-30T09:35:37.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9857e-05 2023-09-30T09:35:37.680 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.680 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.492853+0000", 2023-09-30T09:35:37.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.126136478 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.492918+0000", 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4628000000000005e-05 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:34.497251+0000", 2023-09-30T09:35:37.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004333384 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:145247 7.6 7:6ffe3934:::benchmark_data_smithi019_404601_object31969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.509374+0000", 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 61.119012321, 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.269183332, 2023-09-30T09:35:37.682 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145247 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509374+0000", 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509374+0000", 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509376+0000", 2023-09-30T09:35:37.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5999999999999999e-06 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509382+0000", 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5679999999999996e-06 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509399+0000", 2023-09-30T09:35:37.685 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6305000000000001e-05 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509411+0000", 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1966e-05 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.686 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.516749+0000", 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0073378269999999999 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.516898+0000", 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014945299999999999 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:35:37.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517008+0000", 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000110299 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524212+0000", 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0072039929999999997 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.529282+0000", 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050696489999999999 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.773766+0000", 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.24448402299999999 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.689 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.773827+0000", 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1353999999999996e-05 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.778558+0000", 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0047302949999999998 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.690 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:145256 7.6 7:6d734290:::benchmark_data_smithi019_404601_object31978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.524177+0000", 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 61.104209197000003, 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25938216400000003, 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.691 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145256 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524177+0000", 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524177+0000", 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524179+0000", 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.268e-06 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.693 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524190+0000", 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0905e-05 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524203+0000", 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3406e-05 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.694 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524211+0000", 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2779999999999993e-06 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524257+0000", 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5769999999999997e-05 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524370+0000", 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000112953 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524467+0000", 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6589999999999995e-05 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.529403+0000", 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049365889999999999 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.529872+0000", 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00046835699999999997 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.778631+0000", 2023-09-30T09:35:37.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.248759229 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.778703+0000", 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2402000000000002e-05 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.783560+0000", 2023-09-30T09:35:37.698 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004856417 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:153162 7.6 7:6ea7641f:::benchmark_data_smithi019_404601_object39884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:55.602987+0000", 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 42.025399907000001, 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22311632300000001, 2023-09-30T09:35:37.699 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153162 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.700 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.602987+0000", 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.602987+0000", 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.701 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.602989+0000", 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1169999999999998e-06 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.602996+0000", 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6899999999999992e-06 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.702 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.603020+0000", 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3524e-05 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.603030+0000", 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0349999999999999e-05 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.606796+0000", 2023-09-30T09:35:37.703 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0037654950000000002 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.606975+0000", 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000179345 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.631021+0000", 2023-09-30T09:35:37.704 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.024045703000000002 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.631145+0000", 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000123619 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.631316+0000", 2023-09-30T09:35:37.705 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017185400000000001 2023-09-30T09:35:37.706 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.706 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.706 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.706 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.649110+0000", 2023-09-30T09:35:37.709 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017793384999999998 2023-09-30T09:35:37.709 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.709 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.654218+0000", 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0051086489999999998 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.822278+0000", 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.168059979 2023-09-30T09:35:37.710 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.822344+0000", 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5914999999999995e-05 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.826103+0000", 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003758698 2023-09-30T09:35:37.711 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:153197 7.6 7:6732c287:::benchmark_data_smithi019_404601_object39919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:55.671614+0000", 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 41.956772133000001, 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15853534599999999, 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.712 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153197 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.671614+0000", 2023-09-30T09:35:37.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.671614+0000", 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.714 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.671616+0000", 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5090000000000001e-06 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.671622+0000", 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0939999999999996e-06 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.671632+0000", 2023-09-30T09:35:37.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7459999999999995e-06 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.671639+0000", 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7009999999999999e-06 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.674348+0000", 2023-09-30T09:35:37.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027098500000000002 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.674421+0000", 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2304000000000004e-05 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.674509+0000", 2023-09-30T09:35:37.717 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8226999999999994e-05 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.678139+0000", 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036303149999999998 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.718 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.689388+0000", 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011249034 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.826173+0000", 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.13678453199999999 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.719 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.826232+0000", 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9355999999999999e-05 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:55.830150+0000", 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003917678 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.720 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:153756 7.6 7:6f6a579c:::benchmark_data_smithi019_404601_object40478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:56.657316+0000", 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 40.971070826999998, 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.140509986, 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.721 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.722 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.722 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153756 2023-09-30T09:35:37.722 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.722 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.722 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.722 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.722 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.657316+0000", 2023-09-30T09:35:37.722 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.657316+0000", 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.657318+0000", 2023-09-30T09:35:37.724 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9190000000000002e-06 2023-09-30T09:35:37.724 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.724 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.724 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.724 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.657326+0000", 2023-09-30T09:35:37.724 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0169999999999996e-06 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.657342+0000", 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6351e-05 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.657353+0000", 2023-09-30T09:35:37.725 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0668e-05 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.664472+0000", 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0071196569999999997 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.664613+0000", 2023-09-30T09:35:37.726 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014074899999999999 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.686009+0000", 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.021396057 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.686134+0000", 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012473200000000001 2023-09-30T09:35:37.727 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.686289+0000", 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015554800000000001 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.696799+0000", 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010509539 2023-09-30T09:35:37.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.707536+0000", 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010737217 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.794353+0000", 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.086816386999999995 2023-09-30T09:35:37.729 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.794410+0000", 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7071000000000003e-05 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:56.797826+0000", 2023-09-30T09:35:37.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034160739999999998 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:158085 7.6 7:66b58169:::benchmark_data_smithi019_404601_object44807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:04.273391+0000", 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 33.354995328999998, 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.152536433, 2023-09-30T09:35:37.731 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 158085 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.732 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.273391+0000", 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.273391+0000", 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.273393+0000", 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6160000000000001e-06 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.273396+0000", 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.252e-06 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.273404+0000", 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1790000000000008e-06 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.273410+0000", 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.254e-06 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.286929+0000", 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013519305000000001 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.287077+0000", 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014853499999999999 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.736 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.295507+0000", 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0084296100000000006 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.295590+0000", 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3343000000000005e-05 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.295733+0000", 2023-09-30T09:35:37.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014277699999999999 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.313601+0000", 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017867471999999999 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.313635+0000", 2023-09-30T09:35:37.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4393000000000001e-05 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.422247+0000", 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.108611557 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.422308+0000", 2023-09-30T09:35:37.739 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0983999999999998e-05 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:04.425928+0000", 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036201559999999998 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:159171 7.6 7:62be09e7:::benchmark_data_smithi019_404601_object45893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:06.251595+0000", 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 31.376791259000001, 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.221514084, 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.741 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 159171 2023-09-30T09:35:37.742 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.742 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.742 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.742 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.742 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.251595+0000", 2023-09-30T09:35:37.742 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.742 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.251595+0000", 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.251597+0000", 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1579999999999999e-06 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.251605+0000", 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4950000000000002e-06 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.251620+0000", 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4995e-05 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.744 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.251629+0000", 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3919999999999992e-06 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.260444+0000", 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088141650000000005 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.745 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.260519+0000", 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5915999999999996e-05 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.260660+0000", 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014034299999999999 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.746 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.265346+0000", 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0046864649999999999 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.279350+0000", 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014004121 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.747 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.470298+0000", 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.19094729799999999 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.470368+0000", 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9894000000000005e-05 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.748 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:06.473109+0000", 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002741842 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:167932 7.6 7:65c894ba:::benchmark_data_smithi019_404601_object54654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-30T09:35:37.749 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:24.442138+0000", 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 13.186249049000001, 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.140258986, 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 167932 2023-09-30T09:35:37.750 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442138+0000", 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442138+0000", 2023-09-30T09:35:37.751 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442139+0000", 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5889999999999999e-06 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442145+0000", 2023-09-30T09:35:37.752 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1410000000000002e-06 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442159+0000", 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3339000000000001e-05 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442168+0000", 2023-09-30T09:35:37.753 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0319999999999992e-06 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.453280+0000", 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011112294 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.453376+0000", 2023-09-30T09:35:37.754 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5721000000000001e-05 2023-09-30T09:35:37.755 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.755 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.755 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.755 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.453903+0000", 2023-09-30T09:35:37.755 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00052685899999999996 2023-09-30T09:35:37.755 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.755 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.755 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.459670+0000", 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0057671490000000001 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.484149+0000", 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.02447914 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.756 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.579073+0000", 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.094924295000000006 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.579142+0000", 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9393999999999993e-05 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.757 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.582397+0000", 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0032540329999999999 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:167934 7.6 7:623ff86e:::benchmark_data_smithi019_404601_object54656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-30T09:35:37.758 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:24.442182+0000", 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 13.186204431, 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.143594692, 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 167934 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.759 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442182+0000", 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442182+0000", 2023-09-30T09:35:37.760 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442182+0000", 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8000000000000002e-07 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442190+0000", 2023-09-30T09:35:37.761 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6420000000000007e-06 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442197+0000", 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5050000000000004e-06 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.442202+0000", 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.023e-06 2023-09-30T09:35:37.762 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.456395+0000", 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014193771000000001 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.456503+0000", 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000107899 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.763 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.456593+0000", 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9460000000000001e-05 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.459675+0000", 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030817980000000002 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.764 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.484201+0000", 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.024526334 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.582462+0000", 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.098261181000000003 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.765 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.582531+0000", 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9431999999999995e-05 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.585777+0000", 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0032453669999999999 2023-09-30T09:35:37.766 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:167942 7.6 7:68e27886:::benchmark_data_smithi019_404601_object54664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:24.454989+0000", 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 13.173397951, 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.13748147199999999, 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.767 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.768 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.768 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.768 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.768 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 167942 2023-09-30T09:35:37.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.768 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.768 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.768 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.454989+0000", 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.454989+0000", 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.769 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.454990+0000", 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9000000000000005e-07 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.454994+0000", 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1200000000000004e-06 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.770 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.771 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.455005+0000", 2023-09-30T09:35:37.771 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.096e-05 2023-09-30T09:35:37.771 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.771 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.771 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.771 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.455012+0000", 2023-09-30T09:35:37.771 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3370000000000002e-06 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.459568+0000", 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0045554819999999996 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:35:37.772 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.459653+0000", 2023-09-30T09:35:37.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5066000000000002e-05 2023-09-30T09:35:37.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.773 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.773 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.472039+0000", 2023-09-30T09:35:37.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012386549 2023-09-30T09:35:37.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.472115+0000", 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6171e-05 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.472228+0000", 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011292 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.484215+0000", 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011986613 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.484272+0000", 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7336999999999997e-05 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.589180+0000", 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.104908034 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.589259+0000", 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8592999999999998e-05 2023-09-30T09:35:37.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.592470+0000", 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0032112999999999998 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:168065 7.6 7:65e4b022:::benchmark_data_smithi019_404601_object54787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-30T09:35:37.777 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:24.818421+0000", 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 12.809965121999999, 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.142399463, 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 168065 2023-09-30T09:35:37.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.818421+0000", 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.818421+0000", 2023-09-30T09:35:37.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.818424+0000", 2023-09-30T09:35:37.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1490000000000001e-06 2023-09-30T09:35:37.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.818432+0000", 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4169999999999992e-06 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.818447+0000", 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4868999999999999e-05 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.781 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.818459+0000", 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1819e-05 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.828296+0000", 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0098368559999999997 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.828401+0000", 2023-09-30T09:35:37.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000105593 2023-09-30T09:35:37.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.828552+0000", 2023-09-30T09:35:37.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015065900000000001 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.831634+0000", 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030825879999999998 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.937783+0000", 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.106148701 2023-09-30T09:35:37.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.957460+0000", 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.019677324 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.957508+0000", 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7712e-05 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:24.960821+0000", 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033127759999999999 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:172463 7.6 7:6d1a067d:::benchmark_data_smithi019_404601_object59185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:35.224113+0000", 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.4042740010000001, 2023-09-30T09:35:37.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21989881, 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172463 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.224113+0000", 2023-09-30T09:35:37.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.224113+0000", 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.224114+0000", 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2410000000000001e-06 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.224117+0000", 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5899999999999999e-06 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.224126+0000", 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1130000000000001e-06 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.224132+0000", 2023-09-30T09:35:37.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3110000000000002e-06 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.230153+0000", 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0060213580000000001 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.230240+0000", 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6618000000000003e-05 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.230334+0000", 2023-09-30T09:35:37.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4450999999999994e-05 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.233345+0000", 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030111830000000002 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.247690+0000", 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014344753999999999 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.440808+0000", 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.19311778299999999 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.440868+0000", 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9709e-05 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.444011+0000", 2023-09-30T09:35:37.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031436989999999998 2023-09-30T09:35:37.794 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.794 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.794 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.794 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:172472 7.6 7:6644bbfd:::benchmark_data_smithi019_404601_object59194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:35.236303+0000", 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.3920838569999998, 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21149442900000001, 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172472 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236303+0000", 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236303+0000", 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236305+0000", 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0710000000000002e-06 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236311+0000", 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3969999999999999e-06 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236322+0000", 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1287000000000001e-05 2023-09-30T09:35:37.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236330+0000", 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5850000000000002e-06 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236385+0000", 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5000000000000002e-05 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236461+0000", 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6089999999999998e-05 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.236554+0000", 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3129999999999998e-05 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.239065+0000", 2023-09-30T09:35:37.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025105370000000002 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.247746+0000", 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0086811070000000004 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.444701+0000", 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.19695539000000001 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.444766+0000", 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5006999999999996e-05 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.447797+0000", 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030308280000000002 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.801 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPushReply(7.6 335/320 [PushReplyOp(7:6a83edf6:::benchmark_data_smithi019_404601_object59612:head)])", 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:35.274633+0000", 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.3537531949999999, 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15414275199999999, 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.274633+0000", 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.274633+0000", 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.274635+0000", 2023-09-30T09:35:37.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9e-06 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.274646+0000", 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0443e-05 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.274670+0000", 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4147000000000001e-05 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.274688+0000", 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7785999999999999e-05 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.274827+0000", 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000138863 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.428776+0000", 2023-09-30T09:35:37.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15394961300000001 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:172476 7.4 e335/327 7:235f10eb:::benchmark_data_smithi019_404601_object59198:head v 335'10856, mlcod=0'0)", 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:35.280763+0000", 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.3476232939999999, 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14984061800000001, 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:35:37.806 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.280763+0000", 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.280763+0000", 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.280764+0000", 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1090000000000001e-06 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.280774+0000", 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.961e-06 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:35:37.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.428691+0000", 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.14791655200000001 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.428702+0000", 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1129e-05 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:35:37.809 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.428765+0000", 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3132000000000005e-05 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.428788+0000", 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2918e-05 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.430551+0000", 2023-09-30T09:35:37.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0017626880000000001 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:35.430604+0000", 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3128999999999999e-05 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:35:37.811 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:35:37.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:37.812 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:35:37.812 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:35:37.813 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:37.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:37.834 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:35:37.914 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:37.935 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:35:38.003 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:35:38.040 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:38.063 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:35:38.108 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:35:38.177 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:35:38.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:35:38.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:35:38.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:35:38.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:35:38.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:35:38.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:35:38.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:35:38.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:35:38.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:38 smithi019 ceph-mon[216815]: pgmap v835: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 26494/163785 objects degraded (16.176%); 50493/163785 objects misplaced (30.829%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:35:38.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1687449283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:38.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 26494/163785 objects degraded (16.176%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:38.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:38 smithi019 ceph-mon[224043]: pgmap v835: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 26494/163785 objects degraded (16.176%); 50493/163785 objects misplaced (30.829%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:35:38.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1687449283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:38.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 26494/163785 objects degraded (16.176%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:38 smithi167 ceph-mon[192880]: pgmap v835: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 26494/163785 objects degraded (16.176%); 50493/163785 objects misplaced (30.829%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:35:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1687449283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 26494/163785 objects degraded (16.176%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:39.444 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:39.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:39.550 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:39.667 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:39.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:39 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:35:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:39 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:35:40.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:39 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:35:40.403 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:35:40.412 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:40.588 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:40.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:40.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:40.697 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:40.749 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:40.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:40 smithi019 ceph-mon[216815]: pgmap v836: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 25477/157527 objects degraded (16.173%); 48695/157527 objects misplaced (30.912%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:40.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:40 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:35:40.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:40 smithi019 ceph-mon[224043]: pgmap v836: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 25477/157527 objects degraded (16.173%); 48695/157527 objects misplaced (30.912%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:40.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:40 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:35:40.873 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:40.874 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:40.876 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:40 smithi167 ceph-mon[192880]: pgmap v836: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 25477/157527 objects degraded (16.173%); 48695/157527 objects misplaced (30.912%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:40.876 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:40 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:35:40.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:40.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:40.978 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:41.092 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:41.255 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:41.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:41.369 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:35:41.449 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:41.518 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:35:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:41.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:35:41.627 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:41.667 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:41.719 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:41.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:41.828 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:41.867 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:42.010 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:42.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:42.117 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:42.197 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:42.451 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:35:42.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:42 smithi019 ceph-mon[216815]: pgmap v837: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 25477/157086 objects degraded (16.219%); 48534/157086 objects misplaced (30.896%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:35:42.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:42 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:35:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:42 smithi019 ceph-mon[224043]: pgmap v837: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 25477/157086 objects degraded (16.219%); 48534/157086 objects misplaced (30.896%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:35:42.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:42 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:35:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:42 smithi167 ceph-mon[192880]: pgmap v837: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 25477/157086 objects degraded (16.219%); 48534/157086 objects misplaced (30.896%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:35:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:42 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:35:43.231 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:43.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:43.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:43.338 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:43.391 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:43.481 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:43.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:43.578 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 25477/157086 objects degraded (16.219%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:43.583 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:43.706 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 25477/157086 objects degraded (16.219%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:43.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:35:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:35:43] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.43.0" 2023-09-30T09:35:43.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 25477/157086 objects degraded (16.219%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:44.675 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:44.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:44.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:44 smithi019 ceph-mon[216815]: pgmap v838: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 25477/157086 objects degraded (16.219%); 48534/157086 objects misplaced (30.896%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:35:44.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:44 smithi019 ceph-mon[224043]: pgmap v838: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 25477/157086 objects degraded (16.219%); 48534/157086 objects misplaced (30.896%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:35:44.784 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:44.860 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:44 smithi167 ceph-mon[192880]: pgmap v838: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 25477/157086 objects degraded (16.219%); 48534/157086 objects misplaced (30.896%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:35:45.076 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:45.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:45.186 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:45.305 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:45.506 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:45.565 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:45.631 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:45.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:45.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:45.741 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:45.870 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:46.023 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:46.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:46.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:46.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:35:46.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:46.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:46.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[224043]: pgmap v839: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 24418/150639 objects degraded (16.210%); 46563/150639 objects misplaced (30.910%); 17 MiB/s, 279 objects/s recovering 2023-09-30T09:35:46.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2729326957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:35:46.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:35:46.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:46.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:46.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[216815]: pgmap v839: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 24418/150639 objects degraded (16.210%); 46563/150639 objects misplaced (30.910%); 17 MiB/s, 279 objects/s recovering 2023-09-30T09:35:46.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2729326957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:35:46.153 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:46.185 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:35:46.445 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:35:46.481 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:35:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:46 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:35:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:46 smithi167 ceph-mon[192880]: pgmap v839: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 24418/150639 objects degraded (16.210%); 46563/150639 objects misplaced (30.910%); 17 MiB/s, 279 objects/s recovering 2023-09-30T09:35:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2729326957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:35:46.656 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:46.751 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:46.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:46.857 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:46.895 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:46.957 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:46.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:47.064 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:35:47.091 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:47.159 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:35:47.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:47.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:35:47.269 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2729326957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: osdmap e336: 8 total, 8 up, 7 in 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: failed to encode map e336 with expected crc 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: failed to encode map e336 with expected crc 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:47 smithi167 ceph-mon[192880]: from='client.55459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:35:47.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:35:47.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:47.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2729326957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:35:47.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: osdmap e336: 8 total, 8 up, 7 in 2023-09-30T09:35:47.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: failed to encode map e336 with expected crc 2023-09-30T09:35:47.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: failed to encode map e336 with expected crc 2023-09-30T09:35:47.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:35:47.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:35:47.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:47.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[224043]: from='client.55459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:35:47.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:35:47.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:35:47.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2729326957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:35:47.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: osdmap e336: 8 total, 8 up, 7 in 2023-09-30T09:35:47.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: failed to encode map e336 with expected crc 2023-09-30T09:35:47.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: failed to encode map e336 with expected crc 2023-09-30T09:35:47.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:35:47.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:35:47.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:47.361 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:47 smithi019 ceph-mon[216815]: from='client.55459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:35:47.377 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:48.506 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:48 smithi019 ceph-mon[216815]: pgmap v841: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 24418/150639 objects degraded (16.210%); 46563/150639 objects misplaced (30.910%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:48.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:48.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:48.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:48 smithi019 ceph-mon[224043]: pgmap v841: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 24418/150639 objects degraded (16.210%); 46563/150639 objects misplaced (30.910%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:48.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:48.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:48 smithi167 ceph-mon[192880]: pgmap v841: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 24418/150639 objects degraded (16.210%); 46563/150639 objects misplaced (30.910%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:35:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:35:49.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:49 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 24418/150639 objects degraded (16.210%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:49.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:49 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 24418/150639 objects degraded (16.210%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:49.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:49 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 24418/150639 objects degraded (16.210%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:50.140 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:50.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:50.251 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:50.374 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:50 smithi019 ceph-mon[224043]: pgmap v842: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 23453/145254 objects degraded (16.146%); 45004/145254 objects misplaced (30.983%); 14 MiB/s, 226 objects/s recovering 2023-09-30T09:35:50.379 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:50 smithi019 ceph-mon[216815]: pgmap v842: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 23453/145254 objects degraded (16.146%); 45004/145254 objects misplaced (30.983%); 14 MiB/s, 226 objects/s recovering 2023-09-30T09:35:50.379 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:50.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:50 smithi167 ceph-mon[192880]: pgmap v842: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 102 active+clean; 3.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 23453/145254 objects degraded (16.146%); 45004/145254 objects misplaced (30.983%); 14 MiB/s, 226 objects/s recovering 2023-09-30T09:35:51.003 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":840,"stamp":"2023-09-30T09:35:46.130883+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3293789028,"num_objects":50213,"num_object_clones":0,"num_object_copies":150639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24418,"num_objects_misplaced":46563,"num_objects_unfound":0,"num_objects_dirty":50213,"num_whiteouts":0,"num_read":6114,"num_read_kb":12467,"num_write":177356,"num_write_kb":7263166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16106,"num_bytes_recovered":1050386904,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86254,"ondisk_log_size":86254,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":397,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8782412,"kb_used_data":6581780,"kb_used_omap":23,"kb_used_meta":2200488,"kb_avail":647433652,"statfs":{"total":671965249536,"available":662972059648,"internally_reserved":0,"allocated":6739742720,"data_stored":6728563709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2253299971},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-287178752,"num_objects":-4382,"num_object_clones":0,"num_object_copies":-13146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2076,"num_objects_misplaced":-3930,"num_objects_unfound":0,"num_objects_dirty":-4382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4382,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2311,"num_bytes_recovered":151453696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.662398"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":799,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842721+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:35:23.842721+0000","last_peered":"2023-09-30T09:35:23.842721+0000","last_clean":"2023-09-30T09:35:23.842721+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:35:23.842721+0000","last_undegraded":"2023-09-30T09:35:23.842721+0000","last_fullsized":"2023-09-30T09:35:23.842721+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":353,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847228+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:35:23.847228+0000","last_peered":"2023-09-30T09:35:23.847228+0000","last_clean":"2023-09-30T09:35:23.847228+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:35:23.847228+0000","last_undegraded":"2023-09-30T09:35:23.847228+0000","last_fullsized":"2023-09-30T09:35:23.847228+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":315,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833647+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:35:23.833647+0000","last_peered":"2023-09-30T09:35:23.833647+0000","last_clean":"2023-09-30T09:35:23.833647+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:35:23.833647+0000","last_undegraded":"2023-09-30T09:35:23.833647+0000","last_fullsized":"2023-09-30T09:35:23.833647+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":779,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842127+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:35:23.842127+0000","last_peered":"2023-09-30T09:35:23.842127+0000","last_clean":"2023-09-30T09:35:23.842127+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:35:23.842127+0000","last_undegraded":"2023-09-30T09:35:23.842127+0000","last_fullsized":"2023-09-30T09:35:23.842127+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.832727+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:35:23.832727+0000","last_peered":"2023-09-30T09:35:23.832727+0000","last_clean":"2023-09-30T09:35:23.832727+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:35:23.832727+0000","last_undegraded":"2023-09-30T09:35:23.832727+0000","last_fullsized":"2023-09-30T09:35:23.832727+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":239,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.831689+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:35:23.831689+0000","last_peered":"2023-09-30T09:35:23.831689+0000","last_clean":"2023-09-30T09:35:23.831689+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:35:23.831689+0000","last_undegraded":"2023-09-30T09:35:23.831689+0000","last_fullsized":"2023-09-30T09:35:23.831689+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":548,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847136+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:35:23.847136+0000","last_peered":"2023-09-30T09:35:23.847136+0000","last_clean":"2023-09-30T09:35:23.847136+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:35:23.847136+0000","last_undegraded":"2023-09-30T09:35:23.847136+0000","last_fullsized":"2023-09-30T09:35:23.847136+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":381,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.830001+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:35:23.830001+0000","last_peered":"2023-09-30T09:35:23.830001+0000","last_clean":"2023-09-30T09:35:23.830001+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:35:23.830001+0000","last_undegraded":"2023-09-30T09:35:23.830001+0000","last_fullsized":"2023-09-30T09:35:23.830001+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":311,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.837016+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:35:23.837016+0000","last_peered":"2023-09-30T09:35:23.837016+0000","last_clean":"2023-09-30T09:35:23.837016+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:35:23.837016+0000","last_undegraded":"2023-09-30T09:35:23.837016+0000","last_fullsized":"2023-09-30T09:35:23.837016+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":758,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833466+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:35:23.833466+0000","last_peered":"2023-09-30T09:35:23.833466+0000","last_clean":"2023-09-30T09:35:23.833466+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:35:23.833466+0000","last_undegraded":"2023-09-30T09:35:23.833466+0000","last_fullsized":"2023-09-30T09:35:23.833466+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":230,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.829588+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:35:23.829588+0000","last_peered":"2023-09-30T09:35:23.829588+0000","last_clean":"2023-09-30T09:35:23.829588+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:35:23.829588+0000","last_undegraded":"2023-09-30T09:35:23.829588+0000","last_fullsized":"2023-09-30T09:35:23.829588+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":316,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:24.249782+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:35:24.249782+0000","last_peered":"2023-09-30T09:35:24.249782+0000","last_clean":"2023-09-30T09:35:24.249782+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:35:24.249782+0000","last_undegraded":"2023-09-30T09:35:24.249782+0000","last_fullsized":"2023-09-30T09:35:24.249782+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":498,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.877954+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:35:27.877954+0000","last_peered":"2023-09-30T09:35:27.877954+0000","last_clean":"2023-09-30T09:35:27.877954+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:35:27.877954+0000","last_undegraded":"2023-09-30T09:35:27.877954+0000","last_fullsized":"2023-09-30T09:35:27.877954+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841644+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:35:23.841644+0000","last_peered":"2023-09-30T09:35:23.841644+0000","last_clean":"2023-09-30T09:35:23.841644+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:35:23.841644+0000","last_undegraded":"2023-09-30T09:35:23.841644+0000","last_fullsized":"2023-09-30T09:35:23.841644+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":262,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.891032+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:35:27.891032+0000","last_peered":"2023-09-30T09:35:27.891032+0000","last_clean":"2023-09-30T09:35:27.891032+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:35:27.891032+0000","last_undegraded":"2023-09-30T09:35:27.891032+0000","last_fullsized":"2023-09-30T09:35:27.891032+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":501,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840662+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:35:23.840662+0000","last_peered":"2023-09-30T09:35:23.840662+0000","last_clean":"2023-09-30T09:35:23.840662+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:35:23.840662+0000","last_undegraded":"2023-09-30T09:35:23.840662+0000","last_fullsized":"2023-09-30T09:35:23.840662+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":56,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833723+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:35:23.833723+0000","last_peered":"2023-09-30T09:35:23.833723+0000","last_clean":"2023-09-30T09:35:23.833723+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:35:23.833723+0000","last_undegraded":"2023-09-30T09:35:23.833723+0000","last_fullsized":"2023-09-30T09:35:23.833723+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":248,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.836572+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:35:23.836572+0000","last_peered":"2023-09-30T09:35:23.836572+0000","last_clean":"2023-09-30T09:35:23.836572+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:35:23.836572+0000","last_undegraded":"2023-09-30T09:35:23.836572+0000","last_fullsized":"2023-09-30T09:35:23.836572+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":494,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835413+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:35:23.835413+0000","last_peered":"2023-09-30T09:35:23.835413+0000","last_clean":"2023-09-30T09:35:23.835413+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:35:23.835413+0000","last_undegraded":"2023-09-30T09:35:23.835413+0000","last_fullsized":"2023-09-30T09:35:23.835413+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":319,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848910+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:35:23.848910+0000","last_peered":"2023-09-30T09:35:23.848910+0000","last_clean":"2023-09-30T09:35:23.848910+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:35:23.848910+0000","last_undegraded":"2023-09-30T09:35:23.848910+0000","last_fullsized":"2023-09-30T09:35:23.848910+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":330,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847450+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:35:23.847450+0000","last_peered":"2023-09-30T09:35:23.847450+0000","last_clean":"2023-09-30T09:35:23.847450+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:35:23.847450+0000","last_undegraded":"2023-09-30T09:35:23.847450+0000","last_fullsized":"2023-09-30T09:35:23.847450+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":612,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.829170+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:35:23.829170+0000","last_peered":"2023-09-30T09:35:23.829170+0000","last_clean":"2023-09-30T09:35:23.829170+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:35:23.829170+0000","last_undegraded":"2023-09-30T09:35:23.829170+0000","last_fullsized":"2023-09-30T09:35:23.829170+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":57,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847331+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:35:23.847331+0000","last_peered":"2023-09-30T09:35:23.847331+0000","last_clean":"2023-09-30T09:35:23.847331+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:35:23.847331+0000","last_undegraded":"2023-09-30T09:35:23.847331+0000","last_fullsized":"2023-09-30T09:35:23.847331+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":346,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.883365+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:35:27.883365+0000","last_peered":"2023-09-30T09:35:27.883365+0000","last_clean":"2023-09-30T09:35:27.883365+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:35:27.883365+0000","last_undegraded":"2023-09-30T09:35:27.883365+0000","last_fullsized":"2023-09-30T09:35:27.883365+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":681,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842879+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:35:23.842879+0000","last_peered":"2023-09-30T09:35:23.842879+0000","last_clean":"2023-09-30T09:35:23.842879+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:35:23.842879+0000","last_undegraded":"2023-09-30T09:35:23.842879+0000","last_fullsized":"2023-09-30T09:35:23.842879+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":238,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.838816+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:35:23.838816+0000","last_peered":"2023-09-30T09:35:23.838816+0000","last_clean":"2023-09-30T09:35:23.838816+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:35:23.838816+0000","last_undegraded":"2023-09-30T09:35:23.838816+0000","last_fullsized":"2023-09-30T09:35:23.838816+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":24,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841282+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:35:23.841282+0000","last_peered":"2023-09-30T09:35:23.841282+0000","last_clean":"2023-09-30T09:35:23.841282+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:35:23.841282+0000","last_undegraded":"2023-09-30T09:35:23.841282+0000","last_fullsized":"2023-09-30T09:35:23.841282+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":773,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.830784+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:35:23.830784+0000","last_peered":"2023-09-30T09:35:23.830784+0000","last_clean":"2023-09-30T09:35:23.830784+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:35:23.830784+0000","last_undegraded":"2023-09-30T09:35:23.830784+0000","last_fullsized":"2023-09-30T09:35:23.830784+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":40,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848161+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:35:23.848161+0000","last_peered":"2023-09-30T09:35:23.848161+0000","last_clean":"2023-09-30T09:35:23.848161+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:35:23.848161+0000","last_undegraded":"2023-09-30T09:35:23.848161+0000","last_fullsized":"2023-09-30T09:35:23.848161+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":238,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835272+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:35:23.835272+0000","last_peered":"2023-09-30T09:35:23.835272+0000","last_clean":"2023-09-30T09:35:23.835272+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:35:23.835272+0000","last_undegraded":"2023-09-30T09:35:23.835272+0000","last_fullsized":"2023-09-30T09:35:23.835272+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":800,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840933+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:35:23.840933+0000","last_peered":"2023-09-30T09:35:23.840933+0000","last_clean":"2023-09-30T09:35:23.840933+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:35:23.840933+0000","last_undegraded":"2023-09-30T09:35:23.840933+0000","last_fullsized":"2023-09-30T09:35:23.840933+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":494,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834041+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:35:23.834041+0000","last_peered":"2023-09-30T09:35:23.834041+0000","last_clean":"2023-09-30T09:35:23.834041+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:35:23.834041+0000","last_undegraded":"2023-09-30T09:35:23.834041+0000","last_fullsized":"2023-09-30T09:35:23.834041+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":343,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835402+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:35:23.835402+0000","last_peered":"2023-09-30T09:35:23.835402+0000","last_clean":"2023-09-30T09:35:23.835402+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:35:23.835402+0000","last_undegraded":"2023-09-30T09:35:23.835402+0000","last_fullsized":"2023-09-30T09:35:23.835402+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"335'10810","reported_seq":14786,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:43.895791+0000","last_change":"2023-09-30T09:34:48.517696+0000","last_active":"2023-09-30T09:35:43.895791+0000","last_peered":"2023-09-30T09:35:43.895791+0000","last_clean":"2023-09-30T09:32:35.457744+0000","last_became_active":"2023-09-30T09:34:48.499533+0000","last_became_peered":"2023-09-30T09:34:48.499533+0000","last_unstale":"2023-09-30T09:35:43.895791+0000","last_undegraded":"2023-09-30T09:34:48.469688+0000","last_fullsized":"2023-09-30T09:35:43.895791+0000","mapping_epoch":321,"log_start":"298'6021","ondisk_log_start":"298'6021","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4789,"log_dups_size":0,"ondisk_log_size":4789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:05:42.164341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202375168,"num_objects":3088,"num_object_clones":0,"num_object_copies":9264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1608,"num_objects_misplaced":3088,"num_objects_unfound":0,"num_objects_dirty":3088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10810,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1934,"num_bytes_recovered":126746624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[6,1,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"1,4,6","objects":1480},{"shards":"4,6","objects":1608}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":796,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:34.689092+0000","last_change":"2023-09-30T09:35:34.689092+0000","last_active":"2023-09-30T09:35:34.689092+0000","last_peered":"2023-09-30T09:35:34.689092+0000","last_clean":"2023-09-30T09:35:34.689092+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:35:34.689092+0000","last_undegraded":"2023-09-30T09:35:34.689092+0000","last_fullsized":"2023-09-30T09:35:34.689092+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":69,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848317+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:35:23.848317+0000","last_peered":"2023-09-30T09:35:23.848317+0000","last_clean":"2023-09-30T09:35:23.848317+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:35:23.848317+0000","last_undegraded":"2023-09-30T09:35:23.848317+0000","last_fullsized":"2023-09-30T09:35:23.848317+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":24,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842632+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:35:23.842632+0000","last_peered":"2023-09-30T09:35:23.842632+0000","last_clean":"2023-09-30T09:35:23.842632+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:35:23.842632+0000","last_undegraded":"2023-09-30T09:35:23.842632+0000","last_fullsized":"2023-09-30T09:35:23.842632+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"335'11210","reported_seq":13268,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:44.228271+0000","last_change":"2023-09-30T09:35:10.766663+0000","last_active":"2023-09-30T09:35:44.228271+0000","last_peered":"2023-09-30T09:35:44.228271+0000","last_clean":"2023-09-30T09:32:35.454324+0000","last_became_active":"2023-09-30T09:33:33.826036+0000","last_became_peered":"2023-09-30T09:33:33.826036+0000","last_unstale":"2023-09-30T09:35:44.228271+0000","last_undegraded":"2023-09-30T09:33:33.732722+0000","last_fullsized":"2023-09-30T09:35:44.228271+0000","mapping_epoch":302,"log_start":"299'6699","ondisk_log_start":"299'6699","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4511,"log_dups_size":0,"ondisk_log_size":4511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:42.597216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209059840,"num_objects":3190,"num_object_clones":0,"num_object_copies":9570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11210,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":983,"num_bytes_recovered":64421888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,5,1],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1},{"shards":"2,3,5","objects":223},{"shards":"3,5","objects":2966}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":784,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834320+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:35:23.834320+0000","last_peered":"2023-09-30T09:35:23.834320+0000","last_clean":"2023-09-30T09:35:23.834320+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:35:23.834320+0000","last_undegraded":"2023-09-30T09:35:23.834320+0000","last_fullsized":"2023-09-30T09:35:23.834320+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":47,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847002+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:35:23.847002+0000","last_peered":"2023-09-30T09:35:23.847002+0000","last_clean":"2023-09-30T09:35:23.847002+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:35:23.847002+0000","last_undegraded":"2023-09-30T09:35:23.847002+0000","last_fullsized":"2023-09-30T09:35:23.847002+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":327,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848092+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:35:23.848092+0000","last_peered":"2023-09-30T09:35:23.848092+0000","last_clean":"2023-09-30T09:35:23.848092+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:35:23.848092+0000","last_undegraded":"2023-09-30T09:35:23.848092+0000","last_fullsized":"2023-09-30T09:35:23.848092+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"335'10826","reported_seq":11850,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:43.937080+0000","last_change":"2023-09-30T09:35:10.879711+0000","last_active":"2023-09-30T09:35:43.937080+0000","last_peered":"2023-09-30T09:35:43.937080+0000","last_clean":"2023-09-30T09:32:35.467048+0000","last_became_active":"2023-09-30T09:35:10.845361+0000","last_became_peered":"2023-09-30T09:35:10.845361+0000","last_unstale":"2023-09-30T09:35:43.937080+0000","last_undegraded":"2023-09-30T09:35:10.791338+0000","last_fullsized":"2023-09-30T09:35:43.937080+0000","mapping_epoch":327,"log_start":"298'5976","ondisk_log_start":"298'5976","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:41.089234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206438400,"num_objects":3150,"num_object_clones":0,"num_object_copies":9450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3150,"num_objects_misplaced":6300,"num_objects_unfound":0,"num_objects_dirty":3150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10826,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":458,"num_bytes_recovered":29884416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[6,1,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":3150}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":618,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.836939+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:35:23.836939+0000","last_peered":"2023-09-30T09:35:23.836939+0000","last_clean":"2023-09-30T09:35:23.836939+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:35:23.836939+0000","last_undegraded":"2023-09-30T09:35:23.836939+0000","last_fullsized":"2023-09-30T09:35:23.836939+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":28,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841215+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:35:23.841215+0000","last_peered":"2023-09-30T09:35:23.841215+0000","last_clean":"2023-09-30T09:35:23.841215+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:35:23.841215+0000","last_undegraded":"2023-09-30T09:35:23.841215+0000","last_fullsized":"2023-09-30T09:35:23.841215+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":314,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.836435+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:35:23.836435+0000","last_peered":"2023-09-30T09:35:23.836435+0000","last_clean":"2023-09-30T09:35:23.836435+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:35:23.836435+0000","last_undegraded":"2023-09-30T09:35:23.836435+0000","last_fullsized":"2023-09-30T09:35:23.836435+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"335'11153","reported_seq":11230,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:35:44.168636+0000","last_change":"2023-09-30T09:34:08.527633+0000","last_active":"2023-09-30T09:35:44.168636+0000","last_peered":"2023-09-30T09:35:44.168636+0000","last_clean":"2023-09-30T09:33:09.318417+0000","last_became_active":"2023-09-30T09:34:08.510746+0000","last_became_peered":"2023-09-30T09:34:08.510746+0000","last_unstale":"2023-09-30T09:35:44.168636+0000","last_undegraded":"2023-09-30T09:35:44.168636+0000","last_fullsized":"2023-09-30T09:35:44.168636+0000","mapping_epoch":308,"log_start":"299'6576","ondisk_log_start":"299'6576","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4577,"log_dups_size":0,"ondisk_log_size":4577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:07.374969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206110720,"num_objects":3145,"num_object_clones":0,"num_object_copies":9435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3145,"num_objects_unfound":0,"num_objects_dirty":3145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11153,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":3145}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":473,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:41.706133+0000","last_change":"2023-09-30T09:35:41.706133+0000","last_active":"2023-09-30T09:35:41.706133+0000","last_peered":"2023-09-30T09:35:41.706133+0000","last_clean":"2023-09-30T09:35:41.706133+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:35:41.706133+0000","last_undegraded":"2023-09-30T09:35:41.706133+0000","last_fullsized":"2023-09-30T09:35:41.706133+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":294,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841827+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:35:23.841827+0000","last_peered":"2023-09-30T09:35:23.841827+0000","last_clean":"2023-09-30T09:35:23.841827+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:35:23.841827+0000","last_undegraded":"2023-09-30T09:35:23.841827+0000","last_fullsized":"2023-09-30T09:35:23.841827+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":240,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839270+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:35:23.839270+0000","last_peered":"2023-09-30T09:35:23.839270+0000","last_clean":"2023-09-30T09:35:23.839270+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:35:23.839270+0000","last_undegraded":"2023-09-30T09:35:23.839270+0000","last_fullsized":"2023-09-30T09:35:23.839270+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"335'10964","reported_seq":11053,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:35:44.074316+0000","last_change":"2023-09-30T09:35:10.779076+0000","last_active":"2023-09-30T09:35:44.074316+0000","last_peered":"2023-09-30T09:35:44.074316+0000","last_clean":"2023-09-30T09:34:06.422096+0000","last_became_active":"2023-09-30T09:35:10.747716+0000","last_became_peered":"2023-09-30T09:35:10.747716+0000","last_unstale":"2023-09-30T09:35:44.074316+0000","last_undegraded":"2023-09-30T09:35:44.074316+0000","last_fullsized":"2023-09-30T09:35:44.074316+0000","mapping_epoch":327,"log_start":"299'6420","ondisk_log_start":"299'6420","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4544,"log_dups_size":0,"ondisk_log_size":4544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:09.191024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201981952,"num_objects":3082,"num_object_clones":0,"num_object_copies":9246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3082,"num_objects_unfound":0,"num_objects_dirty":3082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10964,"num_write_kb":449472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":3082}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.832454+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:35:23.832454+0000","last_peered":"2023-09-30T09:35:23.832454+0000","last_clean":"2023-09-30T09:35:23.832454+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:35:23.832454+0000","last_undegraded":"2023-09-30T09:35:23.832454+0000","last_fullsized":"2023-09-30T09:35:23.832454+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":362,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833922+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:35:23.833922+0000","last_peered":"2023-09-30T09:35:23.833922+0000","last_clean":"2023-09-30T09:35:23.833922+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:35:23.833922+0000","last_undegraded":"2023-09-30T09:35:23.833922+0000","last_fullsized":"2023-09-30T09:35:23.833922+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":293,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848983+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:35:23.848983+0000","last_peered":"2023-09-30T09:35:23.848983+0000","last_clean":"2023-09-30T09:35:23.848983+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:35:23.848983+0000","last_undegraded":"2023-09-30T09:35:23.848983+0000","last_fullsized":"2023-09-30T09:35:23.848983+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"335'11097","reported_seq":12192,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:44.260727+0000","last_change":"2023-09-30T09:35:10.871065+0000","last_active":"2023-09-30T09:35:44.260727+0000","last_peered":"2023-09-30T09:35:44.260727+0000","last_clean":"2023-09-30T09:32:35.449714+0000","last_became_active":"2023-09-30T09:35:10.838732+0000","last_became_peered":"2023-09-30T09:35:10.838732+0000","last_unstale":"2023-09-30T09:35:44.260727+0000","last_undegraded":"2023-09-30T09:35:10.818868+0000","last_fullsized":"2023-09-30T09:35:44.260727+0000","mapping_epoch":327,"log_start":"299'6606","ondisk_log_start":"299'6606","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4491,"log_dups_size":0,"ondisk_log_size":4491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:44.318381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206766080,"num_objects":3155,"num_object_clones":0,"num_object_copies":9465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3152,"num_objects_misplaced":6313,"num_objects_unfound":0,"num_objects_dirty":3155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11097,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":496,"num_bytes_recovered":32505856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[3,7,1],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"1,3,7","objects":3},{"shards":"3,7","objects":3152}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":756,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835385+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:35:23.835385+0000","last_peered":"2023-09-30T09:35:23.835385+0000","last_clean":"2023-09-30T09:35:23.835385+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:35:23.835385+0000","last_undegraded":"2023-09-30T09:35:23.835385+0000","last_fullsized":"2023-09-30T09:35:23.835385+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":312,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834334+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:35:23.834334+0000","last_peered":"2023-09-30T09:35:23.834334+0000","last_clean":"2023-09-30T09:35:23.834334+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:35:23.834334+0000","last_undegraded":"2023-09-30T09:35:23.834334+0000","last_fullsized":"2023-09-30T09:35:23.834334+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":29,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847481+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:35:23.847481+0000","last_peered":"2023-09-30T09:35:23.847481+0000","last_clean":"2023-09-30T09:35:23.847481+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:35:23.847481+0000","last_undegraded":"2023-09-30T09:35:23.847481+0000","last_fullsized":"2023-09-30T09:35:23.847481+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"335'10929","reported_seq":15459,"reported_epoch":335,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:43.973484+0000","last_change":"2023-09-30T09:35:10.766317+0000","last_active":"2023-09-30T09:35:43.973484+0000","last_peered":"2023-09-30T09:35:43.973484+0000","last_clean":"2023-09-30T09:32:35.473843+0000","last_became_active":"2023-09-30T09:33:33.842621+0000","last_became_peered":"2023-09-30T09:33:33.842621+0000","last_unstale":"2023-09-30T09:35:43.973484+0000","last_undegraded":"2023-09-30T09:33:33.676280+0000","last_fullsized":"2023-09-30T09:33:33.596470+0000","mapping_epoch":302,"log_start":"298'6121","ondisk_log_start":"298'6121","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4808,"log_dups_size":0,"ondisk_log_size":4808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:19:47.448341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204079128,"num_objects":3115,"num_object_clones":0,"num_object_copies":9345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1340,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3115,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10929,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2222,"num_bytes_recovered":145620992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":1775}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":770,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839084+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:35:23.839084+0000","last_peered":"2023-09-30T09:35:23.839084+0000","last_clean":"2023-09-30T09:35:23.839084+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:35:23.839084+0000","last_undegraded":"2023-09-30T09:35:23.839084+0000","last_fullsized":"2023-09-30T09:35:23.839084+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":331,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847962+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:35:23.847962+0000","last_peered":"2023-09-30T09:35:23.847962+0000","last_clean":"2023-09-30T09:35:23.847962+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:35:23.847962+0000","last_undegraded":"2023-09-30T09:35:23.847962+0000","last_fullsized":"2023-09-30T09:35:23.847962+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":310,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848663+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:35:23.848663+0000","last_peered":"2023-09-30T09:35:23.848663+0000","last_clean":"2023-09-30T09:35:23.848663+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:35:23.848663+0000","last_undegraded":"2023-09-30T09:35:23.848663+0000","last_fullsized":"2023-09-30T09:35:23.848663+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"335'11012","reported_seq":11093,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:35:43.973338+0000","last_change":"2023-09-30T09:34:47.630136+0000","last_active":"2023-09-30T09:35:43.973338+0000","last_peered":"2023-09-30T09:35:43.973338+0000","last_clean":"2023-09-30T09:34:06.422409+0000","last_became_active":"2023-09-30T09:34:08.626702+0000","last_became_peered":"2023-09-30T09:34:08.626702+0000","last_unstale":"2023-09-30T09:35:43.973338+0000","last_undegraded":"2023-09-30T09:35:43.973338+0000","last_fullsized":"2023-09-30T09:35:43.973338+0000","mapping_epoch":308,"log_start":"298'6206","ondisk_log_start":"298'6206","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4806,"log_dups_size":0,"ondisk_log_size":4806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:53.753813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207355904,"num_objects":3164,"num_object_clones":0,"num_object_copies":9492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3164,"num_objects_unfound":0,"num_objects_dirty":3164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11012,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3164}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":480,"reported_epoch":335,"state":"active+clean+scrubbing","last_fresh":"2023-09-30T09:35:38.724263+0000","last_change":"2023-09-30T09:35:28.678924+0000","last_active":"2023-09-30T09:35:38.724263+0000","last_peered":"2023-09-30T09:35:38.724263+0000","last_clean":"2023-09-30T09:35:38.724263+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:35:38.724263+0000","last_undegraded":"2023-09-30T09:35:38.724263+0000","last_fullsized":"2023-09-30T09:35:38.724263+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":325,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.829843+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:35:23.829843+0000","last_peered":"2023-09-30T09:35:23.829843+0000","last_clean":"2023-09-30T09:35:23.829843+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:35:23.829843+0000","last_undegraded":"2023-09-30T09:35:23.829843+0000","last_fullsized":"2023-09-30T09:35:23.829843+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":102,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835254+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:35:23.835254+0000","last_peered":"2023-09-30T09:35:23.835254+0000","last_clean":"2023-09-30T09:35:23.835254+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:35:23.835254+0000","last_undegraded":"2023-09-30T09:35:23.835254+0000","last_fullsized":"2023-09-30T09:35:23.835254+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"335'10935","reported_seq":11860,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:44.255087+0000","last_change":"2023-09-30T09:34:39.166112+0000","last_active":"2023-09-30T09:35:44.255087+0000","last_peered":"2023-09-30T09:35:44.255087+0000","last_clean":"2023-09-30T09:32:35.465008+0000","last_became_active":"2023-09-30T09:34:39.150257+0000","last_became_peered":"2023-09-30T09:34:39.150257+0000","last_unstale":"2023-09-30T09:35:44.255087+0000","last_undegraded":"2023-09-30T09:34:38.934740+0000","last_fullsized":"2023-09-30T09:35:44.255087+0000","mapping_epoch":316,"log_start":"299'6378","ondisk_log_start":"299'6378","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4557,"log_dups_size":0,"ondisk_log_size":4557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:50.805799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199950336,"num_objects":3051,"num_object_clones":0,"num_object_copies":9153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3050,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":3051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10935,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":419,"num_bytes_recovered":27394048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6,1],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1},{"shards":"3,6","objects":3050}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":629,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.837097+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:35:23.837097+0000","last_peered":"2023-09-30T09:35:23.837097+0000","last_clean":"2023-09-30T09:35:23.837097+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:35:23.837097+0000","last_undegraded":"2023-09-30T09:35:23.837097+0000","last_fullsized":"2023-09-30T09:35:23.837097+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":526,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839730+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:35:23.839730+0000","last_peered":"2023-09-30T09:35:23.839730+0000","last_clean":"2023-09-30T09:35:23.839730+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:35:23.839730+0000","last_undegraded":"2023-09-30T09:35:23.839730+0000","last_fullsized":"2023-09-30T09:35:23.839730+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":331,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841491+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:35:23.841491+0000","last_peered":"2023-09-30T09:35:23.841491+0000","last_clean":"2023-09-30T09:35:23.841491+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:35:23.841491+0000","last_undegraded":"2023-09-30T09:35:23.841491+0000","last_fullsized":"2023-09-30T09:35:23.841491+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840670+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:35:23.840670+0000","last_peered":"2023-09-30T09:35:23.840670+0000","last_clean":"2023-09-30T09:35:23.840670+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:35:23.840670+0000","last_undegraded":"2023-09-30T09:35:23.840670+0000","last_fullsized":"2023-09-30T09:35:23.840670+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"335'11089","reported_seq":11153,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:43.989474+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:35:43.989474+0000","last_peered":"2023-09-30T09:35:43.989474+0000","last_clean":"2023-09-30T09:35:43.989474+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:35:43.989474+0000","last_undegraded":"2023-09-30T09:35:43.989474+0000","last_fullsized":"2023-09-30T09:35:43.989474+0000","mapping_epoch":279,"log_start":"298'6257","ondisk_log_start":"298'6257","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4832,"log_dups_size":0,"ondisk_log_size":4832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207552512,"num_objects":3167,"num_object_clones":0,"num_object_copies":9501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11089,"num_write_kb":456192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":797,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840585+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:35:23.840585+0000","last_peered":"2023-09-30T09:35:23.840585+0000","last_clean":"2023-09-30T09:35:23.840585+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:35:23.840585+0000","last_undegraded":"2023-09-30T09:35:23.840585+0000","last_fullsized":"2023-09-30T09:35:23.840585+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":326,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834286+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:35:23.834286+0000","last_peered":"2023-09-30T09:35:23.834286+0000","last_clean":"2023-09-30T09:35:23.834286+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:35:23.834286+0000","last_undegraded":"2023-09-30T09:35:23.834286+0000","last_fullsized":"2023-09-30T09:35:23.834286+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":314,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847526+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:35:23.847526+0000","last_peered":"2023-09-30T09:35:23.847526+0000","last_clean":"2023-09-30T09:35:23.847526+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:35:23.847526+0000","last_undegraded":"2023-09-30T09:35:23.847526+0000","last_fullsized":"2023-09-30T09:35:23.847526+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":52,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848589+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:35:23.848589+0000","last_peered":"2023-09-30T09:35:23.848589+0000","last_clean":"2023-09-30T09:35:23.848589+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:35:23.848589+0000","last_undegraded":"2023-09-30T09:35:23.848589+0000","last_fullsized":"2023-09-30T09:35:23.848589+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"335'11095","reported_seq":12117,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:43.967112+0000","last_change":"2023-09-30T09:35:10.808625+0000","last_active":"2023-09-30T09:35:43.967112+0000","last_peered":"2023-09-30T09:35:43.967112+0000","last_clean":"2023-09-30T09:32:35.460821+0000","last_became_active":"2023-09-30T09:35:10.778683+0000","last_became_peered":"2023-09-30T09:35:10.778683+0000","last_unstale":"2023-09-30T09:35:43.967112+0000","last_undegraded":"2023-09-30T09:35:10.766189+0000","last_fullsized":"2023-09-30T09:35:43.967112+0000","mapping_epoch":327,"log_start":"298'6333","ondisk_log_start":"298'6333","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4762,"log_dups_size":0,"ondisk_log_size":4762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:45.905717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213450752,"num_objects":3257,"num_object_clones":0,"num_object_copies":9771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3257,"num_objects_misplaced":6514,"num_objects_unfound":0,"num_objects_dirty":3257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11095,"num_write_kb":459264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":459,"num_bytes_recovered":30081024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[6,1,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":3257}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":694,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.838890+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:35:23.838890+0000","last_peered":"2023-09-30T09:35:23.838890+0000","last_clean":"2023-09-30T09:35:23.838890+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:35:23.838890+0000","last_undegraded":"2023-09-30T09:35:23.838890+0000","last_fullsized":"2023-09-30T09:35:23.838890+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":513,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835690+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:35:23.835690+0000","last_peered":"2023-09-30T09:35:23.835690+0000","last_clean":"2023-09-30T09:35:23.835690+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:35:23.835690+0000","last_undegraded":"2023-09-30T09:35:23.835690+0000","last_fullsized":"2023-09-30T09:35:23.835690+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":60,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835579+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:35:23.835579+0000","last_peered":"2023-09-30T09:35:23.835579+0000","last_clean":"2023-09-30T09:35:23.835579+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:35:23.835579+0000","last_undegraded":"2023-09-30T09:35:23.835579+0000","last_fullsized":"2023-09-30T09:35:23.835579+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":254,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.840835+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:35:23.840835+0000","last_peered":"2023-09-30T09:35:23.840835+0000","last_clean":"2023-09-30T09:35:23.840835+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:35:23.840835+0000","last_undegraded":"2023-09-30T09:35:23.840835+0000","last_fullsized":"2023-09-30T09:35:23.840835+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"335'10945","reported_seq":14386,"reported_epoch":335,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:45.000883+0000","last_change":"2023-09-30T09:35:23.879286+0000","last_active":"2023-09-30T09:35:45.000883+0000","last_peered":"2023-09-30T09:35:45.000883+0000","last_clean":"2023-09-30T09:34:06.430942+0000","last_became_active":"2023-09-30T09:34:47.755052+0000","last_became_peered":"2023-09-30T09:34:47.755052+0000","last_unstale":"2023-09-30T09:35:45.000883+0000","last_undegraded":"2023-09-30T09:34:47.557422+0000","last_fullsized":"2023-09-30T09:34:47.514378+0000","mapping_epoch":320,"log_start":"290'4750","ondisk_log_start":"290'4750","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":6195,"log_dups_size":0,"ondisk_log_size":6195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:44:56.113242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201916416,"num_objects":3081,"num_object_clones":0,"num_object_copies":9243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":141,"num_objects_misplaced":2969,"num_objects_unfound":0,"num_objects_dirty":3081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10945,"num_write_kb":448832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1680,"num_bytes_recovered":109772800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-29},{"shards":"0,3,7","objects":2969}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":907,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842219+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:35:23.842219+0000","last_peered":"2023-09-30T09:35:23.842219+0000","last_clean":"2023-09-30T09:35:23.842219+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:35:23.842219+0000","last_undegraded":"2023-09-30T09:35:23.842219+0000","last_fullsized":"2023-09-30T09:35:23.842219+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":599,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.838645+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:35:23.838645+0000","last_peered":"2023-09-30T09:35:23.838645+0000","last_clean":"2023-09-30T09:35:23.838645+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:35:23.838645+0000","last_undegraded":"2023-09-30T09:35:23.838645+0000","last_fullsized":"2023-09-30T09:35:23.838645+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":253,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.842053+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:35:23.842053+0000","last_peered":"2023-09-30T09:35:23.842053+0000","last_clean":"2023-09-30T09:35:23.842053+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:35:23.842053+0000","last_undegraded":"2023-09-30T09:35:23.842053+0000","last_fullsized":"2023-09-30T09:35:23.842053+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":287,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835692+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:35:23.835692+0000","last_peered":"2023-09-30T09:35:23.835692+0000","last_clean":"2023-09-30T09:35:23.835692+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:35:23.835692+0000","last_undegraded":"2023-09-30T09:35:23.835692+0000","last_fullsized":"2023-09-30T09:35:23.835692+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"335'11005","reported_seq":19256,"reported_epoch":335,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:35:44.307455+0000","last_change":"2023-09-30T09:35:10.837720+0000","last_active":"2023-09-30T09:35:44.307455+0000","last_peered":"2023-09-30T09:35:44.307455+0000","last_clean":"2023-09-30T09:32:35.481650+0000","last_became_active":"2023-09-30T09:35:10.788829+0000","last_became_peered":"2023-09-30T09:35:10.788829+0000","last_unstale":"2023-09-30T09:35:44.307455+0000","last_undegraded":"2023-09-30T09:35:10.757584+0000","last_fullsized":"2023-09-30T09:35:44.307455+0000","mapping_epoch":327,"log_start":"286'3000","ondisk_log_start":"286'3000","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":8005,"log_dups_size":0,"ondisk_log_size":8005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:05.384343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206635008,"num_objects":3153,"num_object_clones":0,"num_object_copies":9459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1647,"num_objects_misplaced":5749,"num_objects_unfound":0,"num_objects_dirty":3153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11005,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4068,"num_bytes_recovered":266600448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[7,1,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":557},{"shards":"2,6,7","objects":1240},{"shards":"6,7","objects":1356}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":480,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835513+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:35:23.835513+0000","last_peered":"2023-09-30T09:35:23.835513+0000","last_clean":"2023-09-30T09:35:23.835513+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:35:23.835513+0000","last_undegraded":"2023-09-30T09:35:23.835513+0000","last_fullsized":"2023-09-30T09:35:23.835513+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":710,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.838721+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:35:23.838721+0000","last_peered":"2023-09-30T09:35:23.838721+0000","last_clean":"2023-09-30T09:35:23.838721+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:35:23.838721+0000","last_undegraded":"2023-09-30T09:35:23.838721+0000","last_fullsized":"2023-09-30T09:35:23.838721+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":263,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.867893+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:35:27.867893+0000","last_peered":"2023-09-30T09:35:27.867893+0000","last_clean":"2023-09-30T09:35:27.867893+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:35:27.867893+0000","last_undegraded":"2023-09-30T09:35:27.867893+0000","last_fullsized":"2023-09-30T09:35:27.867893+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.831437+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:35:23.831437+0000","last_peered":"2023-09-30T09:35:23.831437+0000","last_clean":"2023-09-30T09:35:23.831437+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:35:23.831437+0000","last_undegraded":"2023-09-30T09:35:23.831437+0000","last_fullsized":"2023-09-30T09:35:23.831437+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"335'11031","reported_seq":15473,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:44.269640+0000","last_change":"2023-09-30T09:34:47.553318+0000","last_active":"2023-09-30T09:35:44.269640+0000","last_peered":"2023-09-30T09:35:44.269640+0000","last_clean":"2023-09-30T09:32:35.469075+0000","last_became_active":"2023-09-30T09:33:33.792569+0000","last_became_peered":"2023-09-30T09:33:33.792569+0000","last_unstale":"2023-09-30T09:35:44.269640+0000","last_undegraded":"2023-09-30T09:33:33.610726+0000","last_fullsized":"2023-09-30T09:33:33.587755+0000","mapping_epoch":302,"log_start":"292'5545","ondisk_log_start":"292'5545","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:16.121133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204800000,"num_objects":3125,"num_object_clones":0,"num_object_copies":9375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11031,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2178,"num_bytes_recovered":142737408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1685}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":609,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834735+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:35:23.834735+0000","last_peered":"2023-09-30T09:35:23.834735+0000","last_clean":"2023-09-30T09:35:23.834735+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:35:23.834735+0000","last_undegraded":"2023-09-30T09:35:23.834735+0000","last_fullsized":"2023-09-30T09:35:23.834735+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":843,"reported_epoch":334,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:23.849441+0000","last_change":"2023-09-30T09:35:11.685841+0000","last_active":"2023-09-30T09:35:23.849441+0000","last_peered":"2023-09-30T09:35:23.849441+0000","last_clean":"2023-09-30T09:26:47.308941+0000","last_became_active":"2023-09-30T09:35:11.685274+0000","last_became_peered":"2023-09-30T09:35:11.685274+0000","last_unstale":"2023-09-30T09:35:23.849441+0000","last_undegraded":"2023-09-30T09:35:11.661789+0000","last_fullsized":"2023-09-30T09:35:11.661213+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:38.516737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[1,4],"avail_no_missing":["1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":60,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848432+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:35:23.848432+0000","last_peered":"2023-09-30T09:35:23.848432+0000","last_clean":"2023-09-30T09:35:23.848432+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:35:23.848432+0000","last_undegraded":"2023-09-30T09:35:23.848432+0000","last_fullsized":"2023-09-30T09:35:23.848432+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":235,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.875601+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:35:27.875601+0000","last_peered":"2023-09-30T09:35:27.875601+0000","last_clean":"2023-09-30T09:35:27.875601+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:35:27.875601+0000","last_undegraded":"2023-09-30T09:35:27.875601+0000","last_fullsized":"2023-09-30T09:35:27.875601+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"335'11129","reported_seq":11294,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:35:44.167513+0000","last_change":"2023-09-30T09:33:32.864041+0000","last_active":"2023-09-30T09:35:44.167513+0000","last_peered":"2023-09-30T09:35:44.167513+0000","last_clean":"2023-09-30T09:33:09.329441+0000","last_became_active":"2023-09-30T09:33:11.506402+0000","last_became_peered":"2023-09-30T09:33:11.506402+0000","last_unstale":"2023-09-30T09:35:44.167513+0000","last_undegraded":"2023-09-30T09:35:44.167513+0000","last_fullsized":"2023-09-30T09:35:44.167513+0000","mapping_epoch":295,"log_start":"299'6598","ondisk_log_start":"299'6598","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4531,"log_dups_size":0,"ondisk_log_size":4531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:51:06.485777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204144640,"num_objects":3115,"num_object_clones":0,"num_object_copies":9345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3092,"num_objects_unfound":0,"num_objects_dirty":3115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11129,"num_write_kb":455808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":3115}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":807,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847716+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:35:23.847716+0000","last_peered":"2023-09-30T09:35:23.847716+0000","last_clean":"2023-09-30T09:35:23.847716+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:35:23.847716+0000","last_undegraded":"2023-09-30T09:35:23.847716+0000","last_fullsized":"2023-09-30T09:35:23.847716+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":335,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.835333+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:35:23.835333+0000","last_peered":"2023-09-30T09:35:23.835333+0000","last_clean":"2023-09-30T09:35:23.835333+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:35:23.835333+0000","last_undegraded":"2023-09-30T09:35:23.835333+0000","last_fullsized":"2023-09-30T09:35:23.835333+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1101,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834573+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:35:23.834573+0000","last_peered":"2023-09-30T09:35:23.834573+0000","last_clean":"2023-09-30T09:35:23.834573+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:35:23.834573+0000","last_undegraded":"2023-09-30T09:35:23.834573+0000","last_fullsized":"2023-09-30T09:35:23.834573+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":352,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.906364+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:35:27.906364+0000","last_peered":"2023-09-30T09:35:27.906364+0000","last_clean":"2023-09-30T09:35:27.906364+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:35:27.906364+0000","last_undegraded":"2023-09-30T09:35:27.906364+0000","last_fullsized":"2023-09-30T09:35:27.906364+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"335'11138","reported_seq":13121,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:35:44.150908+0000","last_change":"2023-09-30T09:34:37.846918+0000","last_active":"2023-09-30T09:35:44.150908+0000","last_peered":"2023-09-30T09:35:44.150908+0000","last_clean":"2023-09-30T09:32:35.469896+0000","last_became_active":"2023-09-30T09:33:32.854592+0000","last_became_peered":"2023-09-30T09:33:32.854592+0000","last_unstale":"2023-09-30T09:35:44.150908+0000","last_undegraded":"2023-09-30T09:33:32.787590+0000","last_fullsized":"2023-09-30T09:33:32.601623+0000","mapping_epoch":301,"log_start":"299'6650","ondisk_log_start":"299'6650","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4488,"log_dups_size":0,"ondisk_log_size":4488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:02.368619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206176256,"num_objects":3146,"num_object_clones":0,"num_object_copies":9438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2653,"num_objects_misplaced":3146,"num_objects_unfound":0,"num_objects_dirty":3146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11138,"num_write_kb":457088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":946,"num_bytes_recovered":61997056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":493}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":800,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.834705+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:35:23.834705+0000","last_peered":"2023-09-30T09:35:23.834705+0000","last_clean":"2023-09-30T09:35:23.834705+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:35:23.834705+0000","last_undegraded":"2023-09-30T09:35:23.834705+0000","last_fullsized":"2023-09-30T09:35:23.834705+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":660,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.837245+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:35:23.837245+0000","last_peered":"2023-09-30T09:35:23.837245+0000","last_clean":"2023-09-30T09:35:23.837245+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:35:23.837245+0000","last_undegraded":"2023-09-30T09:35:23.837245+0000","last_fullsized":"2023-09-30T09:35:23.837245+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":22,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.831528+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:35:23.831528+0000","last_peered":"2023-09-30T09:35:23.831528+0000","last_clean":"2023-09-30T09:35:23.831528+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:35:23.831528+0000","last_undegraded":"2023-09-30T09:35:23.831528+0000","last_fullsized":"2023-09-30T09:35:23.831528+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":276,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.848029+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:35:23.848029+0000","last_peered":"2023-09-30T09:35:23.848029+0000","last_clean":"2023-09-30T09:35:23.848029+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:35:23.848029+0000","last_undegraded":"2023-09-30T09:35:23.848029+0000","last_fullsized":"2023-09-30T09:35:23.848029+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":748,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.832370+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:35:23.832370+0000","last_peered":"2023-09-30T09:35:23.832370+0000","last_clean":"2023-09-30T09:35:23.832370+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:35:23.832370+0000","last_undegraded":"2023-09-30T09:35:23.832370+0000","last_fullsized":"2023-09-30T09:35:23.832370+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":18,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.839344+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:35:23.839344+0000","last_peered":"2023-09-30T09:35:23.839344+0000","last_clean":"2023-09-30T09:35:23.839344+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:35:23.839344+0000","last_undegraded":"2023-09-30T09:35:23.839344+0000","last_fullsized":"2023-09-30T09:35:23.839344+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:25.342082+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:35:25.342082+0000","last_peered":"2023-09-30T09:35:25.342082+0000","last_clean":"2023-09-30T09:35:25.342082+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:35:25.342082+0000","last_undegraded":"2023-09-30T09:35:25.342082+0000","last_fullsized":"2023-09-30T09:35:25.342082+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":519,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.858975+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:35:23.858975+0000","last_peered":"2023-09-30T09:35:23.858975+0000","last_clean":"2023-09-30T09:35:23.858975+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:35:23.858975+0000","last_undegraded":"2023-09-30T09:35:23.858975+0000","last_fullsized":"2023-09-30T09:35:23.858975+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":313,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841765+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:35:23.841765+0000","last_peered":"2023-09-30T09:35:23.841765+0000","last_clean":"2023-09-30T09:35:23.841765+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:35:23.841765+0000","last_undegraded":"2023-09-30T09:35:23.841765+0000","last_fullsized":"2023-09-30T09:35:23.841765+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":29,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.846976+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:35:23.846976+0000","last_peered":"2023-09-30T09:35:23.846976+0000","last_clean":"2023-09-30T09:35:23.846976+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:35:23.846976+0000","last_undegraded":"2023-09-30T09:35:23.846976+0000","last_fullsized":"2023-09-30T09:35:23.846976+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":279,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.841418+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:35:23.841418+0000","last_peered":"2023-09-30T09:35:23.841418+0000","last_clean":"2023-09-30T09:35:23.841418+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:35:23.841418+0000","last_undegraded":"2023-09-30T09:35:23.841418+0000","last_fullsized":"2023-09-30T09:35:23.841418+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":29,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.847308+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:35:23.847308+0000","last_peered":"2023-09-30T09:35:23.847308+0000","last_clean":"2023-09-30T09:35:23.847308+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:35:23.847308+0000","last_undegraded":"2023-09-30T09:35:23.847308+0000","last_fullsized":"2023-09-30T09:35:23.847308+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":463,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.837663+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:35:23.837663+0000","last_peered":"2023-09-30T09:35:23.837663+0000","last_clean":"2023-09-30T09:35:23.837663+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:35:23.837663+0000","last_undegraded":"2023-09-30T09:35:23.837663+0000","last_fullsized":"2023-09-30T09:35:23.837663+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":331,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-30T09:35:27.891275+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:35:27.891275+0000","last_peered":"2023-09-30T09:35:27.891275+0000","last_clean":"2023-09-30T09:35:27.891275+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:35:27.891275+0000","last_undegraded":"2023-09-30T09:35:27.891275+0000","last_fullsized":"2023-09-30T09:35:27.891275+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":355,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.832579+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:35:23.832579+0000","last_peered":"2023-09-30T09:35:23.832579+0000","last_clean":"2023-09-30T09:35:23.832579+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:35:23.832579+0000","last_undegraded":"2023-09-30T09:35:23.832579+0000","last_fullsized":"2023-09-30T09:35:23.832579+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":783,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.833541+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:35:23.833541+0000","last_peered":"2023-09-30T09:35:23.833541+0000","last_clean":"2023-09-30T09:35:23.833541+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:35:23.833541+0000","last_undegraded":"2023-09-30T09:35:23.833541+0000","last_fullsized":"2023-09-30T09:35:23.833541+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":38,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.872212+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:35:23.872212+0000","last_peered":"2023-09-30T09:35:23.872212+0000","last_clean":"2023-09-30T09:35:23.872212+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:35:23.872212+0000","last_undegraded":"2023-09-30T09:35:23.872212+0000","last_fullsized":"2023-09-30T09:35:23.872212+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":310,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-30T09:35:23.846837+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:35:23.846837+0000","last_peered":"2023-09-30T09:35:23.846837+0000","last_clean":"2023-09-30T09:35:23.846837+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:35:23.846837+0000","last_undegraded":"2023-09-30T09:35:23.846837+0000","last_fullsized":"2023-09-30T09:35:23.846837+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3288793112,"num_objects":50184,"num_object_clones":0,"num_object_copies":150552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24404,"num_objects_misplaced":46563,"num_objects_unfound":0,"num_objects_dirty":50184,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":176368,"num_write_kb":7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15887,"num_bytes_recovered":1040580608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8491905024,"data_stored":8491892808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80232,"ondisk_log_size":80232,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234289,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495926,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2195772,"kb_used_data":1821300,"kb_used_omap":0,"kb_used_meta":374464,"kb_avail":91549380,"statfs":{"total":95995035648,"available":93746565120,"internally_reserved":0,"allocated":1865011200,"data_stored":1863847099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":383451136},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:34:48 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.0680000000000001,"15min":1.387},"min":{"1min":0.23699999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.968,"5min":19.882000000000001,"15min":45.927999999999997},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.724,"15min":1.7070000000000001},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.23499999999999999},"max":{"1min":5.8730000000000002,"5min":20.373000000000001,"15min":45.316000000000003},"last":0.28199999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:35:35 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":1.9470000000000001,"15min":1.851},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.7070000000000007,"5min":9.7070000000000007,"15min":9.7070000000000007},"last":1.796},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.1080000000000001,"15min":2.0779999999999998},"min":{"1min":0.35799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.0820000000000007,"5min":9.0820000000000007,"15min":9.0820000000000007},"last":2.7610000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":1.738,"15min":1.7290000000000001},"min":{"1min":0.32800000000000001,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":8.0850000000000009,"5min":22.158000000000001,"15min":37.978000000000002},"last":0.315},{"interface":"front","average":{"1min":2.6360000000000001,"5min":2.0289999999999999,"15min":1.641},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.706,"5min":22.356999999999999,"15min":37.759999999999998},"last":2.6349999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":3.1960000000000002,"15min":1.921},"min":{"1min":0.34699999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1920000000000002,"5min":80.436999999999998,"15min":80.436999999999998},"last":2.6720000000000002},{"interface":"front","average":{"1min":2.827,"5min":3.6720000000000002,"15min":2.1970000000000001},"min":{"1min":0.39000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":13.340999999999999,"5min":80.039000000000001,"15min":80.039000000000001},"last":1.4930000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:34:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.496,"15min":1.554},"min":{"1min":0.31,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.2890000000000001,"5min":8.1059999999999999,"15min":73.656000000000006},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.2330000000000001,"15min":1.3520000000000001},"min":{"1min":0.30099999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.8200000000000001,"5min":7.7160000000000002,"15min":73.882999999999996},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":3.0289999999999999,"15min":1.7809999999999999},"min":{"1min":0.33200000000000002,"5min":0.29099999999999998,"15min":0.28499999999999998},"max":{"1min":5.4880000000000004,"5min":76.613,"15min":76.613},"last":0.438},{"interface":"front","average":{"1min":0.54400000000000004,"5min":2.9039999999999999,"15min":1.7250000000000001},"min":{"1min":0.36199999999999999,"5min":0.31,"15min":0.31},"max":{"1min":1.246,"5min":76.329999999999998,"15min":76.329999999999998},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:35:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.712,"15min":2.1320000000000001},"min":{"1min":0.38,"5min":0.377,"15min":0.32100000000000001},"max":{"1min":8.7400000000000002,"5min":8.7400000000000002,"15min":91.120999999999995},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.306,"5min":1.448,"15min":2.0710000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.9489999999999998,"5min":9.9489999999999998,"15min":91.733999999999995},"last":0.45400000000000001}]}]},{"osd":1,"up_from":301,"seq":1292785156124,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":538920,"kb_used_data":388448,"kb_used_omap":9,"kb_used_meta":150454,"kb_avail":93206232,"statfs":{"total":95995035648,"available":95443181568,"internally_reserved":0,"allocated":397770752,"data_stored":395876385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":154064914},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.6549999999999998,"15min":2.5790000000000002},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.699999999999999,"5min":13.699999999999999,"15min":13.699999999999999},"last":6.5730000000000004},{"interface":"front","average":{"1min":4.0659999999999998,"5min":3.2320000000000002,"15min":3.093},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.25},"max":{"1min":13.779999999999999,"5min":13.779999999999999,"15min":13.779999999999999},"last":3.0569999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":2.7120000000000002,"15min":2.7679999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":16.437000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":6.3209999999999997},{"interface":"front","average":{"1min":3.3740000000000001,"5min":2.839,"15min":2.75},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.121,"5min":12.121,"15min":12.121},"last":0.53200000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":4.4589999999999996,"5min":8.4489999999999998,"15min":9.1140000000000008},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.837999999999999,"5min":77.959000000000003,"15min":77.959000000000003},"last":3.2869999999999999},{"interface":"front","average":{"1min":3.2690000000000001,"5min":8.7949999999999999,"15min":9.7159999999999993},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":11.657,"5min":77.828999999999994,"15min":77.828999999999994},"last":26.402999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":2.0249999999999999,"15min":1.9990000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":13.602,"5min":13.602,"15min":13.602},"last":2.9079999999999999},{"interface":"front","average":{"1min":2.1920000000000002,"5min":1.472,"15min":1.3520000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":10.773,"5min":10.773,"15min":10.773},"last":3.1150000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":2.2309999999999999,"15min":2.149},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":13.666,"5min":13.666,"15min":13.666},"last":2.9649999999999999},{"interface":"front","average":{"1min":2.218,"5min":1.724,"15min":1.6419999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":3.3250000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":14.346,"15min":16.18},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":13.728,"5min":209.221,"15min":209.221},"last":2.8639999999999999},{"interface":"front","average":{"1min":2.5110000000000001,"5min":14.115,"15min":16.048999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":13.755000000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":3.012}]},{"osd":7,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":10.254,"15min":11.714},"min":{"1min":0.46000000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":4.5279999999999996,"5min":165.59399999999999,"15min":165.59399999999999},"last":3.8580000000000001},{"interface":"front","average":{"1min":1.429,"5min":10.429,"15min":11.929},"min":{"1min":0.47299999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":4.5640000000000001,"5min":165.833,"15min":165.833},"last":3.8959999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926023,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":931000,"kb_used_data":607984,"kb_used_omap":5,"kb_used_meta":323002,"kb_avail":92814152,"statfs":{"total":95995035648,"available":95041691648,"internally_reserved":0,"allocated":622575616,"data_stored":620551392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":330754301},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":2.8250000000000002,"15min":2.9399999999999999},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":11.175000000000001,"5min":17.966000000000001,"15min":17.966000000000001},"last":0.252},{"interface":"front","average":{"1min":4.1660000000000004,"5min":8.3729999999999993,"15min":9.0739999999999998},"min":{"1min":0.22500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":14.097,"5min":138.92400000000001,"15min":138.92400000000001},"last":7.4169999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:35:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7639999999999998,"5min":1.756,"15min":1.5720000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.6769999999999996,"5min":11.218999999999999,"15min":55.823999999999998},"last":7.1970000000000001},{"interface":"front","average":{"1min":5.6159999999999997,"5min":2.0830000000000002,"15min":1.748},"min":{"1min":0.29299999999999998,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":40.128999999999998,"5min":40.128999999999998,"15min":55.790999999999997},"last":7.2220000000000004}]},{"osd":3,"last update":"Sat Sep 30 09:34:53 2023","interfaces":[{"interface":"back","average":{"1min":26.254999999999999,"5min":6.7469999999999999,"15min":3.419},"min":{"1min":0.35599999999999998,"5min":0.221,"15min":0.221},"max":{"1min":218.06999999999999,"5min":218.06999999999999,"15min":218.06999999999999},"last":1.1060000000000001},{"interface":"front","average":{"1min":18.782,"5min":5.2729999999999997,"15min":2.8730000000000002},"min":{"1min":0.46100000000000002,"5min":0.245,"15min":0.245},"max":{"1min":218.63200000000001,"5min":218.63200000000001,"15min":218.63200000000001},"last":0.27800000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:34:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.0580000000000001,"15min":1.4530000000000001},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":4.4370000000000003,"5min":7.2830000000000004,"15min":121.486},"last":7.165},{"interface":"front","average":{"1min":9.1699999999999999,"5min":2.8980000000000001,"15min":2.242},"min":{"1min":0.40300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":136.262,"5min":136.262,"15min":136.262},"last":1.1850000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:34:45 2023","interfaces":[{"interface":"back","average":{"1min":9.3550000000000004,"5min":5.2199999999999998,"15min":2.6360000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":136.34,"5min":136.34,"15min":136.34},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.9019999999999999,"5min":3.609,"15min":2.0830000000000002},"min":{"1min":0.41799999999999998,"5min":0.39900000000000002,"15min":0.33700000000000002},"max":{"1min":8.7430000000000003,"5min":135.381,"15min":135.381},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:34:45 2023","interfaces":[{"interface":"back","average":{"1min":22.914000000000001,"5min":6.7789999999999999,"15min":3.0179999999999998},"min":{"1min":0.39100000000000001,"5min":0.36199999999999999,"15min":0.33300000000000002},"max":{"1min":214.785,"5min":214.785,"15min":214.785},"last":0.503},{"interface":"front","average":{"1min":23.077000000000002,"5min":6.7469999999999999,"15min":2.9580000000000002},"min":{"1min":0.40600000000000003,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":214.93899999999999,"5min":214.93899999999999,"15min":214.93899999999999},"last":7.0350000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:34:45 2023","interfaces":[{"interface":"back","average":{"1min":7.4210000000000003,"5min":5.0599999999999996,"15min":2.9569999999999999},"min":{"1min":0.432,"5min":0.373,"15min":0.373},"max":{"1min":97.382000000000005,"5min":150.96899999999999,"15min":150.96899999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":6.8760000000000003,"5min":4.8369999999999997,"15min":2.7789999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":97.015000000000001,"5min":150.328,"15min":150.328},"last":7.109}]}]},{"osd":2,"up_from":104,"seq":446676598964,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":404960,"kb_used_data":116760,"kb_used_omap":5,"kb_used_meta":288186,"kb_avail":93340192,"statfs":{"total":95995035648,"available":95580356608,"internally_reserved":0,"allocated":119562240,"data_stored":117737589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":295103117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":2.9449999999999998,"15min":1.9179999999999999},"min":{"1min":0.22800000000000001,"5min":0.187,"15min":0.187},"max":{"1min":77.379000000000005,"5min":77.379000000000005,"15min":77.379000000000005},"last":13.365},{"interface":"front","average":{"1min":6.2549999999999999,"5min":2.9660000000000002,"15min":1.8140000000000001},"min":{"1min":0.25600000000000001,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":76.796999999999997,"5min":77.066999999999993,"15min":77.066999999999993},"last":8.1850000000000005}]},{"osd":1,"last update":"Sat Sep 30 09:35:36 2023","interfaces":[{"interface":"back","average":{"1min":4.4779999999999998,"5min":3.7530000000000001,"15min":3.6320000000000001},"min":{"1min":0.28199999999999997,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.728999999999999,"5min":14.707000000000001,"15min":14.707000000000001},"last":8.3330000000000002},{"interface":"front","average":{"1min":3.819,"5min":3.2090000000000001,"15min":3.1070000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.968,"5min":10.179,"15min":10.179},"last":8.1519999999999992}]},{"osd":3,"last update":"Sat Sep 30 09:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.4169999999999998,"5min":3.883,"15min":2.085},"min":{"1min":0.35199999999999998,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":77.370000000000005,"5min":78.915999999999997,"15min":78.915999999999997},"last":8.2089999999999996},{"interface":"front","average":{"1min":8.0079999999999991,"5min":3.9199999999999999,"15min":2.2669999999999999},"min":{"1min":0.30199999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":80.834000000000003,"5min":80.834000000000003,"15min":80.834000000000003},"last":13.335000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.8029999999999999,"15min":1.25},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":5.524,"5min":35.972000000000001,"15min":35.972000000000001},"last":8.1240000000000006},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.7470000000000001,"15min":1.4810000000000001},"min":{"1min":0.34799999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":5.8789999999999996,"5min":36.518000000000001,"15min":36.518000000000001},"last":13.224}]},{"osd":5,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.732,"15min":1.9410000000000001},"min":{"1min":0.377,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":12.651999999999999,"5min":12.651999999999999,"15min":155.477},"last":8.298},{"interface":"front","average":{"1min":1.794,"5min":1.4119999999999999,"15min":1.8660000000000001},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":4.4560000000000004,"5min":6.048,"15min":155.46000000000001},"last":8.0839999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":6.7009999999999996,"15min":3.1899999999999999},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.9729999999999999,"5min":154.417,"15min":154.417},"last":13.273},{"interface":"front","average":{"1min":3.1589999999999998,"5min":6.9749999999999996,"15min":3.1960000000000002},"min":{"1min":0.47399999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":12.710000000000001,"5min":154.04400000000001,"15min":154.04400000000001},"last":8.25}]},{"osd":7,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":3.665,"15min":3},"min":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8399999999999999,"5min":123.294,"15min":213.404},"last":13.304},{"interface":"front","average":{"1min":2.234,"5min":3.9769999999999999,"15min":3.0219999999999998},"min":{"1min":0.438,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":12.686999999999999,"5min":123.16,"15min":213.27199999999999},"last":8.2750000000000004}]}]},{"osd":3,"up_from":109,"seq":468151435436,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2781544,"kb_used_data":2220896,"kb_used_omap":2,"kb_used_meta":560573,"kb_avail":90963608,"statfs":{"total":95995035648,"available":93146734592,"internally_reserved":0,"allocated":2274197504,"data_stored":2272242100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":574027111},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":3.1619999999999999,"15min":2.395},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":17.241,"5min":102.57299999999999,"15min":130.946},"last":60.726999999999997},{"interface":"front","average":{"1min":4.3620000000000001,"5min":3.516,"15min":2.476},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":26.687000000000001,"5min":102.745,"15min":130.96899999999999},"last":3.9729999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:35:33 2023","interfaces":[{"interface":"back","average":{"1min":4.625,"5min":17.937999999999999,"15min":20.157},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":21.709,"5min":273.99599999999998,"15min":273.99599999999998},"last":4.0419999999999998},{"interface":"front","average":{"1min":5.2140000000000004,"5min":16.260000000000002,"15min":18.100999999999999},"min":{"1min":0.38800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":24.771999999999998,"5min":274.21600000000001,"15min":274.21600000000001},"last":0.77800000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":7.2629999999999999,"5min":2.851,"15min":2.6669999999999998},"min":{"1min":0.378,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":76.974000000000004,"5min":76.974000000000004,"15min":145.75899999999999},"last":0.754},{"interface":"front","average":{"1min":6.0830000000000002,"5min":2.5299999999999998,"15min":2.496},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":77.593999999999994,"5min":77.593999999999994,"15min":150.358},"last":60.561999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.788,"15min":1.2589999999999999},"min":{"1min":0.32400000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":17.300999999999998,"5min":17.300999999999998,"15min":17.300999999999998},"last":60.637},{"interface":"front","average":{"1min":3.9079999999999999,"5min":1.7569999999999999,"15min":1.3640000000000001},"min":{"1min":0.378,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":10.731,"5min":10.731,"15min":15.638},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":3.7839999999999998,"5min":2.4350000000000001,"15min":1.6399999999999999},"min":{"1min":0.52200000000000002,"5min":0.33100000000000002,"15min":0.308},"max":{"1min":10.77,"5min":37.485999999999997,"15min":37.485999999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":3.8730000000000002,"5min":2.7120000000000002,"15min":1.79},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.33500000000000002},"max":{"1min":24.654,"5min":37.448,"15min":37.448},"last":0.67100000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":1.905,"15min":1.3400000000000001},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":17.344999999999999,"5min":17.344999999999999,"15min":17.344999999999999},"last":60.698},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.0259999999999998,"15min":1.5469999999999999},"min":{"1min":0.49399999999999999,"5min":0.38900000000000001,"15min":0.34799999999999998},"max":{"1min":10.818,"5min":10.818,"15min":14.69},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:34:55 2023","interfaces":[{"interface":"back","average":{"1min":4.3120000000000003,"5min":2.4529999999999998,"15min":2.3809999999999998},"min":{"1min":0.504,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":24.687000000000001,"5min":26.469000000000001,"15min":226.03999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.0640000000000001,"15min":2.1110000000000002},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":17.379000000000001,"5min":26.428000000000001,"15min":226.06100000000001},"last":60.670000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019199,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1042036,"kb_used_data":801644,"kb_used_omap":0,"kb_used_meta":240384,"kb_avail":92703116,"statfs":{"total":95995035648,"available":94927990784,"internally_reserved":0,"allocated":820883456,"data_stored":819727050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":246153216},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":1.415,"15min":2.7999999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":14.353,"5min":14.353,"15min":128.79300000000001},"last":4.5940000000000003},{"interface":"front","average":{"1min":2.798,"5min":1.659,"15min":2.7509999999999999},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.248},"max":{"1min":10.618,"5min":11.744,"15min":128.38800000000001},"last":6.9939999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":8.2840000000000007,"15min":9.0969999999999995},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.957999999999998,"5min":160.69999999999999,"15min":160.69999999999999},"last":2.6669999999999998},{"interface":"front","average":{"1min":2.024,"5min":8.0559999999999992,"15min":9.0619999999999994},"min":{"1min":0.26800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":10.91,"5min":160.50700000000001,"15min":160.50700000000001},"last":4.6319999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:35:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":1.8200000000000001,"15min":2.8410000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.096,"5min":19.391999999999999,"15min":140.601},"last":2.6509999999999998},{"interface":"front","average":{"1min":2.089,"5min":1.865,"15min":2.8130000000000002},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.309},"max":{"1min":11.057,"5min":19.422999999999998,"15min":140.54900000000001},"last":2.6190000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:34:44 2023","interfaces":[{"interface":"back","average":{"1min":5.1029999999999998,"5min":3.1320000000000001,"15min":3.2770000000000001},"min":{"1min":0.315,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":52.762,"5min":70.241,"15min":281.42200000000003},"last":2.4860000000000002},{"interface":"front","average":{"1min":4.7389999999999999,"5min":2.9169999999999998,"15min":3.218},"min":{"1min":0.32800000000000001,"5min":0.312,"15min":0.312},"max":{"1min":52.790999999999997,"5min":70.433999999999997,"15min":281.44999999999999},"last":2.504}]},{"osd":5,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.254,"15min":1.6899999999999999},"min":{"1min":0.27600000000000002,"5min":0.219,"15min":0.219},"max":{"1min":8.6509999999999998,"5min":11.808,"15min":134.30600000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.161,"15min":1.766},"min":{"1min":0.29599999999999999,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":6.2149999999999999,"5min":6.3929999999999998,"15min":134.35300000000001},"last":0.379}]},{"osd":6,"last update":"Sat Sep 30 09:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":3.081,"15min":1.514},"min":{"1min":0.30499999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.4470000000000001,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.95699999999999996,"5min":2.9609999999999999,"15min":1.5469999999999999},"min":{"1min":0.30199999999999999,"5min":0.249,"15min":0.249},"max":{"1min":7.8079999999999998,"5min":202.42400000000001,"15min":202.42400000000001},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:35:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":4.6449999999999996,"15min":2.9969999999999999},"min":{"1min":0.35099999999999998,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":7.5220000000000002,"5min":230.90100000000001,"15min":230.90100000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.67500000000000004,"5min":4.2539999999999996,"15min":2.5510000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":3.7360000000000002,"5min":229.97,"15min":229.97},"last":0.51600000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757563,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888180,"kb_used_data":624748,"kb_used_omap":0,"kb_used_meta":263424,"kb_avail":92856972,"statfs":{"total":95995035648,"available":95085539328,"internally_reserved":0,"allocated":639741952,"data_stored":638582094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":269746176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:42 2023","interfaces":[{"interface":"back","average":{"1min":3.8999999999999999,"5min":1.99,"15min":2.355},"min":{"1min":0.24199999999999999,"5min":0.224,"15min":0.22},"max":{"1min":56.563000000000002,"5min":56.563000000000002,"15min":148.79900000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":5.0019999999999998,"5min":2.3290000000000002,"15min":2.681},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.23499999999999999},"max":{"1min":57.241999999999997,"5min":57.241999999999997,"15min":148.779},"last":2.762}]},{"osd":1,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":5.3440000000000003,"5min":2.5390000000000001,"15min":2.0710000000000002},"min":{"1min":0.28399999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":60.799999999999997,"5min":60.799999999999997,"15min":60.799999999999997},"last":0.34799999999999998},{"interface":"front","average":{"1min":5.407,"5min":3.6110000000000002,"15min":3.3109999999999999},"min":{"1min":0.33900000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":63.798999999999999,"5min":63.798999999999999,"15min":63.798999999999999},"last":0.44}]},{"osd":2,"last update":"Sat Sep 30 09:35:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2679999999999998,"5min":1.8089999999999999,"15min":3.0049999999999999},"min":{"1min":0.42799999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":30.613,"5min":30.613,"15min":165.768},"last":0.75600000000000001},{"interface":"front","average":{"1min":3.484,"5min":1.6910000000000001,"15min":3.4540000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":36.488,"5min":36.488,"15min":166.24000000000001},"last":2.8889999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:34:51 2023","interfaces":[{"interface":"back","average":{"1min":3.7850000000000001,"5min":7.5519999999999996,"15min":3.5590000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":35.737000000000002,"5min":226.34700000000001,"15min":226.34700000000001},"last":3.6509999999999998},{"interface":"front","average":{"1min":4.3319999999999999,"5min":7.5780000000000003,"15min":3.4220000000000002},"min":{"1min":0.39100000000000001,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":35.689,"5min":226.36000000000001,"15min":226.36000000000001},"last":3.6589999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.92000000000000004,"15min":0.85699999999999998},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":1.6200000000000001,"5min":7.4299999999999997,"15min":8.5359999999999996},"last":0.312},{"interface":"front","average":{"1min":0.998,"5min":1.097,"15min":0.93700000000000006},"min":{"1min":0.28799999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1580000000000004,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.33400000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:35:02 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":3.2669999999999999,"15min":1.716},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":5.7539999999999996,"5min":155.602,"15min":155.602},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":3.2749999999999999,"15min":1.841},"min":{"1min":0.36499999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.8620000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.36499999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.766,"15min":1.633},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":2.8260000000000001,"5min":133.40700000000001,"15min":133.40700000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.3939999999999999,"5min":2.8260000000000001,"15min":1.5629999999999999},"min":{"1min":0.35599999999999998,"5min":0.313,"15min":0.29999999999999999},"max":{"1min":5.6829999999999998,"5min":132.797,"15min":132.797},"last":0.45500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":609554432,"data_stored":609550360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":386994176,"data_stored":386990104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":114229248,"data_stored":114229248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2255486976,"data_stored":2255486976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":815333376,"data_stored":815333376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":633274368,"data_stored":633274368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1853952000,"data_stored":1853947928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1823080448,"data_stored":1823080448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:35:51.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:35:51.123 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:51.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:51.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:51.252 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:35:51.298 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:51.363 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:35:51.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:51.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:35:51.474 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:51.565 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:51.855 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:35:51.856 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:35:51.870 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:51.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:51.981 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:52.082 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:52.298 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:52.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:52.380 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:35:52.403 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:52.447 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:35:52.482 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:35:52.503 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:52.580 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:52.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:52.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:52.602 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:35:52.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:35:52.685 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:52.718 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:35:52.753 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:35:52.763 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:52.794 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:35:52.800 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:35:52.800 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:35:52.800 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:35:52.800 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:35:52.801 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:35:52.801 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:35:52.801 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:35:52.801 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:35:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:52 smithi019 ceph-mon[224043]: pgmap v843: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 23453/144894 objects degraded (16.186%); 44807/144894 objects misplaced (30.924%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:52.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1737229048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:52.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:52 smithi019 ceph-mon[216815]: pgmap v843: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 23453/144894 objects degraded (16.186%); 44807/144894 objects misplaced (30.924%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:52.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1737229048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:52.887 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:35:52.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:52.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:35:52.991 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:53.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:52 smithi167 ceph-mon[192880]: pgmap v843: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 23453/144894 objects degraded (16.186%); 44807/144894 objects misplaced (30.924%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:53.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1737229048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:35:53.071 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:53.175 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:35:53.558 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23453/144894 objects degraded (16.186%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:53.810 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23453/144894 objects degraded (16.186%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:54.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23453/144894 objects degraded (16.186%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:54.090 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:35:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:35:53] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.43.0" 2023-09-30T09:35:54.720 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:54.742 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:35:54.752 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:54.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:54.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:54.814 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:54 smithi019 ceph-mon[216815]: pgmap v844: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 23453/144894 objects degraded (16.186%); 44807/144894 objects misplaced (30.924%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:54.815 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:54 smithi019 ceph-mon[224043]: pgmap v844: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 23453/144894 objects degraded (16.186%); 44807/144894 objects misplaced (30.924%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:54.861 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:35:54.965 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:55.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:54 smithi167 ceph-mon[192880]: pgmap v844: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 3.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 23453/144894 objects degraded (16.186%); 44807/144894 objects misplaced (30.924%); 14 MiB/s, 230 objects/s recovering 2023-09-30T09:35:55.164 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:35:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:55.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:35:55.270 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:55.365 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:55.508 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:55.649 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:55.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:55.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:55.768 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:35:55.812 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:55.901 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:35:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:55.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:35:56.009 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:56.091 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:56.764 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:35:56.832 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:56 smithi019 ceph-mon[224043]: pgmap v845: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 22391/138318 objects degraded (16.188%); 42827/138318 objects misplaced (30.963%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:35:56.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:56 smithi019 ceph-mon[216815]: pgmap v845: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 22391/138318 objects degraded (16.188%); 42827/138318 objects misplaced (30.963%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:35:57.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:56 smithi167 ceph-mon[192880]: pgmap v845: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 22391/138318 objects degraded (16.188%); 42827/138318 objects misplaced (30.963%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:35:57.255 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:57.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:57.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:57.368 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:57.494 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:57.555 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:57.778 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:57.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:57.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:57.893 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:35:57.996 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:58 smithi019 ceph-mon[216815]: pgmap v846: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 22391/138318 objects degraded (16.188%); 42827/138318 objects misplaced (30.963%); 13 MiB/s, 201 objects/s recovering 2023-09-30T09:35:58.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:35:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22391/138318 objects degraded (16.188%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:58 smithi019 ceph-mon[224043]: pgmap v846: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 22391/138318 objects degraded (16.188%); 42827/138318 objects misplaced (30.963%); 13 MiB/s, 201 objects/s recovering 2023-09-30T09:35:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:35:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22391/138318 objects degraded (16.188%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:58.931 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:35:58.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:58.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:35:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:58 smithi167 ceph-mon[192880]: pgmap v846: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 22391/138318 objects degraded (16.188%); 42827/138318 objects misplaced (30.963%); 13 MiB/s, 201 objects/s recovering 2023-09-30T09:35:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:35:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22391/138318 objects degraded (16.188%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:35:59.042 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:35:59.145 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:59.357 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:35:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:35:59.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:35:59.474 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:35:59.592 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:35:59.651 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:35:59.758 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:35:59.759 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 21296/132558 objects degraded (16.065%), 12 pgs degraded, 4 pgs undersized","count":16},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1098,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":336,"num_osds":8,"num_up_osds":8,"osd_up_since":1696066411,"num_in_osds":7,"osd_in_since":1696066446,"num_remapped_pgs":16},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":103},{"state_name":"active+recovery_wait+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":4},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":119,"num_pools":5,"num_objects":44186,"data_bytes":2898803556,"bytes_used":8236613632,"bytes_avail":663728635904,"bytes_total":671965249536,"degraded_objects":21296,"degraded_total":132558,"degraded_ratio":0.16065420419740792,"misplaced_objects":41199,"misplaced_total":132558,"misplaced_ratio":0.31079980084189562,"recovering_objects_per_sec":277,"recovering_bytes_per_sec":18154408,"recovering_keys_per_sec":0,"num_objects_recovered":3326,"num_bytes_recovered":217972736,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":501},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-30T09:34:31.415542+0000","services":{}},"progress_events":{"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"bf6cdc82-6db1-4694-b9dc-04effc0cdb2e":{"message":"Global Recovery Event (3m)\n [========================....] (remaining: 30s)","progress":0.86554622650146484,"add_to_ceph_s":true}}} 2023-09-30T09:36:00.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:00 smithi019 ceph-mon[216815]: pgmap v847: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 21296/132558 objects degraded (16.065%); 41199/132558 objects misplaced (31.080%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:36:00.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1362181012' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:36:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:00 smithi019 ceph-mon[224043]: pgmap v847: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 21296/132558 objects degraded (16.065%); 41199/132558 objects misplaced (31.080%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:36:00.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1362181012' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:36:01.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:00 smithi167 ceph-mon[192880]: pgmap v847: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 21296/132558 objects degraded (16.065%); 41199/132558 objects misplaced (31.080%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:36:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1362181012' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:36:01.462 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:01.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:01.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:01.569 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:36:01.602 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:01.667 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:36:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:01.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:36:01.772 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:01.814 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:01.871 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:01.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:01.980 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:02.053 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:02.336 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:36:02.337 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:36:02.416 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:02.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:02.524 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:02.553 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:02.604 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:02.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:02.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:02.707 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:02.839 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:03.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:02 smithi167 ceph-mon[192880]: pgmap v848: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 21296/132174 objects degraded (16.112%); 40981/132174 objects misplaced (31.005%); 14 MiB/s, 218 objects/s recovering 2023-09-30T09:36:03.060 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:36:03.061 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:36:03.061 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:36:03.061 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:36:03.061 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:36:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:02 smithi019 ceph-mon[216815]: pgmap v848: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 21296/132174 objects degraded (16.112%); 40981/132174 objects misplaced (31.005%); 14 MiB/s, 218 objects/s recovering 2023-09-30T09:36:03.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:02 smithi019 ceph-mon[224043]: pgmap v848: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 21296/132174 objects degraded (16.112%); 40981/132174 objects misplaced (31.005%); 14 MiB/s, 218 objects/s recovering 2023-09-30T09:36:03.508 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:36:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3665424575' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 21296/132174 objects degraded (16.112%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:03 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3665424575' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:36:04.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:03 smithi167 ceph-mon[192880]: osdmap e337: 8 total, 8 up, 7 in 2023-09-30T09:36:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:36:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:36:03] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-09-30T09:36:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3665424575' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 21296/132174 objects degraded (16.112%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3665424575' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:36:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[224043]: osdmap e337: 8 total, 8 up, 7 in 2023-09-30T09:36:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3665424575' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 21296/132174 objects degraded (16.112%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3665424575' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:36:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:03 smithi019 ceph-mon[216815]: osdmap e337: 8 total, 8 up, 7 in 2023-09-30T09:36:04.872 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[216815]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[216815]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[216815]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[216815]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[216815]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[216815]: pgmap v850: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 21296/132174 objects degraded (16.112%); 40981/132174 objects misplaced (31.005%); 15 MiB/s, 242 objects/s recovering 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[216815]: osdmap e338: 8 total, 8 up, 7 in 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[224043]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[224043]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.873 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[224043]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.874 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[224043]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.874 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[224043]: failed to encode map e337 with expected crc 2023-09-30T09:36:04.874 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[224043]: pgmap v850: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 21296/132174 objects degraded (16.112%); 40981/132174 objects misplaced (31.005%); 15 MiB/s, 242 objects/s recovering 2023-09-30T09:36:04.874 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:04 smithi019 ceph-mon[224043]: osdmap e338: 8 total, 8 up, 7 in 2023-09-30T09:36:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:04 smithi167 ceph-mon[192880]: failed to encode map e337 with expected crc 2023-09-30T09:36:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:04 smithi167 ceph-mon[192880]: failed to encode map e337 with expected crc 2023-09-30T09:36:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:04 smithi167 ceph-mon[192880]: failed to encode map e337 with expected crc 2023-09-30T09:36:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:04 smithi167 ceph-mon[192880]: failed to encode map e337 with expected crc 2023-09-30T09:36:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:04 smithi167 ceph-mon[192880]: failed to encode map e337 with expected crc 2023-09-30T09:36:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:04 smithi167 ceph-mon[192880]: pgmap v850: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 21296/132174 objects degraded (16.112%); 40981/132174 objects misplaced (31.005%); 15 MiB/s, 242 objects/s recovering 2023-09-30T09:36:05.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:04 smithi167 ceph-mon[192880]: osdmap e338: 8 total, 8 up, 7 in 2023-09-30T09:36:05.420 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:05.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:05.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:05.535 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:05.572 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:05.679 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:05.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:05.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:05.788 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:05.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:05 smithi019 ceph-mon[216815]: failed to encode map e338 with expected crc 2023-09-30T09:36:05.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:05 smithi019 ceph-mon[216815]: failed to encode map e338 with expected crc 2023-09-30T09:36:05.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:05 smithi019 ceph-mon[216815]: failed to encode map e338 with expected crc 2023-09-30T09:36:05.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:05 smithi019 ceph-mon[216815]: failed to encode map e338 with expected crc 2023-09-30T09:36:05.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:05 smithi019 ceph-mon[224043]: failed to encode map e338 with expected crc 2023-09-30T09:36:05.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:05 smithi019 ceph-mon[224043]: failed to encode map e338 with expected crc 2023-09-30T09:36:05.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:05 smithi019 ceph-mon[224043]: failed to encode map e338 with expected crc 2023-09-30T09:36:05.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:05 smithi019 ceph-mon[224043]: failed to encode map e338 with expected crc 2023-09-30T09:36:05.897 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:36:05.942 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:06.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:05 smithi167 ceph-mon[192880]: failed to encode map e338 with expected crc 2023-09-30T09:36:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:05 smithi167 ceph-mon[192880]: failed to encode map e338 with expected crc 2023-09-30T09:36:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:05 smithi167 ceph-mon[192880]: failed to encode map e338 with expected crc 2023-09-30T09:36:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:05 smithi167 ceph-mon[192880]: failed to encode map e338 with expected crc 2023-09-30T09:36:06.864 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:06.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:06.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:06.973 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:07.007 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:06 smithi167 ceph-mon[192880]: pgmap v852: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.6 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 20228/125802 objects degraded (16.079%); 38993/125802 objects misplaced (30.996%); 19 MiB/s, 309 objects/s recovering 2023-09-30T09:36:07.010 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:07.065 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:06 smithi019 ceph-mon[224043]: pgmap v852: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.6 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 20228/125802 objects degraded (16.079%); 38993/125802 objects misplaced (30.996%); 19 MiB/s, 309 objects/s recovering 2023-09-30T09:36:07.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:06 smithi019 ceph-mon[216815]: pgmap v852: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.6 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 20228/125802 objects degraded (16.079%); 38993/125802 objects misplaced (30.996%); 19 MiB/s, 309 objects/s recovering 2023-09-30T09:36:07.124 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:07.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:07.230 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:07.311 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:07.414 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:07.474 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:07.512 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:07.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:07.614 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:07.617 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:36:07.676 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:07.679 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:36:07.735 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:36:07.770 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:36:07.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:07 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3850968102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:07.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:07 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3850968102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:07.823 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:36:07.875 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:36:07.905 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:36:07.944 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:36:07.969 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:07.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:07.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:07.994 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:36:07.994 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:36:07.994 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:36:07.994 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:36:07.994 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:36:07.995 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:36:07.995 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:36:07.995 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:36:08.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:07 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3850968102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:08.074 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:08.167 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:08.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:08 smithi019 ceph-mon[216815]: pgmap v853: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.6 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 20228/125802 objects degraded (16.079%); 38993/125802 objects misplaced (30.996%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:36:08.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 20228/125802 objects degraded (16.079%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:08.862 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:08.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:08.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:08.971 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:09.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:08 smithi167 ceph-mon[192880]: pgmap v853: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.6 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 20228/125802 objects degraded (16.079%); 38993/125802 objects misplaced (30.996%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:36:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 20228/125802 objects degraded (16.079%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:09.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:08 smithi019 ceph-mon[224043]: pgmap v853: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 103 active+clean; 2.6 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 20228/125802 objects degraded (16.079%); 38993/125802 objects misplaced (30.996%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:36:09.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 20228/125802 objects degraded (16.079%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:09.065 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:09.066 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:36:10.267 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:10.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:10.381 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:10.483 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:10.813 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:10.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:10.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:10.917 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:10.993 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:10 smithi019 ceph-mon[216815]: pgmap v854: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 19410/120192 objects degraded (16.149%); 37312/120192 objects misplaced (31.044%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:36:10.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:10 smithi019 ceph-mon[224043]: pgmap v854: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 19410/120192 objects degraded (16.149%); 37312/120192 objects misplaced (31.044%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:36:10.999 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:11.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:10 smithi167 ceph-mon[192880]: pgmap v854: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 19410/120192 objects degraded (16.149%); 37312/120192 objects misplaced (31.044%); 16 MiB/s, 257 objects/s recovering 2023-09-30T09:36:12.171 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:12.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:12.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:12.301 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:12.333 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:12.349 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:12.396 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:12.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:12.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:12.500 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:12.605 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:12.707 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:12 smithi167 ceph-mon[192880]: pgmap v855: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 19410/119745 objects degraded (16.209%); 37108/119745 objects misplaced (30.989%); 18 MiB/s, 282 objects/s recovering 2023-09-30T09:36:12.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:12 smithi019 ceph-mon[216815]: pgmap v855: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 19410/119745 objects degraded (16.209%); 37108/119745 objects misplaced (30.989%); 18 MiB/s, 282 objects/s recovering 2023-09-30T09:36:12.757 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:12 smithi019 ceph-mon[224043]: pgmap v855: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 19410/119745 objects degraded (16.209%); 37108/119745 objects misplaced (30.989%); 18 MiB/s, 282 objects/s recovering 2023-09-30T09:36:12.882 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:36:12.882 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:36:13.116 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:13.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:13.227 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:36:13.258 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:13.316 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:36:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:13.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:36:13.429 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:13.523 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:13.780 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:36:13.802 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:36:14.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 19410/119745 objects degraded (16.209%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 19410/119745 objects degraded (16.209%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 19410/119745 objects degraded (16.209%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:36:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:36:13] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-09-30T09:36:14.110 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:36:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:14 smithi167 ceph-mon[192880]: pgmap v856: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 19410/119745 objects degraded (16.209%); 37108/119745 objects misplaced (30.989%); 14 MiB/s, 226 objects/s recovering 2023-09-30T09:36:15.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:14 smithi167 ceph-mon[192880]: from='client.55474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:15.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:14 smithi019 ceph-mon[224043]: pgmap v856: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 19410/119745 objects degraded (16.209%); 37108/119745 objects misplaced (30.989%); 14 MiB/s, 226 objects/s recovering 2023-09-30T09:36:15.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:14 smithi019 ceph-mon[224043]: from='client.55474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:15.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:14 smithi019 ceph-mon[216815]: pgmap v856: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 19410/119745 objects degraded (16.209%); 37108/119745 objects misplaced (30.989%); 14 MiB/s, 226 objects/s recovering 2023-09-30T09:36:15.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:14 smithi019 ceph-mon[216815]: from='client.55474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:15.149 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:15.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:15.259 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:15.300 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:15.366 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:15.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:15.472 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:15.581 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:16.398 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:16.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:16.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:16.445 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:36:16.512 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:16.610 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:16.744 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:16 smithi019 ceph-mon[216815]: pgmap v857: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 18530/113208 objects degraded (16.368%); 34967/113208 objects misplaced (30.887%); 14 MiB/s, 222 objects/s recovering 2023-09-30T09:36:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:16 smithi167 ceph-mon[192880]: pgmap v857: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 18530/113208 objects degraded (16.368%); 34967/113208 objects misplaced (30.887%); 14 MiB/s, 222 objects/s recovering 2023-09-30T09:36:17.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:16 smithi019 ceph-mon[224043]: pgmap v857: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 18530/113208 objects degraded (16.368%); 34967/113208 objects misplaced (30.887%); 14 MiB/s, 222 objects/s recovering 2023-09-30T09:36:17.357 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:17.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:17.477 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:17.508 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":856,"stamp":"2023-09-30T09:36:13.499265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2618899300,"num_objects":39915,"num_object_clones":0,"num_object_copies":119745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19410,"num_objects_misplaced":37108,"num_objects_unfound":0,"num_objects_dirty":39915,"num_whiteouts":0,"num_read":6114,"num_read_kb":12467,"num_write":187654,"num_write_kb":7263166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21915,"num_bytes_recovered":1431085528,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86652,"ondisk_log_size":86652,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":397,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7493684,"kb_used_data":5241980,"kb_used_omap":23,"kb_used_meta":2251368,"kb_avail":648722380,"statfs":{"total":671965249536,"available":664291717120,"internally_reserved":0,"allocated":5367787520,"data_stored":5356488662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2305401091},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,5,2,4,8,1,3,4,2,3,4],"upper_bound":2048},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-271515648,"num_objects":-4143,"num_object_clones":0,"num_object_copies":-12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1886,"num_objects_misplaced":-3873,"num_objects_unfound":0,"num_objects_dirty":-4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4143,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2265,"num_bytes_recovered":148439040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.012914"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":802,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.494208+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:36:03.494208+0000","last_peered":"2023-09-30T09:36:03.494208+0000","last_clean":"2023-09-30T09:36:03.494208+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:36:03.494208+0000","last_undegraded":"2023-09-30T09:36:03.494208+0000","last_fullsized":"2023-09-30T09:36:03.494208+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":356,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485991+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:36:03.485991+0000","last_peered":"2023-09-30T09:36:03.485991+0000","last_clean":"2023-09-30T09:36:03.485991+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:36:03.485991+0000","last_undegraded":"2023-09-30T09:36:03.485991+0000","last_fullsized":"2023-09-30T09:36:03.485991+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":318,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.472099+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:36:03.472099+0000","last_peered":"2023-09-30T09:36:03.472099+0000","last_clean":"2023-09-30T09:36:03.472099+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:36:03.472099+0000","last_undegraded":"2023-09-30T09:36:03.472099+0000","last_fullsized":"2023-09-30T09:36:03.472099+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":782,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.495514+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:36:03.495514+0000","last_peered":"2023-09-30T09:36:03.495514+0000","last_clean":"2023-09-30T09:36:03.495514+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:36:03.495514+0000","last_undegraded":"2023-09-30T09:36:03.495514+0000","last_fullsized":"2023-09-30T09:36:03.495514+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":211,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.481413+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:36:03.481413+0000","last_peered":"2023-09-30T09:36:03.481413+0000","last_clean":"2023-09-30T09:36:03.481413+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:36:03.481413+0000","last_undegraded":"2023-09-30T09:36:03.481413+0000","last_fullsized":"2023-09-30T09:36:03.481413+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":242,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.508493+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:36:03.508493+0000","last_peered":"2023-09-30T09:36:03.508493+0000","last_clean":"2023-09-30T09:36:03.508493+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:36:03.508493+0000","last_undegraded":"2023-09-30T09:36:03.508493+0000","last_fullsized":"2023-09-30T09:36:03.508493+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":551,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485894+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:36:03.485894+0000","last_peered":"2023-09-30T09:36:03.485894+0000","last_clean":"2023-09-30T09:36:03.485894+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:36:03.485894+0000","last_undegraded":"2023-09-30T09:36:03.485894+0000","last_fullsized":"2023-09-30T09:36:03.485894+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":385,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471090+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:36:03.471090+0000","last_peered":"2023-09-30T09:36:03.471090+0000","last_clean":"2023-09-30T09:36:03.471090+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:36:03.471090+0000","last_undegraded":"2023-09-30T09:36:03.471090+0000","last_fullsized":"2023-09-30T09:36:03.471090+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":314,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.488668+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:36:03.488668+0000","last_peered":"2023-09-30T09:36:03.488668+0000","last_clean":"2023-09-30T09:36:03.488668+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:36:03.488668+0000","last_undegraded":"2023-09-30T09:36:03.488668+0000","last_fullsized":"2023-09-30T09:36:03.488668+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":761,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471902+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:36:03.471902+0000","last_peered":"2023-09-30T09:36:03.471902+0000","last_clean":"2023-09-30T09:36:03.471902+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:36:03.471902+0000","last_undegraded":"2023-09-30T09:36:03.471902+0000","last_fullsized":"2023-09-30T09:36:03.471902+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":233,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.506129+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:36:03.506129+0000","last_peered":"2023-09-30T09:36:03.506129+0000","last_clean":"2023-09-30T09:36:03.506129+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:36:03.506129+0000","last_undegraded":"2023-09-30T09:36:03.506129+0000","last_fullsized":"2023-09-30T09:36:03.506129+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":319,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471160+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:36:03.471160+0000","last_peered":"2023-09-30T09:36:03.471160+0000","last_clean":"2023-09-30T09:36:03.471160+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:36:03.471160+0000","last_undegraded":"2023-09-30T09:36:03.471160+0000","last_fullsized":"2023-09-30T09:36:03.471160+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":501,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.513358+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:36:03.513358+0000","last_peered":"2023-09-30T09:36:03.513358+0000","last_clean":"2023-09-30T09:36:03.513358+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:36:03.513358+0000","last_undegraded":"2023-09-30T09:36:03.513358+0000","last_fullsized":"2023-09-30T09:36:03.513358+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":216,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.496608+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:36:03.496608+0000","last_peered":"2023-09-30T09:36:03.496608+0000","last_clean":"2023-09-30T09:36:03.496608+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:36:03.496608+0000","last_undegraded":"2023-09-30T09:36:03.496608+0000","last_fullsized":"2023-09-30T09:36:03.496608+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":265,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.481317+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:36:03.481317+0000","last_peered":"2023-09-30T09:36:03.481317+0000","last_clean":"2023-09-30T09:36:03.481317+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:36:03.481317+0000","last_undegraded":"2023-09-30T09:36:03.481317+0000","last_fullsized":"2023-09-30T09:36:03.481317+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":504,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.496537+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:36:03.496537+0000","last_peered":"2023-09-30T09:36:03.496537+0000","last_clean":"2023-09-30T09:36:03.496537+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:36:03.496537+0000","last_undegraded":"2023-09-30T09:36:03.496537+0000","last_fullsized":"2023-09-30T09:36:03.496537+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":59,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.472192+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:36:03.472192+0000","last_peered":"2023-09-30T09:36:03.472192+0000","last_clean":"2023-09-30T09:36:03.472192+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:36:03.472192+0000","last_undegraded":"2023-09-30T09:36:03.472192+0000","last_fullsized":"2023-09-30T09:36:03.472192+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":251,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485581+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:36:03.485581+0000","last_peered":"2023-09-30T09:36:03.485581+0000","last_clean":"2023-09-30T09:36:03.485581+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:36:03.485581+0000","last_undegraded":"2023-09-30T09:36:03.485581+0000","last_fullsized":"2023-09-30T09:36:03.485581+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":497,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.509322+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:36:03.509322+0000","last_peered":"2023-09-30T09:36:03.509322+0000","last_clean":"2023-09-30T09:36:03.509322+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:36:03.509322+0000","last_undegraded":"2023-09-30T09:36:03.509322+0000","last_fullsized":"2023-09-30T09:36:03.509322+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":322,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486990+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:36:03.486990+0000","last_peered":"2023-09-30T09:36:03.486990+0000","last_clean":"2023-09-30T09:36:03.486990+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:36:03.486990+0000","last_undegraded":"2023-09-30T09:36:03.486990+0000","last_fullsized":"2023-09-30T09:36:03.486990+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":333,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486331+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:36:03.486331+0000","last_peered":"2023-09-30T09:36:03.486331+0000","last_clean":"2023-09-30T09:36:03.486331+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:36:03.486331+0000","last_undegraded":"2023-09-30T09:36:03.486331+0000","last_fullsized":"2023-09-30T09:36:03.486331+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":615,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.492693+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:36:03.492693+0000","last_peered":"2023-09-30T09:36:03.492693+0000","last_clean":"2023-09-30T09:36:03.492693+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:36:03.492693+0000","last_undegraded":"2023-09-30T09:36:03.492693+0000","last_fullsized":"2023-09-30T09:36:03.492693+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":60,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486259+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:36:03.486259+0000","last_peered":"2023-09-30T09:36:03.486259+0000","last_clean":"2023-09-30T09:36:03.486259+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:36:03.486259+0000","last_undegraded":"2023-09-30T09:36:03.486259+0000","last_fullsized":"2023-09-30T09:36:03.486259+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":349,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471549+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:36:03.471549+0000","last_peered":"2023-09-30T09:36:03.471549+0000","last_clean":"2023-09-30T09:36:03.471549+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:36:03.471549+0000","last_undegraded":"2023-09-30T09:36:03.471549+0000","last_fullsized":"2023-09-30T09:36:03.471549+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":684,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.494381+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:36:03.494381+0000","last_peered":"2023-09-30T09:36:03.494381+0000","last_clean":"2023-09-30T09:36:03.494381+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:36:03.494381+0000","last_undegraded":"2023-09-30T09:36:03.494381+0000","last_fullsized":"2023-09-30T09:36:03.494381+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":241,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.492365+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:36:03.492365+0000","last_peered":"2023-09-30T09:36:03.492365+0000","last_clean":"2023-09-30T09:36:03.492365+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:36:03.492365+0000","last_undegraded":"2023-09-30T09:36:03.492365+0000","last_fullsized":"2023-09-30T09:36:03.492365+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":27,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.494890+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:36:03.494890+0000","last_peered":"2023-09-30T09:36:03.494890+0000","last_clean":"2023-09-30T09:36:03.494890+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:36:03.494890+0000","last_undegraded":"2023-09-30T09:36:03.494890+0000","last_fullsized":"2023-09-30T09:36:03.494890+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":776,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471004+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:36:03.471004+0000","last_peered":"2023-09-30T09:36:03.471004+0000","last_clean":"2023-09-30T09:36:03.471004+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:36:03.471004+0000","last_undegraded":"2023-09-30T09:36:03.471004+0000","last_fullsized":"2023-09-30T09:36:03.471004+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.487915+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:36:03.487915+0000","last_peered":"2023-09-30T09:36:03.487915+0000","last_clean":"2023-09-30T09:36:03.487915+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:36:03.487915+0000","last_undegraded":"2023-09-30T09:36:03.487915+0000","last_fullsized":"2023-09-30T09:36:03.487915+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":241,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.509181+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:36:03.509181+0000","last_peered":"2023-09-30T09:36:03.509181+0000","last_clean":"2023-09-30T09:36:03.509181+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:36:03.509181+0000","last_undegraded":"2023-09-30T09:36:03.509181+0000","last_fullsized":"2023-09-30T09:36:03.509181+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":803,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.496559+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:36:03.496559+0000","last_peered":"2023-09-30T09:36:03.496559+0000","last_clean":"2023-09-30T09:36:03.496559+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:36:03.496559+0000","last_undegraded":"2023-09-30T09:36:03.496559+0000","last_fullsized":"2023-09-30T09:36:03.496559+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":497,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485011+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:36:03.485011+0000","last_peered":"2023-09-30T09:36:03.485011+0000","last_clean":"2023-09-30T09:36:03.485011+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:36:03.485011+0000","last_undegraded":"2023-09-30T09:36:03.485011+0000","last_fullsized":"2023-09-30T09:36:03.485011+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":347,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.473338+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:36:03.473338+0000","last_peered":"2023-09-30T09:36:03.473338+0000","last_clean":"2023-09-30T09:36:03.473338+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:36:03.473338+0000","last_undegraded":"2023-09-30T09:36:03.473338+0000","last_fullsized":"2023-09-30T09:36:03.473338+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"338'11409","reported_seq":15388,"reported_epoch":338,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:08.970324+0000","last_change":"2023-09-30T09:34:48.517696+0000","last_active":"2023-09-30T09:36:08.970324+0000","last_peered":"2023-09-30T09:36:08.970324+0000","last_clean":"2023-09-30T09:32:35.457744+0000","last_became_active":"2023-09-30T09:34:48.499533+0000","last_became_peered":"2023-09-30T09:34:48.499533+0000","last_unstale":"2023-09-30T09:36:08.970324+0000","last_undegraded":"2023-09-30T09:34:48.469688+0000","last_fullsized":"2023-09-30T09:36:08.970324+0000","mapping_epoch":321,"log_start":"299'6621","ondisk_log_start":"299'6621","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4788,"log_dups_size":0,"ondisk_log_size":4788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:05:42.164341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1608,"num_objects_misplaced":2489,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11409,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1934,"num_bytes_recovered":126746624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[6,1,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"1,4,6","objects":881},{"shards":"4,6","objects":1608}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":802,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486368+0000","last_change":"2023-09-30T09:35:44.763111+0000","last_active":"2023-09-30T09:36:03.486368+0000","last_peered":"2023-09-30T09:36:03.486368+0000","last_clean":"2023-09-30T09:36:03.486368+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:36:03.486368+0000","last_undegraded":"2023-09-30T09:36:03.486368+0000","last_fullsized":"2023-09-30T09:36:03.486368+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":72,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486437+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:36:03.486437+0000","last_peered":"2023-09-30T09:36:03.486437+0000","last_clean":"2023-09-30T09:36:03.486437+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:36:03.486437+0000","last_undegraded":"2023-09-30T09:36:03.486437+0000","last_fullsized":"2023-09-30T09:36:03.486437+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":27,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.494119+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:36:03.494119+0000","last_peered":"2023-09-30T09:36:03.494119+0000","last_clean":"2023-09-30T09:36:03.494119+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:36:03.494119+0000","last_undegraded":"2023-09-30T09:36:03.494119+0000","last_fullsized":"2023-09-30T09:36:03.494119+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"338'11869","reported_seq":15246,"reported_epoch":338,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:09.340309+0000","last_change":"2023-09-30T09:35:10.766663+0000","last_active":"2023-09-30T09:36:09.340309+0000","last_peered":"2023-09-30T09:36:09.340309+0000","last_clean":"2023-09-30T09:32:35.454324+0000","last_became_active":"2023-09-30T09:33:33.826036+0000","last_became_peered":"2023-09-30T09:33:33.826036+0000","last_unstale":"2023-09-30T09:36:09.340309+0000","last_undegraded":"2023-09-30T09:33:33.732722+0000","last_fullsized":"2023-09-30T09:36:09.340309+0000","mapping_epoch":302,"log_start":"303'7299","ondisk_log_start":"303'7299","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4570,"log_dups_size":0,"ondisk_log_size":4570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:42.597216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165871616,"num_objects":2531,"num_object_clones":0,"num_object_copies":7593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2355,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11869,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1641,"num_bytes_recovered":107544576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,5,1],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":175},{"shards":"3,5","objects":2356}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":787,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471477+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:36:03.471477+0000","last_peered":"2023-09-30T09:36:03.471477+0000","last_clean":"2023-09-30T09:36:03.471477+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:36:03.471477+0000","last_undegraded":"2023-09-30T09:36:03.471477+0000","last_fullsized":"2023-09-30T09:36:03.471477+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":50,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485898+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:36:03.485898+0000","last_peered":"2023-09-30T09:36:03.485898+0000","last_clean":"2023-09-30T09:36:03.485898+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:36:03.485898+0000","last_undegraded":"2023-09-30T09:36:03.485898+0000","last_fullsized":"2023-09-30T09:36:03.485898+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":330,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.487836+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:36:03.487836+0000","last_peered":"2023-09-30T09:36:03.487836+0000","last_clean":"2023-09-30T09:36:03.487836+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:36:03.487836+0000","last_undegraded":"2023-09-30T09:36:03.487836+0000","last_fullsized":"2023-09-30T09:36:03.487836+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"338'11441","reported_seq":13698,"reported_epoch":338,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:08.972497+0000","last_change":"2023-09-30T09:35:10.879711+0000","last_active":"2023-09-30T09:36:08.972497+0000","last_peered":"2023-09-30T09:36:08.972497+0000","last_clean":"2023-09-30T09:32:35.467048+0000","last_became_active":"2023-09-30T09:35:10.845361+0000","last_became_peered":"2023-09-30T09:35:10.845361+0000","last_unstale":"2023-09-30T09:36:08.972497+0000","last_undegraded":"2023-09-30T09:35:10.791338+0000","last_fullsized":"2023-09-30T09:36:08.972497+0000","mapping_epoch":327,"log_start":"299'6676","ondisk_log_start":"299'6676","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4765,"log_dups_size":0,"ondisk_log_size":4765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:41.089234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166133760,"num_objects":2535,"num_object_clones":0,"num_object_copies":7605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2535,"num_objects_misplaced":5070,"num_objects_unfound":0,"num_objects_dirty":2535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11441,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1073,"num_bytes_recovered":70189056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[6,1,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":2535}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":621,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.488586+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:36:03.488586+0000","last_peered":"2023-09-30T09:36:03.488586+0000","last_clean":"2023-09-30T09:36:03.488586+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:36:03.488586+0000","last_undegraded":"2023-09-30T09:36:03.488586+0000","last_fullsized":"2023-09-30T09:36:03.488586+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":31,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.494808+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:36:03.494808+0000","last_peered":"2023-09-30T09:36:03.494808+0000","last_clean":"2023-09-30T09:36:03.494808+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:36:03.494808+0000","last_undegraded":"2023-09-30T09:36:03.494808+0000","last_fullsized":"2023-09-30T09:36:03.494808+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":317,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485445+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:36:03.485445+0000","last_peered":"2023-09-30T09:36:03.485445+0000","last_clean":"2023-09-30T09:36:03.485445+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:36:03.485445+0000","last_undegraded":"2023-09-30T09:36:03.485445+0000","last_fullsized":"2023-09-30T09:36:03.485445+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"338'11825","reported_seq":11905,"reported_epoch":338,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:09.329713+0000","last_change":"2023-09-30T09:34:08.527633+0000","last_active":"2023-09-30T09:36:09.329713+0000","last_peered":"2023-09-30T09:36:09.329713+0000","last_clean":"2023-09-30T09:33:09.318417+0000","last_became_active":"2023-09-30T09:34:08.510746+0000","last_became_peered":"2023-09-30T09:34:08.510746+0000","last_unstale":"2023-09-30T09:36:09.329713+0000","last_undegraded":"2023-09-30T09:36:09.329713+0000","last_fullsized":"2023-09-30T09:36:09.329713+0000","mapping_epoch":308,"log_start":"303'7276","ondisk_log_start":"303'7276","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4549,"log_dups_size":0,"ondisk_log_size":4549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:07.374969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162070528,"num_objects":2473,"num_object_clones":0,"num_object_copies":7419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2473,"num_objects_unfound":0,"num_objects_dirty":2473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11825,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":2473}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":479,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485980+0000","last_change":"2023-09-30T09:35:45.753329+0000","last_active":"2023-09-30T09:36:03.485980+0000","last_peered":"2023-09-30T09:36:03.485980+0000","last_clean":"2023-09-30T09:36:03.485980+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:36:03.485980+0000","last_undegraded":"2023-09-30T09:36:03.485980+0000","last_fullsized":"2023-09-30T09:36:03.485980+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":297,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.496031+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:36:03.496031+0000","last_peered":"2023-09-30T09:36:03.496031+0000","last_clean":"2023-09-30T09:36:03.496031+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:36:03.496031+0000","last_undegraded":"2023-09-30T09:36:03.496031+0000","last_fullsized":"2023-09-30T09:36:03.496031+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":243,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.493231+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:36:03.493231+0000","last_peered":"2023-09-30T09:36:03.493231+0000","last_clean":"2023-09-30T09:36:03.493231+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:36:03.493231+0000","last_undegraded":"2023-09-30T09:36:03.493231+0000","last_fullsized":"2023-09-30T09:36:03.493231+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"338'11612","reported_seq":11704,"reported_epoch":338,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:09.334180+0000","last_change":"2023-09-30T09:35:10.779076+0000","last_active":"2023-09-30T09:36:09.334180+0000","last_peered":"2023-09-30T09:36:09.334180+0000","last_clean":"2023-09-30T09:34:06.422096+0000","last_became_active":"2023-09-30T09:35:10.747716+0000","last_became_peered":"2023-09-30T09:35:10.747716+0000","last_unstale":"2023-09-30T09:36:09.334180+0000","last_undegraded":"2023-09-30T09:36:09.334180+0000","last_fullsized":"2023-09-30T09:36:09.334180+0000","mapping_epoch":327,"log_start":"303'7120","ondisk_log_start":"303'7120","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4492,"log_dups_size":0,"ondisk_log_size":4492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:09.191024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2434,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11612,"num_write_kb":449472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":2434}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":683,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.481064+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:36:03.481064+0000","last_peered":"2023-09-30T09:36:03.481064+0000","last_clean":"2023-09-30T09:36:03.481064+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:36:03.481064+0000","last_undegraded":"2023-09-30T09:36:03.481064+0000","last_fullsized":"2023-09-30T09:36:03.481064+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":365,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471286+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:36:03.471286+0000","last_peered":"2023-09-30T09:36:03.471286+0000","last_clean":"2023-09-30T09:36:03.471286+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:36:03.471286+0000","last_undegraded":"2023-09-30T09:36:03.471286+0000","last_fullsized":"2023-09-30T09:36:03.471286+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":296,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.487066+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:36:03.487066+0000","last_peered":"2023-09-30T09:36:03.487066+0000","last_clean":"2023-09-30T09:36:03.487066+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:36:03.487066+0000","last_undegraded":"2023-09-30T09:36:03.487066+0000","last_fullsized":"2023-09-30T09:36:03.487066+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"338'11730","reported_seq":14092,"reported_epoch":338,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:09.363891+0000","last_change":"2023-09-30T09:35:10.871065+0000","last_active":"2023-09-30T09:36:09.363891+0000","last_peered":"2023-09-30T09:36:09.363891+0000","last_clean":"2023-09-30T09:32:35.449714+0000","last_became_active":"2023-09-30T09:35:10.838732+0000","last_became_peered":"2023-09-30T09:35:10.838732+0000","last_unstale":"2023-09-30T09:36:09.363891+0000","last_undegraded":"2023-09-30T09:35:10.818868+0000","last_fullsized":"2023-09-30T09:36:09.363891+0000","mapping_epoch":327,"log_start":"303'7206","ondisk_log_start":"303'7206","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4524,"log_dups_size":0,"ondisk_log_size":4524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:44.318381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165281792,"num_objects":2522,"num_object_clones":0,"num_object_copies":7566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2520,"num_objects_misplaced":5046,"num_objects_unfound":0,"num_objects_dirty":2522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11730,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1128,"num_bytes_recovered":73924608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[3,7,1],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"1,3,7","objects":2},{"shards":"3,7","objects":2520}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":759,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.508742+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:36:03.508742+0000","last_peered":"2023-09-30T09:36:03.508742+0000","last_clean":"2023-09-30T09:36:03.508742+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:36:03.508742+0000","last_undegraded":"2023-09-30T09:36:03.508742+0000","last_fullsized":"2023-09-30T09:36:03.508742+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":315,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485653+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:36:03.485653+0000","last_peered":"2023-09-30T09:36:03.485653+0000","last_clean":"2023-09-30T09:36:03.485653+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:36:03.485653+0000","last_undegraded":"2023-09-30T09:36:03.485653+0000","last_fullsized":"2023-09-30T09:36:03.485653+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":32,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486221+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:36:03.486221+0000","last_peered":"2023-09-30T09:36:03.486221+0000","last_clean":"2023-09-30T09:36:03.486221+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:36:03.486221+0000","last_undegraded":"2023-09-30T09:36:03.486221+0000","last_fullsized":"2023-09-30T09:36:03.486221+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"338'11584","reported_seq":18807,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:09.024803+0000","last_change":"2023-09-30T09:36:04.573419+0000","last_active":"2023-09-30T09:36:09.024803+0000","last_peered":"2023-09-30T09:36:09.024803+0000","last_clean":"2023-09-30T09:36:09.024803+0000","last_became_active":"2023-09-30T09:36:04.549763+0000","last_became_peered":"2023-09-30T09:36:04.549763+0000","last_unstale":"2023-09-30T09:36:09.024803+0000","last_undegraded":"2023-09-30T09:36:09.024803+0000","last_fullsized":"2023-09-30T09:36:09.024803+0000","mapping_epoch":337,"log_start":"299'6821","ondisk_log_start":"299'6821","created":279,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4763,"log_dups_size":0,"ondisk_log_size":4763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:27:18.185457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161153048,"num_objects":2460,"num_object_clones":0,"num_object_copies":7380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2460,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11584,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3562,"num_bytes_recovered":233439232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":773,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.493000+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:36:03.493000+0000","last_peered":"2023-09-30T09:36:03.493000+0000","last_clean":"2023-09-30T09:36:03.493000+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:36:03.493000+0000","last_undegraded":"2023-09-30T09:36:03.493000+0000","last_fullsized":"2023-09-30T09:36:03.493000+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":334,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.487696+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:36:03.487696+0000","last_peered":"2023-09-30T09:36:03.487696+0000","last_clean":"2023-09-30T09:36:03.487696+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:36:03.487696+0000","last_undegraded":"2023-09-30T09:36:03.487696+0000","last_fullsized":"2023-09-30T09:36:03.487696+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":313,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486161+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:36:03.486161+0000","last_peered":"2023-09-30T09:36:03.486161+0000","last_clean":"2023-09-30T09:36:03.486161+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:36:03.486161+0000","last_undegraded":"2023-09-30T09:36:03.486161+0000","last_fullsized":"2023-09-30T09:36:03.486161+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"338'11684","reported_seq":11768,"reported_epoch":338,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:08.840990+0000","last_change":"2023-09-30T09:34:47.630136+0000","last_active":"2023-09-30T09:36:08.840990+0000","last_peered":"2023-09-30T09:36:08.840990+0000","last_clean":"2023-09-30T09:34:06.422409+0000","last_became_active":"2023-09-30T09:34:08.626702+0000","last_became_peered":"2023-09-30T09:34:08.626702+0000","last_unstale":"2023-09-30T09:36:08.840990+0000","last_undegraded":"2023-09-30T09:36:08.840990+0000","last_fullsized":"2023-09-30T09:36:08.840990+0000","mapping_epoch":308,"log_start":"299'6906","ondisk_log_start":"299'6906","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4778,"log_dups_size":0,"ondisk_log_size":4778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:53.753813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163315712,"num_objects":2492,"num_object_clones":0,"num_object_copies":7476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2492,"num_objects_unfound":0,"num_objects_dirty":2492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11684,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2492}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":483,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.514258+0000","last_change":"2023-09-30T09:35:46.150279+0000","last_active":"2023-09-30T09:36:03.514258+0000","last_peered":"2023-09-30T09:36:03.514258+0000","last_clean":"2023-09-30T09:36:03.514258+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:36:03.514258+0000","last_undegraded":"2023-09-30T09:36:03.514258+0000","last_fullsized":"2023-09-30T09:36:03.514258+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":328,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.470939+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:36:03.470939+0000","last_peered":"2023-09-30T09:36:03.470939+0000","last_clean":"2023-09-30T09:36:03.470939+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:36:03.470939+0000","last_undegraded":"2023-09-30T09:36:03.470939+0000","last_fullsized":"2023-09-30T09:36:03.470939+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":105,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.473165+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:36:03.473165+0000","last_peered":"2023-09-30T09:36:03.473165+0000","last_clean":"2023-09-30T09:36:03.473165+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:36:03.473165+0000","last_undegraded":"2023-09-30T09:36:03.473165+0000","last_fullsized":"2023-09-30T09:36:03.473165+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"338'11565","reported_seq":13748,"reported_epoch":338,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:09.374179+0000","last_change":"2023-09-30T09:34:39.166112+0000","last_active":"2023-09-30T09:36:09.374179+0000","last_peered":"2023-09-30T09:36:09.374179+0000","last_clean":"2023-09-30T09:32:35.465008+0000","last_became_active":"2023-09-30T09:34:39.150257+0000","last_became_peered":"2023-09-30T09:34:39.150257+0000","last_unstale":"2023-09-30T09:36:09.374179+0000","last_undegraded":"2023-09-30T09:34:38.934740+0000","last_fullsized":"2023-09-30T09:36:09.374179+0000","mapping_epoch":316,"log_start":"303'7078","ondisk_log_start":"303'7078","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:50.805799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2421,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11565,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1048,"num_bytes_recovered":68616192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6,1],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":2421}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":632,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.488751+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:36:03.488751+0000","last_peered":"2023-09-30T09:36:03.488751+0000","last_clean":"2023-09-30T09:36:03.488751+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:36:03.488751+0000","last_undegraded":"2023-09-30T09:36:03.488751+0000","last_fullsized":"2023-09-30T09:36:03.488751+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":529,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.493432+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:36:03.493432+0000","last_peered":"2023-09-30T09:36:03.493432+0000","last_clean":"2023-09-30T09:36:03.493432+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:36:03.493432+0000","last_undegraded":"2023-09-30T09:36:03.493432+0000","last_fullsized":"2023-09-30T09:36:03.493432+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":334,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.495119+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:36:03.495119+0000","last_peered":"2023-09-30T09:36:03.495119+0000","last_clean":"2023-09-30T09:36:03.495119+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:36:03.495119+0000","last_undegraded":"2023-09-30T09:36:03.495119+0000","last_fullsized":"2023-09-30T09:36:03.495119+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":27,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.495556+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:36:03.495556+0000","last_peered":"2023-09-30T09:36:03.495556+0000","last_clean":"2023-09-30T09:36:03.495556+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:36:03.495556+0000","last_undegraded":"2023-09-30T09:36:03.495556+0000","last_fullsized":"2023-09-30T09:36:03.495556+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"338'11749","reported_seq":11816,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:09.028060+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:36:09.028060+0000","last_peered":"2023-09-30T09:36:09.028060+0000","last_clean":"2023-09-30T09:36:09.028060+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:36:09.028060+0000","last_undegraded":"2023-09-30T09:36:09.028060+0000","last_fullsized":"2023-09-30T09:36:09.028060+0000","mapping_epoch":279,"log_start":"299'6957","ondisk_log_start":"299'6957","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4792,"log_dups_size":0,"ondisk_log_size":4792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164298752,"num_objects":2507,"num_object_clones":0,"num_object_copies":7521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11749,"num_write_kb":456192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":800,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.495468+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:36:03.495468+0000","last_peered":"2023-09-30T09:36:03.495468+0000","last_clean":"2023-09-30T09:36:03.495468+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:36:03.495468+0000","last_undegraded":"2023-09-30T09:36:03.495468+0000","last_fullsized":"2023-09-30T09:36:03.495468+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":329,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485349+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:36:03.485349+0000","last_peered":"2023-09-30T09:36:03.485349+0000","last_clean":"2023-09-30T09:36:03.485349+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:36:03.485349+0000","last_undegraded":"2023-09-30T09:36:03.485349+0000","last_fullsized":"2023-09-30T09:36:03.485349+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":317,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486408+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:36:03.486408+0000","last_peered":"2023-09-30T09:36:03.486408+0000","last_clean":"2023-09-30T09:36:03.486408+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:36:03.486408+0000","last_undegraded":"2023-09-30T09:36:03.486408+0000","last_fullsized":"2023-09-30T09:36:03.486408+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":55,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.513438+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:36:03.513438+0000","last_peered":"2023-09-30T09:36:03.513438+0000","last_clean":"2023-09-30T09:36:03.513438+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:36:03.513438+0000","last_undegraded":"2023-09-30T09:36:03.513438+0000","last_fullsized":"2023-09-30T09:36:03.513438+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"338'11769","reported_seq":14142,"reported_epoch":338,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:08.994019+0000","last_change":"2023-09-30T09:35:10.808625+0000","last_active":"2023-09-30T09:36:08.994019+0000","last_peered":"2023-09-30T09:36:08.994019+0000","last_clean":"2023-09-30T09:32:35.460821+0000","last_became_active":"2023-09-30T09:35:10.778683+0000","last_became_peered":"2023-09-30T09:35:10.778683+0000","last_unstale":"2023-09-30T09:36:08.994019+0000","last_undegraded":"2023-09-30T09:35:10.766189+0000","last_fullsized":"2023-09-30T09:36:08.994019+0000","mapping_epoch":327,"log_start":"299'6933","ondisk_log_start":"299'6933","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4836,"log_dups_size":0,"ondisk_log_size":4836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:45.905717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169279488,"num_objects":2583,"num_object_clones":0,"num_object_copies":7749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2583,"num_objects_misplaced":5166,"num_objects_unfound":0,"num_objects_dirty":2583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11769,"num_write_kb":459264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1133,"num_bytes_recovered":74252288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[6,1,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":2583}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":697,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.492502+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:36:03.492502+0000","last_peered":"2023-09-30T09:36:03.492502+0000","last_clean":"2023-09-30T09:36:03.492502+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:36:03.492502+0000","last_undegraded":"2023-09-30T09:36:03.492502+0000","last_fullsized":"2023-09-30T09:36:03.492502+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":516,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.509777+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:36:03.509777+0000","last_peered":"2023-09-30T09:36:03.509777+0000","last_clean":"2023-09-30T09:36:03.509777+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:36:03.509777+0000","last_undegraded":"2023-09-30T09:36:03.509777+0000","last_fullsized":"2023-09-30T09:36:03.509777+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":63,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.508654+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:36:03.508654+0000","last_peered":"2023-09-30T09:36:03.508654+0000","last_clean":"2023-09-30T09:36:03.508654+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:36:03.508654+0000","last_undegraded":"2023-09-30T09:36:03.508654+0000","last_fullsized":"2023-09-30T09:36:03.508654+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":257,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.496744+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:36:03.496744+0000","last_peered":"2023-09-30T09:36:03.496744+0000","last_clean":"2023-09-30T09:36:03.496744+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:36:03.496744+0000","last_undegraded":"2023-09-30T09:36:03.496744+0000","last_fullsized":"2023-09-30T09:36:03.496744+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"338'11618","reported_seq":17077,"reported_epoch":338,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:10.090559+0000","last_change":"2023-09-30T09:35:23.879286+0000","last_active":"2023-09-30T09:36:10.090559+0000","last_peered":"2023-09-30T09:36:10.090559+0000","last_clean":"2023-09-30T09:34:06.430942+0000","last_became_active":"2023-09-30T09:34:47.755052+0000","last_became_peered":"2023-09-30T09:34:47.755052+0000","last_unstale":"2023-09-30T09:36:10.090559+0000","last_undegraded":"2023-09-30T09:34:47.557422+0000","last_fullsized":"2023-09-30T09:34:47.514378+0000","mapping_epoch":320,"log_start":"292'5450","ondisk_log_start":"292'5450","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":6168,"log_dups_size":0,"ondisk_log_size":6168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:44:56.113242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157810688,"num_objects":2408,"num_object_clones":0,"num_object_copies":7224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":141,"num_objects_misplaced":1962,"num_objects_unfound":0,"num_objects_dirty":2408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11618,"num_write_kb":448832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2687,"num_bytes_recovered":175767552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":305},{"shards":"0,3,7","objects":1962}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":910,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.495593+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:36:03.495593+0000","last_peered":"2023-09-30T09:36:03.495593+0000","last_clean":"2023-09-30T09:36:03.495593+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:36:03.495593+0000","last_undegraded":"2023-09-30T09:36:03.495593+0000","last_fullsized":"2023-09-30T09:36:03.495593+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":602,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.492367+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:36:03.492367+0000","last_peered":"2023-09-30T09:36:03.492367+0000","last_clean":"2023-09-30T09:36:03.492367+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:36:03.492367+0000","last_undegraded":"2023-09-30T09:36:03.492367+0000","last_fullsized":"2023-09-30T09:36:03.492367+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":257,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.341720+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:36:04.341720+0000","last_peered":"2023-09-30T09:36:04.341720+0000","last_clean":"2023-09-30T09:36:04.341720+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:36:04.341720+0000","last_undegraded":"2023-09-30T09:36:04.341720+0000","last_fullsized":"2023-09-30T09:36:04.341720+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":290,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.487429+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:36:03.487429+0000","last_peered":"2023-09-30T09:36:03.487429+0000","last_clean":"2023-09-30T09:36:03.487429+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:36:03.487429+0000","last_undegraded":"2023-09-30T09:36:03.487429+0000","last_fullsized":"2023-09-30T09:36:03.487429+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"338'11628","reported_seq":20017,"reported_epoch":338,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:09.383009+0000","last_change":"2023-09-30T09:35:10.837720+0000","last_active":"2023-09-30T09:36:09.383009+0000","last_peered":"2023-09-30T09:36:09.383009+0000","last_clean":"2023-09-30T09:32:35.481650+0000","last_became_active":"2023-09-30T09:35:10.788829+0000","last_became_peered":"2023-09-30T09:35:10.788829+0000","last_unstale":"2023-09-30T09:36:09.383009+0000","last_undegraded":"2023-09-30T09:35:10.757584+0000","last_fullsized":"2023-09-30T09:36:09.383009+0000","mapping_epoch":327,"log_start":"286'3000","ondisk_log_start":"286'3000","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":8628,"log_dups_size":0,"ondisk_log_size":8628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:05.384343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165806080,"num_objects":2530,"num_object_clones":0,"num_object_copies":7590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1327,"num_objects_misplaced":5059,"num_objects_unfound":0,"num_objects_dirty":2530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11628,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4135,"num_bytes_recovered":270991360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[7,1,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":1},{"shards":"2,6,7","objects":1203},{"shards":"6,7","objects":1326}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":483,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.488509+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:36:03.488509+0000","last_peered":"2023-09-30T09:36:03.488509+0000","last_clean":"2023-09-30T09:36:03.488509+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:36:03.488509+0000","last_undegraded":"2023-09-30T09:36:03.488509+0000","last_fullsized":"2023-09-30T09:36:03.488509+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":714,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.484470+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:36:04.484470+0000","last_peered":"2023-09-30T09:36:04.484470+0000","last_clean":"2023-09-30T09:36:04.484470+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:36:04.484470+0000","last_undegraded":"2023-09-30T09:36:04.484470+0000","last_fullsized":"2023-09-30T09:36:04.484470+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":266,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.492617+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:36:03.492617+0000","last_peered":"2023-09-30T09:36:03.492617+0000","last_clean":"2023-09-30T09:36:03.492617+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:36:03.492617+0000","last_undegraded":"2023-09-30T09:36:03.492617+0000","last_fullsized":"2023-09-30T09:36:03.492617+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.508278+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:36:03.508278+0000","last_peered":"2023-09-30T09:36:03.508278+0000","last_clean":"2023-09-30T09:36:03.508278+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:36:03.508278+0000","last_undegraded":"2023-09-30T09:36:03.508278+0000","last_fullsized":"2023-09-30T09:36:03.508278+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"338'11645","reported_seq":16090,"reported_epoch":338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:09.227769+0000","last_change":"2023-09-30T09:34:47.553318+0000","last_active":"2023-09-30T09:36:09.227769+0000","last_peered":"2023-09-30T09:36:09.227769+0000","last_clean":"2023-09-30T09:32:35.469075+0000","last_became_active":"2023-09-30T09:33:33.792569+0000","last_became_peered":"2023-09-30T09:33:33.792569+0000","last_unstale":"2023-09-30T09:36:09.227769+0000","last_undegraded":"2023-09-30T09:33:33.610726+0000","last_fullsized":"2023-09-30T09:33:33.587755+0000","mapping_epoch":302,"log_start":"298'6145","ondisk_log_start":"298'6145","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":5500,"log_dups_size":0,"ondisk_log_size":5500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:16.121133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164560896,"num_objects":2511,"num_object_clones":0,"num_object_copies":7533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11645,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2178,"num_bytes_recovered":142737408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1071}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":612,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485569+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:36:03.485569+0000","last_peered":"2023-09-30T09:36:03.485569+0000","last_clean":"2023-09-30T09:36:03.485569+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:36:03.485569+0000","last_undegraded":"2023-09-30T09:36:03.485569+0000","last_fullsized":"2023-09-30T09:36:03.485569+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":846,"reported_epoch":337,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:03.497317+0000","last_change":"2023-09-30T09:35:11.685841+0000","last_active":"2023-09-30T09:36:03.497317+0000","last_peered":"2023-09-30T09:36:03.497317+0000","last_clean":"2023-09-30T09:26:47.308941+0000","last_became_active":"2023-09-30T09:35:11.685274+0000","last_became_peered":"2023-09-30T09:35:11.685274+0000","last_unstale":"2023-09-30T09:36:03.497317+0000","last_undegraded":"2023-09-30T09:35:11.661789+0000","last_fullsized":"2023-09-30T09:35:11.661213+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:38.516737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[1,4],"avail_no_missing":["1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":63,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.513268+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:36:03.513268+0000","last_peered":"2023-09-30T09:36:03.513268+0000","last_clean":"2023-09-30T09:36:03.513268+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:36:03.513268+0000","last_undegraded":"2023-09-30T09:36:03.513268+0000","last_fullsized":"2023-09-30T09:36:03.513268+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":238,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.493315+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:36:03.493315+0000","last_peered":"2023-09-30T09:36:03.493315+0000","last_clean":"2023-09-30T09:36:03.493315+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:36:03.493315+0000","last_undegraded":"2023-09-30T09:36:03.493315+0000","last_fullsized":"2023-09-30T09:36:03.493315+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"338'11776","reported_seq":11945,"reported_epoch":338,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:09.334737+0000","last_change":"2023-09-30T09:33:32.864041+0000","last_active":"2023-09-30T09:36:09.334737+0000","last_peered":"2023-09-30T09:36:09.334737+0000","last_clean":"2023-09-30T09:33:09.329441+0000","last_became_active":"2023-09-30T09:33:11.506402+0000","last_became_peered":"2023-09-30T09:33:11.506402+0000","last_unstale":"2023-09-30T09:36:09.334737+0000","last_undegraded":"2023-09-30T09:36:09.334737+0000","last_fullsized":"2023-09-30T09:36:09.334737+0000","mapping_epoch":295,"log_start":"303'7298","ondisk_log_start":"303'7298","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4478,"log_dups_size":0,"ondisk_log_size":4478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:51:06.485777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161742848,"num_objects":2468,"num_object_clones":0,"num_object_copies":7404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2451,"num_objects_unfound":0,"num_objects_dirty":2468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11776,"num_write_kb":455808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2468}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":810,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486462+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:36:03.486462+0000","last_peered":"2023-09-30T09:36:03.486462+0000","last_clean":"2023-09-30T09:36:03.486462+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:36:03.486462+0000","last_undegraded":"2023-09-30T09:36:03.486462+0000","last_fullsized":"2023-09-30T09:36:03.486462+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":338,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.473261+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:36:03.473261+0000","last_peered":"2023-09-30T09:36:03.473261+0000","last_clean":"2023-09-30T09:36:03.473261+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:36:03.473261+0000","last_undegraded":"2023-09-30T09:36:03.473261+0000","last_fullsized":"2023-09-30T09:36:03.473261+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1104,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485410+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:36:03.485410+0000","last_peered":"2023-09-30T09:36:03.485410+0000","last_clean":"2023-09-30T09:36:03.485410+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:36:03.485410+0000","last_undegraded":"2023-09-30T09:36:03.485410+0000","last_fullsized":"2023-09-30T09:36:03.485410+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":355,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471015+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:36:03.471015+0000","last_peered":"2023-09-30T09:36:03.471015+0000","last_clean":"2023-09-30T09:36:03.471015+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:36:03.471015+0000","last_undegraded":"2023-09-30T09:36:03.471015+0000","last_fullsized":"2023-09-30T09:36:03.471015+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"338'11762","reported_seq":14123,"reported_epoch":338,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:09.377660+0000","last_change":"2023-09-30T09:36:02.717804+0000","last_active":"2023-09-30T09:36:09.377660+0000","last_peered":"2023-09-30T09:36:09.377660+0000","last_clean":"2023-09-30T09:32:35.469896+0000","last_became_active":"2023-09-30T09:33:32.854592+0000","last_became_peered":"2023-09-30T09:33:32.854592+0000","last_unstale":"2023-09-30T09:36:09.377660+0000","last_undegraded":"2023-09-30T09:33:32.787590+0000","last_fullsized":"2023-09-30T09:33:32.601623+0000","mapping_epoch":301,"log_start":"303'7250","ondisk_log_start":"303'7250","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4512,"log_dups_size":0,"ondisk_log_size":4512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:02.368619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165281792,"num_objects":2522,"num_object_clones":0,"num_object_copies":7566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2466,"num_objects_misplaced":2466,"num_objects_unfound":0,"num_objects_dirty":2522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11762,"num_write_kb":457088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1133,"num_bytes_recovered":74252288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":56}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":803,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471809+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:36:03.471809+0000","last_peered":"2023-09-30T09:36:03.471809+0000","last_clean":"2023-09-30T09:36:03.471809+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:36:03.471809+0000","last_undegraded":"2023-09-30T09:36:03.471809+0000","last_fullsized":"2023-09-30T09:36:03.471809+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":663,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.488896+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:36:03.488896+0000","last_peered":"2023-09-30T09:36:03.488896+0000","last_clean":"2023-09-30T09:36:03.488896+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:36:03.488896+0000","last_undegraded":"2023-09-30T09:36:03.488896+0000","last_fullsized":"2023-09-30T09:36:03.488896+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":25,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.508358+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:36:03.508358+0000","last_peered":"2023-09-30T09:36:03.508358+0000","last_clean":"2023-09-30T09:36:03.508358+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:36:03.508358+0000","last_undegraded":"2023-09-30T09:36:03.508358+0000","last_fullsized":"2023-09-30T09:36:03.508358+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":279,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.487766+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:36:03.487766+0000","last_peered":"2023-09-30T09:36:03.487766+0000","last_clean":"2023-09-30T09:36:03.487766+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:36:03.487766+0000","last_undegraded":"2023-09-30T09:36:03.487766+0000","last_fullsized":"2023-09-30T09:36:03.487766+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":751,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.480950+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:36:03.480950+0000","last_peered":"2023-09-30T09:36:03.480950+0000","last_clean":"2023-09-30T09:36:03.480950+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:36:03.480950+0000","last_undegraded":"2023-09-30T09:36:03.480950+0000","last_fullsized":"2023-09-30T09:36:03.480950+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":21,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.493474+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:36:03.493474+0000","last_peered":"2023-09-30T09:36:03.493474+0000","last_clean":"2023-09-30T09:36:03.493474+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:36:03.493474+0000","last_undegraded":"2023-09-30T09:36:03.493474+0000","last_fullsized":"2023-09-30T09:36:03.493474+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":15,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.513158+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:36:03.513158+0000","last_peered":"2023-09-30T09:36:03.513158+0000","last_clean":"2023-09-30T09:36:03.513158+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:36:03.513158+0000","last_undegraded":"2023-09-30T09:36:03.513158+0000","last_fullsized":"2023-09-30T09:36:03.513158+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":523,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.472366+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:36:03.472366+0000","last_peered":"2023-09-30T09:36:03.472366+0000","last_clean":"2023-09-30T09:36:03.472366+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:36:03.472366+0000","last_undegraded":"2023-09-30T09:36:03.472366+0000","last_fullsized":"2023-09-30T09:36:03.472366+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":316,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.495970+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:36:03.495970+0000","last_peered":"2023-09-30T09:36:03.495970+0000","last_clean":"2023-09-30T09:36:03.495970+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:36:03.495970+0000","last_undegraded":"2023-09-30T09:36:03.495970+0000","last_fullsized":"2023-09-30T09:36:03.495970+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":32,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485886+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:36:03.485886+0000","last_peered":"2023-09-30T09:36:03.485886+0000","last_clean":"2023-09-30T09:36:03.485886+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:36:03.485886+0000","last_undegraded":"2023-09-30T09:36:03.485886+0000","last_fullsized":"2023-09-30T09:36:03.485886+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":282,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.495039+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:36:03.495039+0000","last_peered":"2023-09-30T09:36:03.495039+0000","last_clean":"2023-09-30T09:36:03.495039+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:36:03.495039+0000","last_undegraded":"2023-09-30T09:36:03.495039+0000","last_fullsized":"2023-09-30T09:36:03.495039+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":32,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486075+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:36:03.486075+0000","last_peered":"2023-09-30T09:36:03.486075+0000","last_clean":"2023-09-30T09:36:03.486075+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:36:03.486075+0000","last_undegraded":"2023-09-30T09:36:03.486075+0000","last_fullsized":"2023-09-30T09:36:03.486075+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":466,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.486508+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:36:03.486508+0000","last_peered":"2023-09-30T09:36:03.486508+0000","last_clean":"2023-09-30T09:36:03.486508+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:36:03.486508+0000","last_undegraded":"2023-09-30T09:36:03.486508+0000","last_fullsized":"2023-09-30T09:36:03.486508+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":334,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.472788+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:36:03.472788+0000","last_peered":"2023-09-30T09:36:03.472788+0000","last_clean":"2023-09-30T09:36:03.472788+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:36:03.472788+0000","last_undegraded":"2023-09-30T09:36:03.472788+0000","last_fullsized":"2023-09-30T09:36:03.472788+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":359,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.481226+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:36:03.481226+0000","last_peered":"2023-09-30T09:36:03.481226+0000","last_clean":"2023-09-30T09:36:03.481226+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:36:03.481226+0000","last_undegraded":"2023-09-30T09:36:03.481226+0000","last_fullsized":"2023-09-30T09:36:03.481226+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":786,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.471979+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:36:03.471979+0000","last_peered":"2023-09-30T09:36:03.471979+0000","last_clean":"2023-09-30T09:36:03.471979+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:36:03.471979+0000","last_undegraded":"2023-09-30T09:36:03.471979+0000","last_fullsized":"2023-09-30T09:36:03.471979+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":41,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.506332+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:36:03.506332+0000","last_peered":"2023-09-30T09:36:03.506332+0000","last_clean":"2023-09-30T09:36:03.506332+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:36:03.506332+0000","last_undegraded":"2023-09-30T09:36:03.506332+0000","last_fullsized":"2023-09-30T09:36:03.506332+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":313,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-30T09:36:03.485693+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:36:03.485693+0000","last_peered":"2023-09-30T09:36:03.485693+0000","last_clean":"2023-09-30T09:36:03.485693+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:36:03.485693+0000","last_undegraded":"2023-09-30T09:36:03.485693+0000","last_fullsized":"2023-09-30T09:36:03.485693+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2613903384,"num_objects":39886,"num_object_clones":0,"num_object_copies":119658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19396,"num_objects_misplaced":37108,"num_objects_unfound":0,"num_objects_dirty":39886,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186666,"num_write_kb":7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21696,"num_bytes_recovered":1421279232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6820737024,"data_stored":6820724808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80630,"ondisk_log_size":80630,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234294,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495931,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1822688,"kb_used_data":1453208,"kb_used_omap":0,"kb_used_meta":369472,"kb_avail":91922464,"statfs":{"total":95995035648,"available":94128603136,"internally_reserved":0,"allocated":1488084992,"data_stored":1486909712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":378339328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,0,0,0,3,2,3,4],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5779999999999998,"5min":1.706,"15min":1.583},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":14.795,"5min":19.882000000000001,"15min":45.927999999999997},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.853,"5min":1.859,"15min":1.7869999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.0939999999999994,"5min":20.373000000000001,"15min":45.316000000000003},"last":2.0680000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:35:35 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":1.9470000000000001,"15min":1.851},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.7070000000000007,"5min":9.7070000000000007,"15min":9.7070000000000007},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.1080000000000001,"15min":2.0779999999999998},"min":{"1min":0.35799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.0820000000000007,"5min":9.0820000000000007,"15min":9.0820000000000007},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":1.738,"15min":1.7290000000000001},"min":{"1min":0.32800000000000001,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":8.0850000000000009,"5min":22.158000000000001,"15min":37.978000000000002},"last":3.4449999999999998},{"interface":"front","average":{"1min":2.6360000000000001,"5min":2.0289999999999999,"15min":1.641},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.706,"5min":22.356999999999999,"15min":37.759999999999998},"last":3.3159999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:35:52 2023","interfaces":[{"interface":"back","average":{"1min":17.585000000000001,"5min":6.2119999999999997,"15min":3.0529999999999999},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":212.44800000000001,"5min":212.44800000000001,"15min":212.44800000000001},"last":1.113},{"interface":"front","average":{"1min":18.085000000000001,"5min":6.7649999999999997,"15min":3.363},"min":{"1min":0.39500000000000002,"5min":0.313,"15min":0.313},"max":{"1min":210.84299999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":5.6109999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.486,"15min":1.591},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.23400000000000001},"max":{"1min":7.234,"5min":8.1059999999999999,"15min":73.656000000000006},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.4590000000000001,"5min":1.3260000000000001,"15min":1.415},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.23000000000000001},"max":{"1min":9.6590000000000007,"5min":9.6590000000000007,"15min":73.882999999999996},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":1.399,"15min":1.823},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":4.3010000000000002,"5min":10.103999999999999,"15min":76.613},"last":0.44},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.45,"15min":1.786},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.31},"max":{"1min":9.7750000000000004,"5min":10.054,"15min":76.329999999999998},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:35:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.712,"15min":2.1320000000000001},"min":{"1min":0.38,"5min":0.377,"15min":0.32100000000000001},"max":{"1min":8.7400000000000002,"5min":8.7400000000000002,"15min":91.120999999999995},"last":1.6100000000000001},{"interface":"front","average":{"1min":1.306,"5min":1.448,"15min":2.0710000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.9489999999999998,"5min":9.9489999999999998,"15min":91.733999999999995},"last":0.496}]}]},{"osd":1,"up_from":301,"seq":1292785156129,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":472532,"kb_used_data":288972,"kb_used_omap":9,"kb_used_meta":183542,"kb_avail":93272620,"statfs":{"total":95995035648,"available":95511162880,"internally_reserved":0,"allocated":295907328,"data_stored":293994223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":187947026},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.6549999999999998,"15min":2.5790000000000002},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.699999999999999,"5min":13.699999999999999,"15min":13.699999999999999},"last":6.218},{"interface":"front","average":{"1min":4.0659999999999998,"5min":3.2320000000000002,"15min":3.093},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.25},"max":{"1min":13.779999999999999,"5min":13.779999999999999,"15min":13.779999999999999},"last":2.1539999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":2.7120000000000002,"15min":2.7679999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":16.437000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":1.268},{"interface":"front","average":{"1min":3.3740000000000001,"5min":2.839,"15min":2.75},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.121,"5min":12.121,"15min":12.121},"last":2.9569999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":4.4589999999999996,"5min":8.4489999999999998,"15min":9.1140000000000008},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.837999999999999,"5min":77.959000000000003,"15min":77.959000000000003},"last":7.0149999999999997},{"interface":"front","average":{"1min":3.2690000000000001,"5min":8.7949999999999999,"15min":9.7159999999999993},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":11.657,"5min":77.828999999999994,"15min":77.828999999999994},"last":9.1560000000000006}]},{"osd":4,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":2.0249999999999999,"15min":1.9990000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":13.602,"5min":13.602,"15min":13.602},"last":1.9790000000000001},{"interface":"front","average":{"1min":2.1920000000000002,"5min":1.472,"15min":1.3520000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":10.773,"5min":10.773,"15min":10.773},"last":3.46}]},{"osd":5,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":2.2309999999999999,"15min":2.149},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":13.666,"5min":13.666,"15min":13.666},"last":1.8779999999999999},{"interface":"front","average":{"1min":2.218,"5min":1.724,"15min":1.6419999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":3.4969999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":14.346,"15min":16.18},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":13.728,"5min":209.221,"15min":209.221},"last":1.917},{"interface":"front","average":{"1min":2.5110000000000001,"5min":14.115,"15min":16.048999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":13.755000000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":1.946}]},{"osd":7,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":10.254,"15min":11.714},"min":{"1min":0.46000000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":4.5279999999999996,"5min":165.59399999999999,"15min":165.59399999999999},"last":1.3049999999999999},{"interface":"front","average":{"1min":1.429,"5min":10.429,"15min":11.929},"min":{"1min":0.47299999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":4.5640000000000001,"5min":165.833,"15min":165.833},"last":1.337}]}]},{"osd":0,"up_from":94,"seq":403726926028,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":804456,"kb_used_data":481440,"kb_used_omap":5,"kb_used_meta":323002,"kb_avail":92940696,"statfs":{"total":95995035648,"available":95171272704,"internally_reserved":0,"allocated":492994560,"data_stored":490947812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":330754301},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":2.8250000000000002,"15min":2.9399999999999999},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":11.175000000000001,"5min":17.966000000000001,"15min":17.966000000000001},"last":7.6260000000000003},{"interface":"front","average":{"1min":4.1660000000000004,"5min":8.3729999999999993,"15min":9.0739999999999998},"min":{"1min":0.22500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":14.097,"5min":138.92400000000001,"15min":138.92400000000001},"last":8.2919999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:35:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7639999999999998,"5min":1.756,"15min":1.5720000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.6769999999999996,"5min":11.218999999999999,"15min":55.823999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":5.6159999999999997,"5min":2.0830000000000002,"15min":1.748},"min":{"1min":0.29299999999999998,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":40.128999999999998,"5min":40.128999999999998,"15min":55.790999999999997},"last":0.56399999999999995}]},{"osd":3,"last update":"Sat Sep 30 09:35:53 2023","interfaces":[{"interface":"back","average":{"1min":8.9269999999999996,"5min":8.3209999999999997,"15min":3.9820000000000002},"min":{"1min":0.311,"5min":0.221,"15min":0.221},"max":{"1min":47.277999999999999,"5min":218.06999999999999,"15min":218.06999999999999},"last":3.7789999999999999},{"interface":"front","average":{"1min":6.1849999999999996,"5min":6.2850000000000001,"15min":3.2570000000000001},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":47.643000000000001,"5min":218.63200000000001,"15min":218.63200000000001},"last":1.401}]},{"osd":4,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":1.3560000000000001,"15min":1.5509999999999999},"min":{"1min":0.34300000000000003,"5min":0.32800000000000001,"15min":0.29099999999999998},"max":{"1min":8.6110000000000007,"5min":8.6110000000000007,"15min":121.486},"last":0.499},{"interface":"front","average":{"1min":4.1219999999999999,"5min":3.4329999999999998,"15min":2.464},"min":{"1min":0.318,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":15.587999999999999,"5min":136.262,"15min":136.262},"last":0.40100000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":4.4989999999999997,"5min":3.7730000000000001,"15min":2.8809999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":15.647,"5min":136.34,"15min":136.34},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.1030000000000002,"5min":1.698,"15min":2.1669999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.33700000000000002},"max":{"1min":12.717000000000001,"5min":12.717000000000001,"15min":135.381},"last":0.38}]},{"osd":6,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":6.9829999999999997,"15min":3.1070000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.33300000000000002},"max":{"1min":12.661,"5min":214.785,"15min":214.785},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.0430000000000001,"5min":6.9880000000000004,"15min":3.036},"min":{"1min":0.436,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":8.5679999999999996,"5min":214.93899999999999,"15min":214.93899999999999},"last":0.73699999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":2.8999999999999999,"15min":3.0470000000000002},"min":{"1min":0.42999999999999999,"5min":0.373,"15min":0.373},"max":{"1min":12.749000000000001,"5min":97.382000000000005,"15min":150.96899999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.7040000000000002,"15min":2.8580000000000001},"min":{"1min":0.36799999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.7080000000000002,"5min":97.015000000000001,"15min":150.328},"last":0.77300000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598969,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":382160,"kb_used_data":93960,"kb_used_omap":5,"kb_used_meta":288186,"kb_avail":93362992,"statfs":{"total":95995035648,"available":95603703808,"internally_reserved":0,"allocated":96215040,"data_stored":94367865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":295103117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":2.9449999999999998,"15min":1.9179999999999999},"min":{"1min":0.22800000000000001,"5min":0.187,"15min":0.187},"max":{"1min":77.379000000000005,"5min":77.379000000000005,"15min":77.379000000000005},"last":15.063000000000001},{"interface":"front","average":{"1min":6.2549999999999999,"5min":2.9660000000000002,"15min":1.8140000000000001},"min":{"1min":0.25600000000000001,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":76.796999999999997,"5min":77.066999999999993,"15min":77.066999999999993},"last":15.151}]},{"osd":1,"last update":"Sat Sep 30 09:35:36 2023","interfaces":[{"interface":"back","average":{"1min":4.4779999999999998,"5min":3.7530000000000001,"15min":3.6320000000000001},"min":{"1min":0.28199999999999997,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.728999999999999,"5min":14.707000000000001,"15min":14.707000000000001},"last":23.486999999999998},{"interface":"front","average":{"1min":3.819,"5min":3.2090000000000001,"15min":3.1070000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.968,"5min":10.179,"15min":10.179},"last":21.826000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:35:49 2023","interfaces":[{"interface":"back","average":{"1min":6.5060000000000002,"5min":4.5499999999999998,"15min":2.4889999999999999},"min":{"1min":0.38100000000000001,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":31.509,"5min":78.915999999999997,"15min":78.915999999999997},"last":15.118},{"interface":"front","average":{"1min":5.1230000000000002,"5min":4.3230000000000004,"15min":2.5750000000000002},"min":{"1min":0.28899999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":34.329000000000001,"5min":80.834000000000003,"15min":80.834000000000003},"last":14.337}]},{"osd":4,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.8029999999999999,"15min":1.25},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":5.524,"5min":35.972000000000001,"15min":35.972000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.7470000000000001,"15min":1.4810000000000001},"min":{"1min":0.34799999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":5.8789999999999996,"5min":36.518000000000001,"15min":36.518000000000001},"last":0.62}]},{"osd":5,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.732,"15min":1.9410000000000001},"min":{"1min":0.377,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":12.651999999999999,"5min":12.651999999999999,"15min":155.477},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.794,"5min":1.4119999999999999,"15min":1.8660000000000001},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":4.4560000000000004,"5min":6.048,"15min":155.46000000000001},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":6.7009999999999996,"15min":3.1899999999999999},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.9729999999999999,"5min":154.417,"15min":154.417},"last":0.55300000000000005},{"interface":"front","average":{"1min":3.1589999999999998,"5min":6.9749999999999996,"15min":3.1960000000000002},"min":{"1min":0.47399999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":12.710000000000001,"5min":154.04400000000001,"15min":154.04400000000001},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":3.665,"15min":3},"min":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8399999999999999,"5min":123.294,"15min":213.404},"last":0.59899999999999998},{"interface":"front","average":{"1min":2.234,"5min":3.9769999999999999,"15min":3.0219999999999998},"min":{"1min":0.438,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":12.686999999999999,"5min":123.16,"15min":213.27199999999999},"last":0.51300000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435441,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2352600,"kb_used_data":1765392,"kb_used_omap":2,"kb_used_meta":586941,"kb_avail":91392552,"statfs":{"total":95995035648,"available":93585973248,"internally_reserved":0,"allocated":1807761408,"data_stored":1805783480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":601027943},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,3,8,0,3,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.8040000000000003,"5min":4.1310000000000002,"15min":2.7559999999999998},"min":{"1min":0.23999999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":60.726999999999997,"5min":102.57299999999999,"15min":130.946},"last":0.39900000000000002},{"interface":"front","average":{"1min":3.9049999999999998,"5min":3.9729999999999999,"15min":2.7069999999999999},"min":{"1min":0.34599999999999997,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":12.477,"5min":102.745,"15min":130.96899999999999},"last":5.3029999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:35:33 2023","interfaces":[{"interface":"back","average":{"1min":4.625,"5min":17.937999999999999,"15min":20.157},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":21.709,"5min":273.99599999999998,"15min":273.99599999999998},"last":8.5969999999999995},{"interface":"front","average":{"1min":5.2140000000000004,"5min":16.260000000000002,"15min":18.100999999999999},"min":{"1min":0.38800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":24.771999999999998,"5min":274.21600000000001,"15min":274.21600000000001},"last":8.6910000000000007}]},{"osd":2,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":4.7089999999999996,"5min":3.4580000000000002,"15min":2.9510000000000001},"min":{"1min":0.50800000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.339,"5min":76.974000000000004,"15min":145.75899999999999},"last":6.2290000000000001},{"interface":"front","average":{"1min":5.6070000000000002,"5min":3.415,"15min":2.8410000000000002},"min":{"1min":0.35699999999999998,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":60.561999999999998,"5min":77.593999999999994,"15min":150.358},"last":1.05}]},{"osd":4,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.282,"5min":2.5979999999999999,"15min":1.5700000000000001},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":60.637,"5min":60.637,"15min":60.637},"last":0.33600000000000002},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.9990000000000001,"15min":1.4670000000000001},"min":{"1min":0.441,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":7.7990000000000004,"5min":10.731,"15min":15.638},"last":5.2590000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.629,"15min":1.74},"min":{"1min":0.41499999999999998,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":7.7519999999999998,"5min":37.485999999999997,"15min":37.485999999999997},"last":5.234},{"interface":"front","average":{"1min":3.3300000000000001,"5min":3.044,"15min":1.9650000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":8.5980000000000008,"5min":37.448,"15min":37.448},"last":6.0309999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.5650000000000004,"5min":2.6960000000000002,"15min":1.6659999999999999},"min":{"1min":0.36799999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":60.698,"5min":60.698,"15min":60.698},"last":0.373},{"interface":"front","average":{"1min":2.778,"5min":2.3090000000000002,"15min":1.6859999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":9.5860000000000003,"5min":10.818,"15min":14.69},"last":5.2850000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5590000000000002,"5min":2.855,"15min":2.573},"min":{"1min":0.39100000000000001,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":8.6660000000000004,"5min":26.469000000000001,"15min":226.03999999999999},"last":6.0789999999999997},{"interface":"front","average":{"1min":5.8849999999999998,"5min":3.008,"15min":2.4580000000000002},"min":{"1min":0.45400000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":60.670000000000002,"5min":60.670000000000002,"15min":226.06100000000001},"last":0.58199999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019204,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":884440,"kb_used_data":638224,"kb_used_omap":0,"kb_used_meta":246208,"kb_avail":92860712,"statfs":{"total":95995035648,"available":95089369088,"internally_reserved":0,"allocated":653541376,"data_stored":652373791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":252116992},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":1.415,"15min":2.7999999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":14.353,"5min":14.353,"15min":128.79300000000001},"last":2.2829999999999999},{"interface":"front","average":{"1min":2.798,"5min":1.659,"15min":2.7509999999999999},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.248},"max":{"1min":10.618,"5min":11.744,"15min":128.38800000000001},"last":6.8879999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":8.2840000000000007,"15min":9.0969999999999995},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.957999999999998,"5min":160.69999999999999,"15min":160.69999999999999},"last":2.1589999999999998},{"interface":"front","average":{"1min":2.024,"5min":8.0559999999999992,"15min":9.0619999999999994},"min":{"1min":0.26800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":10.91,"5min":160.50700000000001,"15min":160.50700000000001},"last":7}]},{"osd":2,"last update":"Sat Sep 30 09:35:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":1.8200000000000001,"15min":2.8410000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.096,"5min":19.391999999999999,"15min":140.601},"last":4.2329999999999997},{"interface":"front","average":{"1min":2.089,"5min":1.865,"15min":2.8130000000000002},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.309},"max":{"1min":11.057,"5min":19.422999999999998,"15min":140.54900000000001},"last":4.25}]},{"osd":3,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":3.3109999999999999,"15min":3.4220000000000002},"min":{"1min":0.33300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.7629999999999999,"5min":70.241,"15min":281.42200000000003},"last":1.627},{"interface":"front","average":{"1min":3.1190000000000002,"5min":3.2090000000000001,"15min":3.3849999999999998},"min":{"1min":0.442,"5min":0.312,"15min":0.312},"max":{"1min":9.7349999999999994,"5min":70.433999999999997,"15min":281.44999999999999},"last":1.639}]},{"osd":5,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.254,"15min":1.6899999999999999},"min":{"1min":0.27600000000000002,"5min":0.219,"15min":0.219},"max":{"1min":8.6509999999999998,"5min":11.808,"15min":134.30600000000001},"last":0.309},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.161,"15min":1.766},"min":{"1min":0.29599999999999999,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":6.2149999999999999,"5min":6.3929999999999998,"15min":134.35300000000001},"last":0.34799999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.9409999999999998,"15min":1.522},"min":{"1min":0.32500000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.609,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.927,"15min":1.575},"min":{"1min":0.312,"5min":0.249,"15min":0.249},"max":{"1min":3.964,"5min":202.42400000000001,"15min":202.42400000000001},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:35:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":4.6449999999999996,"15min":2.9969999999999999},"min":{"1min":0.35099999999999998,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":7.5220000000000002,"5min":230.90100000000001,"15min":230.90100000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.67500000000000004,"5min":4.2539999999999996,"15min":2.5510000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":3.7360000000000002,"5min":229.97,"15min":229.97},"last":0.39900000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757568,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":774808,"kb_used_data":520784,"kb_used_omap":0,"kb_used_meta":254016,"kb_avail":92970344,"statfs":{"total":95995035648,"available":95201632256,"internally_reserved":0,"allocated":533282816,"data_stored":532111779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":260112384},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:42 2023","interfaces":[{"interface":"back","average":{"1min":3.8999999999999999,"5min":1.99,"15min":2.355},"min":{"1min":0.24199999999999999,"5min":0.224,"15min":0.22},"max":{"1min":56.563000000000002,"5min":56.563000000000002,"15min":148.79900000000001},"last":3.0699999999999998},{"interface":"front","average":{"1min":5.0019999999999998,"5min":2.3290000000000002,"15min":2.681},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.23499999999999999},"max":{"1min":57.241999999999997,"5min":57.241999999999997,"15min":148.779},"last":1.462}]},{"osd":1,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":5.3440000000000003,"5min":2.5390000000000001,"15min":2.0710000000000002},"min":{"1min":0.28399999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":60.799999999999997,"5min":60.799999999999997,"15min":60.799999999999997},"last":8.484},{"interface":"front","average":{"1min":5.407,"5min":3.6110000000000002,"15min":3.3109999999999999},"min":{"1min":0.33900000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":63.798999999999999,"5min":63.798999999999999,"15min":63.798999999999999},"last":4.5330000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:35:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2679999999999998,"5min":1.8089999999999999,"15min":3.0049999999999999},"min":{"1min":0.42799999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":30.613,"5min":30.613,"15min":165.768},"last":1.643},{"interface":"front","average":{"1min":3.484,"5min":1.6910000000000001,"15min":3.4540000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":36.488,"5min":36.488,"15min":166.24000000000001},"last":0.70899999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:35:53 2023","interfaces":[{"interface":"back","average":{"1min":5.1660000000000004,"5min":8.2040000000000006,"15min":3.8610000000000002},"min":{"1min":0.39200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":38.280000000000001,"5min":226.34700000000001,"15min":226.34700000000001},"last":2.2999999999999998},{"interface":"front","average":{"1min":4.9379999999999997,"5min":8.1809999999999992,"15min":3.7080000000000002},"min":{"1min":0.39100000000000001,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":38.262999999999998,"5min":226.36000000000001,"15min":226.36000000000001},"last":2.3149999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.92000000000000004,"15min":0.85699999999999998},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":1.6200000000000001,"5min":7.4299999999999997,"15min":8.5359999999999996},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.998,"5min":1.097,"15min":0.93700000000000006},"min":{"1min":0.28799999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1580000000000004,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":1.2330000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":3.1749999999999998,"15min":1.7390000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":4.6639999999999997,"5min":155.602,"15min":155.602},"last":1.03},{"interface":"front","average":{"1min":1.014,"5min":3.2309999999999999,"15min":1.8680000000000001},"min":{"1min":0.36499999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.1600000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":2.5630000000000002,"15min":1.639},"min":{"1min":0.33000000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":3.2000000000000002,"5min":133.40700000000001,"15min":133.40700000000001},"last":0.37},{"interface":"front","average":{"1min":0.56200000000000006,"5min":2.6549999999999998,"15min":1.5609999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":2.274,"5min":132.797,"15min":132.797},"last":1.1080000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":479203328,"data_stored":479199256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":282660864,"data_stored":282656792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":90308608,"data_stored":90308608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1788149760,"data_stored":1788149760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":647954432,"data_stored":647954432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":527695872,"data_stored":527695872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1475481600,"data_stored":1475477528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1529282560,"data_stored":1529282560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:36:17.580 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:17.709 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:17.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:17.817 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:17.885 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:17.929 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:17.946 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:18.245 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:18.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:18.361 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:18.445 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:18.811 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:36:18.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[224043]: pgmap v858: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 18530/113208 objects degraded (16.368%); 34967/113208 objects misplaced (30.887%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:36:18.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:18.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3458574122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:36:18.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:36:18.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18530/113208 objects degraded (16.368%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:18.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[216815]: pgmap v858: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 18530/113208 objects degraded (16.368%); 34967/113208 objects misplaced (30.887%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:36:18.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:18.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3458574122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:36:18.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:36:18.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18530/113208 objects degraded (16.368%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:18 smithi167 ceph-mon[192880]: pgmap v858: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 18530/113208 objects degraded (16.368%); 34967/113208 objects misplaced (30.887%); 12 MiB/s, 185 objects/s recovering 2023-09-30T09:36:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3458574122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:36:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:18 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:36:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18530/113208 objects degraded (16.368%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:19.807 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:19.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:19.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:19.921 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:20.010 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:20.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:19 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:19 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:36:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:19 smithi167 ceph-mon[192880]: osdmap e339: 8 total, 8 up, 7 in 2023-09-30T09:36:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:19 smithi167 ceph-mon[192880]: failed to encode map e339 with expected crc 2023-09-30T09:36:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:19 smithi167 ceph-mon[192880]: failed to encode map e339 with expected crc 2023-09-30T09:36:20.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:20.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:36:20.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[224043]: osdmap e339: 8 total, 8 up, 7 in 2023-09-30T09:36:20.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[224043]: failed to encode map e339 with expected crc 2023-09-30T09:36:20.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[224043]: failed to encode map e339 with expected crc 2023-09-30T09:36:20.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:20.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:36:20.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[216815]: osdmap e339: 8 total, 8 up, 7 in 2023-09-30T09:36:20.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[216815]: failed to encode map e339 with expected crc 2023-09-30T09:36:20.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:19 smithi019 ceph-mon[216815]: failed to encode map e339 with expected crc 2023-09-30T09:36:20.518 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:36:21.074 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:20 smithi019 ceph-mon[216815]: pgmap v860: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 18205/110067 objects degraded (16.540%); 33902/110067 objects misplaced (30.801%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:36:21.075 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:20 smithi019 ceph-mon[224043]: pgmap v860: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 18205/110067 objects degraded (16.540%); 33902/110067 objects misplaced (30.801%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:36:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:20 smithi167 ceph-mon[192880]: pgmap v860: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 18205/110067 objects degraded (16.540%); 33902/110067 objects misplaced (30.801%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:36:22.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:21 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:36:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:21 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:36:22.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:21 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:36:22.430 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:22.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:22.544 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:22.645 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:22.769 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:22.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:22.876 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:22.909 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:22.984 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:22.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:23.007 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:22 smithi019 ceph-mon[216815]: pgmap v861: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 17662/106578 objects degraded (16.572%); 32792/106578 objects misplaced (30.768%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:36:23.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:22 smithi019 ceph-mon[224043]: pgmap v861: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 17662/106578 objects degraded (16.572%); 32792/106578 objects misplaced (30.768%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:36:23.089 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:36:23.123 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:22 smithi167 ceph-mon[192880]: pgmap v861: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 17662/106578 objects degraded (16.572%); 32792/106578 objects misplaced (30.768%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:36:23.126 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:23.201 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:36:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:23.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:36:23.254 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:23.306 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:23.382 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:36:23.382 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:36:23.409 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:23.708 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:23.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:23.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:23.819 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:23.861 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:36:23.866 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:23.912 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:36:23.929 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:23.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:23.959 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:36:24.006 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:36:24.040 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:24.068 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:24.075 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:36:24.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 17662/106578 objects degraded (16.572%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:24.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2567163117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:24.092 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:36:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:36:23] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-30T09:36:24.093 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 17662/106578 objects degraded (16.572%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:24.093 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2567163117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:24.135 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:36:24.135 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:24.141 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 17662/106578 objects degraded (16.572%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:24.142 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2567163117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:24.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:24.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:24.177 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:36:24.218 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:36:24.236 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:36:24.236 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:36:24.236 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:36:24.236 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:36:24.236 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:36:24.236 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:36:24.236 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:36:24.236 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:36:24.244 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:24.278 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:24.347 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:24.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:24.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:24.457 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:24.580 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:24.719 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:36:25.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:24 smithi019 ceph-mon[216815]: pgmap v862: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 17662/106578 objects degraded (16.572%); 32792/106578 objects misplaced (30.768%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:36:25.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:24 smithi019 ceph-mon[224043]: pgmap v862: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 17662/106578 objects degraded (16.572%); 32792/106578 objects misplaced (30.768%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:36:25.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:24 smithi167 ceph-mon[192880]: pgmap v862: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 17662/106578 objects degraded (16.572%); 32792/106578 objects misplaced (30.768%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:36:25.763 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:25.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:25.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:25.792 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:36:25.868 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:25.929 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:25.993 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:26.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:26.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:26.102 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:26.227 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:27.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:26 smithi167 ceph-mon[192880]: pgmap v863: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.0 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 16780/100242 objects degraded (16.739%); 30652/100242 objects misplaced (30.578%); 15 MiB/s, 238 objects/s recovering 2023-09-30T09:36:27.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:26 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:36:27.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:26 smithi019 ceph-mon[216815]: pgmap v863: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.0 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 16780/100242 objects degraded (16.739%); 30652/100242 objects misplaced (30.578%); 15 MiB/s, 238 objects/s recovering 2023-09-30T09:36:27.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:26 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:36:27.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:26 smithi019 ceph-mon[224043]: pgmap v863: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.0 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 16780/100242 objects degraded (16.739%); 30652/100242 objects misplaced (30.578%); 15 MiB/s, 238 objects/s recovering 2023-09-30T09:36:27.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:26 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:36:27.564 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:27.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:27.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:27.673 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:27.700 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:28.045 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:28.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:28.171 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:28.251 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:28.263 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:28.349 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:28.628 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:28.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:28.741 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:28.869 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:28 smithi019 ceph-mon[216815]: pgmap v864: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.0 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 16529/98919 objects degraded (16.710%); 30267/98919 objects misplaced (30.598%); 17 MiB/s, 264 objects/s recovering 2023-09-30T09:36:28.869 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 16529/98919 objects degraded (16.710%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:28.879 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:29.033 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:29.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:29.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:29.142 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:29.195 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:28 smithi019 ceph-mon[224043]: pgmap v864: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.0 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 16529/98919 objects degraded (16.710%); 30267/98919 objects misplaced (30.598%); 17 MiB/s, 264 objects/s recovering 2023-09-30T09:36:29.195 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 16529/98919 objects degraded (16.710%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:29.242 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:28 smithi167 ceph-mon[192880]: pgmap v864: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 2.0 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 16529/98919 objects degraded (16.710%); 30267/98919 objects misplaced (30.598%); 17 MiB/s, 264 objects/s recovering 2023-09-30T09:36:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 16529/98919 objects degraded (16.710%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:29.433 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:36:29.436 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:36:29.448 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:29.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:29.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:29.565 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:29.600 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:29.686 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:29.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:29.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:29.795 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:29.832 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:29.894 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:29.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:29.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:30.001 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:30.003 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:36:30.101 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:30.355 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:30.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:30.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:30.469 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:30.574 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:31.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:30 smithi167 ceph-mon[192880]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:31.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:30 smithi167 ceph-mon[192880]: pgmap v865: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 16169/95505 objects degraded (16.930%); 29152/95505 objects misplaced (30.524%); 18 MiB/s, 285 objects/s recovering 2023-09-30T09:36:31.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:30 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:36:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:30 smithi019 ceph-mon[216815]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:30 smithi019 ceph-mon[216815]: pgmap v865: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 16169/95505 objects degraded (16.930%); 29152/95505 objects misplaced (30.524%); 18 MiB/s, 285 objects/s recovering 2023-09-30T09:36:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:30 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:36:31.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:30 smithi019 ceph-mon[224043]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:31.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:30 smithi019 ceph-mon[224043]: pgmap v865: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 16169/95505 objects degraded (16.930%); 29152/95505 objects misplaced (30.524%); 18 MiB/s, 285 objects/s recovering 2023-09-30T09:36:31.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:30 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:36:32.127 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:32.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:32.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:32.236 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:32.270 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:31 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:36:32.274 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:32.326 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:32.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:32.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:31 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:36:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:31 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:36:32.432 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:32.472 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:32.532 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:32.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:32.640 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:32.681 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:32.746 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:32.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:32.853 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:32.885 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:32.960 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:32.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:32.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:33.072 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:33.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:32 smithi019 ceph-mon[216815]: pgmap v866: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 16169/95184 objects degraded (16.987%); 28851/95184 objects misplaced (30.311%); 15 MiB/s, 247 objects/s recovering 2023-09-30T09:36:33.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:32 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:36:33.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:32 smithi019 ceph-mon[224043]: pgmap v866: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 16169/95184 objects degraded (16.987%); 28851/95184 objects misplaced (30.311%); 15 MiB/s, 247 objects/s recovering 2023-09-30T09:36:33.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:32 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:36:33.174 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:32 smithi167 ceph-mon[192880]: pgmap v866: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 16169/95184 objects degraded (16.987%); 28851/95184 objects misplaced (30.311%); 15 MiB/s, 247 objects/s recovering 2023-09-30T09:36:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:32 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:36:33.311 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":864,"stamp":"2023-09-30T09:36:27.514995+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2163948388,"num_objects":32973,"num_object_clones":0,"num_object_copies":98919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16529,"num_objects_misplaced":30267,"num_objects_unfound":0,"num_objects_dirty":32973,"num_whiteouts":0,"num_read":6114,"num_read_kb":12467,"num_write":194596,"num_write_kb":7263166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25756,"num_bytes_recovered":1682809304,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87294,"ondisk_log_size":87294,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":397,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6668888,"kb_used_data":4339360,"kb_used_omap":23,"kb_used_meta":2329192,"kb_avail":649547176,"statfs":{"total":671965249536,"available":665136308224,"internally_reserved":0,"allocated":4443504640,"data_stored":4432164454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2385092867},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,6,0,2,6,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-312147968,"num_objects":-4763,"num_object_clones":0,"num_object_copies":-14289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2001,"num_objects_misplaced":-4700,"num_objects_unfound":0,"num_objects_dirty":-4763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4763,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2644,"num_bytes_recovered":173277184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009033"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":803,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.501819+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:36:04.501819+0000","last_peered":"2023-09-30T09:36:04.501819+0000","last_clean":"2023-09-30T09:36:04.501819+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:36:04.501819+0000","last_undegraded":"2023-09-30T09:36:04.501819+0000","last_fullsized":"2023-09-30T09:36:04.501819+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":357,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554732+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:36:04.554732+0000","last_peered":"2023-09-30T09:36:04.554732+0000","last_clean":"2023-09-30T09:36:04.554732+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:36:04.554732+0000","last_undegraded":"2023-09-30T09:36:04.554732+0000","last_fullsized":"2023-09-30T09:36:04.554732+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":319,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488748+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:36:04.488748+0000","last_peered":"2023-09-30T09:36:04.488748+0000","last_clean":"2023-09-30T09:36:04.488748+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:36:04.488748+0000","last_undegraded":"2023-09-30T09:36:04.488748+0000","last_fullsized":"2023-09-30T09:36:04.488748+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":783,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502144+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:36:04.502144+0000","last_peered":"2023-09-30T09:36:04.502144+0000","last_clean":"2023-09-30T09:36:04.502144+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:36:04.502144+0000","last_undegraded":"2023-09-30T09:36:04.502144+0000","last_fullsized":"2023-09-30T09:36:04.502144+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":212,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488804+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:36:04.488804+0000","last_peered":"2023-09-30T09:36:04.488804+0000","last_clean":"2023-09-30T09:36:04.488804+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:36:04.488804+0000","last_undegraded":"2023-09-30T09:36:04.488804+0000","last_fullsized":"2023-09-30T09:36:04.488804+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":243,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.548765+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:36:04.548765+0000","last_peered":"2023-09-30T09:36:04.548765+0000","last_clean":"2023-09-30T09:36:04.548765+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:36:04.548765+0000","last_undegraded":"2023-09-30T09:36:04.548765+0000","last_fullsized":"2023-09-30T09:36:04.548765+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":552,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554671+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:36:04.554671+0000","last_peered":"2023-09-30T09:36:04.554671+0000","last_clean":"2023-09-30T09:36:04.554671+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:36:04.554671+0000","last_undegraded":"2023-09-30T09:36:04.554671+0000","last_fullsized":"2023-09-30T09:36:04.554671+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":386,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488072+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:36:04.488072+0000","last_peered":"2023-09-30T09:36:04.488072+0000","last_clean":"2023-09-30T09:36:04.488072+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:36:04.488072+0000","last_undegraded":"2023-09-30T09:36:04.488072+0000","last_fullsized":"2023-09-30T09:36:04.488072+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":315,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.493254+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:36:04.493254+0000","last_peered":"2023-09-30T09:36:04.493254+0000","last_clean":"2023-09-30T09:36:04.493254+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:36:04.493254+0000","last_undegraded":"2023-09-30T09:36:04.493254+0000","last_fullsized":"2023-09-30T09:36:04.493254+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":762,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488512+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:36:04.488512+0000","last_peered":"2023-09-30T09:36:04.488512+0000","last_clean":"2023-09-30T09:36:04.488512+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:36:04.488512+0000","last_undegraded":"2023-09-30T09:36:04.488512+0000","last_fullsized":"2023-09-30T09:36:04.488512+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":234,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.548500+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:36:04.548500+0000","last_peered":"2023-09-30T09:36:04.548500+0000","last_clean":"2023-09-30T09:36:04.548500+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:36:04.548500+0000","last_undegraded":"2023-09-30T09:36:04.548500+0000","last_fullsized":"2023-09-30T09:36:04.548500+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":320,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488138+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:36:04.488138+0000","last_peered":"2023-09-30T09:36:04.488138+0000","last_clean":"2023-09-30T09:36:04.488138+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:36:04.488138+0000","last_undegraded":"2023-09-30T09:36:04.488138+0000","last_fullsized":"2023-09-30T09:36:04.488138+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":502,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.500199+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:36:04.500199+0000","last_peered":"2023-09-30T09:36:04.500199+0000","last_clean":"2023-09-30T09:36:04.500199+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:36:04.500199+0000","last_undegraded":"2023-09-30T09:36:04.500199+0000","last_fullsized":"2023-09-30T09:36:04.500199+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":217,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502591+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:36:04.502591+0000","last_peered":"2023-09-30T09:36:04.502591+0000","last_clean":"2023-09-30T09:36:04.502591+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:36:04.502591+0000","last_undegraded":"2023-09-30T09:36:04.502591+0000","last_fullsized":"2023-09-30T09:36:04.502591+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":266,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488721+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:36:04.488721+0000","last_peered":"2023-09-30T09:36:04.488721+0000","last_clean":"2023-09-30T09:36:04.488721+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:36:04.488721+0000","last_undegraded":"2023-09-30T09:36:04.488721+0000","last_fullsized":"2023-09-30T09:36:04.488721+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":505,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502754+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:36:04.502754+0000","last_peered":"2023-09-30T09:36:04.502754+0000","last_clean":"2023-09-30T09:36:04.502754+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:36:04.502754+0000","last_undegraded":"2023-09-30T09:36:04.502754+0000","last_fullsized":"2023-09-30T09:36:04.502754+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":60,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488840+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:36:04.488840+0000","last_peered":"2023-09-30T09:36:04.488840+0000","last_clean":"2023-09-30T09:36:04.488840+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:36:04.488840+0000","last_undegraded":"2023-09-30T09:36:04.488840+0000","last_fullsized":"2023-09-30T09:36:04.488840+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":252,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.492824+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:36:04.492824+0000","last_peered":"2023-09-30T09:36:04.492824+0000","last_clean":"2023-09-30T09:36:04.492824+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:36:04.492824+0000","last_undegraded":"2023-09-30T09:36:04.492824+0000","last_fullsized":"2023-09-30T09:36:04.492824+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":498,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.549451+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:36:04.549451+0000","last_peered":"2023-09-30T09:36:04.549451+0000","last_clean":"2023-09-30T09:36:04.549451+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:36:04.549451+0000","last_undegraded":"2023-09-30T09:36:04.549451+0000","last_fullsized":"2023-09-30T09:36:04.549451+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":323,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555061+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:36:04.555061+0000","last_peered":"2023-09-30T09:36:04.555061+0000","last_clean":"2023-09-30T09:36:04.555061+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:36:04.555061+0000","last_undegraded":"2023-09-30T09:36:04.555061+0000","last_fullsized":"2023-09-30T09:36:04.555061+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":334,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555104+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:36:04.555104+0000","last_peered":"2023-09-30T09:36:04.555104+0000","last_clean":"2023-09-30T09:36:04.555104+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:36:04.555104+0000","last_undegraded":"2023-09-30T09:36:04.555104+0000","last_fullsized":"2023-09-30T09:36:04.555104+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":616,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.484815+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:36:04.484815+0000","last_peered":"2023-09-30T09:36:04.484815+0000","last_clean":"2023-09-30T09:36:04.484815+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:36:04.484815+0000","last_undegraded":"2023-09-30T09:36:04.484815+0000","last_fullsized":"2023-09-30T09:36:04.484815+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":61,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555087+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:36:04.555087+0000","last_peered":"2023-09-30T09:36:04.555087+0000","last_clean":"2023-09-30T09:36:04.555087+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:36:04.555087+0000","last_undegraded":"2023-09-30T09:36:04.555087+0000","last_fullsized":"2023-09-30T09:36:04.555087+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":350,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.489432+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:36:04.489432+0000","last_peered":"2023-09-30T09:36:04.489432+0000","last_clean":"2023-09-30T09:36:04.489432+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:36:04.489432+0000","last_undegraded":"2023-09-30T09:36:04.489432+0000","last_fullsized":"2023-09-30T09:36:04.489432+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":685,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.501990+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:36:04.501990+0000","last_peered":"2023-09-30T09:36:04.501990+0000","last_clean":"2023-09-30T09:36:04.501990+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:36:04.501990+0000","last_undegraded":"2023-09-30T09:36:04.501990+0000","last_fullsized":"2023-09-30T09:36:04.501990+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":242,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.484618+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:36:04.484618+0000","last_peered":"2023-09-30T09:36:04.484618+0000","last_clean":"2023-09-30T09:36:04.484618+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:36:04.484618+0000","last_undegraded":"2023-09-30T09:36:04.484618+0000","last_fullsized":"2023-09-30T09:36:04.484618+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502057+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:36:04.502057+0000","last_peered":"2023-09-30T09:36:04.502057+0000","last_clean":"2023-09-30T09:36:04.502057+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:36:04.502057+0000","last_undegraded":"2023-09-30T09:36:04.502057+0000","last_fullsized":"2023-09-30T09:36:04.502057+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":777,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488106+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:36:04.488106+0000","last_peered":"2023-09-30T09:36:04.488106+0000","last_clean":"2023-09-30T09:36:04.488106+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:36:04.488106+0000","last_undegraded":"2023-09-30T09:36:04.488106+0000","last_fullsized":"2023-09-30T09:36:04.488106+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554674+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:36:04.554674+0000","last_peered":"2023-09-30T09:36:04.554674+0000","last_clean":"2023-09-30T09:36:04.554674+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:36:04.554674+0000","last_undegraded":"2023-09-30T09:36:04.554674+0000","last_fullsized":"2023-09-30T09:36:04.554674+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":242,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.549345+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:36:04.549345+0000","last_peered":"2023-09-30T09:36:04.549345+0000","last_clean":"2023-09-30T09:36:04.549345+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:36:04.549345+0000","last_undegraded":"2023-09-30T09:36:04.549345+0000","last_fullsized":"2023-09-30T09:36:04.549345+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":804,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502298+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:36:04.502298+0000","last_peered":"2023-09-30T09:36:04.502298+0000","last_clean":"2023-09-30T09:36:04.502298+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:36:04.502298+0000","last_undegraded":"2023-09-30T09:36:04.502298+0000","last_fullsized":"2023-09-30T09:36:04.502298+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":498,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.492541+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:36:04.492541+0000","last_peered":"2023-09-30T09:36:04.492541+0000","last_clean":"2023-09-30T09:36:04.492541+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:36:04.492541+0000","last_undegraded":"2023-09-30T09:36:04.492541+0000","last_fullsized":"2023-09-30T09:36:04.492541+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":348,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.489624+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:36:04.489624+0000","last_peered":"2023-09-30T09:36:04.489624+0000","last_clean":"2023-09-30T09:36:04.489624+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:36:04.489624+0000","last_undegraded":"2023-09-30T09:36:04.489624+0000","last_fullsized":"2023-09-30T09:36:04.489624+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"339'11814","reported_seq":15794,"reported_epoch":339,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:24.045239+0000","last_change":"2023-09-30T09:34:48.517696+0000","last_active":"2023-09-30T09:36:24.045239+0000","last_peered":"2023-09-30T09:36:24.045239+0000","last_clean":"2023-09-30T09:32:35.457744+0000","last_became_active":"2023-09-30T09:34:48.499533+0000","last_became_peered":"2023-09-30T09:34:48.499533+0000","last_unstale":"2023-09-30T09:36:24.045239+0000","last_undegraded":"2023-09-30T09:34:48.469688+0000","last_fullsized":"2023-09-30T09:36:24.045239+0000","mapping_epoch":321,"log_start":"303'7021","ondisk_log_start":"303'7021","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4793,"log_dups_size":0,"ondisk_log_size":4793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:05:42.164341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136577024,"num_objects":2084,"num_object_clones":0,"num_object_copies":6252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1608,"num_objects_misplaced":2084,"num_objects_unfound":0,"num_objects_dirty":2084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11814,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1934,"num_bytes_recovered":126746624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[6,1,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"1,4,6","objects":476},{"shards":"4,6","objects":1608}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":806,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-09-30T09:36:20.837581+0000","last_change":"2023-09-30T09:36:20.837581+0000","last_active":"2023-09-30T09:36:20.837581+0000","last_peered":"2023-09-30T09:36:20.837581+0000","last_clean":"2023-09-30T09:36:20.837581+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:36:20.837581+0000","last_undegraded":"2023-09-30T09:36:20.837581+0000","last_fullsized":"2023-09-30T09:36:20.837581+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":73,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555638+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:36:04.555638+0000","last_peered":"2023-09-30T09:36:04.555638+0000","last_clean":"2023-09-30T09:36:04.555638+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:36:04.555638+0000","last_undegraded":"2023-09-30T09:36:04.555638+0000","last_fullsized":"2023-09-30T09:36:04.555638+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.501728+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:36:04.501728+0000","last_peered":"2023-09-30T09:36:04.501728+0000","last_clean":"2023-09-30T09:36:04.501728+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:36:04.501728+0000","last_undegraded":"2023-09-30T09:36:04.501728+0000","last_fullsized":"2023-09-30T09:36:04.501728+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"339'12353","reported_seq":16700,"reported_epoch":339,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:26.947754+0000","last_change":"2023-09-30T09:35:10.766663+0000","last_active":"2023-09-30T09:36:26.947754+0000","last_peered":"2023-09-30T09:36:26.947754+0000","last_clean":"2023-09-30T09:32:35.454324+0000","last_became_active":"2023-09-30T09:33:33.826036+0000","last_became_peered":"2023-09-30T09:33:33.826036+0000","last_unstale":"2023-09-30T09:36:26.947754+0000","last_undegraded":"2023-09-30T09:33:33.732722+0000","last_fullsized":"2023-09-30T09:36:26.947754+0000","mapping_epoch":302,"log_start":"304'7799","ondisk_log_start":"304'7799","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4554,"log_dups_size":0,"ondisk_log_size":4554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:42.597216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134152192,"num_objects":2047,"num_object_clones":0,"num_object_copies":6141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1902,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12353,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2125,"num_bytes_recovered":139264000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,5,1],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":144},{"shards":"3,5","objects":1903}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":788,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.489353+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:36:04.489353+0000","last_peered":"2023-09-30T09:36:04.489353+0000","last_clean":"2023-09-30T09:36:04.489353+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:36:04.489353+0000","last_undegraded":"2023-09-30T09:36:04.489353+0000","last_fullsized":"2023-09-30T09:36:04.489353+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":51,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555173+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:36:04.555173+0000","last_peered":"2023-09-30T09:36:04.555173+0000","last_clean":"2023-09-30T09:36:04.555173+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:36:04.555173+0000","last_undegraded":"2023-09-30T09:36:04.555173+0000","last_fullsized":"2023-09-30T09:36:04.555173+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":331,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554612+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:36:04.554612+0000","last_peered":"2023-09-30T09:36:04.554612+0000","last_clean":"2023-09-30T09:36:04.554612+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:36:04.554612+0000","last_undegraded":"2023-09-30T09:36:04.554612+0000","last_fullsized":"2023-09-30T09:36:04.554612+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"339'11852","reported_seq":14932,"reported_epoch":339,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:24.035340+0000","last_change":"2023-09-30T09:35:10.879711+0000","last_active":"2023-09-30T09:36:24.035340+0000","last_peered":"2023-09-30T09:36:24.035340+0000","last_clean":"2023-09-30T09:32:35.467048+0000","last_became_active":"2023-09-30T09:35:10.845361+0000","last_became_peered":"2023-09-30T09:35:10.845361+0000","last_unstale":"2023-09-30T09:36:24.035340+0000","last_undegraded":"2023-09-30T09:35:10.791338+0000","last_fullsized":"2023-09-30T09:36:24.035340+0000","mapping_epoch":327,"log_start":"303'7076","ondisk_log_start":"303'7076","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4776,"log_dups_size":0,"ondisk_log_size":4776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:41.089234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139198464,"num_objects":2124,"num_object_clones":0,"num_object_copies":6372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2124,"num_objects_misplaced":4248,"num_objects_unfound":0,"num_objects_dirty":2124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11852,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1484,"num_bytes_recovered":97124352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[6,1,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":2124}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":622,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.493186+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:36:04.493186+0000","last_peered":"2023-09-30T09:36:04.493186+0000","last_clean":"2023-09-30T09:36:04.493186+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:36:04.493186+0000","last_undegraded":"2023-09-30T09:36:04.493186+0000","last_fullsized":"2023-09-30T09:36:04.493186+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":32,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.501980+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:36:04.501980+0000","last_peered":"2023-09-30T09:36:04.501980+0000","last_clean":"2023-09-30T09:36:04.501980+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:36:04.501980+0000","last_undegraded":"2023-09-30T09:36:04.501980+0000","last_fullsized":"2023-09-30T09:36:04.501980+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":318,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.492738+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:36:04.492738+0000","last_peered":"2023-09-30T09:36:04.492738+0000","last_clean":"2023-09-30T09:36:04.492738+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:36:04.492738+0000","last_undegraded":"2023-09-30T09:36:04.492738+0000","last_fullsized":"2023-09-30T09:36:04.492738+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"339'12296","reported_seq":12377,"reported_epoch":339,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:26.983144+0000","last_change":"2023-09-30T09:34:08.527633+0000","last_active":"2023-09-30T09:36:26.983144+0000","last_peered":"2023-09-30T09:36:26.983144+0000","last_clean":"2023-09-30T09:33:09.318417+0000","last_became_active":"2023-09-30T09:34:08.510746+0000","last_became_peered":"2023-09-30T09:34:08.510746+0000","last_unstale":"2023-09-30T09:36:26.983144+0000","last_undegraded":"2023-09-30T09:36:26.983144+0000","last_fullsized":"2023-09-30T09:36:26.983144+0000","mapping_epoch":308,"log_start":"304'7776","ondisk_log_start":"304'7776","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4520,"log_dups_size":0,"ondisk_log_size":4520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:07.374969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131203072,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2002,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12296,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":2002}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":480,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555261+0000","last_change":"2023-09-30T09:35:45.753329+0000","last_active":"2023-09-30T09:36:04.555261+0000","last_peered":"2023-09-30T09:36:04.555261+0000","last_clean":"2023-09-30T09:36:04.555261+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:36:04.555261+0000","last_undegraded":"2023-09-30T09:36:04.555261+0000","last_fullsized":"2023-09-30T09:36:04.555261+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":298,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502606+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:36:04.502606+0000","last_peered":"2023-09-30T09:36:04.502606+0000","last_clean":"2023-09-30T09:36:04.502606+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:36:04.502606+0000","last_undegraded":"2023-09-30T09:36:04.502606+0000","last_fullsized":"2023-09-30T09:36:04.502606+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":244,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.548622+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:36:04.548622+0000","last_peered":"2023-09-30T09:36:04.548622+0000","last_clean":"2023-09-30T09:36:04.548622+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:36:04.548622+0000","last_undegraded":"2023-09-30T09:36:04.548622+0000","last_fullsized":"2023-09-30T09:36:04.548622+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"339'12057","reported_seq":12150,"reported_epoch":339,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:26.979405+0000","last_change":"2023-09-30T09:35:10.779076+0000","last_active":"2023-09-30T09:36:26.979405+0000","last_peered":"2023-09-30T09:36:26.979405+0000","last_clean":"2023-09-30T09:34:06.422096+0000","last_became_active":"2023-09-30T09:35:10.747716+0000","last_became_peered":"2023-09-30T09:35:10.747716+0000","last_unstale":"2023-09-30T09:36:26.979405+0000","last_undegraded":"2023-09-30T09:36:26.979405+0000","last_fullsized":"2023-09-30T09:36:26.979405+0000","mapping_epoch":327,"log_start":"304'7520","ondisk_log_start":"304'7520","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4537,"log_dups_size":0,"ondisk_log_size":4537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:09.191024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130351104,"num_objects":1989,"num_object_clones":0,"num_object_copies":5967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1989,"num_objects_unfound":0,"num_objects_dirty":1989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12057,"num_write_kb":449472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1989}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":685,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-09-30T09:36:18.790124+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:36:18.790124+0000","last_peered":"2023-09-30T09:36:18.790124+0000","last_clean":"2023-09-30T09:36:18.790124+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:36:18.790124+0000","last_undegraded":"2023-09-30T09:36:18.790124+0000","last_fullsized":"2023-09-30T09:36:18.790124+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":366,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488408+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:36:04.488408+0000","last_peered":"2023-09-30T09:36:04.488408+0000","last_clean":"2023-09-30T09:36:04.488408+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:36:04.488408+0000","last_undegraded":"2023-09-30T09:36:04.488408+0000","last_fullsized":"2023-09-30T09:36:04.488408+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":297,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555179+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:36:04.555179+0000","last_peered":"2023-09-30T09:36:04.555179+0000","last_clean":"2023-09-30T09:36:04.555179+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:36:04.555179+0000","last_undegraded":"2023-09-30T09:36:04.555179+0000","last_fullsized":"2023-09-30T09:36:04.555179+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"339'12222","reported_seq":15567,"reported_epoch":339,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:26.937075+0000","last_change":"2023-09-30T09:35:10.871065+0000","last_active":"2023-09-30T09:36:26.937075+0000","last_peered":"2023-09-30T09:36:26.937075+0000","last_clean":"2023-09-30T09:32:35.449714+0000","last_became_active":"2023-09-30T09:35:10.838732+0000","last_became_peered":"2023-09-30T09:35:10.838732+0000","last_unstale":"2023-09-30T09:36:26.937075+0000","last_undegraded":"2023-09-30T09:35:10.818868+0000","last_fullsized":"2023-09-30T09:36:26.937075+0000","mapping_epoch":327,"log_start":"304'7706","ondisk_log_start":"304'7706","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4516,"log_dups_size":0,"ondisk_log_size":4516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:44.318381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133038080,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2029,"num_objects_misplaced":4061,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12222,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1619,"num_bytes_recovered":106102784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[3,7,1],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"1,3,7","objects":1},{"shards":"3,7","objects":2029}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":760,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.499926+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:36:04.499926+0000","last_peered":"2023-09-30T09:36:04.499926+0000","last_clean":"2023-09-30T09:36:04.499926+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:36:04.499926+0000","last_undegraded":"2023-09-30T09:36:04.499926+0000","last_fullsized":"2023-09-30T09:36:04.499926+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":316,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.492659+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:36:04.492659+0000","last_peered":"2023-09-30T09:36:04.492659+0000","last_clean":"2023-09-30T09:36:04.492659+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:36:04.492659+0000","last_undegraded":"2023-09-30T09:36:04.492659+0000","last_fullsized":"2023-09-30T09:36:04.492659+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":33,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555064+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:36:04.555064+0000","last_peered":"2023-09-30T09:36:04.555064+0000","last_clean":"2023-09-30T09:36:04.555064+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:36:04.555064+0000","last_undegraded":"2023-09-30T09:36:04.555064+0000","last_fullsized":"2023-09-30T09:36:04.555064+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"339'11968","reported_seq":19192,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-09-30T09:36:23.991311+0000","last_change":"2023-09-30T09:36:04.573419+0000","last_active":"2023-09-30T09:36:23.991311+0000","last_peered":"2023-09-30T09:36:23.991311+0000","last_clean":"2023-09-30T09:36:23.991311+0000","last_became_active":"2023-09-30T09:36:04.549763+0000","last_became_peered":"2023-09-30T09:36:04.549763+0000","last_unstale":"2023-09-30T09:36:23.991311+0000","last_undegraded":"2023-09-30T09:36:23.991311+0000","last_fullsized":"2023-09-30T09:36:23.991311+0000","mapping_epoch":337,"log_start":"303'7121","ondisk_log_start":"303'7121","created":279,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4847,"log_dups_size":0,"ondisk_log_size":4847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:27:18.185457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135987224,"num_objects":2076,"num_object_clones":0,"num_object_copies":6228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2076,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11968,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3562,"num_bytes_recovered":233439232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":774,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.486469+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:36:04.486469+0000","last_peered":"2023-09-30T09:36:04.486469+0000","last_clean":"2023-09-30T09:36:04.486469+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:36:04.486469+0000","last_undegraded":"2023-09-30T09:36:04.486469+0000","last_fullsized":"2023-09-30T09:36:04.486469+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":335,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554598+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:36:04.554598+0000","last_peered":"2023-09-30T09:36:04.554598+0000","last_clean":"2023-09-30T09:36:04.554598+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:36:04.554598+0000","last_undegraded":"2023-09-30T09:36:04.554598+0000","last_fullsized":"2023-09-30T09:36:04.554598+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":314,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555449+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:36:04.555449+0000","last_peered":"2023-09-30T09:36:04.555449+0000","last_clean":"2023-09-30T09:36:04.555449+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:36:04.555449+0000","last_undegraded":"2023-09-30T09:36:04.555449+0000","last_fullsized":"2023-09-30T09:36:04.555449+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"339'12084","reported_seq":12169,"reported_epoch":339,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:23.956918+0000","last_change":"2023-09-30T09:34:47.630136+0000","last_active":"2023-09-30T09:36:23.956918+0000","last_peered":"2023-09-30T09:36:23.956918+0000","last_clean":"2023-09-30T09:34:06.422409+0000","last_became_active":"2023-09-30T09:34:08.626702+0000","last_became_peered":"2023-09-30T09:34:08.626702+0000","last_unstale":"2023-09-30T09:36:23.956918+0000","last_undegraded":"2023-09-30T09:36:23.956918+0000","last_fullsized":"2023-09-30T09:36:23.956918+0000","mapping_epoch":308,"log_start":"303'7306","ondisk_log_start":"303'7306","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4778,"log_dups_size":0,"ondisk_log_size":4778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:53.753813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2092,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12084,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2092}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":484,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.500596+0000","last_change":"2023-09-30T09:35:46.150279+0000","last_active":"2023-09-30T09:36:04.500596+0000","last_peered":"2023-09-30T09:36:04.500596+0000","last_clean":"2023-09-30T09:36:04.500596+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:36:04.500596+0000","last_undegraded":"2023-09-30T09:36:04.500596+0000","last_fullsized":"2023-09-30T09:36:04.500596+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":329,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.487916+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:36:04.487916+0000","last_peered":"2023-09-30T09:36:04.487916+0000","last_clean":"2023-09-30T09:36:04.487916+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:36:04.487916+0000","last_undegraded":"2023-09-30T09:36:04.487916+0000","last_fullsized":"2023-09-30T09:36:04.487916+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.489456+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:36:04.489456+0000","last_peered":"2023-09-30T09:36:04.489456+0000","last_clean":"2023-09-30T09:36:04.489456+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:36:04.489456+0000","last_undegraded":"2023-09-30T09:36:04.489456+0000","last_fullsized":"2023-09-30T09:36:04.489456+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"339'12022","reported_seq":15126,"reported_epoch":339,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:27.118531+0000","last_change":"2023-09-30T09:34:39.166112+0000","last_active":"2023-09-30T09:36:27.118531+0000","last_peered":"2023-09-30T09:36:27.118531+0000","last_clean":"2023-09-30T09:32:35.465008+0000","last_became_active":"2023-09-30T09:34:39.150257+0000","last_became_peered":"2023-09-30T09:34:39.150257+0000","last_unstale":"2023-09-30T09:36:27.118531+0000","last_undegraded":"2023-09-30T09:34:38.934740+0000","last_fullsized":"2023-09-30T09:36:27.118531+0000","mapping_epoch":316,"log_start":"304'7478","ondisk_log_start":"304'7478","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4544,"log_dups_size":0,"ondisk_log_size":4544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:50.805799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712704,"num_objects":1964,"num_object_clones":0,"num_object_copies":5892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1963,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12022,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1506,"num_bytes_recovered":98631680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6,1],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1},{"shards":"3,6","objects":1963}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":633,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.493330+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:36:04.493330+0000","last_peered":"2023-09-30T09:36:04.493330+0000","last_clean":"2023-09-30T09:36:04.493330+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:36:04.493330+0000","last_undegraded":"2023-09-30T09:36:04.493330+0000","last_fullsized":"2023-09-30T09:36:04.493330+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":530,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.547983+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:36:04.547983+0000","last_peered":"2023-09-30T09:36:04.547983+0000","last_clean":"2023-09-30T09:36:04.547983+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:36:04.547983+0000","last_undegraded":"2023-09-30T09:36:04.547983+0000","last_fullsized":"2023-09-30T09:36:04.547983+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":335,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502356+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:36:04.502356+0000","last_peered":"2023-09-30T09:36:04.502356+0000","last_clean":"2023-09-30T09:36:04.502356+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:36:04.502356+0000","last_undegraded":"2023-09-30T09:36:04.502356+0000","last_fullsized":"2023-09-30T09:36:04.502356+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502042+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:36:04.502042+0000","last_peered":"2023-09-30T09:36:04.502042+0000","last_clean":"2023-09-30T09:36:04.502042+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:36:04.502042+0000","last_undegraded":"2023-09-30T09:36:04.502042+0000","last_fullsized":"2023-09-30T09:36:04.502042+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"339'12169","reported_seq":12237,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-09-30T09:36:23.975971+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:36:23.975971+0000","last_peered":"2023-09-30T09:36:23.975971+0000","last_clean":"2023-09-30T09:36:23.975971+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:36:23.975971+0000","last_undegraded":"2023-09-30T09:36:23.975971+0000","last_fullsized":"2023-09-30T09:36:23.975971+0000","mapping_epoch":279,"log_start":"303'7357","ondisk_log_start":"303'7357","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4812,"log_dups_size":0,"ondisk_log_size":4812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136773632,"num_objects":2087,"num_object_clones":0,"num_object_copies":6261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12169,"num_write_kb":456192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":801,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.503655+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:36:04.503655+0000","last_peered":"2023-09-30T09:36:04.503655+0000","last_clean":"2023-09-30T09:36:04.503655+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:36:04.503655+0000","last_undegraded":"2023-09-30T09:36:04.503655+0000","last_fullsized":"2023-09-30T09:36:04.503655+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":330,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.492710+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:36:04.492710+0000","last_peered":"2023-09-30T09:36:04.492710+0000","last_clean":"2023-09-30T09:36:04.492710+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:36:04.492710+0000","last_undegraded":"2023-09-30T09:36:04.492710+0000","last_fullsized":"2023-09-30T09:36:04.492710+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":318,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555162+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:36:04.555162+0000","last_peered":"2023-09-30T09:36:04.555162+0000","last_clean":"2023-09-30T09:36:04.555162+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:36:04.555162+0000","last_undegraded":"2023-09-30T09:36:04.555162+0000","last_fullsized":"2023-09-30T09:36:04.555162+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":56,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.500296+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:36:04.500296+0000","last_peered":"2023-09-30T09:36:04.500296+0000","last_clean":"2023-09-30T09:36:04.500296+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:36:04.500296+0000","last_undegraded":"2023-09-30T09:36:04.500296+0000","last_fullsized":"2023-09-30T09:36:04.500296+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"339'12141","reported_seq":15259,"reported_epoch":339,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:23.991930+0000","last_change":"2023-09-30T09:35:10.808625+0000","last_active":"2023-09-30T09:36:23.991930+0000","last_peered":"2023-09-30T09:36:23.991930+0000","last_clean":"2023-09-30T09:32:35.460821+0000","last_became_active":"2023-09-30T09:35:10.778683+0000","last_became_peered":"2023-09-30T09:35:10.778683+0000","last_unstale":"2023-09-30T09:36:23.991930+0000","last_undegraded":"2023-09-30T09:35:10.766189+0000","last_fullsized":"2023-09-30T09:36:23.991930+0000","mapping_epoch":327,"log_start":"303'7333","ondisk_log_start":"303'7333","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4808,"log_dups_size":0,"ondisk_log_size":4808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:45.905717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144900096,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2211,"num_objects_misplaced":4422,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12141,"num_write_kb":459264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1505,"num_bytes_recovered":98631680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[6,1,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":2211}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":699,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:15.153176+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:36:15.153176+0000","last_peered":"2023-09-30T09:36:15.153176+0000","last_clean":"2023-09-30T09:36:15.153176+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:36:15.153176+0000","last_undegraded":"2023-09-30T09:36:15.153176+0000","last_fullsized":"2023-09-30T09:36:15.153176+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":517,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.500665+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:36:04.500665+0000","last_peered":"2023-09-30T09:36:04.500665+0000","last_clean":"2023-09-30T09:36:04.500665+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:36:04.500665+0000","last_undegraded":"2023-09-30T09:36:04.500665+0000","last_fullsized":"2023-09-30T09:36:04.500665+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.499843+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:36:04.499843+0000","last_peered":"2023-09-30T09:36:04.499843+0000","last_clean":"2023-09-30T09:36:04.499843+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:36:04.499843+0000","last_undegraded":"2023-09-30T09:36:04.499843+0000","last_fullsized":"2023-09-30T09:36:04.499843+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":258,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502978+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:36:04.502978+0000","last_peered":"2023-09-30T09:36:04.502978+0000","last_clean":"2023-09-30T09:36:04.502978+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:36:04.502978+0000","last_undegraded":"2023-09-30T09:36:04.502978+0000","last_fullsized":"2023-09-30T09:36:04.502978+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"339'12027","reported_seq":18906,"reported_epoch":339,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:25.171428+0000","last_change":"2023-09-30T09:35:23.879286+0000","last_active":"2023-09-30T09:36:25.171428+0000","last_peered":"2023-09-30T09:36:25.171428+0000","last_clean":"2023-09-30T09:34:06.430942+0000","last_became_active":"2023-09-30T09:34:47.755052+0000","last_became_peered":"2023-09-30T09:34:47.755052+0000","last_unstale":"2023-09-30T09:36:25.171428+0000","last_undegraded":"2023-09-30T09:34:47.557422+0000","last_fullsized":"2023-09-30T09:34:47.514378+0000","mapping_epoch":320,"log_start":"293'5850","ondisk_log_start":"293'5850","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:44:56.113242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131006464,"num_objects":1999,"num_object_clones":0,"num_object_copies":5997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":141,"num_objects_misplaced":1253,"num_objects_unfound":0,"num_objects_dirty":1999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12027,"num_write_kb":448832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3396,"num_bytes_recovered":222232576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":605},{"shards":"0,3,7","objects":1253}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":911,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502236+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:36:04.502236+0000","last_peered":"2023-09-30T09:36:04.502236+0000","last_clean":"2023-09-30T09:36:04.502236+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:36:04.502236+0000","last_undegraded":"2023-09-30T09:36:04.502236+0000","last_fullsized":"2023-09-30T09:36:04.502236+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":603,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.484440+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:36:04.484440+0000","last_peered":"2023-09-30T09:36:04.484440+0000","last_clean":"2023-09-30T09:36:04.484440+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:36:04.484440+0000","last_undegraded":"2023-09-30T09:36:04.484440+0000","last_fullsized":"2023-09-30T09:36:04.484440+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":258,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.490298+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:36:04.490298+0000","last_peered":"2023-09-30T09:36:04.490298+0000","last_clean":"2023-09-30T09:36:04.490298+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:36:04.490298+0000","last_undegraded":"2023-09-30T09:36:04.490298+0000","last_fullsized":"2023-09-30T09:36:04.490298+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":291,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.493767+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:36:04.493767+0000","last_peered":"2023-09-30T09:36:04.493767+0000","last_clean":"2023-09-30T09:36:04.493767+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:36:04.493767+0000","last_undegraded":"2023-09-30T09:36:04.493767+0000","last_fullsized":"2023-09-30T09:36:04.493767+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"339'12072","reported_seq":21350,"reported_epoch":339,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:24.401570+0000","last_change":"2023-09-30T09:35:10.837720+0000","last_active":"2023-09-30T09:36:24.401570+0000","last_peered":"2023-09-30T09:36:24.401570+0000","last_clean":"2023-09-30T09:32:35.481650+0000","last_became_active":"2023-09-30T09:35:10.788829+0000","last_became_peered":"2023-09-30T09:35:10.788829+0000","last_unstale":"2023-09-30T09:36:24.401570+0000","last_undegraded":"2023-09-30T09:35:10.757584+0000","last_fullsized":"2023-09-30T09:36:24.401570+0000","mapping_epoch":327,"log_start":"286'3000","ondisk_log_start":"286'3000","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":9072,"log_dups_size":0,"ondisk_log_size":9072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:05.384343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136708096,"num_objects":2086,"num_object_clones":0,"num_object_copies":6258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1103,"num_objects_misplaced":4171,"num_objects_unfound":0,"num_objects_dirty":2086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12072,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4579,"num_bytes_recovered":300089344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[7,1,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":1},{"shards":"2,6,7","objects":983},{"shards":"6,7","objects":1102}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":484,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.492845+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:36:04.492845+0000","last_peered":"2023-09-30T09:36:04.492845+0000","last_clean":"2023-09-30T09:36:04.492845+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:36:04.492845+0000","last_undegraded":"2023-09-30T09:36:04.492845+0000","last_fullsized":"2023-09-30T09:36:04.492845+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":715,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:10.114139+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:36:10.114139+0000","last_peered":"2023-09-30T09:36:10.114139+0000","last_clean":"2023-09-30T09:36:10.114139+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:36:10.114139+0000","last_undegraded":"2023-09-30T09:36:10.114139+0000","last_fullsized":"2023-09-30T09:36:10.114139+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":267,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.484735+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:36:04.484735+0000","last_peered":"2023-09-30T09:36:04.484735+0000","last_clean":"2023-09-30T09:36:04.484735+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:36:04.484735+0000","last_undegraded":"2023-09-30T09:36:04.484735+0000","last_fullsized":"2023-09-30T09:36:04.484735+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.548500+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:36:04.548500+0000","last_peered":"2023-09-30T09:36:04.548500+0000","last_clean":"2023-09-30T09:36:04.548500+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:36:04.548500+0000","last_undegraded":"2023-09-30T09:36:04.548500+0000","last_fullsized":"2023-09-30T09:36:04.548500+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"339'12039","reported_seq":16484,"reported_epoch":339,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:24.207082+0000","last_change":"2023-09-30T09:34:47.553318+0000","last_active":"2023-09-30T09:36:24.207082+0000","last_peered":"2023-09-30T09:36:24.207082+0000","last_clean":"2023-09-30T09:32:35.469075+0000","last_became_active":"2023-09-30T09:33:33.792569+0000","last_became_peered":"2023-09-30T09:33:33.792569+0000","last_unstale":"2023-09-30T09:36:24.207082+0000","last_undegraded":"2023-09-30T09:33:33.610726+0000","last_fullsized":"2023-09-30T09:33:33.587755+0000","mapping_epoch":302,"log_start":"299'6545","ondisk_log_start":"299'6545","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":5494,"log_dups_size":0,"ondisk_log_size":5494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:16.121133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138739712,"num_objects":2117,"num_object_clones":0,"num_object_copies":6351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12039,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2178,"num_bytes_recovered":142737408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":677}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":613,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.493316+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:36:04.493316+0000","last_peered":"2023-09-30T09:36:04.493316+0000","last_clean":"2023-09-30T09:36:04.493316+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:36:04.493316+0000","last_undegraded":"2023-09-30T09:36:04.493316+0000","last_fullsized":"2023-09-30T09:36:04.493316+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":847,"reported_epoch":338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:04.495293+0000","last_change":"2023-09-30T09:35:11.685841+0000","last_active":"2023-09-30T09:36:04.495293+0000","last_peered":"2023-09-30T09:36:04.495293+0000","last_clean":"2023-09-30T09:26:47.308941+0000","last_became_active":"2023-09-30T09:35:11.685274+0000","last_became_peered":"2023-09-30T09:35:11.685274+0000","last_unstale":"2023-09-30T09:36:04.495293+0000","last_undegraded":"2023-09-30T09:35:11.661789+0000","last_fullsized":"2023-09-30T09:35:11.661213+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:38.516737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[1,4],"avail_no_missing":["1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":64,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.500115+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:36:04.500115+0000","last_peered":"2023-09-30T09:36:04.500115+0000","last_clean":"2023-09-30T09:36:04.500115+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:36:04.500115+0000","last_undegraded":"2023-09-30T09:36:04.500115+0000","last_fullsized":"2023-09-30T09:36:04.500115+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":239,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.548573+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:36:04.548573+0000","last_peered":"2023-09-30T09:36:04.548573+0000","last_clean":"2023-09-30T09:36:04.548573+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:36:04.548573+0000","last_undegraded":"2023-09-30T09:36:04.548573+0000","last_fullsized":"2023-09-30T09:36:04.548573+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"339'12282","reported_seq":12452,"reported_epoch":339,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:26.932530+0000","last_change":"2023-09-30T09:33:32.864041+0000","last_active":"2023-09-30T09:36:26.932530+0000","last_peered":"2023-09-30T09:36:26.932530+0000","last_clean":"2023-09-30T09:33:09.329441+0000","last_became_active":"2023-09-30T09:33:11.506402+0000","last_became_peered":"2023-09-30T09:33:11.506402+0000","last_unstale":"2023-09-30T09:36:26.932530+0000","last_undegraded":"2023-09-30T09:36:26.932530+0000","last_fullsized":"2023-09-30T09:36:26.932530+0000","mapping_epoch":295,"log_start":"305'7798","ondisk_log_start":"305'7798","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4484,"log_dups_size":0,"ondisk_log_size":4484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:51:06.485777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128581632,"num_objects":1962,"num_object_clones":0,"num_object_copies":5886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1950,"num_objects_unfound":0,"num_objects_dirty":1962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12282,"num_write_kb":455808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1962}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":811,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.555310+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:36:04.555310+0000","last_peered":"2023-09-30T09:36:04.555310+0000","last_clean":"2023-09-30T09:36:04.555310+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:36:04.555310+0000","last_undegraded":"2023-09-30T09:36:04.555310+0000","last_fullsized":"2023-09-30T09:36:04.555310+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":339,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.489544+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:36:04.489544+0000","last_peered":"2023-09-30T09:36:04.489544+0000","last_clean":"2023-09-30T09:36:04.489544+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:36:04.489544+0000","last_undegraded":"2023-09-30T09:36:04.489544+0000","last_fullsized":"2023-09-30T09:36:04.489544+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1105,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.493119+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:36:04.493119+0000","last_peered":"2023-09-30T09:36:04.493119+0000","last_clean":"2023-09-30T09:36:04.493119+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:36:04.493119+0000","last_undegraded":"2023-09-30T09:36:04.493119+0000","last_fullsized":"2023-09-30T09:36:04.493119+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":356,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488001+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:36:04.488001+0000","last_peered":"2023-09-30T09:36:04.488001+0000","last_clean":"2023-09-30T09:36:04.488001+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:36:04.488001+0000","last_undegraded":"2023-09-30T09:36:04.488001+0000","last_fullsized":"2023-09-30T09:36:04.488001+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"339'12210","reported_seq":15517,"reported_epoch":339,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:27.118450+0000","last_change":"2023-09-30T09:36:02.717804+0000","last_active":"2023-09-30T09:36:27.118450+0000","last_peered":"2023-09-30T09:36:27.118450+0000","last_clean":"2023-09-30T09:32:35.469896+0000","last_became_active":"2023-09-30T09:33:32.854592+0000","last_became_peered":"2023-09-30T09:33:32.854592+0000","last_unstale":"2023-09-30T09:36:27.118450+0000","last_undegraded":"2023-09-30T09:33:32.787590+0000","last_fullsized":"2023-09-30T09:33:32.601623+0000","mapping_epoch":301,"log_start":"304'7650","ondisk_log_start":"304'7650","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4560,"log_dups_size":0,"ondisk_log_size":4560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:02.368619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135921664,"num_objects":2074,"num_object_clones":0,"num_object_copies":6222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1994,"num_objects_misplaced":1994,"num_objects_unfound":0,"num_objects_dirty":2074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12210,"num_write_kb":457088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1605,"num_bytes_recovered":105185280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":80}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":804,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.489733+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:36:04.489733+0000","last_peered":"2023-09-30T09:36:04.489733+0000","last_clean":"2023-09-30T09:36:04.489733+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:36:04.489733+0000","last_undegraded":"2023-09-30T09:36:04.489733+0000","last_fullsized":"2023-09-30T09:36:04.489733+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":664,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.493478+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:36:04.493478+0000","last_peered":"2023-09-30T09:36:04.493478+0000","last_clean":"2023-09-30T09:36:04.493478+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:36:04.493478+0000","last_undegraded":"2023-09-30T09:36:04.493478+0000","last_fullsized":"2023-09-30T09:36:04.493478+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":26,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.548608+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:36:04.548608+0000","last_peered":"2023-09-30T09:36:04.548608+0000","last_clean":"2023-09-30T09:36:04.548608+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:36:04.548608+0000","last_undegraded":"2023-09-30T09:36:04.548608+0000","last_fullsized":"2023-09-30T09:36:04.548608+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":280,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554567+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:36:04.554567+0000","last_peered":"2023-09-30T09:36:04.554567+0000","last_clean":"2023-09-30T09:36:04.554567+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:36:04.554567+0000","last_undegraded":"2023-09-30T09:36:04.554567+0000","last_fullsized":"2023-09-30T09:36:04.554567+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":752,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488410+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:36:04.488410+0000","last_peered":"2023-09-30T09:36:04.488410+0000","last_clean":"2023-09-30T09:36:04.488410+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:36:04.488410+0000","last_undegraded":"2023-09-30T09:36:04.488410+0000","last_fullsized":"2023-09-30T09:36:04.488410+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":22,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.547999+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:36:04.547999+0000","last_peered":"2023-09-30T09:36:04.547999+0000","last_clean":"2023-09-30T09:36:04.547999+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:36:04.547999+0000","last_undegraded":"2023-09-30T09:36:04.547999+0000","last_fullsized":"2023-09-30T09:36:04.547999+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":16,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.500036+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:36:04.500036+0000","last_peered":"2023-09-30T09:36:04.500036+0000","last_clean":"2023-09-30T09:36:04.500036+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:36:04.500036+0000","last_undegraded":"2023-09-30T09:36:04.500036+0000","last_fullsized":"2023-09-30T09:36:04.500036+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":524,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488483+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:36:04.488483+0000","last_peered":"2023-09-30T09:36:04.488483+0000","last_clean":"2023-09-30T09:36:04.488483+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:36:04.488483+0000","last_undegraded":"2023-09-30T09:36:04.488483+0000","last_fullsized":"2023-09-30T09:36:04.488483+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":317,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502455+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:36:04.502455+0000","last_peered":"2023-09-30T09:36:04.502455+0000","last_clean":"2023-09-30T09:36:04.502455+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:36:04.502455+0000","last_undegraded":"2023-09-30T09:36:04.502455+0000","last_fullsized":"2023-09-30T09:36:04.502455+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":33,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554905+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:36:04.554905+0000","last_peered":"2023-09-30T09:36:04.554905+0000","last_clean":"2023-09-30T09:36:04.554905+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:36:04.554905+0000","last_undegraded":"2023-09-30T09:36:04.554905+0000","last_fullsized":"2023-09-30T09:36:04.554905+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":283,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.502269+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:36:04.502269+0000","last_peered":"2023-09-30T09:36:04.502269+0000","last_clean":"2023-09-30T09:36:04.502269+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:36:04.502269+0000","last_undegraded":"2023-09-30T09:36:04.502269+0000","last_fullsized":"2023-09-30T09:36:04.502269+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":33,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554773+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:36:04.554773+0000","last_peered":"2023-09-30T09:36:04.554773+0000","last_clean":"2023-09-30T09:36:04.554773+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:36:04.554773+0000","last_undegraded":"2023-09-30T09:36:04.554773+0000","last_fullsized":"2023-09-30T09:36:04.554773+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":467,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.493167+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:36:04.493167+0000","last_peered":"2023-09-30T09:36:04.493167+0000","last_clean":"2023-09-30T09:36:04.493167+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:36:04.493167+0000","last_undegraded":"2023-09-30T09:36:04.493167+0000","last_fullsized":"2023-09-30T09:36:04.493167+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":335,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.490501+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:36:04.490501+0000","last_peered":"2023-09-30T09:36:04.490501+0000","last_clean":"2023-09-30T09:36:04.490501+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:36:04.490501+0000","last_undegraded":"2023-09-30T09:36:04.490501+0000","last_fullsized":"2023-09-30T09:36:04.490501+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":360,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488641+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:36:04.488641+0000","last_peered":"2023-09-30T09:36:04.488641+0000","last_clean":"2023-09-30T09:36:04.488641+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:36:04.488641+0000","last_undegraded":"2023-09-30T09:36:04.488641+0000","last_fullsized":"2023-09-30T09:36:04.488641+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":787,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.488604+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:36:04.488604+0000","last_peered":"2023-09-30T09:36:04.488604+0000","last_clean":"2023-09-30T09:36:04.488604+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:36:04.488604+0000","last_undegraded":"2023-09-30T09:36:04.488604+0000","last_fullsized":"2023-09-30T09:36:04.488604+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":42,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.548793+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:36:04.548793+0000","last_peered":"2023-09-30T09:36:04.548793+0000","last_clean":"2023-09-30T09:36:04.548793+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:36:04.548793+0000","last_undegraded":"2023-09-30T09:36:04.548793+0000","last_fullsized":"2023-09-30T09:36:04.548793+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":314,"reported_epoch":338,"state":"active+clean","last_fresh":"2023-09-30T09:36:04.554466+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:36:04.554466+0000","last_peered":"2023-09-30T09:36:04.554466+0000","last_clean":"2023-09-30T09:36:04.554466+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:36:04.554466+0000","last_undegraded":"2023-09-30T09:36:04.554466+0000","last_fullsized":"2023-09-30T09:36:04.554466+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2158952472,"num_objects":32944,"num_object_clones":0,"num_object_copies":98832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16515,"num_objects_misplaced":30267,"num_objects_unfound":0,"num_objects_dirty":32944,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193608,"num_write_kb":7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25537,"num_bytes_recovered":1673003008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5715800064,"data_stored":5715787848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81272,"ondisk_log_size":81272,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234297,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495934,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1591660,"kb_used_data":1217572,"kb_used_omap":0,"kb_used_meta":374080,"kb_avail":92153492,"statfs":{"total":95995035648,"available":94365175808,"internally_reserved":0,"allocated":1246793728,"data_stored":1245614712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":383057920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5779999999999998,"5min":1.706,"15min":1.583},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":14.795,"5min":19.882000000000001,"15min":45.927999999999997},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.853,"5min":1.859,"15min":1.7869999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.0939999999999994,"5min":20.373000000000001,"15min":45.316000000000003},"last":1.8240000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:35:35 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":1.9470000000000001,"15min":1.851},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.7070000000000007,"5min":9.7070000000000007,"15min":9.7070000000000007},"last":3.8290000000000002},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.1080000000000001,"15min":2.0779999999999998},"min":{"1min":0.35799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.0820000000000007,"5min":9.0820000000000007,"15min":9.0820000000000007},"last":3.8460000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":1.738,"15min":1.7290000000000001},"min":{"1min":0.32800000000000001,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":8.0850000000000009,"5min":22.158000000000001,"15min":37.978000000000002},"last":6.2300000000000004},{"interface":"front","average":{"1min":2.6360000000000001,"5min":2.0289999999999999,"15min":1.641},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.706,"5min":22.356999999999999,"15min":37.759999999999998},"last":5.9610000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:35:52 2023","interfaces":[{"interface":"back","average":{"1min":17.585000000000001,"5min":6.2119999999999997,"15min":3.0529999999999999},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":212.44800000000001,"5min":212.44800000000001,"15min":212.44800000000001},"last":2.46},{"interface":"front","average":{"1min":18.085000000000001,"5min":6.7649999999999997,"15min":3.363},"min":{"1min":0.39500000000000002,"5min":0.313,"15min":0.313},"max":{"1min":210.84299999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":0.70399999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.486,"15min":1.591},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.23400000000000001},"max":{"1min":7.234,"5min":8.1059999999999999,"15min":73.656000000000006},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.4590000000000001,"5min":1.3260000000000001,"15min":1.415},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.23000000000000001},"max":{"1min":9.6590000000000007,"5min":9.6590000000000007,"15min":73.882999999999996},"last":0.47299999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":1.399,"15min":1.823},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":4.3010000000000002,"5min":10.103999999999999,"15min":76.613},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.45,"15min":1.786},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.31},"max":{"1min":9.7750000000000004,"5min":10.054,"15min":76.329999999999998},"last":0.501}]},{"osd":7,"last update":"Sat Sep 30 09:35:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.712,"15min":2.1320000000000001},"min":{"1min":0.38,"5min":0.377,"15min":0.32100000000000001},"max":{"1min":8.7400000000000002,"5min":8.7400000000000002,"15min":91.120999999999995},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.306,"5min":1.448,"15min":2.0710000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.9489999999999998,"5min":9.9489999999999998,"15min":91.733999999999995},"last":0.51800000000000002}]}]},{"osd":1,"up_from":301,"seq":1292785156132,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":396836,"kb_used_data":213020,"kb_used_omap":9,"kb_used_meta":183542,"kb_avail":93348316,"statfs":{"total":95995035648,"available":95588675584,"internally_reserved":0,"allocated":218132480,"data_stored":216211845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":187947026},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.6549999999999998,"15min":2.5790000000000002},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.699999999999999,"5min":13.699999999999999,"15min":13.699999999999999},"last":2.363},{"interface":"front","average":{"1min":4.0659999999999998,"5min":3.2320000000000002,"15min":3.093},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.25},"max":{"1min":13.779999999999999,"5min":13.779999999999999,"15min":13.779999999999999},"last":2.5950000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":2.7120000000000002,"15min":2.7679999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":16.437000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":0.254},{"interface":"front","average":{"1min":3.3740000000000001,"5min":2.839,"15min":2.75},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.121,"5min":12.121,"15min":12.121},"last":2.4390000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":4.4589999999999996,"5min":8.4489999999999998,"15min":9.1140000000000008},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.837999999999999,"5min":77.959000000000003,"15min":77.959000000000003},"last":1.5},{"interface":"front","average":{"1min":3.2690000000000001,"5min":8.7949999999999999,"15min":9.7159999999999993},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":11.657,"5min":77.828999999999994,"15min":77.828999999999994},"last":0.439}]},{"osd":4,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":2.0249999999999999,"15min":1.9990000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":13.602,"5min":13.602,"15min":13.602},"last":2.5089999999999999},{"interface":"front","average":{"1min":2.1920000000000002,"5min":1.472,"15min":1.3520000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":10.773,"5min":10.773,"15min":10.773},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":2.2309999999999999,"15min":2.149},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":13.666,"5min":13.666,"15min":13.666},"last":2.4750000000000001},{"interface":"front","average":{"1min":2.218,"5min":1.724,"15min":1.6419999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":14.346,"15min":16.18},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":13.728,"5min":209.221,"15min":209.221},"last":2.5659999999999998},{"interface":"front","average":{"1min":2.5110000000000001,"5min":14.115,"15min":16.048999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":13.755000000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":2.5369999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":10.254,"15min":11.714},"min":{"1min":0.46000000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":4.5279999999999996,"5min":165.59399999999999,"15min":165.59399999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.429,"5min":10.429,"15min":11.929},"min":{"1min":0.47299999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":4.5640000000000001,"5min":165.833,"15min":165.833},"last":0.54700000000000004}]}]},{"osd":0,"up_from":94,"seq":403726926031,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":757240,"kb_used_data":403696,"kb_used_omap":5,"kb_used_meta":353530,"kb_avail":92987912,"statfs":{"total":95995035648,"available":95219621888,"internally_reserved":0,"allocated":413384704,"data_stored":411330426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":362014973},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":2.8250000000000002,"15min":2.9399999999999999},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":11.175000000000001,"5min":17.966000000000001,"15min":17.966000000000001},"last":14.547000000000001},{"interface":"front","average":{"1min":4.1660000000000004,"5min":8.3729999999999993,"15min":9.0739999999999998},"min":{"1min":0.22500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":14.097,"5min":138.92400000000001,"15min":138.92400000000001},"last":7.7430000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.1299999999999999,"15min":1.744},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.887,"5min":11.218999999999999,"15min":55.823999999999998},"last":7.7149999999999999},{"interface":"front","average":{"1min":3.2080000000000002,"5min":2.5350000000000001,"15min":1.9319999999999999},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":10.763,"5min":40.128999999999998,"15min":55.790999999999997},"last":7.6890000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:35:53 2023","interfaces":[{"interface":"back","average":{"1min":8.9269999999999996,"5min":8.3209999999999997,"15min":3.9820000000000002},"min":{"1min":0.311,"5min":0.221,"15min":0.221},"max":{"1min":47.277999999999999,"5min":218.06999999999999,"15min":218.06999999999999},"last":0.5},{"interface":"front","average":{"1min":6.1849999999999996,"5min":6.2850000000000001,"15min":3.2570000000000001},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":47.643000000000001,"5min":218.63200000000001,"15min":218.63200000000001},"last":10.997999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":1.3560000000000001,"15min":1.5509999999999999},"min":{"1min":0.34300000000000003,"5min":0.32800000000000001,"15min":0.29099999999999998},"max":{"1min":8.6110000000000007,"5min":8.6110000000000007,"15min":121.486},"last":7.6230000000000002},{"interface":"front","average":{"1min":4.1219999999999999,"5min":3.4329999999999998,"15min":2.464},"min":{"1min":0.318,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":15.587999999999999,"5min":136.262,"15min":136.262},"last":0.53600000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":4.4989999999999997,"5min":3.7730000000000001,"15min":2.8809999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":15.647,"5min":136.34,"15min":136.34},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.1030000000000002,"5min":1.698,"15min":2.1669999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.33700000000000002},"max":{"1min":12.717000000000001,"5min":12.717000000000001,"15min":135.381},"last":11.092000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":6.9829999999999997,"15min":3.1070000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.33300000000000002},"max":{"1min":12.661,"5min":214.785,"15min":214.785},"last":11.055},{"interface":"front","average":{"1min":2.0430000000000001,"5min":6.9880000000000004,"15min":3.036},"min":{"1min":0.436,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":8.5679999999999996,"5min":214.93899999999999,"15min":214.93899999999999},"last":7.577}]},{"osd":7,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":2.8999999999999999,"15min":3.0470000000000002},"min":{"1min":0.42999999999999999,"5min":0.373,"15min":0.373},"max":{"1min":12.749000000000001,"5min":97.382000000000005,"15min":150.96899999999999},"last":11.127000000000001},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.7040000000000002,"15min":2.8580000000000001},"min":{"1min":0.36799999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.7080000000000002,"5min":97.015000000000001,"15min":150.328},"last":7.6619999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598972,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":392672,"kb_used_data":78104,"kb_used_omap":5,"kb_used_meta":314554,"kb_avail":93352480,"statfs":{"total":95995035648,"available":95592939520,"internally_reserved":0,"allocated":79978496,"data_stored":78123791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":322103949},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":2.9449999999999998,"15min":1.9179999999999999},"min":{"1min":0.22800000000000001,"5min":0.187,"15min":0.187},"max":{"1min":77.379000000000005,"5min":77.379000000000005,"15min":77.379000000000005},"last":1.637},{"interface":"front","average":{"1min":6.2549999999999999,"5min":2.9660000000000002,"15min":1.8140000000000001},"min":{"1min":0.25600000000000001,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":76.796999999999997,"5min":77.066999999999993,"15min":77.066999999999993},"last":0.41599999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:35:36 2023","interfaces":[{"interface":"back","average":{"1min":4.4779999999999998,"5min":3.7530000000000001,"15min":3.6320000000000001},"min":{"1min":0.28199999999999997,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.728999999999999,"5min":14.707000000000001,"15min":14.707000000000001},"last":1.7030000000000001},{"interface":"front","average":{"1min":3.819,"5min":3.2090000000000001,"15min":3.1070000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.968,"5min":10.179,"15min":10.179},"last":1.2130000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:35:49 2023","interfaces":[{"interface":"back","average":{"1min":6.5060000000000002,"5min":4.5499999999999998,"15min":2.4889999999999999},"min":{"1min":0.38100000000000001,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":31.509,"5min":78.915999999999997,"15min":78.915999999999997},"last":1.3140000000000001},{"interface":"front","average":{"1min":5.1230000000000002,"5min":4.3230000000000004,"15min":2.5750000000000002},"min":{"1min":0.28899999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":34.329000000000001,"5min":80.834000000000003,"15min":80.834000000000003},"last":1.153}]},{"osd":4,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.8029999999999999,"15min":1.25},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":5.524,"5min":35.972000000000001,"15min":35.972000000000001},"last":1.2549999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.7470000000000001,"15min":1.4810000000000001},"min":{"1min":0.34799999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":5.8789999999999996,"5min":36.518000000000001,"15min":36.518000000000001},"last":5.1280000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.732,"15min":1.9410000000000001},"min":{"1min":0.377,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":12.651999999999999,"5min":12.651999999999999,"15min":155.477},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.794,"5min":1.4119999999999999,"15min":1.8660000000000001},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":4.4560000000000004,"5min":6.048,"15min":155.46000000000001},"last":1.2829999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":6.7009999999999996,"15min":3.1899999999999999},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.9729999999999999,"5min":154.417,"15min":154.417},"last":1.052},{"interface":"front","average":{"1min":3.1589999999999998,"5min":6.9749999999999996,"15min":3.1960000000000002},"min":{"1min":0.47399999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":12.710000000000001,"5min":154.04400000000001,"15min":154.04400000000001},"last":0.437}]},{"osd":7,"last update":"Sat Sep 30 09:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":3.665,"15min":3},"min":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8399999999999999,"5min":123.294,"15min":213.404},"last":1.1830000000000001},{"interface":"front","average":{"1min":2.234,"5min":3.9769999999999999,"15min":3.0219999999999998},"min":{"1min":0.438,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":12.686999999999999,"5min":123.16,"15min":213.27199999999999},"last":0.496}]}]},{"osd":3,"up_from":109,"seq":468151435445,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2014120,"kb_used_data":1427168,"kb_used_omap":2,"kb_used_meta":586941,"kb_avail":91731032,"statfs":{"total":95995035648,"available":93932576768,"internally_reserved":0,"allocated":1461420032,"data_stored":1459434574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":601027943},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,0,2,6,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.8040000000000003,"5min":4.1310000000000002,"15min":2.7559999999999998},"min":{"1min":0.23999999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":60.726999999999997,"5min":102.57299999999999,"15min":130.946},"last":0.65200000000000002},{"interface":"front","average":{"1min":3.9049999999999998,"5min":3.9729999999999999,"15min":2.7069999999999999},"min":{"1min":0.34599999999999997,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":12.477,"5min":102.745,"15min":130.96899999999999},"last":0.93000000000000005}]},{"osd":1,"last update":"Sat Sep 30 09:35:33 2023","interfaces":[{"interface":"back","average":{"1min":4.625,"5min":17.937999999999999,"15min":20.157},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":21.709,"5min":273.99599999999998,"15min":273.99599999999998},"last":0.68700000000000006},{"interface":"front","average":{"1min":5.2140000000000004,"5min":16.260000000000002,"15min":18.100999999999999},"min":{"1min":0.38800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":24.771999999999998,"5min":274.21600000000001,"15min":274.21600000000001},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":4.7089999999999996,"5min":3.4580000000000002,"15min":2.9510000000000001},"min":{"1min":0.50800000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.339,"5min":76.974000000000004,"15min":145.75899999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":5.6070000000000002,"5min":3.415,"15min":2.8410000000000002},"min":{"1min":0.35699999999999998,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":60.561999999999998,"5min":77.593999999999994,"15min":150.358},"last":1.143}]},{"osd":4,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.282,"5min":2.5979999999999999,"15min":1.5700000000000001},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":60.637,"5min":60.637,"15min":60.637},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.9990000000000001,"15min":1.4670000000000001},"min":{"1min":0.441,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":7.7990000000000004,"5min":10.731,"15min":15.638},"last":0.72699999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.629,"15min":1.74},"min":{"1min":0.41499999999999998,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":7.7519999999999998,"5min":37.485999999999997,"15min":37.485999999999997},"last":0.76100000000000001},{"interface":"front","average":{"1min":3.3300000000000001,"5min":3.044,"15min":1.9650000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":8.5980000000000008,"5min":37.448,"15min":37.448},"last":0.503}]},{"osd":6,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.5650000000000004,"5min":2.6960000000000002,"15min":1.6659999999999999},"min":{"1min":0.36799999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":60.698,"5min":60.698,"15min":60.698},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.778,"5min":2.3090000000000002,"15min":1.6859999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":9.5860000000000003,"5min":10.818,"15min":14.69},"last":0.79600000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5590000000000002,"5min":2.855,"15min":2.573},"min":{"1min":0.39100000000000001,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":8.6660000000000004,"5min":26.469000000000001,"15min":226.03999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":5.8849999999999998,"5min":3.008,"15min":2.4580000000000002},"min":{"1min":0.45400000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":60.670000000000002,"5min":60.670000000000002,"15min":226.06100000000001},"last":0.57499999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019207,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781412,"kb_used_data":535644,"kb_used_omap":0,"kb_used_meta":245760,"kb_avail":92963740,"statfs":{"total":95995035648,"available":95194869760,"internally_reserved":0,"allocated":548499456,"data_stored":547328135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":251658240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":1.415,"15min":2.7999999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":14.353,"5min":14.353,"15min":128.79300000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.798,"5min":1.659,"15min":2.7509999999999999},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.248},"max":{"1min":10.618,"5min":11.744,"15min":128.38800000000001},"last":0.314}]},{"osd":1,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":8.2840000000000007,"15min":9.0969999999999995},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.957999999999998,"5min":160.69999999999999,"15min":160.69999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.024,"5min":8.0559999999999992,"15min":9.0619999999999994},"min":{"1min":0.26800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":10.91,"5min":160.50700000000001,"15min":160.50700000000001},"last":0.64400000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:35:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":1.8200000000000001,"15min":2.8410000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.096,"5min":19.391999999999999,"15min":140.601},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.089,"5min":1.865,"15min":2.8130000000000002},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.309},"max":{"1min":11.057,"5min":19.422999999999998,"15min":140.54900000000001},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 30 09:35:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":3.3109999999999999,"15min":3.4220000000000002},"min":{"1min":0.33300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.7629999999999999,"5min":70.241,"15min":281.42200000000003},"last":6.7130000000000001},{"interface":"front","average":{"1min":3.1190000000000002,"5min":3.2090000000000001,"15min":3.3849999999999998},"min":{"1min":0.442,"5min":0.312,"15min":0.312},"max":{"1min":9.7349999999999994,"5min":70.433999999999997,"15min":281.44999999999999},"last":6.7309999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:35:32 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.254,"15min":1.6899999999999999},"min":{"1min":0.27600000000000002,"5min":0.219,"15min":0.219},"max":{"1min":8.6509999999999998,"5min":11.808,"15min":134.30600000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.161,"15min":1.766},"min":{"1min":0.29599999999999999,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":6.2149999999999999,"5min":6.3929999999999998,"15min":134.35300000000001},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.9409999999999998,"15min":1.522},"min":{"1min":0.32500000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.609,"5min":202.72499999999999,"15min":202.72499999999999},"last":1.625},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.927,"15min":1.575},"min":{"1min":0.312,"5min":0.249,"15min":0.249},"max":{"1min":3.964,"5min":202.42400000000001,"15min":202.42400000000001},"last":1.5880000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:35:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":4.6449999999999996,"15min":2.9969999999999999},"min":{"1min":0.35099999999999998,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":7.5220000000000002,"5min":230.90100000000001,"15min":230.90100000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.67500000000000004,"5min":4.2539999999999996,"15min":2.5510000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":3.7360000000000002,"5min":229.97,"15min":229.97},"last":0.67400000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757571,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":734948,"kb_used_data":464156,"kb_used_omap":0,"kb_used_meta":270784,"kb_avail":93010204,"statfs":{"total":95995035648,"available":95242448896,"internally_reserved":0,"allocated":475295744,"data_stored":474120971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":277282816},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:35:42 2023","interfaces":[{"interface":"back","average":{"1min":3.8999999999999999,"5min":1.99,"15min":2.355},"min":{"1min":0.24199999999999999,"5min":0.224,"15min":0.22},"max":{"1min":56.563000000000002,"5min":56.563000000000002,"15min":148.79900000000001},"last":0.251},{"interface":"front","average":{"1min":5.0019999999999998,"5min":2.3290000000000002,"15min":2.681},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.23499999999999999},"max":{"1min":57.241999999999997,"5min":57.241999999999997,"15min":148.779},"last":1.6739999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:35:34 2023","interfaces":[{"interface":"back","average":{"1min":5.3440000000000003,"5min":2.5390000000000001,"15min":2.0710000000000002},"min":{"1min":0.28399999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":60.799999999999997,"5min":60.799999999999997,"15min":60.799999999999997},"last":2.6699999999999999},{"interface":"front","average":{"1min":5.407,"5min":3.6110000000000002,"15min":3.3109999999999999},"min":{"1min":0.33900000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":63.798999999999999,"5min":63.798999999999999,"15min":63.798999999999999},"last":2.589}]},{"osd":2,"last update":"Sat Sep 30 09:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.8879999999999999,"15min":3.1150000000000002},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.71,"5min":30.613,"15min":165.768},"last":3.0379999999999998},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.0249999999999999,"15min":3.6030000000000002},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.1989999999999998,"5min":36.488,"15min":166.24000000000001},"last":6.2400000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:35:53 2023","interfaces":[{"interface":"back","average":{"1min":5.1660000000000004,"5min":8.2040000000000006,"15min":3.8610000000000002},"min":{"1min":0.39200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":38.280000000000001,"5min":226.34700000000001,"15min":226.34700000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":4.9379999999999997,"5min":8.1809999999999992,"15min":3.7080000000000002},"min":{"1min":0.39100000000000001,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":38.262999999999998,"5min":226.36000000000001,"15min":226.36000000000001},"last":0.40699999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.92000000000000004,"15min":0.85699999999999998},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":1.6200000000000001,"5min":7.4299999999999997,"15min":8.5359999999999996},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.998,"5min":1.097,"15min":0.93700000000000006},"min":{"1min":0.28799999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1580000000000004,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.30099999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":3.1749999999999998,"15min":1.7390000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":4.6639999999999997,"5min":155.602,"15min":155.602},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.014,"5min":3.2309999999999999,"15min":1.8680000000000001},"min":{"1min":0.36499999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.1600000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.41999999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":2.5630000000000002,"15min":1.639},"min":{"1min":0.33000000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":3.2000000000000002,"5min":133.40700000000001,"15min":133.40700000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":2.6549999999999998,"15min":1.5609999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":2.274,"5min":132.797,"15min":132.797},"last":0.42899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":398200832,"data_stored":398196760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":205787136,"data_stored":205783064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":74711040,"data_stored":74711040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1445265408,"data_stored":1445265408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":543227904,"data_stored":543227904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":469893120,"data_stored":469893120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1236078592,"data_stored":1236074520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1342636032,"data_stored":1342636032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:36:33.319 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:36:33.329 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:33.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:33.436 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:33.572 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:33.572 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:33.775 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:33.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:33.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:33.884 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:34.024 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:34.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:34.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 16169/95184 objects degraded (16.987%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:36:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:36:33] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-09-30T09:36:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 16169/95184 objects degraded (16.987%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:34.241 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:34.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:34.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 16169/95184 objects degraded (16.987%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:34.308 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:36:34.308 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:36:34.308 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:36:34.308 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:36:34.308 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:36:34.348 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:34.386 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:34.441 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:34.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:34.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:34.550 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:34.583 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:34.644 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:34.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:34.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:34.750 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:34.786 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:34.841 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:34.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:34.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:34.947 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:34.995 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:35.057 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:35.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:35.168 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:35.212 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:34 smithi167 ceph-mon[192880]: pgmap v867: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 15610/92322 objects degraded (16.908%); 28030/92322 objects misplaced (30.361%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:36:35.212 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/108842425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:35.212 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:34 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:35.214 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:35.300 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:35.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:35.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:34 smithi019 ceph-mon[216815]: pgmap v867: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 15610/92322 objects degraded (16.908%); 28030/92322 objects misplaced (30.361%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:36:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/108842425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:34 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:35.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:34 smithi019 ceph-mon[224043]: pgmap v867: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.9 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 15610/92322 objects degraded (16.908%); 28030/92322 objects misplaced (30.361%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:36:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/108842425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:34 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:36:35.407 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:35.439 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:35.489 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:35.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:35.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:35.521 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:36:35.594 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:35.621 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:35.680 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:35.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:35.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:35.787 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:35.825 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:35.887 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:35.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:35.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:35.992 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:36.030 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:36.096 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:36.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:36.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:36.204 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:36.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:36:36.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:36:36.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:36:36.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[216815]: osdmap e340: 8 total, 8 up, 7 in 2023-09-30T09:36:36.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[216815]: failed to encode map e340 with expected crc 2023-09-30T09:36:36.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[216815]: failed to encode map e340 with expected crc 2023-09-30T09:36:36.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:36:36.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:36:36.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:36:36.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[224043]: osdmap e340: 8 total, 8 up, 7 in 2023-09-30T09:36:36.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[224043]: failed to encode map e340 with expected crc 2023-09-30T09:36:36.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:35 smithi019 ceph-mon[224043]: failed to encode map e340 with expected crc 2023-09-30T09:36:36.235 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:36.235 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:35 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:36:36.235 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:35 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:36:36.235 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:35 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:36:36.235 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:35 smithi167 ceph-mon[192880]: osdmap e340: 8 total, 8 up, 7 in 2023-09-30T09:36:36.236 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:35 smithi167 ceph-mon[192880]: failed to encode map e340 with expected crc 2023-09-30T09:36:36.236 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:35 smithi167 ceph-mon[192880]: failed to encode map e340 with expected crc 2023-09-30T09:36:36.238 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:36.306 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:36.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:36.326 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:36:36.416 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:36.554 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:37.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:36 smithi167 ceph-mon[192880]: pgmap v869: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.8 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 15266/88668 objects degraded (17.217%); 26587/88668 objects misplaced (29.985%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:36:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:36 smithi167 ceph-mon[192880]: failed to encode map e340 with expected crc 2023-09-30T09:36:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:36 smithi167 ceph-mon[192880]: failed to encode map e340 with expected crc 2023-09-30T09:36:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:36 smithi019 ceph-mon[216815]: pgmap v869: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.8 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 15266/88668 objects degraded (17.217%); 26587/88668 objects misplaced (29.985%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:36:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:36 smithi019 ceph-mon[216815]: failed to encode map e340 with expected crc 2023-09-30T09:36:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:36 smithi019 ceph-mon[216815]: failed to encode map e340 with expected crc 2023-09-30T09:36:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:36 smithi019 ceph-mon[224043]: pgmap v869: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.8 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 15266/88668 objects degraded (17.217%); 26587/88668 objects misplaced (29.985%); 14 MiB/s, 229 objects/s recovering 2023-09-30T09:36:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:36 smithi019 ceph-mon[224043]: failed to encode map e340 with expected crc 2023-09-30T09:36:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:36 smithi019 ceph-mon[224043]: failed to encode map e340 with expected crc 2023-09-30T09:36:37.762 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:37.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:37.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:37.867 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:37.900 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:37.946 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:37.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:38.052 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:38.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:37 smithi019 ceph-mon[216815]: 1.d deep-scrub starts 2023-09-30T09:36:38.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:37 smithi019 ceph-mon[224043]: 1.d deep-scrub starts 2023-09-30T09:36:38.106 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:38.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:37 smithi167 ceph-mon[192880]: 1.d deep-scrub starts 2023-09-30T09:36:38.989 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:39.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:39.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:39.104 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:39.151 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:39.209 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:39.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:39.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:39.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:38 smithi167 ceph-mon[192880]: pgmap v870: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.8 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 14692/86016 objects degraded (17.081%); 25734/86016 objects misplaced (29.918%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:36:39.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:38 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:36:39.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 14692/86016 objects degraded (17.081%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:39.218 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:38 smithi019 ceph-mon[216815]: pgmap v870: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.8 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 14692/86016 objects degraded (17.081%); 25734/86016 objects misplaced (29.918%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:36:39.218 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:38 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:36:39.219 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 14692/86016 objects degraded (17.081%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:39.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:38 smithi019 ceph-mon[224043]: pgmap v870: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.8 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 14692/86016 objects degraded (17.081%); 25734/86016 objects misplaced (29.918%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:36:39.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:38 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:36:39.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 14692/86016 objects degraded (17.081%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:39.315 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:36:39.352 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:39.418 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:36:39.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:39.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:36:39.530 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:36:39.574 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:39.639 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:36:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:39.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:36:39.749 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:39.783 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:39.836 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:39.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:39.939 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:39.967 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:40.040 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:40.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:40.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:40.146 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:36:40.185 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:40.257 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:36:40.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:40.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:36:40.368 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:40.404 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:40.467 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:40.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:40.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:40.574 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:36:40.574 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:40.598 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:40.611 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:40.633 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:36:40.664 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:40.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:40.691 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:36:40.765 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:36:40.777 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:40.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:36:40.872 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:40.872 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:36:40.920 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:36:40.961 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:36:40.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:36:40.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:36:40.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:36:40.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:36:40.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:36:40.971 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:36:40.971 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:36:40.971 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:36:41.089 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:36:41.105 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:40 smithi019 ceph-mon[224043]: pgmap v871: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 14369/82827 objects degraded (17.348%); 24678/82827 objects misplaced (29.795%); 16 MiB/s, 260 objects/s recovering 2023-09-30T09:36:41.105 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1812474355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:41.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:40 smithi019 ceph-mon[216815]: pgmap v871: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 14369/82827 objects degraded (17.348%); 24678/82827 objects misplaced (29.795%); 16 MiB/s, 260 objects/s recovering 2023-09-30T09:36:41.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1812474355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:41.259 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:41.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:40 smithi167 ceph-mon[192880]: pgmap v871: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 14369/82827 objects degraded (17.348%); 24678/82827 objects misplaced (29.795%); 16 MiB/s, 260 objects/s recovering 2023-09-30T09:36:41.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1812474355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:41.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:41.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:41.376 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:41.450 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:42.665 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:36:42.665 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 14369/82458 objects degraded (17.426%), 11 pgs degraded, 4 pgs undersized","count":15},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1140,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":340,"num_osds":8,"num_up_osds":8,"osd_up_since":1696066411,"num_in_osds":7,"osd_in_since":1696066446,"num_remapped_pgs":15},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":104},{"state_name":"active+recovery_wait+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":4},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":119,"num_pools":5,"num_objects":27486,"data_bytes":1804352356,"bytes_used":6116601856,"bytes_avail":665848647680,"bytes_total":671965249536,"degraded_objects":14369,"degraded_total":82458,"degraded_ratio":0.1742584103422348,"misplaced_objects":24303,"misplaced_total":82458,"misplaced_ratio":0.29473186349414249,"recovering_objects_per_sec":267,"recovering_bytes_per_sec":17533574,"recovering_keys_per_sec":0,"num_objects_recovered":2679,"num_bytes_recovered":175570944,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":423},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-30T09:34:31.415542+0000","services":{}},"progress_events":{"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"bf6cdc82-6db1-4694-b9dc-04effc0cdb2e":{"message":"Global Recovery Event (3m)\n [========================....] (remaining: 33s)","progress":0.87394958734512329,"add_to_ceph_s":true}}} 2023-09-30T09:36:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:42 smithi167 ceph-mon[192880]: pgmap v872: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.7 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 14369/82458 objects degraded (17.426%); 24303/82458 objects misplaced (29.473%); 17 MiB/s, 267 objects/s recovering 2023-09-30T09:36:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4203989345' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:36:43.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:42 smithi019 ceph-mon[216815]: pgmap v872: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.7 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 14369/82458 objects degraded (17.426%); 24303/82458 objects misplaced (29.473%); 17 MiB/s, 267 objects/s recovering 2023-09-30T09:36:43.326 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4203989345' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:36:43.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:42 smithi019 ceph-mon[224043]: pgmap v872: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.7 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 14369/82458 objects degraded (17.426%); 24303/82458 objects misplaced (29.473%); 17 MiB/s, 267 objects/s recovering 2023-09-30T09:36:43.327 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4203989345' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:36:44.087 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:36:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:36:43] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-09-30T09:36:44.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 14369/82458 objects degraded (17.426%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:44.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 14369/82458 objects degraded (17.426%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:44.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 14369/82458 objects degraded (17.426%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:44.390 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:44.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:44.506 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:44.617 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:44 smithi167 ceph-mon[192880]: pgmap v873: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.6 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 13847/79791 objects degraded (17.354%); 23524/79791 objects misplaced (29.482%); 16 MiB/s, 263 objects/s recovering 2023-09-30T09:36:45.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:44 smithi019 ceph-mon[216815]: pgmap v873: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.6 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 13847/79791 objects degraded (17.354%); 23524/79791 objects misplaced (29.482%); 16 MiB/s, 263 objects/s recovering 2023-09-30T09:36:45.290 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:44 smithi019 ceph-mon[224043]: pgmap v873: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.6 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 13847/79791 objects degraded (17.354%); 23524/79791 objects misplaced (29.482%); 16 MiB/s, 263 objects/s recovering 2023-09-30T09:36:45.377 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:36:45.378 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:36:45.405 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:45.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:45.507 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:45.521 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:45.552 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:45.635 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:45.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:45.742 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:45.777 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:45.833 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:45.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:45.939 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:46.039 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:46.132 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:36:46.133 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:45 smithi019 ceph-mon[216815]: pgmap v874: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.6 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 13454/76134 objects degraded (17.671%); 22173/76134 objects misplaced (29.124%); 16 MiB/s, 274 objects/s recovering 2023-09-30T09:36:46.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:45 smithi019 ceph-mon[224043]: pgmap v874: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.6 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 13454/76134 objects degraded (17.671%); 22173/76134 objects misplaced (29.124%); 16 MiB/s, 274 objects/s recovering 2023-09-30T09:36:46.210 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:46.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:46.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:46.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:45 smithi167 ceph-mon[192880]: pgmap v874: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.6 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 13454/76134 objects degraded (17.671%); 22173/76134 objects misplaced (29.124%); 16 MiB/s, 274 objects/s recovering 2023-09-30T09:36:46.305 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:36:46.305 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:36:46.321 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:36:46.416 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:46.703 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:36:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:46.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:36:46.811 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:46.843 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:46.916 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:46.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:46.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:46.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:46 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:36:47.025 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:47.151 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:46 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:36:47.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:46 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:36:48.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:48 smithi167 ceph-mon[192880]: pgmap v875: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.5 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 12905/73161 objects degraded (17.639%); 21341/73161 objects misplaced (29.170%); 16 MiB/s, 275 objects/s recovering 2023-09-30T09:36:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:48 smithi019 ceph-mon[224043]: pgmap v875: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.5 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 12905/73161 objects degraded (17.639%); 21341/73161 objects misplaced (29.170%); 16 MiB/s, 275 objects/s recovering 2023-09-30T09:36:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:48 smithi019 ceph-mon[216815]: pgmap v875: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.5 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 12905/73161 objects degraded (17.639%); 21341/73161 objects misplaced (29.170%); 16 MiB/s, 275 objects/s recovering 2023-09-30T09:36:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:48.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:36:48.741 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:48.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:48.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:48.850 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:36:48.908 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:48.989 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:36:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:48.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:36:49.096 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:36:49.155 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:49 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:36:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:49 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 12905/73161 objects degraded (17.639%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:49.290 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:36:49.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:49.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:36:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:49 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:36:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:49 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 12905/73161 objects degraded (17.639%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:49.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:49 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:36:49.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:49 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 12905/73161 objects degraded (17.639%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:49.400 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:49.447 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:49.497 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:36:49.530 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:49.625 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:49.708 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:49.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:49.814 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:49.919 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:50.262 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:50.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:50.273 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:50 smithi019 ceph-mon[224043]: pgmap v876: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 12414/69894 objects degraded (17.761%); 20212/69894 objects misplaced (28.918%); 16 MiB/s, 273 objects/s recovering 2023-09-30T09:36:50.280 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:50 smithi019 ceph-mon[216815]: pgmap v876: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 12414/69894 objects degraded (17.761%); 20212/69894 objects misplaced (28.918%); 16 MiB/s, 273 objects/s recovering 2023-09-30T09:36:50.289 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:36:50.374 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:50.477 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:50.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:50 smithi167 ceph-mon[192880]: pgmap v876: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 12414/69894 objects degraded (17.761%); 20212/69894 objects misplaced (28.918%); 16 MiB/s, 273 objects/s recovering 2023-09-30T09:36:50.750 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:50.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:50.856 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:50.960 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:51.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:51 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:36:51.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:51 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:36:51.415 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:51 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:36:52.065 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:36:52.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:52 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:36:52.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1478849291' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:36:52.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:52 smithi167 ceph-mon[192880]: pgmap v877: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 12414/69489 objects degraded (17.865%); 20212/69489 objects misplaced (29.087%); 14 MiB/s, 279 objects/s recovering 2023-09-30T09:36:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:52 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:36:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1478849291' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:36:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:52 smithi019 ceph-mon[216815]: pgmap v877: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 12414/69489 objects degraded (17.865%); 20212/69489 objects misplaced (29.087%); 14 MiB/s, 279 objects/s recovering 2023-09-30T09:36:52.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:52 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:36:52.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1478849291' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:36:52.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:52 smithi019 ceph-mon[224043]: pgmap v877: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 12414/69489 objects degraded (17.865%); 20212/69489 objects misplaced (29.087%); 14 MiB/s, 279 objects/s recovering 2023-09-30T09:36:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1478849291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:36:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[216815]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[216815]: osdmap e341: 8 total, 8 up, 7 in 2023-09-30T09:36:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[216815]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[216815]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1478849291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:36:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[224043]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[224043]: osdmap e341: 8 total, 8 up, 7 in 2023-09-30T09:36:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[224043]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[224043]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:53.441 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:36:53.442 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:36:53.471 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:53 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:36:53.471 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1478849291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:36:53.471 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:53 smithi167 ceph-mon[192880]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.472 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:53 smithi167 ceph-mon[192880]: osdmap e341: 8 total, 8 up, 7 in 2023-09-30T09:36:53.472 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:53 smithi167 ceph-mon[192880]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.472 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:53 smithi167 ceph-mon[192880]: failed to encode map e341 with expected crc 2023-09-30T09:36:53.472 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:53.472 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:36:54.061 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:36:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:36:53] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-09-30T09:36:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:54 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 12414/69489 objects degraded (17.865%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:54 smithi019 ceph-mon[224043]: pgmap v879: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 11823/66546 objects degraded (17.767%); 19293/66546 objects misplaced (28.992%); 15 MiB/s, 305 objects/s recovering 2023-09-30T09:36:54.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:54 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 12414/69489 objects degraded (17.865%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:54.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:54 smithi019 ceph-mon[216815]: pgmap v879: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 11823/66546 objects degraded (17.767%); 19293/66546 objects misplaced (28.992%); 15 MiB/s, 305 objects/s recovering 2023-09-30T09:36:54.373 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:54 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 12414/69489 objects degraded (17.865%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:54.373 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:54 smithi167 ceph-mon[192880]: pgmap v879: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 11823/66546 objects degraded (17.767%); 19293/66546 objects misplaced (28.992%); 15 MiB/s, 305 objects/s recovering 2023-09-30T09:36:54.397 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:36:54.802 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:54.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:54.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:54.839 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:36:54.915 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:54.990 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:55.357 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:55.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:55.469 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:55.566 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:55.647 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:55.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:55.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:55.753 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:55.799 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:56.015 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:56.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:56.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:56.066 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:36:56.085 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:36:56.124 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:56.174 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:56.240 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:56.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:56.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:56.283 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:36:56.344 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:36:56.357 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:36:56.402 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:36:56.458 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:36:56.460 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:56.513 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:36:56.554 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:36:56.595 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:36:56.641 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:36:56.651 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:36:56.651 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:36:56.651 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:36:56.651 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:36:56.651 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:36:56.652 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:36:56.652 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:36:56.652 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:36:56.676 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:36:56.676 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:36:56.676 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:36:56.676 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:36:56.676 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:36:56.699 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:56 smithi167 ceph-mon[192880]: pgmap v880: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.3 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 11348/62841 objects degraded (18.058%); 18194/62841 objects misplaced (28.952%); 14 MiB/s, 328 objects/s recovering 2023-09-30T09:36:56.699 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2608553060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:56.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:56 smithi019 ceph-mon[224043]: pgmap v880: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.3 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 11348/62841 objects degraded (18.058%); 18194/62841 objects misplaced (28.952%); 14 MiB/s, 328 objects/s recovering 2023-09-30T09:36:56.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2608553060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:56 smithi019 ceph-mon[216815]: pgmap v880: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.3 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 11348/62841 objects degraded (18.058%); 18194/62841 objects misplaced (28.952%); 14 MiB/s, 328 objects/s recovering 2023-09-30T09:36:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2608553060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:36:57.194 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:36:57.560 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:36:57.628 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:57 smithi019 ceph-mon[216815]: from='client.55504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:57.628 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:57 smithi019 ceph-mon[224043]: from='client.55504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:57.768 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:36:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:57.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:36:57.881 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:36:57.955 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:57 smithi167 ceph-mon[192880]: from='client.55504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:36:58.057 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:36:58.889 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:58 smithi019 ceph-mon[224043]: pgmap v881: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 10705/60081 objects degraded (17.818%); 17295/60081 objects misplaced (28.786%); 15 MiB/s, 337 objects/s recovering 2023-09-30T09:36:58.889 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:36:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 10705/60081 objects degraded (17.818%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:58.889 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:58 smithi019 ceph-mon[216815]: pgmap v881: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 10705/60081 objects degraded (17.818%); 17295/60081 objects misplaced (28.786%); 15 MiB/s, 337 objects/s recovering 2023-09-30T09:36:58.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:36:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 10705/60081 objects degraded (17.818%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:58.891 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":880,"stamp":"2023-09-30T09:36:55.539739+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1375812452,"num_objects":20947,"num_object_clones":0,"num_object_copies":62841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11348,"num_objects_misplaced":18194,"num_objects_unfound":0,"num_objects_dirty":20947,"num_whiteouts":0,"num_read":6114,"num_read_kb":12467,"num_write":206622,"num_write_kb":7263166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33835,"num_bytes_recovered":2135400920,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88220,"ondisk_log_size":88220,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":397,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5214612,"kb_used_data":2774804,"kb_used_omap":23,"kb_used_meta":2439464,"kb_avail":651001452,"statfs":{"total":671965249536,"available":666625486848,"internally_reserved":0,"allocated":2841399296,"data_stored":2829987778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2498011395},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-290390016,"num_objects":-4431,"num_object_clones":0,"num_object_copies":-13293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2106,"num_objects_misplaced":-3979,"num_objects_unfound":0,"num_objects_dirty":-4431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4431,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3285,"num_bytes_recovered":148766720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006028"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":805,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569422+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:36:35.569422+0000","last_peered":"2023-09-30T09:36:35.569422+0000","last_clean":"2023-09-30T09:36:35.569422+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:36:35.569422+0000","last_undegraded":"2023-09-30T09:36:35.569422+0000","last_fullsized":"2023-09-30T09:36:35.569422+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":359,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.570140+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:36:35.570140+0000","last_peered":"2023-09-30T09:36:35.570140+0000","last_clean":"2023-09-30T09:36:35.570140+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:36:35.570140+0000","last_undegraded":"2023-09-30T09:36:35.570140+0000","last_fullsized":"2023-09-30T09:36:35.570140+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":322,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.522910+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:36:35.522910+0000","last_peered":"2023-09-30T09:36:35.522910+0000","last_clean":"2023-09-30T09:36:35.522910+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:36:35.522910+0000","last_undegraded":"2023-09-30T09:36:35.522910+0000","last_fullsized":"2023-09-30T09:36:35.522910+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":785,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568143+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:36:35.568143+0000","last_peered":"2023-09-30T09:36:35.568143+0000","last_clean":"2023-09-30T09:36:35.568143+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:36:35.568143+0000","last_undegraded":"2023-09-30T09:36:35.568143+0000","last_fullsized":"2023-09-30T09:36:35.568143+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":214,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.521828+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:36:35.521828+0000","last_peered":"2023-09-30T09:36:35.521828+0000","last_clean":"2023-09-30T09:36:35.521828+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:36:35.521828+0000","last_undegraded":"2023-09-30T09:36:35.521828+0000","last_fullsized":"2023-09-30T09:36:35.521828+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":245,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621593+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:36:35.621593+0000","last_peered":"2023-09-30T09:36:35.621593+0000","last_clean":"2023-09-30T09:36:35.621593+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:36:35.621593+0000","last_undegraded":"2023-09-30T09:36:35.621593+0000","last_fullsized":"2023-09-30T09:36:35.621593+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":554,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.570038+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:36:35.570038+0000","last_peered":"2023-09-30T09:36:35.570038+0000","last_clean":"2023-09-30T09:36:35.570038+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:36:35.570038+0000","last_undegraded":"2023-09-30T09:36:35.570038+0000","last_fullsized":"2023-09-30T09:36:35.570038+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":388,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.521732+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:36:35.521732+0000","last_peered":"2023-09-30T09:36:35.521732+0000","last_clean":"2023-09-30T09:36:35.521732+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:36:35.521732+0000","last_undegraded":"2023-09-30T09:36:35.521732+0000","last_fullsized":"2023-09-30T09:36:35.521732+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":317,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569254+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:36:35.569254+0000","last_peered":"2023-09-30T09:36:35.569254+0000","last_clean":"2023-09-30T09:36:35.569254+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:36:35.569254+0000","last_undegraded":"2023-09-30T09:36:35.569254+0000","last_fullsized":"2023-09-30T09:36:35.569254+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":765,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:37.142341+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:36:37.142341+0000","last_peered":"2023-09-30T09:36:37.142341+0000","last_clean":"2023-09-30T09:36:37.142341+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:36:37.142341+0000","last_undegraded":"2023-09-30T09:36:37.142341+0000","last_fullsized":"2023-09-30T09:36:37.142341+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":236,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621456+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:36:35.621456+0000","last_peered":"2023-09-30T09:36:35.621456+0000","last_clean":"2023-09-30T09:36:35.621456+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:36:35.621456+0000","last_undegraded":"2023-09-30T09:36:35.621456+0000","last_fullsized":"2023-09-30T09:36:35.621456+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":322,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.521808+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:36:35.521808+0000","last_peered":"2023-09-30T09:36:35.521808+0000","last_clean":"2023-09-30T09:36:35.521808+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:36:35.521808+0000","last_undegraded":"2023-09-30T09:36:35.521808+0000","last_fullsized":"2023-09-30T09:36:35.521808+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":504,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.627387+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:36:35.627387+0000","last_peered":"2023-09-30T09:36:35.627387+0000","last_clean":"2023-09-30T09:36:35.627387+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:36:35.627387+0000","last_undegraded":"2023-09-30T09:36:35.627387+0000","last_fullsized":"2023-09-30T09:36:35.627387+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568772+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:36:35.568772+0000","last_peered":"2023-09-30T09:36:35.568772+0000","last_clean":"2023-09-30T09:36:35.568772+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:36:35.568772+0000","last_undegraded":"2023-09-30T09:36:35.568772+0000","last_fullsized":"2023-09-30T09:36:35.568772+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":268,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.521750+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:36:35.521750+0000","last_peered":"2023-09-30T09:36:35.521750+0000","last_clean":"2023-09-30T09:36:35.521750+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:36:35.521750+0000","last_undegraded":"2023-09-30T09:36:35.521750+0000","last_fullsized":"2023-09-30T09:36:35.521750+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":507,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568723+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:36:35.568723+0000","last_peered":"2023-09-30T09:36:35.568723+0000","last_clean":"2023-09-30T09:36:35.568723+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:36:35.568723+0000","last_undegraded":"2023-09-30T09:36:35.568723+0000","last_fullsized":"2023-09-30T09:36:35.568723+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":63,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.522991+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:36:35.522991+0000","last_peered":"2023-09-30T09:36:35.522991+0000","last_clean":"2023-09-30T09:36:35.522991+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:36:35.522991+0000","last_undegraded":"2023-09-30T09:36:35.522991+0000","last_fullsized":"2023-09-30T09:36:35.522991+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":254,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569000+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:36:35.569000+0000","last_peered":"2023-09-30T09:36:35.569000+0000","last_clean":"2023-09-30T09:36:35.569000+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:36:35.569000+0000","last_undegraded":"2023-09-30T09:36:35.569000+0000","last_fullsized":"2023-09-30T09:36:35.569000+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":501,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:45.216785+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:36:45.216785+0000","last_peered":"2023-09-30T09:36:45.216785+0000","last_clean":"2023-09-30T09:36:45.216785+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:36:45.216785+0000","last_undegraded":"2023-09-30T09:36:45.216785+0000","last_fullsized":"2023-09-30T09:36:45.216785+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":325,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.570490+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:36:35.570490+0000","last_peered":"2023-09-30T09:36:35.570490+0000","last_clean":"2023-09-30T09:36:35.570490+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:36:35.570490+0000","last_undegraded":"2023-09-30T09:36:35.570490+0000","last_fullsized":"2023-09-30T09:36:35.570490+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":336,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.571334+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:36:35.571334+0000","last_peered":"2023-09-30T09:36:35.571334+0000","last_clean":"2023-09-30T09:36:35.571334+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:36:35.571334+0000","last_undegraded":"2023-09-30T09:36:35.571334+0000","last_fullsized":"2023-09-30T09:36:35.571334+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":618,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621267+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:36:35.621267+0000","last_peered":"2023-09-30T09:36:35.621267+0000","last_clean":"2023-09-30T09:36:35.621267+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:36:35.621267+0000","last_undegraded":"2023-09-30T09:36:35.621267+0000","last_fullsized":"2023-09-30T09:36:35.621267+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":63,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.571291+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:36:35.571291+0000","last_peered":"2023-09-30T09:36:35.571291+0000","last_clean":"2023-09-30T09:36:35.571291+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:36:35.571291+0000","last_undegraded":"2023-09-30T09:36:35.571291+0000","last_fullsized":"2023-09-30T09:36:35.571291+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":352,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.522354+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:36:35.522354+0000","last_peered":"2023-09-30T09:36:35.522354+0000","last_clean":"2023-09-30T09:36:35.522354+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:36:35.522354+0000","last_undegraded":"2023-09-30T09:36:35.522354+0000","last_fullsized":"2023-09-30T09:36:35.522354+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":687,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569546+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:36:35.569546+0000","last_peered":"2023-09-30T09:36:35.569546+0000","last_clean":"2023-09-30T09:36:35.569546+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:36:35.569546+0000","last_undegraded":"2023-09-30T09:36:35.569546+0000","last_fullsized":"2023-09-30T09:36:35.569546+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":244,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.620932+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:36:35.620932+0000","last_peered":"2023-09-30T09:36:35.620932+0000","last_clean":"2023-09-30T09:36:35.620932+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:36:35.620932+0000","last_undegraded":"2023-09-30T09:36:35.620932+0000","last_fullsized":"2023-09-30T09:36:35.620932+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":30,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568448+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:36:35.568448+0000","last_peered":"2023-09-30T09:36:35.568448+0000","last_clean":"2023-09-30T09:36:35.568448+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:36:35.568448+0000","last_undegraded":"2023-09-30T09:36:35.568448+0000","last_fullsized":"2023-09-30T09:36:35.568448+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":779,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.521838+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:36:35.521838+0000","last_peered":"2023-09-30T09:36:35.521838+0000","last_clean":"2023-09-30T09:36:35.521838+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:36:35.521838+0000","last_undegraded":"2023-09-30T09:36:35.521838+0000","last_fullsized":"2023-09-30T09:36:35.521838+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569851+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:36:35.569851+0000","last_peered":"2023-09-30T09:36:35.569851+0000","last_clean":"2023-09-30T09:36:35.569851+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:36:35.569851+0000","last_undegraded":"2023-09-30T09:36:35.569851+0000","last_fullsized":"2023-09-30T09:36:35.569851+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":244,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.622648+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:36:35.622648+0000","last_peered":"2023-09-30T09:36:35.622648+0000","last_clean":"2023-09-30T09:36:35.622648+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:36:35.622648+0000","last_undegraded":"2023-09-30T09:36:35.622648+0000","last_fullsized":"2023-09-30T09:36:35.622648+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":806,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569020+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:36:35.569020+0000","last_peered":"2023-09-30T09:36:35.569020+0000","last_clean":"2023-09-30T09:36:35.569020+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:36:35.569020+0000","last_undegraded":"2023-09-30T09:36:35.569020+0000","last_fullsized":"2023-09-30T09:36:35.569020+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":500,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568219+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:36:35.568219+0000","last_peered":"2023-09-30T09:36:35.568219+0000","last_clean":"2023-09-30T09:36:35.568219+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:36:35.568219+0000","last_undegraded":"2023-09-30T09:36:35.568219+0000","last_fullsized":"2023-09-30T09:36:35.568219+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":350,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.524163+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:36:35.524163+0000","last_peered":"2023-09-30T09:36:35.524163+0000","last_clean":"2023-09-30T09:36:35.524163+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:36:35.524163+0000","last_undegraded":"2023-09-30T09:36:35.524163+0000","last_fullsized":"2023-09-30T09:36:35.524163+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"341'12574","reported_seq":17128,"reported_epoch":341,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:53.989491+0000","last_change":"2023-09-30T09:34:48.517696+0000","last_active":"2023-09-30T09:36:53.989491+0000","last_peered":"2023-09-30T09:36:53.989491+0000","last_clean":"2023-09-30T09:32:35.457744+0000","last_became_active":"2023-09-30T09:34:48.499533+0000","last_became_peered":"2023-09-30T09:34:48.499533+0000","last_unstale":"2023-09-30T09:36:53.989491+0000","last_undegraded":"2023-09-30T09:34:48.469688+0000","last_fullsized":"2023-09-30T09:36:53.989491+0000","mapping_epoch":321,"log_start":"305'7721","ondisk_log_start":"305'7721","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4853,"log_dups_size":0,"ondisk_log_size":4853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:05:42.164341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86769664,"num_objects":1324,"num_object_clones":0,"num_object_copies":3972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1322,"num_objects_misplaced":1324,"num_objects_unfound":0,"num_objects_dirty":1324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12574,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2220,"num_bytes_recovered":145489920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[6,1,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"1,4,6","objects":2},{"shards":"4,6","objects":1322}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":823,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:50.905992+0000","last_change":"2023-09-30T09:36:50.905972+0000","last_active":"2023-09-30T09:36:50.905992+0000","last_peered":"2023-09-30T09:36:50.905992+0000","last_clean":"2023-09-30T09:36:50.905992+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:36:50.905992+0000","last_undegraded":"2023-09-30T09:36:50.905992+0000","last_fullsized":"2023-09-30T09:36:50.905992+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":75,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569987+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:36:35.569987+0000","last_peered":"2023-09-30T09:36:35.569987+0000","last_clean":"2023-09-30T09:36:35.569987+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:36:35.569987+0000","last_undegraded":"2023-09-30T09:36:35.569987+0000","last_fullsized":"2023-09-30T09:36:35.569987+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":30,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569354+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:36:35.569354+0000","last_peered":"2023-09-30T09:36:35.569354+0000","last_clean":"2023-09-30T09:36:35.569354+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:36:35.569354+0000","last_undegraded":"2023-09-30T09:36:35.569354+0000","last_fullsized":"2023-09-30T09:36:35.569354+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"340'13050","reported_seq":18795,"reported_epoch":341,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:52.224286+0000","last_change":"2023-09-30T09:35:10.766663+0000","last_active":"2023-09-30T09:36:52.224286+0000","last_peered":"2023-09-30T09:36:52.224286+0000","last_clean":"2023-09-30T09:32:35.454324+0000","last_became_active":"2023-09-30T09:33:33.826036+0000","last_became_peered":"2023-09-30T09:33:33.826036+0000","last_unstale":"2023-09-30T09:36:52.224286+0000","last_undegraded":"2023-09-30T09:33:33.732722+0000","last_fullsized":"2023-09-30T09:36:52.224286+0000","mapping_epoch":302,"log_start":"311'8499","ondisk_log_start":"311'8499","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4551,"log_dups_size":0,"ondisk_log_size":4551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:42.597216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88473600,"num_objects":1350,"num_object_clones":0,"num_object_copies":4050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1260,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13050,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2823,"num_bytes_recovered":185008128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,5,1],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1},{"shards":"2,3,5","objects":89},{"shards":"3,5","objects":1260}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":790,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.522282+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:36:35.522282+0000","last_peered":"2023-09-30T09:36:35.522282+0000","last_clean":"2023-09-30T09:36:35.522282+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:36:35.522282+0000","last_undegraded":"2023-09-30T09:36:35.522282+0000","last_fullsized":"2023-09-30T09:36:35.522282+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":53,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568713+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:36:35.568713+0000","last_peered":"2023-09-30T09:36:35.568713+0000","last_clean":"2023-09-30T09:36:35.568713+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:36:35.568713+0000","last_undegraded":"2023-09-30T09:36:35.568713+0000","last_fullsized":"2023-09-30T09:36:35.568713+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":333,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569796+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:36:35.569796+0000","last_peered":"2023-09-30T09:36:35.569796+0000","last_clean":"2023-09-30T09:36:35.569796+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:36:35.569796+0000","last_undegraded":"2023-09-30T09:36:35.569796+0000","last_fullsized":"2023-09-30T09:36:35.569796+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"341'12684","reported_seq":17430,"reported_epoch":341,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:53.992652+0000","last_change":"2023-09-30T09:35:10.879711+0000","last_active":"2023-09-30T09:36:53.992652+0000","last_peered":"2023-09-30T09:36:53.992652+0000","last_clean":"2023-09-30T09:32:35.467048+0000","last_became_active":"2023-09-30T09:35:10.845361+0000","last_became_peered":"2023-09-30T09:35:10.845361+0000","last_unstale":"2023-09-30T09:36:53.992652+0000","last_undegraded":"2023-09-30T09:35:10.791338+0000","last_fullsized":"2023-09-30T09:36:53.992652+0000","mapping_epoch":327,"log_start":"305'7876","ondisk_log_start":"305'7876","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4808,"log_dups_size":0,"ondisk_log_size":4808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:41.089234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84672512,"num_objects":1292,"num_object_clones":0,"num_object_copies":3876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1292,"num_objects_misplaced":2584,"num_objects_unfound":0,"num_objects_dirty":1292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12684,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2316,"num_bytes_recovered":151650304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[6,1,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":1292}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":624,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569169+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:36:35.569169+0000","last_peered":"2023-09-30T09:36:35.569169+0000","last_clean":"2023-09-30T09:36:35.569169+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:36:35.569169+0000","last_undegraded":"2023-09-30T09:36:35.569169+0000","last_fullsized":"2023-09-30T09:36:35.569169+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":34,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568371+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:36:35.568371+0000","last_peered":"2023-09-30T09:36:35.568371+0000","last_clean":"2023-09-30T09:36:35.568371+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:36:35.568371+0000","last_undegraded":"2023-09-30T09:36:35.568371+0000","last_fullsized":"2023-09-30T09:36:35.568371+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":320,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568872+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:36:35.568872+0000","last_peered":"2023-09-30T09:36:35.568872+0000","last_clean":"2023-09-30T09:36:35.568872+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:36:35.568872+0000","last_undegraded":"2023-09-30T09:36:35.568872+0000","last_fullsized":"2023-09-30T09:36:35.568872+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"341'12946","reported_seq":13029,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:52.222959+0000","last_change":"2023-09-30T09:34:08.527633+0000","last_active":"2023-09-30T09:36:52.222959+0000","last_peered":"2023-09-30T09:36:52.222959+0000","last_clean":"2023-09-30T09:33:09.318417+0000","last_became_active":"2023-09-30T09:34:08.510746+0000","last_became_peered":"2023-09-30T09:34:08.510746+0000","last_unstale":"2023-09-30T09:36:52.222959+0000","last_undegraded":"2023-09-30T09:36:52.222959+0000","last_fullsized":"2023-09-30T09:36:52.222959+0000","mapping_epoch":308,"log_start":"309'8376","ondisk_log_start":"309'8376","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4570,"log_dups_size":0,"ondisk_log_size":4570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:07.374969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88604672,"num_objects":1352,"num_object_clones":0,"num_object_copies":4056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1352,"num_objects_unfound":0,"num_objects_dirty":1352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12946,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1352}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":497,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:49.891007+0000","last_change":"2023-09-30T09:36:49.890977+0000","last_active":"2023-09-30T09:36:49.891007+0000","last_peered":"2023-09-30T09:36:49.891007+0000","last_clean":"2023-09-30T09:36:49.891007+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:36:49.891007+0000","last_undegraded":"2023-09-30T09:36:49.891007+0000","last_fullsized":"2023-09-30T09:36:49.891007+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":300,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.571559+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:36:35.571559+0000","last_peered":"2023-09-30T09:36:35.571559+0000","last_clean":"2023-09-30T09:36:35.571559+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:36:35.571559+0000","last_undegraded":"2023-09-30T09:36:35.571559+0000","last_fullsized":"2023-09-30T09:36:35.571559+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":247,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:40.211727+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:36:40.211727+0000","last_peered":"2023-09-30T09:36:40.211727+0000","last_clean":"2023-09-30T09:36:40.211727+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:36:40.211727+0000","last_undegraded":"2023-09-30T09:36:40.211727+0000","last_fullsized":"2023-09-30T09:36:40.211727+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"341'12725","reported_seq":12821,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:52.228490+0000","last_change":"2023-09-30T09:35:10.779076+0000","last_active":"2023-09-30T09:36:52.228490+0000","last_peered":"2023-09-30T09:36:52.228490+0000","last_clean":"2023-09-30T09:34:06.422096+0000","last_became_active":"2023-09-30T09:35:10.747716+0000","last_became_peered":"2023-09-30T09:35:10.747716+0000","last_unstale":"2023-09-30T09:36:52.228490+0000","last_undegraded":"2023-09-30T09:36:52.228490+0000","last_fullsized":"2023-09-30T09:36:52.228490+0000","mapping_epoch":327,"log_start":"309'8220","ondisk_log_start":"309'8220","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4505,"log_dups_size":0,"ondisk_log_size":4505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:09.191024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86573056,"num_objects":1321,"num_object_clones":0,"num_object_copies":3963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1321,"num_objects_unfound":0,"num_objects_dirty":1321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12725,"num_write_kb":449472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1321}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":687,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.521535+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:36:35.521535+0000","last_peered":"2023-09-30T09:36:35.521535+0000","last_clean":"2023-09-30T09:36:35.521535+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:36:35.521535+0000","last_undegraded":"2023-09-30T09:36:35.521535+0000","last_fullsized":"2023-09-30T09:36:35.521535+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":368,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.522085+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:36:35.522085+0000","last_peered":"2023-09-30T09:36:35.522085+0000","last_clean":"2023-09-30T09:36:35.522085+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:36:35.522085+0000","last_undegraded":"2023-09-30T09:36:35.522085+0000","last_fullsized":"2023-09-30T09:36:35.522085+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":299,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.570555+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:36:35.570555+0000","last_peered":"2023-09-30T09:36:35.570555+0000","last_clean":"2023-09-30T09:36:35.570555+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:36:35.570555+0000","last_undegraded":"2023-09-30T09:36:35.570555+0000","last_fullsized":"2023-09-30T09:36:35.570555+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"340'12904","reported_seq":17616,"reported_epoch":340,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:52.022122+0000","last_change":"2023-09-30T09:35:10.871065+0000","last_active":"2023-09-30T09:36:52.022122+0000","last_peered":"2023-09-30T09:36:52.022122+0000","last_clean":"2023-09-30T09:32:35.449714+0000","last_became_active":"2023-09-30T09:35:10.838732+0000","last_became_peered":"2023-09-30T09:35:10.838732+0000","last_unstale":"2023-09-30T09:36:52.022122+0000","last_undegraded":"2023-09-30T09:35:10.818868+0000","last_fullsized":"2023-09-30T09:36:52.022122+0000","mapping_epoch":327,"log_start":"311'8406","ondisk_log_start":"311'8406","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4498,"log_dups_size":0,"ondisk_log_size":4498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:44.318381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88342528,"num_objects":1348,"num_object_clones":0,"num_object_copies":4044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1347,"num_objects_misplaced":2697,"num_objects_unfound":0,"num_objects_dirty":1348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12904,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2301,"num_bytes_recovered":150798336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[3,7,1],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"1,3,7","objects":1},{"shards":"3,7","objects":1347}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":762,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.629536+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:36:35.629536+0000","last_peered":"2023-09-30T09:36:35.629536+0000","last_clean":"2023-09-30T09:36:35.629536+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:36:35.629536+0000","last_undegraded":"2023-09-30T09:36:35.629536+0000","last_fullsized":"2023-09-30T09:36:35.629536+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":318,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568718+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:36:35.568718+0000","last_peered":"2023-09-30T09:36:35.568718+0000","last_clean":"2023-09-30T09:36:35.568718+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:36:35.568718+0000","last_undegraded":"2023-09-30T09:36:35.568718+0000","last_fullsized":"2023-09-30T09:36:35.568718+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":35,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568056+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:36:35.568056+0000","last_peered":"2023-09-30T09:36:35.568056+0000","last_clean":"2023-09-30T09:36:35.568056+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:36:35.568056+0000","last_undegraded":"2023-09-30T09:36:35.568056+0000","last_fullsized":"2023-09-30T09:36:35.568056+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"341'12786","reported_seq":20012,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:54.059092+0000","last_change":"2023-09-30T09:36:04.573419+0000","last_active":"2023-09-30T09:36:54.059092+0000","last_peered":"2023-09-30T09:36:54.059092+0000","last_clean":"2023-09-30T09:36:54.059092+0000","last_became_active":"2023-09-30T09:36:04.549763+0000","last_became_peered":"2023-09-30T09:36:04.549763+0000","last_unstale":"2023-09-30T09:36:54.059092+0000","last_undegraded":"2023-09-30T09:36:54.059092+0000","last_fullsized":"2023-09-30T09:36:54.059092+0000","mapping_epoch":337,"log_start":"306'8021","ondisk_log_start":"306'8021","created":279,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4765,"log_dups_size":0,"ondisk_log_size":4765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:27:18.185457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378776,"num_objects":1258,"num_object_clones":0,"num_object_copies":3774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1258,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12786,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3562,"num_bytes_recovered":233439232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":776,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621271+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:36:35.621271+0000","last_peered":"2023-09-30T09:36:35.621271+0000","last_clean":"2023-09-30T09:36:35.621271+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:36:35.621271+0000","last_undegraded":"2023-09-30T09:36:35.621271+0000","last_fullsized":"2023-09-30T09:36:35.621271+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":337,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569716+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:36:35.569716+0000","last_peered":"2023-09-30T09:36:35.569716+0000","last_clean":"2023-09-30T09:36:35.569716+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:36:35.569716+0000","last_undegraded":"2023-09-30T09:36:35.569716+0000","last_fullsized":"2023-09-30T09:36:35.569716+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":316,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568991+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:36:35.568991+0000","last_peered":"2023-09-30T09:36:35.568991+0000","last_clean":"2023-09-30T09:36:35.568991+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:36:35.568991+0000","last_undegraded":"2023-09-30T09:36:35.568991+0000","last_fullsized":"2023-09-30T09:36:35.568991+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"341'12910","reported_seq":12997,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:54.049361+0000","last_change":"2023-09-30T09:34:47.630136+0000","last_active":"2023-09-30T09:36:54.049361+0000","last_peered":"2023-09-30T09:36:54.049361+0000","last_clean":"2023-09-30T09:34:06.422409+0000","last_became_active":"2023-09-30T09:34:08.626702+0000","last_became_peered":"2023-09-30T09:34:08.626702+0000","last_unstale":"2023-09-30T09:36:54.049361+0000","last_undegraded":"2023-09-30T09:36:54.049361+0000","last_fullsized":"2023-09-30T09:36:54.049361+0000","mapping_epoch":308,"log_start":"306'8106","ondisk_log_start":"306'8106","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4804,"log_dups_size":0,"ondisk_log_size":4804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:53.753813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82968576,"num_objects":1266,"num_object_clones":0,"num_object_copies":3798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1266,"num_objects_unfound":0,"num_objects_dirty":1266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12910,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":1266}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":486,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.626086+0000","last_change":"2023-09-30T09:35:46.150279+0000","last_active":"2023-09-30T09:36:35.626086+0000","last_peered":"2023-09-30T09:36:35.626086+0000","last_clean":"2023-09-30T09:36:35.626086+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:36:35.626086+0000","last_undegraded":"2023-09-30T09:36:35.626086+0000","last_fullsized":"2023-09-30T09:36:35.626086+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":332,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:37.142166+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:36:37.142166+0000","last_peered":"2023-09-30T09:36:37.142166+0000","last_clean":"2023-09-30T09:36:37.142166+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:36:37.142166+0000","last_undegraded":"2023-09-30T09:36:37.142166+0000","last_fullsized":"2023-09-30T09:36:37.142166+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":108,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.523974+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:36:35.523974+0000","last_peered":"2023-09-30T09:36:35.523974+0000","last_clean":"2023-09-30T09:36:35.523974+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:36:35.523974+0000","last_undegraded":"2023-09-30T09:36:35.523974+0000","last_fullsized":"2023-09-30T09:36:35.523974+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"340'12659","reported_seq":17037,"reported_epoch":341,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:52.057017+0000","last_change":"2023-09-30T09:34:39.166112+0000","last_active":"2023-09-30T09:36:52.057017+0000","last_peered":"2023-09-30T09:36:52.057017+0000","last_clean":"2023-09-30T09:32:35.465008+0000","last_became_active":"2023-09-30T09:34:39.150257+0000","last_became_peered":"2023-09-30T09:34:39.150257+0000","last_unstale":"2023-09-30T09:36:52.057017+0000","last_undegraded":"2023-09-30T09:34:38.934740+0000","last_fullsized":"2023-09-30T09:36:52.057017+0000","mapping_epoch":316,"log_start":"309'8178","ondisk_log_start":"309'8178","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4481,"log_dups_size":0,"ondisk_log_size":4481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:50.805799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86966272,"num_objects":1327,"num_object_clones":0,"num_object_copies":3981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1327,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12659,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2142,"num_bytes_recovered":140312576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6,1],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":1327}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":635,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569343+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:36:35.569343+0000","last_peered":"2023-09-30T09:36:35.569343+0000","last_clean":"2023-09-30T09:36:35.569343+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:36:35.569343+0000","last_undegraded":"2023-09-30T09:36:35.569343+0000","last_fullsized":"2023-09-30T09:36:35.569343+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":533,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:50.217384+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:36:50.217384+0000","last_peered":"2023-09-30T09:36:50.217384+0000","last_clean":"2023-09-30T09:36:50.217384+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:36:50.217384+0000","last_undegraded":"2023-09-30T09:36:50.217384+0000","last_fullsized":"2023-09-30T09:36:50.217384+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":337,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568653+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:36:35.568653+0000","last_peered":"2023-09-30T09:36:35.568653+0000","last_clean":"2023-09-30T09:36:35.568653+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:36:35.568653+0000","last_undegraded":"2023-09-30T09:36:35.568653+0000","last_fullsized":"2023-09-30T09:36:35.568653+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":30,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568741+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:36:35.568741+0000","last_peered":"2023-09-30T09:36:35.568741+0000","last_clean":"2023-09-30T09:36:35.568741+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:36:35.568741+0000","last_undegraded":"2023-09-30T09:36:35.568741+0000","last_fullsized":"2023-09-30T09:36:35.568741+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"341'12991","reported_seq":13061,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:54.044503+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:36:54.044503+0000","last_peered":"2023-09-30T09:36:54.044503+0000","last_clean":"2023-09-30T09:36:54.044503+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:36:54.044503+0000","last_undegraded":"2023-09-30T09:36:54.044503+0000","last_fullsized":"2023-09-30T09:36:54.044503+0000","mapping_epoch":279,"log_start":"306'8157","ondisk_log_start":"306'8157","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4834,"log_dups_size":0,"ondisk_log_size":4834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82903040,"num_objects":1265,"num_object_clones":0,"num_object_copies":3795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12991,"num_write_kb":456192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":803,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568280+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:36:35.568280+0000","last_peered":"2023-09-30T09:36:35.568280+0000","last_clean":"2023-09-30T09:36:35.568280+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:36:35.568280+0000","last_undegraded":"2023-09-30T09:36:35.568280+0000","last_fullsized":"2023-09-30T09:36:35.568280+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":332,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568798+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:36:35.568798+0000","last_peered":"2023-09-30T09:36:35.568798+0000","last_clean":"2023-09-30T09:36:35.568798+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:36:35.568798+0000","last_undegraded":"2023-09-30T09:36:35.568798+0000","last_fullsized":"2023-09-30T09:36:35.568798+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":320,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.571381+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:36:35.571381+0000","last_peered":"2023-09-30T09:36:35.571381+0000","last_clean":"2023-09-30T09:36:35.571381+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:36:35.571381+0000","last_undegraded":"2023-09-30T09:36:35.571381+0000","last_fullsized":"2023-09-30T09:36:35.571381+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":58,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.627461+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:36:35.627461+0000","last_peered":"2023-09-30T09:36:35.627461+0000","last_clean":"2023-09-30T09:36:35.627461+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:36:35.627461+0000","last_undegraded":"2023-09-30T09:36:35.627461+0000","last_fullsized":"2023-09-30T09:36:35.627461+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"341'12964","reported_seq":17732,"reported_epoch":341,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:54.070452+0000","last_change":"2023-09-30T09:35:10.808625+0000","last_active":"2023-09-30T09:36:54.070452+0000","last_peered":"2023-09-30T09:36:54.070452+0000","last_clean":"2023-09-30T09:32:35.460821+0000","last_became_active":"2023-09-30T09:35:10.778683+0000","last_became_peered":"2023-09-30T09:35:10.778683+0000","last_unstale":"2023-09-30T09:36:54.070452+0000","last_undegraded":"2023-09-30T09:35:10.766189+0000","last_fullsized":"2023-09-30T09:36:54.070452+0000","mapping_epoch":327,"log_start":"306'8133","ondisk_log_start":"306'8133","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4831,"log_dups_size":0,"ondisk_log_size":4831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:45.905717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90963968,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1387,"num_objects_misplaced":2776,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12964,"num_write_kb":459264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2329,"num_bytes_recovered":152633344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[6,1,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"1,5,6","objects":1},{"shards":"5,6","objects":1387}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":701,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621025+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:36:35.621025+0000","last_peered":"2023-09-30T09:36:35.621025+0000","last_clean":"2023-09-30T09:36:35.621025+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:36:35.621025+0000","last_undegraded":"2023-09-30T09:36:35.621025+0000","last_fullsized":"2023-09-30T09:36:35.621025+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":519,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.625491+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:36:35.625491+0000","last_peered":"2023-09-30T09:36:35.625491+0000","last_clean":"2023-09-30T09:36:35.625491+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:36:35.625491+0000","last_undegraded":"2023-09-30T09:36:35.625491+0000","last_fullsized":"2023-09-30T09:36:35.625491+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":66,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.629922+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:36:35.629922+0000","last_peered":"2023-09-30T09:36:35.629922+0000","last_clean":"2023-09-30T09:36:35.629922+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:36:35.629922+0000","last_undegraded":"2023-09-30T09:36:35.629922+0000","last_fullsized":"2023-09-30T09:36:35.629922+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":260,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568930+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:36:35.568930+0000","last_peered":"2023-09-30T09:36:35.568930+0000","last_clean":"2023-09-30T09:36:35.568930+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:36:35.568930+0000","last_undegraded":"2023-09-30T09:36:35.568930+0000","last_fullsized":"2023-09-30T09:36:35.568930+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"341'12795","reported_seq":24528,"reported_epoch":341,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:55.226487+0000","last_change":"2023-09-30T09:35:23.879286+0000","last_active":"2023-09-30T09:36:55.226487+0000","last_peered":"2023-09-30T09:36:55.226487+0000","last_clean":"2023-09-30T09:34:06.430942+0000","last_became_active":"2023-09-30T09:34:47.755052+0000","last_became_peered":"2023-09-30T09:34:47.755052+0000","last_unstale":"2023-09-30T09:36:55.226487+0000","last_undegraded":"2023-09-30T09:34:47.557422+0000","last_fullsized":"2023-09-30T09:34:47.514378+0000","mapping_epoch":320,"log_start":"299'6650","ondisk_log_start":"299'6650","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":6145,"log_dups_size":0,"ondisk_log_size":6145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:44:56.113242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80674816,"num_objects":1231,"num_object_clones":0,"num_object_copies":3693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":141,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12795,"num_write_kb":448832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5822,"num_bytes_recovered":304349184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":1090}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":913,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568233+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:36:35.568233+0000","last_peered":"2023-09-30T09:36:35.568233+0000","last_clean":"2023-09-30T09:36:35.568233+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:36:35.568233+0000","last_undegraded":"2023-09-30T09:36:35.568233+0000","last_fullsized":"2023-09-30T09:36:35.568233+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":605,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.620761+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:36:35.620761+0000","last_peered":"2023-09-30T09:36:35.620761+0000","last_clean":"2023-09-30T09:36:35.620761+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:36:35.620761+0000","last_undegraded":"2023-09-30T09:36:35.620761+0000","last_fullsized":"2023-09-30T09:36:35.620761+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":260,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.523687+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:36:35.523687+0000","last_peered":"2023-09-30T09:36:35.523687+0000","last_clean":"2023-09-30T09:36:35.523687+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:36:35.523687+0000","last_undegraded":"2023-09-30T09:36:35.523687+0000","last_fullsized":"2023-09-30T09:36:35.523687+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":293,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.570010+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:36:35.570010+0000","last_peered":"2023-09-30T09:36:35.570010+0000","last_clean":"2023-09-30T09:36:35.570010+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:36:35.570010+0000","last_undegraded":"2023-09-30T09:36:35.570010+0000","last_fullsized":"2023-09-30T09:36:35.570010+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"341'12932","reported_seq":23932,"reported_epoch":341,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:36:54.417237+0000","last_change":"2023-09-30T09:35:10.837720+0000","last_active":"2023-09-30T09:36:54.417237+0000","last_peered":"2023-09-30T09:36:54.417237+0000","last_clean":"2023-09-30T09:32:35.481650+0000","last_became_active":"2023-09-30T09:35:10.788829+0000","last_became_peered":"2023-09-30T09:35:10.788829+0000","last_unstale":"2023-09-30T09:36:54.417237+0000","last_undegraded":"2023-09-30T09:35:10.757584+0000","last_fullsized":"2023-09-30T09:36:54.417237+0000","mapping_epoch":327,"log_start":"286'3000","ondisk_log_start":"286'3000","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":9932,"log_dups_size":0,"ondisk_log_size":9932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:05.384343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80347136,"num_objects":1226,"num_object_clones":0,"num_object_copies":3678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":659,"num_objects_misplaced":2451,"num_objects_unfound":0,"num_objects_dirty":1226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12932,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5439,"num_bytes_recovered":356450304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[7,1,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":1},{"shards":"2,6,7","objects":567},{"shards":"6,7","objects":658}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":486,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.571807+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:36:35.571807+0000","last_peered":"2023-09-30T09:36:35.571807+0000","last_clean":"2023-09-30T09:36:35.571807+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:36:35.571807+0000","last_undegraded":"2023-09-30T09:36:35.571807+0000","last_fullsized":"2023-09-30T09:36:35.571807+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":717,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621651+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:36:35.621651+0000","last_peered":"2023-09-30T09:36:35.621651+0000","last_clean":"2023-09-30T09:36:35.621651+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:36:35.621651+0000","last_undegraded":"2023-09-30T09:36:35.621651+0000","last_fullsized":"2023-09-30T09:36:35.621651+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":269,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621091+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:36:35.621091+0000","last_peered":"2023-09-30T09:36:35.621091+0000","last_clean":"2023-09-30T09:36:35.621091+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:36:35.621091+0000","last_undegraded":"2023-09-30T09:36:35.621091+0000","last_fullsized":"2023-09-30T09:36:35.621091+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":25,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621370+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:36:35.621370+0000","last_peered":"2023-09-30T09:36:35.621370+0000","last_clean":"2023-09-30T09:36:35.621370+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:36:35.621370+0000","last_undegraded":"2023-09-30T09:36:35.621370+0000","last_fullsized":"2023-09-30T09:36:35.621370+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"341'12857","reported_seq":17305,"reported_epoch":341,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:54.156887+0000","last_change":"2023-09-30T09:34:47.553318+0000","last_active":"2023-09-30T09:36:54.156887+0000","last_peered":"2023-09-30T09:36:54.156887+0000","last_clean":"2023-09-30T09:32:35.469075+0000","last_became_active":"2023-09-30T09:33:33.792569+0000","last_became_peered":"2023-09-30T09:33:33.792569+0000","last_unstale":"2023-09-30T09:36:54.156887+0000","last_undegraded":"2023-09-30T09:33:33.610726+0000","last_fullsized":"2023-09-30T09:33:33.587755+0000","mapping_epoch":302,"log_start":"303'7345","ondisk_log_start":"303'7345","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:16.121133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85131264,"num_objects":1299,"num_object_clones":0,"num_object_copies":3897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12857,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2178,"num_bytes_recovered":142737408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-141}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":615,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569474+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:36:35.569474+0000","last_peered":"2023-09-30T09:36:35.569474+0000","last_clean":"2023-09-30T09:36:35.569474+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:36:35.569474+0000","last_undegraded":"2023-09-30T09:36:35.569474+0000","last_fullsized":"2023-09-30T09:36:35.569474+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":848,"reported_epoch":339,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:18.795419+0000","last_change":"2023-09-30T09:35:11.685841+0000","last_active":"2023-09-30T09:36:18.795419+0000","last_peered":"2023-09-30T09:36:18.795419+0000","last_clean":"2023-09-30T09:26:47.308941+0000","last_became_active":"2023-09-30T09:35:11.685274+0000","last_became_peered":"2023-09-30T09:35:11.685274+0000","last_unstale":"2023-09-30T09:36:18.795419+0000","last_undegraded":"2023-09-30T09:35:11.661789+0000","last_fullsized":"2023-09-30T09:35:11.661213+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:38.516737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[1,4],"avail_no_missing":["1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":66,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.627304+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:36:35.627304+0000","last_peered":"2023-09-30T09:36:35.627304+0000","last_clean":"2023-09-30T09:36:35.627304+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:36:35.627304+0000","last_undegraded":"2023-09-30T09:36:35.627304+0000","last_fullsized":"2023-09-30T09:36:35.627304+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":241,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621588+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:36:35.621588+0000","last_peered":"2023-09-30T09:36:35.621588+0000","last_clean":"2023-09-30T09:36:35.621588+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:36:35.621588+0000","last_undegraded":"2023-09-30T09:36:35.621588+0000","last_fullsized":"2023-09-30T09:36:35.621588+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"340'12972","reported_seq":13144,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:36:52.054861+0000","last_change":"2023-09-30T09:33:32.864041+0000","last_active":"2023-09-30T09:36:52.054861+0000","last_peered":"2023-09-30T09:36:52.054861+0000","last_clean":"2023-09-30T09:33:09.329441+0000","last_became_active":"2023-09-30T09:33:11.506402+0000","last_became_peered":"2023-09-30T09:33:11.506402+0000","last_unstale":"2023-09-30T09:36:52.054861+0000","last_undegraded":"2023-09-30T09:36:52.054861+0000","last_fullsized":"2023-09-30T09:36:52.054861+0000","mapping_epoch":295,"log_start":"311'8398","ondisk_log_start":"311'8398","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4574,"log_dups_size":0,"ondisk_log_size":4574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:51:06.485777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83361792,"num_objects":1272,"num_object_clones":0,"num_object_copies":3816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1264,"num_objects_unfound":0,"num_objects_dirty":1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12972,"num_write_kb":455808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1272}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":813,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568303+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:36:35.568303+0000","last_peered":"2023-09-30T09:36:35.568303+0000","last_clean":"2023-09-30T09:36:35.568303+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:36:35.568303+0000","last_undegraded":"2023-09-30T09:36:35.568303+0000","last_fullsized":"2023-09-30T09:36:35.568303+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":342,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:37.142433+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:36:37.142433+0000","last_peered":"2023-09-30T09:36:37.142433+0000","last_clean":"2023-09-30T09:36:37.142433+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:36:37.142433+0000","last_undegraded":"2023-09-30T09:36:37.142433+0000","last_fullsized":"2023-09-30T09:36:37.142433+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1107,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569258+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:36:35.569258+0000","last_peered":"2023-09-30T09:36:35.569258+0000","last_clean":"2023-09-30T09:36:35.569258+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:36:35.569258+0000","last_undegraded":"2023-09-30T09:36:35.569258+0000","last_fullsized":"2023-09-30T09:36:35.569258+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":358,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.521643+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:36:35.521643+0000","last_peered":"2023-09-30T09:36:35.521643+0000","last_clean":"2023-09-30T09:36:35.521643+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:36:35.521643+0000","last_undegraded":"2023-09-30T09:36:35.521643+0000","last_fullsized":"2023-09-30T09:36:35.521643+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"341'12885","reported_seq":17863,"reported_epoch":341,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:36:52.222259+0000","last_change":"2023-09-30T09:36:02.717804+0000","last_active":"2023-09-30T09:36:52.222259+0000","last_peered":"2023-09-30T09:36:52.222259+0000","last_clean":"2023-09-30T09:32:35.469896+0000","last_became_active":"2023-09-30T09:33:32.854592+0000","last_became_peered":"2023-09-30T09:33:32.854592+0000","last_unstale":"2023-09-30T09:36:52.222259+0000","last_undegraded":"2023-09-30T09:33:32.787590+0000","last_fullsized":"2023-09-30T09:33:32.601623+0000","mapping_epoch":301,"log_start":"309'8350","ondisk_log_start":"309'8350","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4535,"log_dups_size":0,"ondisk_log_size":4535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:02.368619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91684864,"num_objects":1399,"num_object_clones":0,"num_object_copies":4197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1159,"num_objects_misplaced":1159,"num_objects_unfound":0,"num_objects_dirty":1399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12885,"num_write_kb":457088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2440,"num_bytes_recovered":159907840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":240}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":807,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:37.142328+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:36:37.142328+0000","last_peered":"2023-09-30T09:36:37.142328+0000","last_clean":"2023-09-30T09:36:37.142328+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:36:37.142328+0000","last_undegraded":"2023-09-30T09:36:37.142328+0000","last_fullsized":"2023-09-30T09:36:37.142328+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":666,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569504+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:36:35.569504+0000","last_peered":"2023-09-30T09:36:35.569504+0000","last_clean":"2023-09-30T09:36:35.569504+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:36:35.569504+0000","last_undegraded":"2023-09-30T09:36:35.569504+0000","last_fullsized":"2023-09-30T09:36:35.569504+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":28,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621472+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:36:35.621472+0000","last_peered":"2023-09-30T09:36:35.621472+0000","last_clean":"2023-09-30T09:36:35.621472+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:36:35.621472+0000","last_undegraded":"2023-09-30T09:36:35.621472+0000","last_fullsized":"2023-09-30T09:36:35.621472+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":282,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569757+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:36:35.569757+0000","last_peered":"2023-09-30T09:36:35.569757+0000","last_clean":"2023-09-30T09:36:35.569757+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:36:35.569757+0000","last_undegraded":"2023-09-30T09:36:35.569757+0000","last_fullsized":"2023-09-30T09:36:35.569757+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":755,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:47.156743+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:36:47.156743+0000","last_peered":"2023-09-30T09:36:47.156743+0000","last_clean":"2023-09-30T09:36:47.156743+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:36:47.156743+0000","last_undegraded":"2023-09-30T09:36:47.156743+0000","last_fullsized":"2023-09-30T09:36:47.156743+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":24,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621540+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:36:35.621540+0000","last_peered":"2023-09-30T09:36:35.621540+0000","last_clean":"2023-09-30T09:36:35.621540+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:36:35.621540+0000","last_undegraded":"2023-09-30T09:36:35.621540+0000","last_fullsized":"2023-09-30T09:36:35.621540+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.627216+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:36:35.627216+0000","last_peered":"2023-09-30T09:36:35.627216+0000","last_clean":"2023-09-30T09:36:35.627216+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:36:35.627216+0000","last_undegraded":"2023-09-30T09:36:35.627216+0000","last_fullsized":"2023-09-30T09:36:35.627216+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":526,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.522387+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:36:35.522387+0000","last_peered":"2023-09-30T09:36:35.522387+0000","last_clean":"2023-09-30T09:36:35.522387+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:36:35.522387+0000","last_undegraded":"2023-09-30T09:36:35.522387+0000","last_fullsized":"2023-09-30T09:36:35.522387+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":319,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.571489+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:36:35.571489+0000","last_peered":"2023-09-30T09:36:35.571489+0000","last_clean":"2023-09-30T09:36:35.571489+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:36:35.571489+0000","last_undegraded":"2023-09-30T09:36:35.571489+0000","last_fullsized":"2023-09-30T09:36:35.571489+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":35,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.571030+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:36:35.571030+0000","last_peered":"2023-09-30T09:36:35.571030+0000","last_clean":"2023-09-30T09:36:35.571030+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:36:35.571030+0000","last_undegraded":"2023-09-30T09:36:35.571030+0000","last_fullsized":"2023-09-30T09:36:35.571030+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":285,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.568581+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:36:35.568581+0000","last_peered":"2023-09-30T09:36:35.568581+0000","last_clean":"2023-09-30T09:36:35.568581+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:36:35.568581+0000","last_undegraded":"2023-09-30T09:36:35.568581+0000","last_fullsized":"2023-09-30T09:36:35.568581+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":35,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.570217+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:36:35.570217+0000","last_peered":"2023-09-30T09:36:35.570217+0000","last_clean":"2023-09-30T09:36:35.570217+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:36:35.570217+0000","last_undegraded":"2023-09-30T09:36:35.570217+0000","last_fullsized":"2023-09-30T09:36:35.570217+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":469,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.570908+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:36:35.570908+0000","last_peered":"2023-09-30T09:36:35.570908+0000","last_clean":"2023-09-30T09:36:35.570908+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:36:35.570908+0000","last_undegraded":"2023-09-30T09:36:35.570908+0000","last_fullsized":"2023-09-30T09:36:35.570908+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":337,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.522633+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:36:35.522633+0000","last_peered":"2023-09-30T09:36:35.522633+0000","last_clean":"2023-09-30T09:36:35.522633+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:36:35.522633+0000","last_undegraded":"2023-09-30T09:36:35.522633+0000","last_fullsized":"2023-09-30T09:36:35.522633+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":362,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.521680+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:36:35.521680+0000","last_peered":"2023-09-30T09:36:35.521680+0000","last_clean":"2023-09-30T09:36:35.521680+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:36:35.521680+0000","last_undegraded":"2023-09-30T09:36:35.521680+0000","last_fullsized":"2023-09-30T09:36:35.521680+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":789,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.522789+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:36:35.522789+0000","last_peered":"2023-09-30T09:36:35.522789+0000","last_clean":"2023-09-30T09:36:35.522789+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:36:35.522789+0000","last_undegraded":"2023-09-30T09:36:35.522789+0000","last_fullsized":"2023-09-30T09:36:35.522789+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":44,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.621605+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:36:35.621605+0000","last_peered":"2023-09-30T09:36:35.621605+0000","last_clean":"2023-09-30T09:36:35.621605+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:36:35.621605+0000","last_undegraded":"2023-09-30T09:36:35.621605+0000","last_fullsized":"2023-09-30T09:36:35.621605+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":316,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-09-30T09:36:35.569331+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:36:35.569331+0000","last_peered":"2023-09-30T09:36:35.569331+0000","last_clean":"2023-09-30T09:36:35.569331+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:36:35.569331+0000","last_undegraded":"2023-09-30T09:36:35.569331+0000","last_fullsized":"2023-09-30T09:36:35.569331+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1370816536,"num_objects":20918,"num_object_clones":0,"num_object_copies":62754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11334,"num_objects_misplaced":18194,"num_objects_unfound":0,"num_objects_dirty":20918,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":205634,"num_write_kb":7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33616,"num_bytes_recovered":2125594624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3751227392,"data_stored":3751215176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82198,"ondisk_log_size":82198,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234303,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495940,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1140228,"kb_used_data":744700,"kb_used_omap":0,"kb_used_meta":395520,"kb_avail":92604924,"statfs":{"total":95995035648,"available":94827442176,"internally_reserved":0,"allocated":762572800,"data_stored":761386312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405012480},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9380000000000002,"5min":2.2330000000000001,"15min":1.802},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":44.113,"5min":44.113,"15min":45.927999999999997},"last":44.113},{"interface":"front","average":{"1min":4.6150000000000002,"5min":2.3639999999999999,"15min":2.0449999999999999},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":44.164000000000001,"5min":44.164000000000001,"15min":45.316000000000003},"last":44.164000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.0419999999999998,"15min":1.8819999999999999},"min":{"1min":0.39100000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.343999999999999,"5min":13.343999999999999,"15min":13.343999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":3.169,"5min":2.3300000000000001,"15min":2.1520000000000001},"min":{"1min":0.35999999999999999,"5min":0.313,"15min":0.313},"max":{"1min":18.425999999999998,"5min":18.425999999999998,"15min":18.425999999999998},"last":0.47799999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:36:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2210000000000001,"5min":2.226,"15min":1.905},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":17.126000000000001,"5min":22.158000000000001,"15min":37.978000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":3.7589999999999999,"5min":2.5110000000000001,"15min":1.851},"min":{"1min":0.32500000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":25.164999999999999,"5min":25.164999999999999,"15min":37.759999999999998},"last":0.53400000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7839999999999998,"5min":7.1130000000000004,"15min":3.532},"min":{"1min":0.45300000000000001,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":90.147999999999996,"5min":212.44800000000001,"15min":212.44800000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":8.9450000000000003,"5min":7.8339999999999996,"15min":3.9199999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":89.859999999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":0.56100000000000005}]},{"osd":4,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.0640000000000001,"15min":1.593},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23400000000000001},"max":{"1min":3.004,"5min":7.4359999999999999,"15min":73.656000000000006},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.0209999999999999,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.23000000000000001},"max":{"1min":1.8089999999999999,"5min":9.6590000000000007,"15min":73.882999999999996},"last":0.499}]},{"osd":5,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.1930000000000001,"15min":1.819},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":1.024,"5min":8.2040000000000006,"15min":76.613},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.1910000000000001,"15min":1.786},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.0979999999999999,"5min":9.7750000000000004,"15min":76.329999999999998},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":1.458,"15min":2.1419999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":3.9870000000000001,"5min":8.7400000000000002,"15min":91.120999999999995},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.1679999999999999,"15min":2.0910000000000002},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.4870000000000001,"5min":9.9489999999999998,"15min":91.733999999999995},"last":0.56899999999999995}]}]},{"osd":1,"up_from":301,"seq":1292785156137,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329588,"kb_used_data":109676,"kb_used_omap":9,"kb_used_meta":219638,"kb_avail":93415564,"statfs":{"total":95995035648,"available":95657537536,"internally_reserved":0,"allocated":112308224,"data_stored":110380059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":224909330},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":9.7590000000000003,"5min":4.0979999999999999,"15min":3.0600000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":138.43799999999999,"5min":138.43799999999999,"15min":138.43799999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":9.4320000000000004,"5min":4.5140000000000002,"15min":3.52},"min":{"1min":0.27800000000000002,"5min":0.25,"15min":0.25},"max":{"1min":138.482,"5min":138.482,"15min":138.482},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.8100000000000001,"15min":2.7999999999999998},"min":{"1min":0.254,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":13.656000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":6.2670000000000003},{"interface":"front","average":{"1min":3.9609999999999999,"5min":3.0899999999999999,"15min":2.8340000000000001},"min":{"1min":0.309,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":18.978999999999999,"5min":18.978999999999999,"15min":18.978999999999999},"last":0.311}]},{"osd":3,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":6.8879999999999999,"5min":7.9370000000000003,"15min":8.9429999999999996},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":40.606000000000002,"5min":77.959000000000003,"15min":77.959000000000003},"last":0.71099999999999997},{"interface":"front","average":{"1min":7.5449999999999999,"5min":8.2690000000000001,"15min":9.5410000000000004},"min":{"1min":0.439,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":41.247,"5min":77.828999999999994,"15min":77.828999999999994},"last":2.548}]},{"osd":4,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.1059999999999999,"15min":2.0259999999999998},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.0090000000000003,"5min":13.602,"15min":13.602},"last":0.375},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.712,"15min":1.4319999999999999},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.311},"max":{"1min":8.6440000000000001,"5min":10.773,"15min":10.773},"last":1.0880000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":2.351,"15min":2.1890000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.2750000000000004,"5min":13.666,"15min":13.666},"last":1.159},{"interface":"front","average":{"1min":2.9239999999999999,"5min":1.9890000000000001,"15min":1.73},"min":{"1min":0.39100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.177,"5min":10.847,"15min":10.847},"last":0.59899999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":11.52,"15min":15.238},"min":{"1min":0.503,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.1029999999999998,"5min":209.221,"15min":209.221},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.48,"5min":11.207000000000001,"15min":15.079000000000001},"min":{"1min":0.40000000000000002,"5min":0.377,"15min":0.377},"max":{"1min":8.1310000000000002,"5min":208.43600000000001,"15min":208.43600000000001},"last":0.92900000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9950000000000001,"5min":8.3640000000000008,"15min":11.084},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.3559999999999999,"5min":165.59399999999999,"15min":165.59399999999999},"last":2.4740000000000002},{"interface":"front","average":{"1min":3.3719999999999999,"5min":8.5670000000000002,"15min":11.308},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":10.565,"5min":165.833,"15min":165.833},"last":2.512}]}]},{"osd":0,"up_from":94,"seq":403726926037,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":589332,"kb_used_data":248268,"kb_used_omap":5,"kb_used_meta":341050,"kb_avail":93155820,"statfs":{"total":95995035648,"available":95391559680,"internally_reserved":0,"allocated":254226432,"data_stored":252160888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":349235453},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:36:38 2023","interfaces":[{"interface":"back","average":{"1min":4.1920000000000002,"5min":3.0630000000000002,"15min":3.0190000000000001},"min":{"1min":0.24099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":15.401,"5min":17.966000000000001,"15min":17.966000000000001},"last":1.1319999999999999},{"interface":"front","average":{"1min":6.3339999999999996,"5min":7.7549999999999999,"15min":8.8680000000000003},"min":{"1min":0.31,"5min":0.22,"15min":0.22},"max":{"1min":25.869,"5min":138.92400000000001,"15min":138.92400000000001},"last":0.83799999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.1299999999999999,"15min":1.744},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.887,"5min":11.218999999999999,"15min":55.823999999999998},"last":1.024},{"interface":"front","average":{"1min":3.2080000000000002,"5min":2.5350000000000001,"15min":1.9319999999999999},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":10.763,"5min":40.128999999999998,"15min":55.790999999999997},"last":0.70699999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":8.5350000000000001,"15min":4.2089999999999996},"min":{"1min":0.30599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":16.271000000000001,"5min":218.06999999999999,"15min":218.06999999999999},"last":2.8559999999999999},{"interface":"front","average":{"1min":2.8199999999999998,"5min":6.2930000000000001,"15min":3.4159999999999999},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":14.987,"5min":218.63200000000001,"15min":218.63200000000001},"last":1.629}]},{"osd":4,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.6599999999999999,"15min":1.679},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.29099999999999998},"max":{"1min":10.614000000000001,"5min":10.614000000000001,"15min":121.486},"last":0.749},{"interface":"front","average":{"1min":3.3780000000000001,"5min":3.863,"15min":2.649},"min":{"1min":0.39900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":17.044,"5min":136.262,"15min":136.262},"last":2.8279999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":4.1710000000000003,"15min":3.0630000000000002},"min":{"1min":0.439,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.995999999999999,"5min":136.34,"15min":136.34},"last":2.79},{"interface":"front","average":{"1min":2.4129999999999998,"5min":1.9419999999999999,"15min":2.2839999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":11.991,"5min":12.717000000000001,"15min":135.381},"last":0.91300000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":7.1440000000000001,"15min":3.2400000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":12.021000000000001,"5min":214.785,"15min":214.785},"last":0.88},{"interface":"front","average":{"1min":2.8999999999999999,"5min":7.2439999999999998,"15min":3.1850000000000001},"min":{"1min":0.46300000000000002,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":10.695,"5min":214.93899999999999,"15min":214.93899999999999},"last":0.78200000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":2.9870000000000001,"15min":3.1720000000000002},"min":{"1min":0.42999999999999999,"5min":0.38300000000000001,"15min":0.373},"max":{"1min":12.048,"5min":97.382000000000005,"15min":150.96899999999999},"last":2.7559999999999998},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.944,"15min":3.0150000000000001},"min":{"1min":0.43099999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.153,"5min":97.015000000000001,"15min":150.328},"last":0.81200000000000006}]}]},{"osd":2,"up_from":104,"seq":446676598978,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":348996,"kb_used_data":46904,"kb_used_omap":5,"kb_used_meta":302074,"kb_avail":93396156,"statfs":{"total":95995035648,"available":95637663744,"internally_reserved":0,"allocated":48029696,"data_stored":46159931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":309324429},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":4.1130000000000004,"5min":3.4630000000000001,"15min":2.1680000000000001},"min":{"1min":0.24199999999999999,"5min":0.187,"15min":0.187},"max":{"1min":15.063000000000001,"5min":77.379000000000005,"15min":77.379000000000005},"last":0.79400000000000004},{"interface":"front","average":{"1min":2.7509999999999999,"5min":3.2709999999999999,"15min":1.9670000000000001},"min":{"1min":0.31900000000000001,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":15.151,"5min":77.066999999999993,"15min":77.066999999999993},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:36:36 2023","interfaces":[{"interface":"back","average":{"1min":8.9789999999999992,"5min":4.8339999999999996,"15min":3.992},"min":{"1min":0.36399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":64.412999999999997,"5min":64.412999999999997,"15min":64.412999999999997},"last":5.0590000000000002},{"interface":"front","average":{"1min":7.6420000000000003,"5min":4.1260000000000003,"15min":3.4129999999999998},"min":{"1min":0.33700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":63.247999999999998,"5min":63.247999999999998,"15min":63.247999999999998},"last":0.318}]},{"osd":3,"last update":"Sat Sep 30 09:36:51 2023","interfaces":[{"interface":"back","average":{"1min":15.153,"5min":7.4130000000000003,"15min":3.4689999999999999},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":134.078,"5min":134.078,"15min":134.078},"last":0.46300000000000002},{"interface":"front","average":{"1min":15.754,"5min":7.0709999999999997,"15min":3.5920000000000001},"min":{"1min":0.38900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":135.73400000000001,"5min":135.73400000000001,"15min":135.73400000000001},"last":0.55900000000000005}]},{"osd":4,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":2.036,"15min":1.3380000000000001},"min":{"1min":0.39000000000000001,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":8.1240000000000006,"5min":35.972000000000001,"15min":35.972000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.931,"15min":1.597},"min":{"1min":0.42899999999999999,"5min":0.33800000000000002,"15min":0.32700000000000001},"max":{"1min":13.224,"5min":36.518000000000001,"15min":36.518000000000001},"last":1.2789999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.8610000000000002,"5min":2.024,"15min":2.0880000000000001},"min":{"1min":0.39400000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.41,"5min":12.651999999999999,"15min":155.477},"last":5.0949999999999998},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.591,"15min":1.952},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":8.0839999999999996,"5min":8.0839999999999996,"15min":155.46000000000001},"last":4.9450000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":6.6180000000000003,"15min":3.2879999999999998},"min":{"1min":0.48199999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.273,"5min":154.417,"15min":154.417},"last":4.9960000000000004},{"interface":"front","average":{"1min":2.8359999999999999,"5min":7.2729999999999997,"15min":3.3399999999999999},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":10.382,"5min":154.04400000000001,"15min":154.04400000000001},"last":5.1200000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":3.8639999999999999,"15min":3.1480000000000001},"min":{"1min":0.52800000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":13.304,"5min":123.294,"15min":213.404},"last":5.0229999999999997},{"interface":"front","average":{"1min":2.9390000000000001,"5min":4.1269999999999998,"15min":3.1699999999999999},"min":{"1min":0.45100000000000001,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":10.35,"5min":123.16,"15min":213.27199999999999},"last":5.9690000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435450,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1592268,"kb_used_data":948928,"kb_used_omap":2,"kb_used_meta":643325,"kb_avail":92152884,"statfs":{"total":95995035648,"available":94364553216,"internally_reserved":0,"allocated":971702272,"data_stored":969701754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":658765159},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.8040000000000003,"5min":4.1310000000000002,"15min":2.7559999999999998},"min":{"1min":0.23999999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":60.726999999999997,"5min":102.57299999999999,"15min":130.946},"last":6.0030000000000001},{"interface":"front","average":{"1min":3.9049999999999998,"5min":3.9729999999999999,"15min":2.7069999999999999},"min":{"1min":0.34599999999999997,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":12.477,"5min":102.745,"15min":130.96899999999999},"last":2.956}]},{"osd":1,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5489999999999999,"5min":14.395,"15min":18.975999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":9.3919999999999995,"5min":273.99599999999998,"15min":273.99599999999998},"last":3.0720000000000001},{"interface":"front","average":{"1min":4.4950000000000001,"5min":13.355,"15min":17.132999999999999},"min":{"1min":0.34899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.6910000000000007,"5min":274.21600000000001,"15min":274.21600000000001},"last":7.1600000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":4.7089999999999996,"5min":3.4580000000000002,"15min":2.9510000000000001},"min":{"1min":0.50800000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.339,"5min":76.974000000000004,"15min":145.75899999999999},"last":1.3480000000000001},{"interface":"front","average":{"1min":5.6070000000000002,"5min":3.415,"15min":2.8410000000000002},"min":{"1min":0.35699999999999998,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":60.561999999999998,"5min":77.593999999999994,"15min":150.358},"last":1.8120000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.282,"5min":2.5979999999999999,"15min":1.5700000000000001},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":60.637,"5min":60.637,"15min":60.637},"last":1.954},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.9990000000000001,"15min":1.4670000000000001},"min":{"1min":0.441,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":7.7990000000000004,"5min":10.731,"15min":15.638},"last":3.0169999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.629,"15min":1.74},"min":{"1min":0.41499999999999998,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":7.7519999999999998,"5min":37.485999999999997,"15min":37.485999999999997},"last":3.044},{"interface":"front","average":{"1min":3.3300000000000001,"5min":3.044,"15min":1.9650000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":8.5980000000000008,"5min":37.448,"15min":37.448},"last":1.2410000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":5.5650000000000004,"5min":2.6960000000000002,"15min":1.6659999999999999},"min":{"1min":0.36799999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":60.698,"5min":60.698,"15min":60.698},"last":1.887},{"interface":"front","average":{"1min":2.778,"5min":2.3090000000000002,"15min":1.6859999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":9.5860000000000003,"5min":10.818,"15min":14.69},"last":2.9860000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:36:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5590000000000002,"5min":2.855,"15min":2.573},"min":{"1min":0.39100000000000001,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":8.6660000000000004,"5min":26.469000000000001,"15min":226.03999999999999},"last":1.486},{"interface":"front","average":{"1min":5.8849999999999998,"5min":3.008,"15min":2.4580000000000002},"min":{"1min":0.45400000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":60.670000000000002,"5min":60.670000000000002,"15min":226.06100000000001},"last":1.921}]}]},{"osd":4,"up_from":30,"seq":128849019213,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":589756,"kb_used_data":332532,"kb_used_omap":0,"kb_used_meta":257216,"kb_avail":93155396,"statfs":{"total":95995035648,"available":95391125504,"internally_reserved":0,"allocated":340512768,"data_stored":339333975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":263389184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":1.4399999999999999,"15min":2.8780000000000001},"min":{"1min":0.33200000000000002,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":4.5940000000000003,"5min":14.353,"15min":128.79300000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.873,"15min":2.8730000000000002},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.248},"max":{"1min":6.9939999999999998,"5min":10.618,"15min":128.38800000000001},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4169999999999998,"5min":6.8659999999999997,"15min":8.6240000000000006},"min":{"1min":0.26600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.915,"5min":160.69999999999999,"15min":160.69999999999999},"last":4.931},{"interface":"front","average":{"1min":3.2629999999999999,"5min":6.7960000000000003,"15min":8.6419999999999995},"min":{"1min":0.28399999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.7029999999999994,"5min":160.50700000000001,"15min":160.50700000000001},"last":2.0129999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.9850000000000001,"15min":2.9399999999999999},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9790000000000001,"5min":19.391999999999999,"15min":140.601},"last":2.7280000000000002},{"interface":"front","average":{"1min":2.3399999999999999,"5min":2.0169999999999999,"15min":2.9300000000000002},"min":{"1min":0.39300000000000002,"5min":0.309,"15min":0.309},"max":{"1min":4.9900000000000002,"5min":19.422999999999998,"15min":140.54900000000001},"last":2.7589999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:36:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3010000000000002,"5min":2.964,"15min":3.6030000000000002},"min":{"1min":0.42799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.2650000000000006,"5min":52.762,"15min":281.42200000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":3.4359999999999999,"5min":2.992,"15min":3.5720000000000001},"min":{"1min":0.49099999999999999,"5min":0.312,"15min":0.312},"max":{"1min":9.2810000000000006,"5min":52.790999999999997,"15min":281.44999999999999},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.94399999999999995,"15min":1.7130000000000001},"min":{"1min":0.309,"5min":0.219,"15min":0.219},"max":{"1min":4.6120000000000001,"5min":8.6509999999999998,"15min":134.30600000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1,"15min":1.772},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":2.2309999999999999,"5min":6.3929999999999998,"15min":134.35300000000001},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.9409999999999998,"15min":1.522},"min":{"1min":0.32500000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.609,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.927,"15min":1.575},"min":{"1min":0.312,"5min":0.249,"15min":0.249},"max":{"1min":3.964,"5min":202.42400000000001,"15min":202.42400000000001},"last":0.59899999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":4.4320000000000004,"15min":3.004},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":3.8330000000000002,"5min":230.90100000000001,"15min":230.90100000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":4.1289999999999996,"15min":2.5579999999999998},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":2.6779999999999999,"5min":229.97,"15min":229.97},"last":0.58299999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757577,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624444,"kb_used_data":343796,"kb_used_omap":0,"kb_used_meta":280640,"kb_avail":93120708,"statfs":{"total":95995035648,"available":95355604992,"internally_reserved":0,"allocated":352047104,"data_stored":350864859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":287375360},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.2069999999999999,"15min":2.4540000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.22},"max":{"1min":4.6459999999999999,"5min":56.563000000000002,"15min":148.79900000000001},"last":1.851},{"interface":"front","average":{"1min":1.738,"5min":2.3199999999999998,"15min":2.7589999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":6.2969999999999997,"5min":57.241999999999997,"15min":148.779},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:36:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":2.6619999999999999,"15min":2.1120000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.237,"5min":60.799999999999997,"15min":60.799999999999997},"last":1.4430000000000001},{"interface":"front","average":{"1min":3.012,"5min":3.581,"15min":3.3010000000000002},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.2539999999999996,"5min":63.798999999999999,"15min":63.798999999999999},"last":1.488}]},{"osd":2,"last update":"Sat Sep 30 09:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.8879999999999999,"15min":3.1150000000000002},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.71,"5min":30.613,"15min":165.768},"last":1.038},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.0249999999999999,"15min":3.6030000000000002},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.1989999999999998,"5min":36.488,"15min":166.24000000000001},"last":1.133}]},{"osd":3,"last update":"Sat Sep 30 09:35:53 2023","interfaces":[{"interface":"back","average":{"1min":5.1660000000000004,"5min":8.2040000000000006,"15min":3.8610000000000002},"min":{"1min":0.39200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":38.280000000000001,"5min":226.34700000000001,"15min":226.34700000000001},"last":1.869},{"interface":"front","average":{"1min":4.9379999999999997,"5min":8.1809999999999992,"15min":3.7080000000000002},"min":{"1min":0.39100000000000001,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":38.262999999999998,"5min":226.36000000000001,"15min":226.36000000000001},"last":1.8360000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:36:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.76500000000000001,"15min":0.84899999999999998},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":1.0760000000000001,"5min":5.7160000000000002,"15min":8.5359999999999996},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.96999999999999997,"15min":0.94199999999999995},"min":{"1min":0.29599999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.1230000000000002,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.40500000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":3.1749999999999998,"15min":1.7390000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":4.6639999999999997,"5min":155.602,"15min":155.602},"last":0.434},{"interface":"front","average":{"1min":1.014,"5min":3.2309999999999999,"15min":1.8680000000000001},"min":{"1min":0.36499999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.1600000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":2.5630000000000002,"15min":1.639},"min":{"1min":0.33000000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":3.2000000000000002,"5min":133.40700000000001,"15min":133.40700000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":2.6549999999999998,"15min":1.5609999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":2.274,"5min":132.797,"15min":132.797},"last":0.49199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":241242112,"data_stored":241238040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":102305792,"data_stored":102301720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":42270720,"data_stored":42270720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":961609728,"data_stored":961609728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":336396288,"data_stored":336396288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":342949888,"data_stored":342949888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":755568640,"data_stored":755564568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":968884224,"data_stored":968884224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:36:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:58 smithi167 ceph-mon[192880]: pgmap v881: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 10705/60081 objects degraded (17.818%); 17295/60081 objects misplaced (28.786%); 15 MiB/s, 337 objects/s recovering 2023-09-30T09:36:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:36:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 10705/60081 objects degraded (17.818%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:36:59.037 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:36:59.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:59.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:36:59.153 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:36:59.234 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:59.331 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:36:59.392 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:36:59.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:59.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:36:59.501 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:59.559 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:59.652 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:36:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:36:59.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:36:59.757 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:36:59.804 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:36:59.859 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:36:59.859 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:36:59.859 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:36:59.859 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:36:59.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.860 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71271 6.e 6:780eb166:::benchmark_data_smithi019_355417_object71270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:36:59.860 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:36:59.860 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 453.84808590699998, 2023-09-30T09:36:59.860 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.228409374, 2023-09-30T09:36:59.860 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.860 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.860 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.861 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.861 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.861 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71271 2023-09-30T09:36:59.861 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.861 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.861 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006370+0000", 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006369+0000", 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006424+0000", 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5114000000000001e-05 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006439+0000", 2023-09-30T09:36:59.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5241e-05 2023-09-30T09:36:59.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006447+0000", 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6529999999999994e-06 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006492+0000", 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5022999999999999e-05 2023-09-30T09:36:59.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006590+0000", 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7985000000000005e-05 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.006695+0000", 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000104984 2023-09-30T09:36:59.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.010237+0000", 2023-09-30T09:36:59.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003542271 2023-09-30T09:36:59.868 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.868 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.869 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.013024+0000", 2023-09-30T09:36:59.869 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027868979999999999 2023-09-30T09:36:59.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.231353+0000", 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.218328895 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.231406+0000", 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3362000000000002e-05 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.870 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234780+0000", 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003373572 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71277 6.e 6:746a3c45:::benchmark_data_smithi019_355417_object71276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 453.84700415999998, 2023-09-30T09:36:59.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.230281654, 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71277 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007452+0000", 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007451+0000", 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007503+0000", 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2033999999999997e-05 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007515+0000", 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1418e-05 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.007520+0000", 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.938e-06 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008550+0000", 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010299840000000001 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008653+0000", 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103688 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.008752+0000", 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8879e-05 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.010372+0000", 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016195960000000001 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.013054+0000", 2023-09-30T09:36:59.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026824940000000001 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234915+0000", 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22186042 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.234978+0000", 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3447000000000004e-05 2023-09-30T09:36:59.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237734+0000", 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027555909999999999 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71282 6.e 6:707e64e4:::benchmark_data_smithi019_355417_object71281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:36:59.879 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 453.838792308, 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22500474600000001, 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71282 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015664+0000", 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015663+0000", 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:36:59.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015715+0000", 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1873999999999998e-05 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015725+0000", 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0387000000000001e-05 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015730+0000", 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.814e-06 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015768+0000", 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7673000000000003e-05 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015871+0000", 2023-09-30T09:36:59.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103274 2023-09-30T09:36:59.884 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.885 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.885 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.015979+0000", 2023-09-30T09:36:59.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108563 2023-09-30T09:36:59.885 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.885 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.885 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.018931+0000", 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029516809999999998 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020250+0000", 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013193289999999999 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237789+0000", 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21753858700000001 2023-09-30T09:36:59.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.237850+0000", 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0448999999999997e-05 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240669+0000", 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028193039999999999 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.887 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71283 6.e 6:7093bd40:::benchmark_data_smithi019_355417_object71282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 453.83845203499999, 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22746534299999999, 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.888 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71283 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.890 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016004+0000", 2023-09-30T09:36:59.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016074+0000", 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0246999999999993e-05 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016082+0000", 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4610000000000002e-06 2023-09-30T09:36:59.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.892 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.892 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016085+0000", 2023-09-30T09:36:59.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5489999999999998e-06 2023-09-30T09:36:59.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.892 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.892 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017714+0000", 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016290549999999999 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017801+0000", 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7076000000000005e-05 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.017904+0000", 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000102876 2023-09-30T09:36:59.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020212+0000", 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023077570000000001 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.022334+0000", 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021222160000000001 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240736+0000", 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.218401652 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.240791+0000", 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.554e-05 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243470+0000", 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026784019999999999 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:71284 6.e 6:7ce92b46:::benchmark_data_smithi019_355417_object71283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 453.83796365199998, 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22906620999999999, 2023-09-30T09:36:59.896 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 71284 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016493+0000", 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016492+0000", 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:36:59.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016557+0000", 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4941e-05 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016565+0000", 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3669999999999997e-06 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.016570+0000", 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2390000000000004e-06 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.018977+0000", 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024078609999999999 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.019059+0000", 2023-09-30T09:36:59.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1386999999999999e-05 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.019149+0000", 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9938000000000005e-05 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.020349+0000", 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012002530000000001 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.022546+0000", 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021970819999999999 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243549+0000", 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22100276599999999 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.243617+0000", 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8547999999999995e-05 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:26.245559+0000", 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019414879999999999 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.903 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:96725 6.e 6:743dbf6c:::benchmark_data_smithi019_355417_object4421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 423.37127600600002, 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.205057722, 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.904 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 96725 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483180+0000", 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483179+0000", 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483187+0000", 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3750000000000003e-06 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.906 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483203+0000", 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6215999999999999e-05 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483210+0000", 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0180000000000004e-06 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.483254+0000", 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3426000000000001e-05 2023-09-30T09:36:59.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.485269+0000", 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002015644 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488763+0000", 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034939120000000001 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488819+0000", 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6135999999999999e-05 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.488893+0000", 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3734999999999998e-05 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.490570+0000", 2023-09-30T09:36:59.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001677015 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.490591+0000", 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0973000000000001e-05 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.686691+0000", 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.196100203 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.686739+0000", 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7395000000000003e-05 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:56.688238+0000", 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001499301 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:98164 6.1e 6:7b1cf70c:::benchmark_data_smithi019_355417_object5860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 421.79390774000001, 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21486691799999999, 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.912 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 98164 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060549+0000", 2023-09-30T09:36:59.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060547+0000", 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060554+0000", 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4069999999999998e-06 2023-09-30T09:36:59.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060565+0000", 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1336e-05 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060571+0000", 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2219999999999999e-06 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.074775+0000", 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014203604999999999 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.075990+0000", 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012149319999999999 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.092854+0000", 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016863878999999998 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.092955+0000", 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101431 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.093073+0000", 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000117923 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.097188+0000", 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041146919999999997 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.105705+0000", 2023-09-30T09:36:59.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0085167469999999999 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.273965+0000", 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16826055000000001 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.274028+0000", 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2502999999999994e-05 2023-09-30T09:36:59.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.275416+0000", 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013878479999999999 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:98166 6.e 6:75ad8793:::benchmark_data_smithi019_355417_object5862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:36:59.920 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 421.79375422599998, 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.214711604, 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 98166 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.921 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060702+0000", 2023-09-30T09:36:59.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060701+0000", 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060712+0000", 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0445000000000001e-05 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060723+0000", 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1114999999999999e-05 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.060730+0000", 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1130000000000003e-06 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.076100+0000", 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015369651999999999 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.076669+0000", 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00056970499999999999 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095225+0000", 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.018555250999999998 2023-09-30T09:36:59.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095303+0000", 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8380999999999994e-05 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.095431+0000", 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012828399999999999 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.098950+0000", 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035189520000000001 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.105689+0000", 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0067384180000000004 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.927 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.273979+0000", 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.168290369 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.274032+0000", 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2456000000000001e-05 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:29:58.275414+0000", 2023-09-30T09:36:59.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001382203 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:99939 6.1e 6:78215d60:::benchmark_data_smithi019_355417_object7635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 419.336264728, 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.201779021, 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.929 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 99939 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518192+0000", 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518190+0000", 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518198+0000", 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3209999999999999e-06 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518211+0000", 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3689000000000001e-05 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518219+0000", 2023-09-30T09:36:59.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0320000000000003e-06 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518265+0000", 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5704999999999997e-05 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.518919+0000", 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00065372300000000004 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.707844+0000", 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.18892562099999999 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.707924+0000", 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0043999999999995e-05 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.934 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.708033+0000", 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108694 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.711930+0000", 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003896708 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.714999+0000", 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030689319999999999 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.718299+0000", 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033004290000000001 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.718348+0000", 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8959999999999999e-05 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.719971+0000", 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001622472 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:99940 6.e 6:70189e37:::benchmark_data_smithi019_355417_object7636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:36:59.937 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 419.33441935500002, 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20653424000000001, 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 99940 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.938 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520037+0000", 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520036+0000", 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:36:59.939 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520043+0000", 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9550000000000003e-06 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520056+0000", 2023-09-30T09:36:59.940 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3206e-05 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520061+0000", 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.242e-06 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520098+0000", 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6636e-05 2023-09-30T09:36:59.941 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.520966+0000", 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00086822099999999997 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716327+0000", 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.19536082900000001 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716416+0000", 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9463999999999996e-05 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.716529+0000", 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112768 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.721106+0000", 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004577378 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.723760+0000", 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026536900000000002 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.725227+0000", 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001467199 2023-09-30T09:36:59.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.725281+0000", 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3758999999999998e-05 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:00.726571+0000", 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001290177 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.945 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:101192 6.e 6:74a2e37f:::benchmark_data_smithi019_355417_object8888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 417.50488527099998, 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.198305908, 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.946 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101192 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349571+0000", 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349570+0000", 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349577+0000", 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1400000000000002e-06 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349594+0000", 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7044000000000001e-05 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349601+0000", 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3339999999999996e-06 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.949 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.349774+0000", 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017279799999999999 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.350391+0000", 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00061680600000000002 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534413+0000", 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.184022306 2023-09-30T09:36:59.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534506+0000", 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3183000000000006e-05 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.534635+0000", 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012914900000000001 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.537971+0000", 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033357170000000002 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.540045+0000", 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020742669999999999 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.546502+0000", 2023-09-30T09:36:59.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006457043 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.546557+0000", 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4824999999999998e-05 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:02.547877+0000", 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013197580000000001 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.953 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107544 6.e 6:73fbd947:::benchmark_data_smithi019_355417_object15240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 408.12090731199999, 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26128712500000001, 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.954 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107544 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733549+0000", 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733548+0000", 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.956 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733555+0000", 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6210000000000004e-06 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733572+0000", 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7017999999999999e-05 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733580+0000", 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1799999999999996e-06 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.733633+0000", 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2781999999999997e-05 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734357+0000", 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00072381799999999999 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739005+0000", 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046474890000000003 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739086+0000", 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1181999999999996e-05 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.739168+0000", 2023-09-30T09:36:59.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2346000000000001e-05 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745745+0000", 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0065768140000000003 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747298+0000", 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001552994 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.960 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.993625+0000", 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.24632679199999999 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.993663+0000", 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8668000000000003e-05 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994836+0000", 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001172841 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107545 6.e 6:77969f61:::benchmark_data_smithi019_355417_object15241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 408.11971207900001, 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26112492199999998, 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.962 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107545 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.963 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734744+0000", 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734751+0000", 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9609999999999999e-06 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734761+0000", 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0096e-05 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734765+0000", 2023-09-30T09:36:59.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5199999999999999e-06 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.734813+0000", 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7290000000000003e-05 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735419+0000", 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00060657400000000002 2023-09-30T09:36:59.966 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742024+0000", 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0066044190000000003 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742129+0000", 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010515599999999999 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.742280+0000", 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015174899999999999 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747335+0000", 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050547270000000002 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747342+0000", 2023-09-30T09:36:59.968 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4069999999999998e-06 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994874+0000", 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.24753244799999999 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.994933+0000", 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8959000000000003e-05 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995869+0000", 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00093624799999999998 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35130.0:107547 6.e 6:7322664d:::benchmark_data_smithi019_355417_object15243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:36:59.970 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 408.118903617, 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26146208700000001, 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35130", 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/369169467", 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 107547 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735553+0000", 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735552+0000", 2023-09-30T09:36:59.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735558+0000", 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6540000000000004e-06 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735569+0000", 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1029000000000001e-05 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735575+0000", 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4320000000000004e-06 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.735624+0000", 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9038e-05 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:36:59.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.736267+0000", 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00064271499999999997 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745778+0000", 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0095116769999999996 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745840+0000", 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1790000000000003e-05 2023-09-30T09:36:59.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.745925+0000", 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5045999999999999e-05 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.747718+0000", 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017924869999999999 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.748063+0000", 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00034562999999999998 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995923+0000", 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.247859211 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.995976+0000", 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3766e-05 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:30:11.997015+0000", 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001038436 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.978 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:120666 7.8 e304/295 7:1b3e6805:::benchmark_data_smithi019_404601_object7388:head v 304'7632, mlcod=304'7630)", 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:33:46.833857+0000", 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 193.02059985099999, 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.017603285999999999, 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833857+0000", 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833857+0000", 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833856+0000", 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833865+0000", 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4949999999999997e-06 2023-09-30T09:36:59.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833880+0000", 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5274e-05 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.833887+0000", 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4250000000000003e-06 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.836188+0000", 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023008460000000001 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.836216+0000", 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8377e-05 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.983 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.851424+0000", 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015207729 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:33:46.851460+0000", 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5960000000000001e-05 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.984 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:170992 7.3 e335/295 7:c05ea440:::benchmark_data_smithi019_404601_object57714:head v 335'10777, mlcod=335'10775)", 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:35:31.539730+0000", 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 88.314726886000003, 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.093789162999999995, 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.539730+0000", 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.985 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.539730+0000", 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.539729+0000", 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.539737+0000", 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6999999999999997e-06 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.539751+0000", 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3706999999999999e-05 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.539756+0000", 2023-09-30T09:36:59.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2190000000000003e-06 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.539799+0000", 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3071000000000001e-05 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.539816+0000", 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7017e-05 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.988 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.541694+0000", 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018772870000000001 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633519+0000", 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.091825076000000005 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.989 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:170994 7.1 e335/316 7:859557a5:::benchmark_data_smithi019_404601_object57716:head v 335'10625, mlcod=0'0)", 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:35:31.544285+0000", 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 88.310171866000005, 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.094530529000000002, 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.990 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.544285+0000", 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.544285+0000", 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.544283+0000", 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.544297+0000", 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3465999999999999e-05 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633501+0000", 2023-09-30T09:36:59.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.089204653999999994 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633511+0000", 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9080000000000007e-06 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633550+0000", 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8603999999999998e-05 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633572+0000", 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2396999999999999e-05 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.638780+0000", 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005207499 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.638815+0000", 2023-09-30T09:36:59.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5410999999999997e-05 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:171001 7.b e335/321 7:dd22701b:::benchmark_data_smithi019_404601_object57723:head v 335'10475, mlcod=292'5339)", 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:35:31.558907+0000", 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 88.295549627, 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.077442917, 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:36:59.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.558907+0000", 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.558907+0000", 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:36:59.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.558906+0000", 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.558916+0000", 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8789999999999998e-06 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633486+0000", 2023-09-30T09:36:59.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.074570337 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633499+0000", 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2558000000000001e-05 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633552+0000", 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3766e-05 2023-09-30T09:36:59.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.633576+0000", 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4023999999999999e-05 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.636307+0000", 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027309700000000001 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:36:59.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:35:31.636350+0000", 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2373999999999998e-05 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:207428 7.8 e341/308 7:1b6ae993:::benchmark_data_smithi019_404601_object94150:head v 341'13082, mlcod=341'13080)", 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:36:57.600303+0000", 2023-09-30T09:37:00.000 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.2541538280000002, 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014920108, 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.600303+0000", 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.001 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.600303+0000", 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.600302+0000", 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.600312+0000", 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0114e-05 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.600325+0000", 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3353000000000001e-05 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.600333+0000", 2023-09-30T09:37:00.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6869999999999994e-06 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.600414+0000", 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1167000000000004e-05 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.600445+0000", 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1291999999999998e-05 2023-09-30T09:37:00.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.615167+0000", 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01472129 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.615223+0000", 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6059e-05 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:00.005 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:207417 7.9 e341/327 7:93d02fe6:::benchmark_data_smithi019_404601_object94139:head v 341'12781, mlcod=0'0)", 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:36:57.601987+0000", 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.2524689250000001, 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013459422, 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.006 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.601987+0000", 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.601987+0000", 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.601987+0000", 2023-09-30T09:37:00.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.601996+0000", 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1710000000000008e-06 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.602011+0000", 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5449999999999999e-05 2023-09-30T09:37:00.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.602018+0000", 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4189999999999999e-06 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.602107+0000", 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8855000000000002e-05 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:00.009 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.602143+0000", 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6427000000000002e-05 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.615414+0000", 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013271275000000001 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:57.615447+0000", 2023-09-30T09:37:00.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2601000000000002e-05 2023-09-30T09:37:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:00.011 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:00.011 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:37:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:00.012 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:00.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:00.113 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:00.157 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:00.232 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:00.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:00.337 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:00.377 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:00.423 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:00.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:00.527 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:00.566 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:00.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:00 smithi167 ceph-mon[192880]: pgmap v882: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 10219/56889 objects degraded (17.963%); 16165/56889 objects misplaced (28.415%); 15 MiB/s, 336 objects/s recovering 2023-09-30T09:37:00.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:00 smithi019 ceph-mon[216815]: pgmap v882: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 10219/56889 objects degraded (17.963%); 16165/56889 objects misplaced (28.415%); 15 MiB/s, 336 objects/s recovering 2023-09-30T09:37:00.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:00 smithi019 ceph-mon[224043]: pgmap v882: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 10219/56889 objects degraded (17.963%); 16165/56889 objects misplaced (28.415%); 15 MiB/s, 336 objects/s recovering 2023-09-30T09:37:00.842 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:00.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:00.858 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:37:00.954 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:01.029 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:01.186 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:37:01.292 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:01.837 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:01.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:01.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:01.900 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:37:01.945 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:02.065 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:02.117 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:02.608 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:02.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:02.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:02.724 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:02.859 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:02.957 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:02 smithi167 ceph-mon[192880]: pgmap v883: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 10219/56511 objects degraded (18.083%); 16165/56511 objects misplaced (28.605%); 15 MiB/s, 352 objects/s recovering 2023-09-30T09:37:02.999 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:02 smithi019 ceph-mon[216815]: pgmap v883: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 10219/56511 objects degraded (18.083%); 16165/56511 objects misplaced (28.605%); 15 MiB/s, 352 objects/s recovering 2023-09-30T09:37:03.001 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:02 smithi019 ceph-mon[224043]: pgmap v883: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 10219/56511 objects degraded (18.083%); 16165/56511 objects misplaced (28.605%); 15 MiB/s, 352 objects/s recovering 2023-09-30T09:37:03.151 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:03.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:03.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:03.207 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:37:03.207 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:37:03.207 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.207 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35424.0:209995 7.b e341/321 7:d11d0859:::benchmark_data_smithi019_404601_object96717:head v 341'12835, mlcod=292'5339)", 2023-09-30T09:37:03.207 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:37:03.205435+0000", 2023-09-30T09:37:03.207 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.001073241, 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00110122, 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:03.205435+0000", 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:03.208 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:03.205435+0000", 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:03.205434+0000", 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:03.205441+0000", 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4000000000000003e-06 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:03.209 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:03.205453+0000", 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1423000000000001e-05 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:03.205460+0000", 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2740000000000002e-06 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:03.210 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:03.205513+0000", 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3628999999999998e-05 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:03.205538+0000", 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4590999999999999e-05 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:03.211 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:03.212 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:03.212 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:37:03.212 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 1 2023-09-30T09:37:03.212 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:37:03.265 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:03.353 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:04.026 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:37:04.027 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:37:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:37:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:37:03] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-09-30T09:37:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/7505192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 10219/56511 objects degraded (18.083%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/7505192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:04.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 10219/56511 objects degraded (18.083%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/7505192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 10219/56511 objects degraded (18.083%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:04.524 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:04.993 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:05.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:05.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:05.093 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: pgmap v884: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.1 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 9614/53526 objects degraded (17.961%); 15299/53526 objects misplaced (28.582%); 16 MiB/s, 360 objects/s recovering 2023-09-30T09:37:05.093 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:05.093 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/7505192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:05.093 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: osdmap e342: 8 total, 8 up, 7 in 2023-09-30T09:37:05.093 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.093 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.094 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.094 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.094 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:04 smithi167 ceph-mon[192880]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.103 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:05.190 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: pgmap v884: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.1 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 9614/53526 objects degraded (17.961%); 15299/53526 objects misplaced (28.582%); 16 MiB/s, 360 objects/s recovering 2023-09-30T09:37:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/7505192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: osdmap e342: 8 total, 8 up, 7 in 2023-09-30T09:37:05.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[216815]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: pgmap v884: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.1 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 9614/53526 objects degraded (17.961%); 15299/53526 objects misplaced (28.582%); 16 MiB/s, 360 objects/s recovering 2023-09-30T09:37:05.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:05.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/7505192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:05.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: osdmap e342: 8 total, 8 up, 7 in 2023-09-30T09:37:05.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:04 smithi019 ceph-mon[224043]: failed to encode map e342 with expected crc 2023-09-30T09:37:05.702 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:05.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:05.810 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:05.849 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:05.949 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:05.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:05.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:06.055 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:06.091 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:06.184 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:06.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:06.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:06.289 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:06.311 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:06.351 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:06.385 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:37:06.410 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:37:06.438 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:37:06.459 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:37:06.471 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:37:06.471 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:37:06.471 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:37:06.471 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:37:06.471 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:37:06.481 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:37:06.504 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:37:06.530 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:37:06.563 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:37:06.566 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:06.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:37:06.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:37:06.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:37:06.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:37:06.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:37:06.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:37:06.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:37:06.568 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:37:06.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:06.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:06.602 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: osdmap e343: 8 total, 8 up, 7 in 2023-09-30T09:37:06.602 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: pgmap v887: 119 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.0 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 8914/49389 objects degraded (18.049%); 14067/49389 objects misplaced (28.482%); 14 MiB/s, 372 objects/s recovering 2023-09-30T09:37:06.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1795122931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: osdmap e343: 8 total, 8 up, 7 in 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: pgmap v887: 119 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.0 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 8914/49389 objects degraded (18.049%); 14067/49389 objects misplaced (28.482%); 14 MiB/s, 372 objects/s recovering 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1795122931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:06.671 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: osdmap e343: 8 total, 8 up, 7 in 2023-09-30T09:37:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: pgmap v887: 119 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 1.0 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 8914/49389 objects degraded (18.049%); 14067/49389 objects misplaced (28.482%); 14 MiB/s, 372 objects/s recovering 2023-09-30T09:37:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: failed to encode map e343 with expected crc 2023-09-30T09:37:06.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1795122931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:06.795 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:07.340 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:37:07.477 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:07.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:07.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:07.585 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:07.612 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:07 smithi167 ceph-mon[192880]: failed to encode map e344 with expected crc 2023-09-30T09:37:07.612 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:07 smithi167 ceph-mon[192880]: osdmap e344: 8 total, 8 up, 7 in 2023-09-30T09:37:07.612 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:07 smithi167 ceph-mon[192880]: failed to encode map e344 with expected crc 2023-09-30T09:37:07.612 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:07.612 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:07.614 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:07.674 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:07.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:07.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:07.783 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:07.811 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[216815]: failed to encode map e344 with expected crc 2023-09-30T09:37:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[216815]: osdmap e344: 8 total, 8 up, 7 in 2023-09-30T09:37:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[216815]: failed to encode map e344 with expected crc 2023-09-30T09:37:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:07.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[224043]: failed to encode map e344 with expected crc 2023-09-30T09:37:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[224043]: osdmap e344: 8 total, 8 up, 7 in 2023-09-30T09:37:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[224043]: failed to encode map e344 with expected crc 2023-09-30T09:37:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:07.839 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:07.856 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:07.904 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:37:08.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[216815]: pgmap v889: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 969 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 8126/46377 objects degraded (17.522%); 12965/46377 objects misplaced (27.956%); 21 MiB/s, 432 objects/s recovering 2023-09-30T09:37:08.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:37:08.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:37:08.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:08.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 8914/49389 objects degraded (18.049%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:08.610 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:08.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:08.715 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:08.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:08 smithi167 ceph-mon[192880]: pgmap v889: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 969 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 8126/46377 objects degraded (17.522%); 12965/46377 objects misplaced (27.956%); 21 MiB/s, 432 objects/s recovering 2023-09-30T09:37:08.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:37:08.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:37:08.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:08.748 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 8914/49389 objects degraded (18.049%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:08.750 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:08.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[224043]: pgmap v889: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 969 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 8126/46377 objects degraded (17.522%); 12965/46377 objects misplaced (27.956%); 21 MiB/s, 432 objects/s recovering 2023-09-30T09:37:08.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:37:08.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:37:08.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:08.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 8914/49389 objects degraded (18.049%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:08.813 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:08.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:08.917 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:09.017 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:09.127 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:09.331 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:09.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:09.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:09.441 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:09.496 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:09.570 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:09.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:09.643 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:37:09.644 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:37:09.684 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:09.728 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:09.851 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:09.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:09.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:09.964 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:10.054 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:10.161 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:37:10.165 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:37:10.196 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:10.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:10.305 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:10.378 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:10.559 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:37:10.832 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:37:10.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:10 smithi019 ceph-mon[216815]: pgmap v890: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 902 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 7663/43179 objects degraded (17.747%); 11944/43179 objects misplaced (27.662%); 20 MiB/s, 416 objects/s recovering 2023-09-30T09:37:10.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:10 smithi019 ceph-mon[224043]: pgmap v890: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 902 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 7663/43179 objects degraded (17.747%); 11944/43179 objects misplaced (27.662%); 20 MiB/s, 416 objects/s recovering 2023-09-30T09:37:10.891 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:10.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:10.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:11.001 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:11.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:10 smithi167 ceph-mon[192880]: pgmap v890: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 902 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 7663/43179 objects degraded (17.747%); 11944/43179 objects misplaced (27.662%); 20 MiB/s, 416 objects/s recovering 2023-09-30T09:37:11.063 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:11.091 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:11.681 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:11.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:11.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:11.712 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":890,"stamp":"2023-09-30T09:37:09.547643+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":946289508,"num_objects":14393,"num_object_clones":0,"num_object_copies":43179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7663,"num_objects_misplaced":11944,"num_objects_unfound":0,"num_objects_dirty":14393,"num_whiteouts":0,"num_read":6114,"num_read_kb":12467,"num_write":213176,"num_write_kb":7263166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38777,"num_bytes_recovered":2381160920,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88274,"ondisk_log_size":88274,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":397,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4429360,"kb_used_data":1956012,"kb_used_omap":23,"kb_used_meta":2473192,"kb_avail":651786704,"statfs":{"total":671965249536,"available":667429584896,"internally_reserved":0,"allocated":2002956288,"data_stored":1991462035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2532548867},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,0,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-226033664,"num_objects":-3449,"num_object_clones":0,"num_object_copies":-10347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1951,"num_objects_misplaced":-3355,"num_objects_unfound":0,"num_objects_dirty":-3449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3449,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2502,"num_bytes_recovered":126681088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.003869"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":808,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.597977+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:37:05.597977+0000","last_peered":"2023-09-30T09:37:05.597977+0000","last_clean":"2023-09-30T09:37:05.597977+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:37:05.597977+0000","last_undegraded":"2023-09-30T09:37:05.597977+0000","last_fullsized":"2023-09-30T09:37:05.597977+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":362,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610711+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:37:05.610711+0000","last_peered":"2023-09-30T09:37:05.610711+0000","last_clean":"2023-09-30T09:37:05.610711+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:37:05.610711+0000","last_undegraded":"2023-09-30T09:37:05.610711+0000","last_fullsized":"2023-09-30T09:37:05.610711+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":325,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521722+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:37:05.521722+0000","last_peered":"2023-09-30T09:37:05.521722+0000","last_clean":"2023-09-30T09:37:05.521722+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:37:05.521722+0000","last_undegraded":"2023-09-30T09:37:05.521722+0000","last_fullsized":"2023-09-30T09:37:05.521722+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":788,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598500+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:37:05.598500+0000","last_peered":"2023-09-30T09:37:05.598500+0000","last_clean":"2023-09-30T09:37:05.598500+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:37:05.598500+0000","last_undegraded":"2023-09-30T09:37:05.598500+0000","last_fullsized":"2023-09-30T09:37:05.598500+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":217,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.522106+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:37:05.522106+0000","last_peered":"2023-09-30T09:37:05.522106+0000","last_clean":"2023-09-30T09:37:05.522106+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:37:05.522106+0000","last_undegraded":"2023-09-30T09:37:05.522106+0000","last_fullsized":"2023-09-30T09:37:05.522106+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":246,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.060006+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:36:52.060006+0000","last_peered":"2023-09-30T09:36:52.060006+0000","last_clean":"2023-09-30T09:36:52.060006+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:36:52.060006+0000","last_undegraded":"2023-09-30T09:36:52.060006+0000","last_fullsized":"2023-09-30T09:36:52.060006+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":557,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610625+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:37:05.610625+0000","last_peered":"2023-09-30T09:37:05.610625+0000","last_clean":"2023-09-30T09:37:05.610625+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:37:05.610625+0000","last_undegraded":"2023-09-30T09:37:05.610625+0000","last_fullsized":"2023-09-30T09:37:05.610625+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":391,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.520604+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:37:05.520604+0000","last_peered":"2023-09-30T09:37:05.520604+0000","last_clean":"2023-09-30T09:37:05.520604+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:37:05.520604+0000","last_undegraded":"2023-09-30T09:37:05.520604+0000","last_fullsized":"2023-09-30T09:37:05.520604+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":320,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.617164+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:37:05.617164+0000","last_peered":"2023-09-30T09:37:05.617164+0000","last_clean":"2023-09-30T09:37:05.617164+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:37:05.617164+0000","last_undegraded":"2023-09-30T09:37:05.617164+0000","last_fullsized":"2023-09-30T09:37:05.617164+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":768,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521432+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:37:05.521432+0000","last_peered":"2023-09-30T09:37:05.521432+0000","last_clean":"2023-09-30T09:37:05.521432+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:37:05.521432+0000","last_undegraded":"2023-09-30T09:37:05.521432+0000","last_fullsized":"2023-09-30T09:37:05.521432+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":237,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059862+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:36:52.059862+0000","last_peered":"2023-09-30T09:36:52.059862+0000","last_clean":"2023-09-30T09:36:52.059862+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:36:52.059862+0000","last_undegraded":"2023-09-30T09:36:52.059862+0000","last_fullsized":"2023-09-30T09:36:52.059862+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":325,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.520618+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:37:05.520618+0000","last_peered":"2023-09-30T09:37:05.520618+0000","last_clean":"2023-09-30T09:37:05.520618+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:37:05.520618+0000","last_undegraded":"2023-09-30T09:37:05.520618+0000","last_fullsized":"2023-09-30T09:37:05.520618+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":505,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.217766+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:36:52.217766+0000","last_peered":"2023-09-30T09:36:52.217766+0000","last_clean":"2023-09-30T09:36:52.217766+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:36:52.217766+0000","last_undegraded":"2023-09-30T09:36:52.217766+0000","last_fullsized":"2023-09-30T09:36:52.217766+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":222,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.599644+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:37:05.599644+0000","last_peered":"2023-09-30T09:37:05.599644+0000","last_clean":"2023-09-30T09:37:05.599644+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:37:05.599644+0000","last_undegraded":"2023-09-30T09:37:05.599644+0000","last_fullsized":"2023-09-30T09:37:05.599644+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":271,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.522016+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:37:05.522016+0000","last_peered":"2023-09-30T09:37:05.522016+0000","last_clean":"2023-09-30T09:37:05.522016+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:37:05.522016+0000","last_undegraded":"2023-09-30T09:37:05.522016+0000","last_fullsized":"2023-09-30T09:37:05.522016+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":510,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.599579+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:37:05.599579+0000","last_peered":"2023-09-30T09:37:05.599579+0000","last_clean":"2023-09-30T09:37:05.599579+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:37:05.599579+0000","last_undegraded":"2023-09-30T09:37:05.599579+0000","last_fullsized":"2023-09-30T09:37:05.599579+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":66,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521851+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:37:05.521851+0000","last_peered":"2023-09-30T09:37:05.521851+0000","last_clean":"2023-09-30T09:37:05.521851+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:37:05.521851+0000","last_undegraded":"2023-09-30T09:37:05.521851+0000","last_fullsized":"2023-09-30T09:37:05.521851+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":257,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.615029+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:37:05.615029+0000","last_peered":"2023-09-30T09:37:05.615029+0000","last_clean":"2023-09-30T09:37:05.615029+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:37:05.615029+0000","last_undegraded":"2023-09-30T09:37:05.615029+0000","last_fullsized":"2023-09-30T09:37:05.615029+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":502,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.061769+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:36:52.061769+0000","last_peered":"2023-09-30T09:36:52.061769+0000","last_clean":"2023-09-30T09:36:52.061769+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:36:52.061769+0000","last_undegraded":"2023-09-30T09:36:52.061769+0000","last_fullsized":"2023-09-30T09:36:52.061769+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":328,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611120+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:37:05.611120+0000","last_peered":"2023-09-30T09:37:05.611120+0000","last_clean":"2023-09-30T09:37:05.611120+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:37:05.611120+0000","last_undegraded":"2023-09-30T09:37:05.611120+0000","last_fullsized":"2023-09-30T09:37:05.611120+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":339,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.612007+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:37:05.612007+0000","last_peered":"2023-09-30T09:37:05.612007+0000","last_clean":"2023-09-30T09:37:05.612007+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:37:05.612007+0000","last_undegraded":"2023-09-30T09:37:05.612007+0000","last_fullsized":"2023-09-30T09:37:05.612007+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":619,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059315+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:36:52.059315+0000","last_peered":"2023-09-30T09:36:52.059315+0000","last_clean":"2023-09-30T09:36:52.059315+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:36:52.059315+0000","last_undegraded":"2023-09-30T09:36:52.059315+0000","last_fullsized":"2023-09-30T09:36:52.059315+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":66,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611924+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:37:05.611924+0000","last_peered":"2023-09-30T09:37:05.611924+0000","last_clean":"2023-09-30T09:37:05.611924+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:37:05.611924+0000","last_undegraded":"2023-09-30T09:37:05.611924+0000","last_fullsized":"2023-09-30T09:37:05.611924+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":355,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521405+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:37:05.521405+0000","last_peered":"2023-09-30T09:37:05.521405+0000","last_clean":"2023-09-30T09:37:05.521405+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:37:05.521405+0000","last_undegraded":"2023-09-30T09:37:05.521405+0000","last_fullsized":"2023-09-30T09:37:05.521405+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":690,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598074+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:37:05.598074+0000","last_peered":"2023-09-30T09:37:05.598074+0000","last_clean":"2023-09-30T09:37:05.598074+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:37:05.598074+0000","last_undegraded":"2023-09-30T09:37:05.598074+0000","last_fullsized":"2023-09-30T09:37:05.598074+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":245,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059378+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:36:52.059378+0000","last_peered":"2023-09-30T09:36:52.059378+0000","last_clean":"2023-09-30T09:36:52.059378+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:36:52.059378+0000","last_undegraded":"2023-09-30T09:36:52.059378+0000","last_fullsized":"2023-09-30T09:36:52.059378+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":33,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598823+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:37:05.598823+0000","last_peered":"2023-09-30T09:37:05.598823+0000","last_clean":"2023-09-30T09:37:05.598823+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:37:05.598823+0000","last_undegraded":"2023-09-30T09:37:05.598823+0000","last_fullsized":"2023-09-30T09:37:05.598823+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":782,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521330+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:37:05.521330+0000","last_peered":"2023-09-30T09:37:05.521330+0000","last_clean":"2023-09-30T09:37:05.521330+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:37:05.521330+0000","last_undegraded":"2023-09-30T09:37:05.521330+0000","last_fullsized":"2023-09-30T09:37:05.521330+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":49,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611825+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:37:05.611825+0000","last_peered":"2023-09-30T09:37:05.611825+0000","last_clean":"2023-09-30T09:37:05.611825+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:37:05.611825+0000","last_undegraded":"2023-09-30T09:37:05.611825+0000","last_fullsized":"2023-09-30T09:37:05.611825+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":245,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.061612+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:36:52.061612+0000","last_peered":"2023-09-30T09:36:52.061612+0000","last_clean":"2023-09-30T09:36:52.061612+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:36:52.061612+0000","last_undegraded":"2023-09-30T09:36:52.061612+0000","last_fullsized":"2023-09-30T09:36:52.061612+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":809,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598267+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:37:05.598267+0000","last_peered":"2023-09-30T09:37:05.598267+0000","last_clean":"2023-09-30T09:37:05.598267+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:37:05.598267+0000","last_undegraded":"2023-09-30T09:37:05.598267+0000","last_fullsized":"2023-09-30T09:37:05.598267+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":503,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.614602+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:37:05.614602+0000","last_peered":"2023-09-30T09:37:05.614602+0000","last_clean":"2023-09-30T09:37:05.614602+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:37:05.614602+0000","last_undegraded":"2023-09-30T09:37:05.614602+0000","last_fullsized":"2023-09-30T09:37:05.614602+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":353,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.522338+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:37:05.522338+0000","last_peered":"2023-09-30T09:37:05.522338+0000","last_clean":"2023-09-30T09:37:05.522338+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:37:05.522338+0000","last_undegraded":"2023-09-30T09:37:05.522338+0000","last_fullsized":"2023-09-30T09:37:05.522338+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"344'13006","reported_seq":18427,"reported_epoch":344,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:09.034647+0000","last_change":"2023-09-30T09:34:48.517696+0000","last_active":"2023-09-30T09:37:09.034647+0000","last_peered":"2023-09-30T09:37:09.034647+0000","last_clean":"2023-09-30T09:32:35.457744+0000","last_became_active":"2023-09-30T09:34:48.499533+0000","last_became_peered":"2023-09-30T09:34:48.499533+0000","last_unstale":"2023-09-30T09:37:09.034647+0000","last_undegraded":"2023-09-30T09:34:48.469688+0000","last_fullsized":"2023-09-30T09:37:09.034647+0000","mapping_epoch":321,"log_start":"311'8221","ondisk_log_start":"311'8221","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4785,"log_dups_size":0,"ondisk_log_size":4785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:05:42.164341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58458112,"num_objects":892,"num_object_clones":0,"num_object_copies":2676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":890,"num_objects_misplaced":892,"num_objects_unfound":0,"num_objects_dirty":892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13006,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2652,"num_bytes_recovered":173801472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[6,1,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"1,4,6","objects":2},{"shards":"4,6","objects":890}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":826,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610131+0000","last_change":"2023-09-30T09:36:50.905972+0000","last_active":"2023-09-30T09:37:05.610131+0000","last_peered":"2023-09-30T09:37:05.610131+0000","last_clean":"2023-09-30T09:37:05.610131+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:37:05.610131+0000","last_undegraded":"2023-09-30T09:37:05.610131+0000","last_fullsized":"2023-09-30T09:37:05.610131+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":78,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611030+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:37:05.611030+0000","last_peered":"2023-09-30T09:37:05.611030+0000","last_clean":"2023-09-30T09:37:05.611030+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:37:05.611030+0000","last_undegraded":"2023-09-30T09:37:05.611030+0000","last_fullsized":"2023-09-30T09:37:05.611030+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":33,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.519912+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:37:05.519912+0000","last_peered":"2023-09-30T09:37:05.519912+0000","last_clean":"2023-09-30T09:37:05.519912+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:37:05.519912+0000","last_undegraded":"2023-09-30T09:37:05.519912+0000","last_fullsized":"2023-09-30T09:37:05.519912+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"344'13440","reported_seq":19966,"reported_epoch":344,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:07.232351+0000","last_change":"2023-09-30T09:35:10.766663+0000","last_active":"2023-09-30T09:37:07.232351+0000","last_peered":"2023-09-30T09:37:07.232351+0000","last_clean":"2023-09-30T09:32:35.454324+0000","last_became_active":"2023-09-30T09:33:33.826036+0000","last_became_peered":"2023-09-30T09:33:33.826036+0000","last_unstale":"2023-09-30T09:37:07.232351+0000","last_undegraded":"2023-09-30T09:33:33.732722+0000","last_fullsized":"2023-09-30T09:37:07.232351+0000","mapping_epoch":302,"log_start":"312'8899","ondisk_log_start":"312'8899","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4541,"log_dups_size":0,"ondisk_log_size":4541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:42.597216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62914560,"num_objects":960,"num_object_clones":0,"num_object_copies":2880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":893,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13440,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3212,"num_bytes_recovered":210501632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,5,1],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":66},{"shards":"3,5","objects":894}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":793,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521346+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:37:05.521346+0000","last_peered":"2023-09-30T09:37:05.521346+0000","last_clean":"2023-09-30T09:37:05.521346+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:37:05.521346+0000","last_undegraded":"2023-09-30T09:37:05.521346+0000","last_fullsized":"2023-09-30T09:37:05.521346+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":56,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610008+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:37:05.610008+0000","last_peered":"2023-09-30T09:37:05.610008+0000","last_clean":"2023-09-30T09:37:05.610008+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:37:05.610008+0000","last_undegraded":"2023-09-30T09:37:05.610008+0000","last_fullsized":"2023-09-30T09:37:05.610008+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":336,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611793+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:37:05.611793+0000","last_peered":"2023-09-30T09:37:05.611793+0000","last_clean":"2023-09-30T09:37:05.611793+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:37:05.611793+0000","last_undegraded":"2023-09-30T09:37:05.611793+0000","last_fullsized":"2023-09-30T09:37:05.611793+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"344'13089","reported_seq":18650,"reported_epoch":344,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:09.078925+0000","last_change":"2023-09-30T09:35:10.879711+0000","last_active":"2023-09-30T09:37:09.078925+0000","last_peered":"2023-09-30T09:37:09.078925+0000","last_clean":"2023-09-30T09:32:35.467048+0000","last_became_active":"2023-09-30T09:35:10.845361+0000","last_became_peered":"2023-09-30T09:35:10.845361+0000","last_unstale":"2023-09-30T09:37:09.078925+0000","last_undegraded":"2023-09-30T09:35:10.791338+0000","last_fullsized":"2023-09-30T09:37:09.078925+0000","mapping_epoch":327,"log_start":"311'8276","ondisk_log_start":"311'8276","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4813,"log_dups_size":0,"ondisk_log_size":4813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:41.089234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58130432,"num_objects":887,"num_object_clones":0,"num_object_copies":2661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":886,"num_objects_misplaced":1774,"num_objects_unfound":0,"num_objects_dirty":887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13089,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2722,"num_bytes_recovered":178257920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[6,1,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"1,6,7","objects":1},{"shards":"6,7","objects":886}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":627,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.617090+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:37:05.617090+0000","last_peered":"2023-09-30T09:37:05.617090+0000","last_clean":"2023-09-30T09:37:05.617090+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:37:05.617090+0000","last_undegraded":"2023-09-30T09:37:05.617090+0000","last_fullsized":"2023-09-30T09:37:05.617090+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":37,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598433+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:37:05.598433+0000","last_peered":"2023-09-30T09:37:05.598433+0000","last_clean":"2023-09-30T09:37:05.598433+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:37:05.598433+0000","last_undegraded":"2023-09-30T09:37:05.598433+0000","last_fullsized":"2023-09-30T09:37:05.598433+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":323,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.614948+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:37:05.614948+0000","last_peered":"2023-09-30T09:37:05.614948+0000","last_clean":"2023-09-30T09:37:05.614948+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:37:05.614948+0000","last_undegraded":"2023-09-30T09:37:05.614948+0000","last_fullsized":"2023-09-30T09:37:05.614948+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"344'13361","reported_seq":13445,"reported_epoch":344,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:07.239955+0000","last_change":"2023-09-30T09:34:08.527633+0000","last_active":"2023-09-30T09:37:07.239955+0000","last_peered":"2023-09-30T09:37:07.239955+0000","last_clean":"2023-09-30T09:33:09.318417+0000","last_became_active":"2023-09-30T09:34:08.510746+0000","last_became_peered":"2023-09-30T09:34:08.510746+0000","last_unstale":"2023-09-30T09:37:07.239955+0000","last_undegraded":"2023-09-30T09:37:07.239955+0000","last_fullsized":"2023-09-30T09:37:07.239955+0000","mapping_epoch":308,"log_start":"312'8876","ondisk_log_start":"312'8876","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4485,"log_dups_size":0,"ondisk_log_size":4485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:07.374969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61407232,"num_objects":937,"num_object_clones":0,"num_object_copies":2811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":937,"num_objects_unfound":0,"num_objects_dirty":937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13361,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":937}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":500,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610090+0000","last_change":"2023-09-30T09:36:49.890977+0000","last_active":"2023-09-30T09:37:05.610090+0000","last_peered":"2023-09-30T09:37:05.610090+0000","last_clean":"2023-09-30T09:37:05.610090+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:37:05.610090+0000","last_undegraded":"2023-09-30T09:37:05.610090+0000","last_fullsized":"2023-09-30T09:37:05.610090+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":303,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598525+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:37:05.598525+0000","last_peered":"2023-09-30T09:37:05.598525+0000","last_clean":"2023-09-30T09:37:05.598525+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:37:05.598525+0000","last_undegraded":"2023-09-30T09:37:05.598525+0000","last_fullsized":"2023-09-30T09:37:05.598525+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":248,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059912+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:36:52.059912+0000","last_peered":"2023-09-30T09:36:52.059912+0000","last_clean":"2023-09-30T09:36:52.059912+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:36:52.059912+0000","last_undegraded":"2023-09-30T09:36:52.059912+0000","last_fullsized":"2023-09-30T09:36:52.059912+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"344'13133","reported_seq":13232,"reported_epoch":344,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:07.180269+0000","last_change":"2023-09-30T09:35:10.779076+0000","last_active":"2023-09-30T09:37:07.180269+0000","last_peered":"2023-09-30T09:37:07.180269+0000","last_clean":"2023-09-30T09:34:06.422096+0000","last_became_active":"2023-09-30T09:35:10.747716+0000","last_became_peered":"2023-09-30T09:35:10.747716+0000","last_unstale":"2023-09-30T09:37:07.180269+0000","last_undegraded":"2023-09-30T09:37:07.180269+0000","last_fullsized":"2023-09-30T09:37:07.180269+0000","mapping_epoch":327,"log_start":"312'8620","ondisk_log_start":"312'8620","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4513,"log_dups_size":0,"ondisk_log_size":4513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:09.191024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59834368,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":913,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13133,"num_write_kb":449472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":913}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":690,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521881+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:37:05.521881+0000","last_peered":"2023-09-30T09:37:05.521881+0000","last_clean":"2023-09-30T09:37:05.521881+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:37:05.521881+0000","last_undegraded":"2023-09-30T09:37:05.521881+0000","last_fullsized":"2023-09-30T09:37:05.521881+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":372,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-09-30T09:37:06.523046+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:37:06.523046+0000","last_peered":"2023-09-30T09:37:06.523046+0000","last_clean":"2023-09-30T09:37:06.523046+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:37:06.523046+0000","last_undegraded":"2023-09-30T09:37:06.523046+0000","last_fullsized":"2023-09-30T09:37:06.523046+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":302,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611193+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:37:05.611193+0000","last_peered":"2023-09-30T09:37:05.611193+0000","last_clean":"2023-09-30T09:37:05.611193+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:37:05.611193+0000","last_undegraded":"2023-09-30T09:37:05.611193+0000","last_fullsized":"2023-09-30T09:37:05.611193+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"344'13325","reported_seq":18884,"reported_epoch":344,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:07.259081+0000","last_change":"2023-09-30T09:35:10.871065+0000","last_active":"2023-09-30T09:37:07.259081+0000","last_peered":"2023-09-30T09:37:07.259081+0000","last_clean":"2023-09-30T09:32:35.449714+0000","last_became_active":"2023-09-30T09:35:10.838732+0000","last_became_peered":"2023-09-30T09:35:10.838732+0000","last_unstale":"2023-09-30T09:37:07.259081+0000","last_undegraded":"2023-09-30T09:35:10.818868+0000","last_fullsized":"2023-09-30T09:37:07.259081+0000","mapping_epoch":327,"log_start":"312'8806","ondisk_log_start":"312'8806","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4519,"log_dups_size":0,"ondisk_log_size":4519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:44.318381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60751872,"num_objects":927,"num_object_clones":0,"num_object_copies":2781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":925,"num_objects_misplaced":1856,"num_objects_unfound":0,"num_objects_dirty":927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13325,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":178454528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[3,7,1],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"1,3,7","objects":2},{"shards":"3,7","objects":925}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":763,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.216592+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:36:52.216592+0000","last_peered":"2023-09-30T09:36:52.216592+0000","last_clean":"2023-09-30T09:36:52.216592+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:36:52.216592+0000","last_undegraded":"2023-09-30T09:36:52.216592+0000","last_fullsized":"2023-09-30T09:36:52.216592+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":321,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.614796+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:37:05.614796+0000","last_peered":"2023-09-30T09:37:05.614796+0000","last_clean":"2023-09-30T09:37:05.614796+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:37:05.614796+0000","last_undegraded":"2023-09-30T09:37:05.614796+0000","last_fullsized":"2023-09-30T09:37:05.614796+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":38,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.609947+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:37:05.609947+0000","last_peered":"2023-09-30T09:37:05.609947+0000","last_clean":"2023-09-30T09:37:05.609947+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:37:05.609947+0000","last_undegraded":"2023-09-30T09:37:05.609947+0000","last_fullsized":"2023-09-30T09:37:05.609947+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"344'13202","reported_seq":20431,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-09-30T09:37:09.006919+0000","last_change":"2023-09-30T09:36:04.573419+0000","last_active":"2023-09-30T09:37:09.006919+0000","last_peered":"2023-09-30T09:37:09.006919+0000","last_clean":"2023-09-30T09:37:09.006919+0000","last_became_active":"2023-09-30T09:36:04.549763+0000","last_became_peered":"2023-09-30T09:36:04.549763+0000","last_unstale":"2023-09-30T09:37:09.006919+0000","last_undegraded":"2023-09-30T09:37:09.006919+0000","last_fullsized":"2023-09-30T09:37:09.006919+0000","mapping_epoch":337,"log_start":"311'8421","ondisk_log_start":"311'8421","created":279,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4781,"log_dups_size":0,"ondisk_log_size":4781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:27:18.185457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55115800,"num_objects":842,"num_object_clones":0,"num_object_copies":2526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":842,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13202,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3562,"num_bytes_recovered":233439232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":777,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059704+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:36:52.059704+0000","last_peered":"2023-09-30T09:36:52.059704+0000","last_clean":"2023-09-30T09:36:52.059704+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:36:52.059704+0000","last_undegraded":"2023-09-30T09:36:52.059704+0000","last_fullsized":"2023-09-30T09:36:52.059704+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":340,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611671+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:37:05.611671+0000","last_peered":"2023-09-30T09:37:05.611671+0000","last_clean":"2023-09-30T09:37:05.611671+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:37:05.611671+0000","last_undegraded":"2023-09-30T09:37:05.611671+0000","last_fullsized":"2023-09-30T09:37:05.611671+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":319,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610309+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:37:05.610309+0000","last_peered":"2023-09-30T09:37:05.610309+0000","last_clean":"2023-09-30T09:37:05.610309+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:37:05.610309+0000","last_undegraded":"2023-09-30T09:37:05.610309+0000","last_fullsized":"2023-09-30T09:37:05.610309+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"344'13341","reported_seq":13431,"reported_epoch":344,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:08.926263+0000","last_change":"2023-09-30T09:34:47.630136+0000","last_active":"2023-09-30T09:37:08.926263+0000","last_peered":"2023-09-30T09:37:08.926263+0000","last_clean":"2023-09-30T09:34:06.422409+0000","last_became_active":"2023-09-30T09:34:08.626702+0000","last_became_peered":"2023-09-30T09:34:08.626702+0000","last_unstale":"2023-09-30T09:37:08.926263+0000","last_undegraded":"2023-09-30T09:37:08.926263+0000","last_fullsized":"2023-09-30T09:37:08.926263+0000","mapping_epoch":308,"log_start":"311'8506","ondisk_log_start":"311'8506","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4835,"log_dups_size":0,"ondisk_log_size":4835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:53.753813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54722560,"num_objects":835,"num_object_clones":0,"num_object_copies":2505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":835,"num_objects_unfound":0,"num_objects_dirty":835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13341,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":835}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":487,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.216620+0000","last_change":"2023-09-30T09:35:46.150279+0000","last_active":"2023-09-30T09:36:52.216620+0000","last_peered":"2023-09-30T09:36:52.216620+0000","last_clean":"2023-09-30T09:36:52.216620+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:36:52.216620+0000","last_undegraded":"2023-09-30T09:36:52.216620+0000","last_fullsized":"2023-09-30T09:36:52.216620+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":335,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.520512+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:37:05.520512+0000","last_peered":"2023-09-30T09:37:05.520512+0000","last_clean":"2023-09-30T09:37:05.520512+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:37:05.520512+0000","last_undegraded":"2023-09-30T09:37:05.520512+0000","last_fullsized":"2023-09-30T09:37:05.520512+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.522197+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:37:05.522197+0000","last_peered":"2023-09-30T09:37:05.522197+0000","last_clean":"2023-09-30T09:37:05.522197+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:37:05.522197+0000","last_undegraded":"2023-09-30T09:37:05.522197+0000","last_fullsized":"2023-09-30T09:37:05.522197+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"344'13081","reported_seq":18306,"reported_epoch":344,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:07.211420+0000","last_change":"2023-09-30T09:34:39.166112+0000","last_active":"2023-09-30T09:37:07.211420+0000","last_peered":"2023-09-30T09:37:07.211420+0000","last_clean":"2023-09-30T09:32:35.465008+0000","last_became_active":"2023-09-30T09:34:39.150257+0000","last_became_peered":"2023-09-30T09:34:39.150257+0000","last_unstale":"2023-09-30T09:37:07.211420+0000","last_undegraded":"2023-09-30T09:34:38.934740+0000","last_fullsized":"2023-09-30T09:37:07.211420+0000","mapping_epoch":316,"log_start":"312'8578","ondisk_log_start":"312'8578","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4503,"log_dups_size":0,"ondisk_log_size":4503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:50.805799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59310080,"num_objects":905,"num_object_clones":0,"num_object_copies":2715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":905,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13081,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2564,"num_bytes_recovered":167968768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6,1],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":905}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":638,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.617293+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:37:05.617293+0000","last_peered":"2023-09-30T09:37:05.617293+0000","last_clean":"2023-09-30T09:37:05.617293+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:37:05.617293+0000","last_undegraded":"2023-09-30T09:37:05.617293+0000","last_fullsized":"2023-09-30T09:37:05.617293+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":534,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059506+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:36:52.059506+0000","last_peered":"2023-09-30T09:36:52.059506+0000","last_clean":"2023-09-30T09:36:52.059506+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:36:52.059506+0000","last_undegraded":"2023-09-30T09:36:52.059506+0000","last_fullsized":"2023-09-30T09:36:52.059506+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":340,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.599588+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:37:05.599588+0000","last_peered":"2023-09-30T09:37:05.599588+0000","last_clean":"2023-09-30T09:37:05.599588+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:37:05.599588+0000","last_undegraded":"2023-09-30T09:37:05.599588+0000","last_fullsized":"2023-09-30T09:37:05.599588+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598134+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:37:05.598134+0000","last_peered":"2023-09-30T09:37:05.598134+0000","last_clean":"2023-09-30T09:37:05.598134+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:37:05.598134+0000","last_undegraded":"2023-09-30T09:37:05.598134+0000","last_fullsized":"2023-09-30T09:37:05.598134+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"344'13389","reported_seq":13462,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-09-30T09:37:09.054684+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:37:09.054684+0000","last_peered":"2023-09-30T09:37:09.054684+0000","last_clean":"2023-09-30T09:37:09.054684+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:37:09.054684+0000","last_undegraded":"2023-09-30T09:37:09.054684+0000","last_fullsized":"2023-09-30T09:37:09.054684+0000","mapping_epoch":279,"log_start":"311'8557","ondisk_log_start":"311'8557","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4832,"log_dups_size":0,"ondisk_log_size":4832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56819712,"num_objects":867,"num_object_clones":0,"num_object_copies":2601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13389,"num_write_kb":456192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":806,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598356+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:37:05.598356+0000","last_peered":"2023-09-30T09:37:05.598356+0000","last_clean":"2023-09-30T09:37:05.598356+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:37:05.598356+0000","last_undegraded":"2023-09-30T09:37:05.598356+0000","last_fullsized":"2023-09-30T09:37:05.598356+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":335,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.614866+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:37:05.614866+0000","last_peered":"2023-09-30T09:37:05.614866+0000","last_clean":"2023-09-30T09:37:05.614866+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:37:05.614866+0000","last_undegraded":"2023-09-30T09:37:05.614866+0000","last_fullsized":"2023-09-30T09:37:05.614866+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":323,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.612125+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:37:05.612125+0000","last_peered":"2023-09-30T09:37:05.612125+0000","last_clean":"2023-09-30T09:37:05.612125+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:37:05.612125+0000","last_undegraded":"2023-09-30T09:37:05.612125+0000","last_fullsized":"2023-09-30T09:37:05.612125+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":59,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.217839+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:36:52.217839+0000","last_peered":"2023-09-30T09:36:52.217839+0000","last_clean":"2023-09-30T09:36:52.217839+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:36:52.217839+0000","last_undegraded":"2023-09-30T09:36:52.217839+0000","last_fullsized":"2023-09-30T09:36:52.217839+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"344'13442","reported_seq":19164,"reported_epoch":344,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:09.081958+0000","last_change":"2023-09-30T09:35:10.808625+0000","last_active":"2023-09-30T09:37:09.081958+0000","last_peered":"2023-09-30T09:37:09.081958+0000","last_clean":"2023-09-30T09:32:35.460821+0000","last_became_active":"2023-09-30T09:35:10.778683+0000","last_became_peered":"2023-09-30T09:35:10.778683+0000","last_unstale":"2023-09-30T09:37:09.081958+0000","last_undegraded":"2023-09-30T09:35:10.766189+0000","last_fullsized":"2023-09-30T09:37:09.081958+0000","mapping_epoch":327,"log_start":"312'8633","ondisk_log_start":"312'8633","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4809,"log_dups_size":0,"ondisk_log_size":4809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:45.905717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59637760,"num_objects":910,"num_object_clones":0,"num_object_copies":2730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":910,"num_objects_misplaced":1820,"num_objects_unfound":0,"num_objects_dirty":910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13442,"num_write_kb":459264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2806,"num_bytes_recovered":183894016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[6,1,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":910}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":702,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059472+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:36:52.059472+0000","last_peered":"2023-09-30T09:36:52.059472+0000","last_clean":"2023-09-30T09:36:52.059472+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:36:52.059472+0000","last_undegraded":"2023-09-30T09:36:52.059472+0000","last_fullsized":"2023-09-30T09:36:52.059472+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":520,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.216465+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:36:52.216465+0000","last_peered":"2023-09-30T09:36:52.216465+0000","last_clean":"2023-09-30T09:36:52.216465+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:36:52.216465+0000","last_undegraded":"2023-09-30T09:36:52.216465+0000","last_fullsized":"2023-09-30T09:36:52.216465+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.216513+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:36:52.216513+0000","last_peered":"2023-09-30T09:36:52.216513+0000","last_clean":"2023-09-30T09:36:52.216513+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:36:52.216513+0000","last_undegraded":"2023-09-30T09:36:52.216513+0000","last_fullsized":"2023-09-30T09:36:52.216513+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":263,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.599745+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:37:05.599745+0000","last_peered":"2023-09-30T09:37:05.599745+0000","last_clean":"2023-09-30T09:37:05.599745+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:37:05.599745+0000","last_undegraded":"2023-09-30T09:37:05.599745+0000","last_fullsized":"2023-09-30T09:37:05.599745+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"342'13067","reported_seq":27186,"reported_epoch":342,"state":"active+undersized+remapped","last_fresh":"2023-09-30T09:37:05.171650+0000","last_change":"2023-09-30T09:37:04.693173+0000","last_active":"2023-09-30T09:37:05.171650+0000","last_peered":"2023-09-30T09:37:05.171650+0000","last_clean":"2023-09-30T09:34:06.430942+0000","last_became_active":"2023-09-30T09:34:47.755052+0000","last_became_peered":"2023-09-30T09:34:47.755052+0000","last_unstale":"2023-09-30T09:37:05.171650+0000","last_undegraded":"2023-09-30T09:37:05.171650+0000","last_fullsized":"2023-09-30T09:34:47.514378+0000","mapping_epoch":320,"log_start":"299'6850","ondisk_log_start":"299'6850","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":6217,"log_dups_size":0,"ondisk_log_size":6217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:44:56.113242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62849024,"num_objects":959,"num_object_clones":0,"num_object_copies":2877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13067,"num_write_kb":448832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7014,"num_bytes_recovered":304349184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","5","7"],"object_location_counts":[{"shards":"0,3","objects":959}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":916,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598592+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:37:05.598592+0000","last_peered":"2023-09-30T09:37:05.598592+0000","last_clean":"2023-09-30T09:37:05.598592+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:37:05.598592+0000","last_undegraded":"2023-09-30T09:37:05.598592+0000","last_fullsized":"2023-09-30T09:37:05.598592+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":606,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059181+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:36:52.059181+0000","last_peered":"2023-09-30T09:36:52.059181+0000","last_clean":"2023-09-30T09:36:52.059181+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:36:52.059181+0000","last_undegraded":"2023-09-30T09:36:52.059181+0000","last_fullsized":"2023-09-30T09:36:52.059181+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":263,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.522600+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:37:05.522600+0000","last_peered":"2023-09-30T09:37:05.522600+0000","last_clean":"2023-09-30T09:37:05.522600+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:37:05.522600+0000","last_undegraded":"2023-09-30T09:37:05.522600+0000","last_fullsized":"2023-09-30T09:37:05.522600+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":296,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.617933+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:37:05.617933+0000","last_peered":"2023-09-30T09:37:05.617933+0000","last_clean":"2023-09-30T09:37:05.617933+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:37:05.617933+0000","last_undegraded":"2023-09-30T09:37:05.617933+0000","last_fullsized":"2023-09-30T09:37:05.617933+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"344'13309","reported_seq":25064,"reported_epoch":344,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:09.423565+0000","last_change":"2023-09-30T09:35:10.837720+0000","last_active":"2023-09-30T09:37:09.423565+0000","last_peered":"2023-09-30T09:37:09.423565+0000","last_clean":"2023-09-30T09:32:35.481650+0000","last_became_active":"2023-09-30T09:35:10.788829+0000","last_became_peered":"2023-09-30T09:35:10.788829+0000","last_unstale":"2023-09-30T09:37:09.423565+0000","last_undegraded":"2023-09-30T09:35:10.757584+0000","last_fullsized":"2023-09-30T09:37:09.423565+0000","mapping_epoch":327,"log_start":"286'3300","ondisk_log_start":"286'3300","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":10009,"log_dups_size":0,"ondisk_log_size":10009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:05.384343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55640064,"num_objects":849,"num_object_clones":0,"num_object_copies":2547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":466,"num_objects_misplaced":1697,"num_objects_unfound":0,"num_objects_dirty":849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13309,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5816,"num_bytes_recovered":381157376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[7,1,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":1},{"shards":"2,6,7","objects":383},{"shards":"6,7","objects":465}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":489,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.615647+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:37:05.615647+0000","last_peered":"2023-09-30T09:37:05.615647+0000","last_clean":"2023-09-30T09:37:05.615647+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:37:05.615647+0000","last_undegraded":"2023-09-30T09:37:05.615647+0000","last_fullsized":"2023-09-30T09:37:05.615647+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":718,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059273+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:36:52.059273+0000","last_peered":"2023-09-30T09:36:52.059273+0000","last_clean":"2023-09-30T09:36:52.059273+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:36:52.059273+0000","last_undegraded":"2023-09-30T09:36:52.059273+0000","last_fullsized":"2023-09-30T09:36:52.059273+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":270,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059577+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:36:52.059577+0000","last_peered":"2023-09-30T09:36:52.059577+0000","last_clean":"2023-09-30T09:36:52.059577+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:36:52.059577+0000","last_undegraded":"2023-09-30T09:36:52.059577+0000","last_fullsized":"2023-09-30T09:36:52.059577+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":26,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059753+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:36:52.059753+0000","last_peered":"2023-09-30T09:36:52.059753+0000","last_clean":"2023-09-30T09:36:52.059753+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:36:52.059753+0000","last_undegraded":"2023-09-30T09:36:52.059753+0000","last_fullsized":"2023-09-30T09:36:52.059753+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"344'13283","reported_seq":17734,"reported_epoch":344,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:37:09.284106+0000","last_change":"2023-09-30T09:34:47.553318+0000","last_active":"2023-09-30T09:37:09.284106+0000","last_peered":"2023-09-30T09:37:09.284106+0000","last_clean":"2023-09-30T09:32:35.469075+0000","last_became_active":"2023-09-30T09:33:33.792569+0000","last_became_peered":"2023-09-30T09:33:33.792569+0000","last_unstale":"2023-09-30T09:37:09.284106+0000","last_undegraded":"2023-09-30T09:33:33.610726+0000","last_fullsized":"2023-09-30T09:33:33.587755+0000","mapping_epoch":302,"log_start":"305'7745","ondisk_log_start":"305'7745","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:16.121133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57212928,"num_objects":873,"num_object_clones":0,"num_object_copies":2619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13283,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2178,"num_bytes_recovered":142737408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-567}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":618,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.615861+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:37:05.615861+0000","last_peered":"2023-09-30T09:37:05.615861+0000","last_clean":"2023-09-30T09:37:05.615861+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:37:05.615861+0000","last_undegraded":"2023-09-30T09:37:05.615861+0000","last_fullsized":"2023-09-30T09:37:05.615861+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":851,"reported_epoch":342,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:37:04.035671+0000","last_change":"2023-09-30T09:35:11.685841+0000","last_active":"2023-09-30T09:37:04.035671+0000","last_peered":"2023-09-30T09:37:04.035671+0000","last_clean":"2023-09-30T09:26:47.308941+0000","last_became_active":"2023-09-30T09:35:11.685274+0000","last_became_peered":"2023-09-30T09:35:11.685274+0000","last_unstale":"2023-09-30T09:37:04.035671+0000","last_undegraded":"2023-09-30T09:35:11.661789+0000","last_fullsized":"2023-09-30T09:35:11.661213+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:38.516737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[1,4],"avail_no_missing":["1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":67,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.217688+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:36:52.217688+0000","last_peered":"2023-09-30T09:36:52.217688+0000","last_clean":"2023-09-30T09:36:52.217688+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:36:52.217688+0000","last_undegraded":"2023-09-30T09:36:52.217688+0000","last_fullsized":"2023-09-30T09:36:52.217688+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":242,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.060091+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:36:52.060091+0000","last_peered":"2023-09-30T09:36:52.060091+0000","last_clean":"2023-09-30T09:36:52.060091+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:36:52.060091+0000","last_undegraded":"2023-09-30T09:36:52.060091+0000","last_fullsized":"2023-09-30T09:36:52.060091+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"344'13354","reported_seq":13529,"reported_epoch":344,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:07.228964+0000","last_change":"2023-09-30T09:33:32.864041+0000","last_active":"2023-09-30T09:37:07.228964+0000","last_peered":"2023-09-30T09:37:07.228964+0000","last_clean":"2023-09-30T09:33:09.329441+0000","last_became_active":"2023-09-30T09:33:11.506402+0000","last_became_peered":"2023-09-30T09:33:11.506402+0000","last_unstale":"2023-09-30T09:37:07.228964+0000","last_undegraded":"2023-09-30T09:37:07.228964+0000","last_fullsized":"2023-09-30T09:37:07.228964+0000","mapping_epoch":295,"log_start":"312'8798","ondisk_log_start":"312'8798","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4556,"log_dups_size":0,"ondisk_log_size":4556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:51:06.485777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58327040,"num_objects":890,"num_object_clones":0,"num_object_copies":2670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":886,"num_objects_unfound":0,"num_objects_dirty":890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13354,"num_write_kb":455808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":890}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":816,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610212+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:37:05.610212+0000","last_peered":"2023-09-30T09:37:05.610212+0000","last_clean":"2023-09-30T09:37:05.610212+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:37:05.610212+0000","last_undegraded":"2023-09-30T09:37:05.610212+0000","last_fullsized":"2023-09-30T09:37:05.610212+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":345,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.522240+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:37:05.522240+0000","last_peered":"2023-09-30T09:37:05.522240+0000","last_clean":"2023-09-30T09:37:05.522240+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:37:05.522240+0000","last_undegraded":"2023-09-30T09:37:05.522240+0000","last_fullsized":"2023-09-30T09:37:05.522240+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1110,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.615584+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:37:05.615584+0000","last_peered":"2023-09-30T09:37:05.615584+0000","last_clean":"2023-09-30T09:37:05.615584+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:37:05.615584+0000","last_undegraded":"2023-09-30T09:37:05.615584+0000","last_fullsized":"2023-09-30T09:37:05.615584+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":361,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.520544+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:37:05.520544+0000","last_peered":"2023-09-30T09:37:05.520544+0000","last_clean":"2023-09-30T09:37:05.520544+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:37:05.520544+0000","last_undegraded":"2023-09-30T09:37:05.520544+0000","last_fullsized":"2023-09-30T09:37:05.520544+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"344'13366","reported_seq":19998,"reported_epoch":344,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:37:07.254580+0000","last_change":"2023-09-30T09:36:02.717804+0000","last_active":"2023-09-30T09:37:07.254580+0000","last_peered":"2023-09-30T09:37:07.254580+0000","last_clean":"2023-09-30T09:32:35.469896+0000","last_became_active":"2023-09-30T09:33:32.854592+0000","last_became_peered":"2023-09-30T09:33:32.854592+0000","last_unstale":"2023-09-30T09:37:07.254580+0000","last_undegraded":"2023-09-30T09:33:32.787590+0000","last_fullsized":"2023-09-30T09:33:32.601623+0000","mapping_epoch":301,"log_start":"312'8850","ondisk_log_start":"312'8850","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4516,"log_dups_size":0,"ondisk_log_size":4516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:02.368619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60162048,"num_objects":918,"num_object_clones":0,"num_object_copies":2754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":334,"num_objects_misplaced":334,"num_objects_unfound":0,"num_objects_dirty":918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13366,"num_write_kb":457088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3265,"num_bytes_recovered":213975040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":584}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":810,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521717+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:37:05.521717+0000","last_peered":"2023-09-30T09:37:05.521717+0000","last_clean":"2023-09-30T09:37:05.521717+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:37:05.521717+0000","last_undegraded":"2023-09-30T09:37:05.521717+0000","last_fullsized":"2023-09-30T09:37:05.521717+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":669,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.617431+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:37:05.617431+0000","last_peered":"2023-09-30T09:37:05.617431+0000","last_clean":"2023-09-30T09:37:05.617431+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:37:05.617431+0000","last_undegraded":"2023-09-30T09:37:05.617431+0000","last_fullsized":"2023-09-30T09:37:05.617431+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":29,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059850+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:36:52.059850+0000","last_peered":"2023-09-30T09:36:52.059850+0000","last_clean":"2023-09-30T09:36:52.059850+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:36:52.059850+0000","last_undegraded":"2023-09-30T09:36:52.059850+0000","last_fullsized":"2023-09-30T09:36:52.059850+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":285,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611749+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:37:05.611749+0000","last_peered":"2023-09-30T09:37:05.611749+0000","last_clean":"2023-09-30T09:37:05.611749+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:37:05.611749+0000","last_undegraded":"2023-09-30T09:37:05.611749+0000","last_fullsized":"2023-09-30T09:37:05.611749+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":758,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521795+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:37:05.521795+0000","last_peered":"2023-09-30T09:37:05.521795+0000","last_clean":"2023-09-30T09:37:05.521795+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:37:05.521795+0000","last_undegraded":"2023-09-30T09:37:05.521795+0000","last_fullsized":"2023-09-30T09:37:05.521795+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":25,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.060005+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:36:52.060005+0000","last_peered":"2023-09-30T09:36:52.060005+0000","last_clean":"2023-09-30T09:36:52.060005+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:36:52.060005+0000","last_undegraded":"2023-09-30T09:36:52.060005+0000","last_fullsized":"2023-09-30T09:36:52.060005+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":19,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.217607+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:36:52.217607+0000","last_peered":"2023-09-30T09:36:52.217607+0000","last_clean":"2023-09-30T09:36:52.217607+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:36:52.217607+0000","last_undegraded":"2023-09-30T09:36:52.217607+0000","last_fullsized":"2023-09-30T09:36:52.217607+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":529,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.522321+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:37:05.522321+0000","last_peered":"2023-09-30T09:37:05.522321+0000","last_clean":"2023-09-30T09:37:05.522321+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:37:05.522321+0000","last_undegraded":"2023-09-30T09:37:05.522321+0000","last_fullsized":"2023-09-30T09:37:05.522321+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":322,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598415+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:37:05.598415+0000","last_peered":"2023-09-30T09:37:05.598415+0000","last_clean":"2023-09-30T09:37:05.598415+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:37:05.598415+0000","last_undegraded":"2023-09-30T09:37:05.598415+0000","last_fullsized":"2023-09-30T09:37:05.598415+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.611486+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:37:05.611486+0000","last_peered":"2023-09-30T09:37:05.611486+0000","last_clean":"2023-09-30T09:37:05.611486+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:37:05.611486+0000","last_undegraded":"2023-09-30T09:37:05.611486+0000","last_fullsized":"2023-09-30T09:37:05.611486+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":288,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.598845+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:37:05.598845+0000","last_peered":"2023-09-30T09:37:05.598845+0000","last_clean":"2023-09-30T09:37:05.598845+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:37:05.598845+0000","last_undegraded":"2023-09-30T09:37:05.598845+0000","last_fullsized":"2023-09-30T09:37:05.598845+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":38,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610785+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:37:05.610785+0000","last_peered":"2023-09-30T09:37:05.610785+0000","last_clean":"2023-09-30T09:37:05.610785+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:37:05.610785+0000","last_undegraded":"2023-09-30T09:37:05.610785+0000","last_fullsized":"2023-09-30T09:37:05.610785+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":472,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.615540+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:37:05.615540+0000","last_peered":"2023-09-30T09:37:05.615540+0000","last_clean":"2023-09-30T09:37:05.615540+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:37:05.615540+0000","last_undegraded":"2023-09-30T09:37:05.615540+0000","last_fullsized":"2023-09-30T09:37:05.615540+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":340,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.522538+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:37:05.522538+0000","last_peered":"2023-09-30T09:37:05.522538+0000","last_clean":"2023-09-30T09:37:05.522538+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:37:05.522538+0000","last_undegraded":"2023-09-30T09:37:05.522538+0000","last_fullsized":"2023-09-30T09:37:05.522538+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":365,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521951+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:37:05.521951+0000","last_peered":"2023-09-30T09:37:05.521951+0000","last_clean":"2023-09-30T09:37:05.521951+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:37:05.521951+0000","last_undegraded":"2023-09-30T09:37:05.521951+0000","last_fullsized":"2023-09-30T09:37:05.521951+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":792,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.521521+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:37:05.521521+0000","last_peered":"2023-09-30T09:37:05.521521+0000","last_clean":"2023-09-30T09:37:05.521521+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:37:05.521521+0000","last_undegraded":"2023-09-30T09:37:05.521521+0000","last_fullsized":"2023-09-30T09:37:05.521521+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":45,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-09-30T09:36:52.059979+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:36:52.059979+0000","last_peered":"2023-09-30T09:36:52.059979+0000","last_clean":"2023-09-30T09:36:52.059979+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:36:52.059979+0000","last_undegraded":"2023-09-30T09:36:52.059979+0000","last_fullsized":"2023-09-30T09:36:52.059979+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":319,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-09-30T09:37:05.610727+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:37:05.610727+0000","last_peered":"2023-09-30T09:37:05.610727+0000","last_clean":"2023-09-30T09:37:05.610727+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:37:05.610727+0000","last_undegraded":"2023-09-30T09:37:05.610727+0000","last_fullsized":"2023-09-30T09:37:05.610727+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":941293592,"num_objects":14364,"num_object_clones":0,"num_object_copies":43092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7649,"num_objects_misplaced":11944,"num_objects_unfound":0,"num_objects_dirty":14364,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":212188,"num_write_kb":7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38558,"num_bytes_recovered":2371354624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2717528064,"data_stored":2717515848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82252,"ondisk_log_size":82252,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234306,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495943,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906664,"kb_used_data":510560,"kb_used_omap":0,"kb_used_meta":396096,"kb_avail":92838488,"statfs":{"total":95995035648,"available":95066611712,"internally_reserved":0,"allocated":522813440,"data_stored":521615686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405602304},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9380000000000002,"5min":2.2330000000000001,"15min":1.802},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":44.113,"5min":44.113,"15min":45.927999999999997},"last":2.4780000000000002},{"interface":"front","average":{"1min":4.6150000000000002,"5min":2.3639999999999999,"15min":2.0449999999999999},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":44.164000000000001,"5min":44.164000000000001,"15min":45.316000000000003},"last":2.556}]},{"osd":1,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.0419999999999998,"15min":1.8819999999999999},"min":{"1min":0.39100000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.343999999999999,"5min":13.343999999999999,"15min":13.343999999999999},"last":1.466},{"interface":"front","average":{"1min":3.169,"5min":2.3300000000000001,"15min":2.1520000000000001},"min":{"1min":0.35999999999999999,"5min":0.313,"15min":0.313},"max":{"1min":18.425999999999998,"5min":18.425999999999998,"15min":18.425999999999998},"last":1.5}]},{"osd":2,"last update":"Sat Sep 30 09:36:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2210000000000001,"5min":2.226,"15min":1.905},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":17.126000000000001,"5min":22.158000000000001,"15min":37.978000000000002},"last":0.442},{"interface":"front","average":{"1min":3.7589999999999999,"5min":2.5110000000000001,"15min":1.851},"min":{"1min":0.32500000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":25.164999999999999,"5min":25.164999999999999,"15min":37.759999999999998},"last":0.51700000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7839999999999998,"5min":7.1130000000000004,"15min":3.532},"min":{"1min":0.45300000000000001,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":90.147999999999996,"5min":212.44800000000001,"15min":212.44800000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":8.9450000000000003,"5min":7.8339999999999996,"15min":3.9199999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":89.859999999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.0640000000000001,"15min":1.593},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23400000000000001},"max":{"1min":3.004,"5min":7.4359999999999999,"15min":73.656000000000006},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.0209999999999999,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.23000000000000001},"max":{"1min":1.8089999999999999,"5min":9.6590000000000007,"15min":73.882999999999996},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.1930000000000001,"15min":1.819},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":1.024,"5min":8.2040000000000006,"15min":76.613},"last":1.486},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.1910000000000001,"15min":1.786},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.0979999999999999,"5min":9.7750000000000004,"15min":76.329999999999998},"last":0.33700000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":1.458,"15min":2.1419999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":3.9870000000000001,"5min":8.7400000000000002,"15min":91.120999999999995},"last":0.501},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.1679999999999999,"15min":2.0910000000000002},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.4870000000000001,"5min":9.9489999999999998,"15min":91.733999999999995},"last":0.45900000000000002}]}]},{"osd":1,"up_from":301,"seq":1292785156140,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":338064,"kb_used_data":94792,"kb_used_omap":9,"kb_used_meta":243254,"kb_avail":93407088,"statfs":{"total":95995035648,"available":95648858112,"internally_reserved":0,"allocated":97067008,"data_stored":95127577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":249092114},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":9.7590000000000003,"5min":4.0979999999999999,"15min":3.0600000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":138.43799999999999,"5min":138.43799999999999,"15min":138.43799999999999},"last":1.4059999999999999},{"interface":"front","average":{"1min":9.4320000000000004,"5min":4.5140000000000002,"15min":3.52},"min":{"1min":0.27800000000000002,"5min":0.25,"15min":0.25},"max":{"1min":138.482,"5min":138.482,"15min":138.482},"last":5.7640000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.8100000000000001,"15min":2.7999999999999998},"min":{"1min":0.254,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":13.656000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":2.5640000000000001},{"interface":"front","average":{"1min":3.9609999999999999,"5min":3.0899999999999999,"15min":2.8340000000000001},"min":{"1min":0.309,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":18.978999999999999,"5min":18.978999999999999,"15min":18.978999999999999},"last":3.968}]},{"osd":3,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":6.8879999999999999,"5min":7.9370000000000003,"15min":8.9429999999999996},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":40.606000000000002,"5min":77.959000000000003,"15min":77.959000000000003},"last":2.1480000000000001},{"interface":"front","average":{"1min":7.5449999999999999,"5min":8.2690000000000001,"15min":9.5410000000000004},"min":{"1min":0.439,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":41.247,"5min":77.828999999999994,"15min":77.828999999999994},"last":2.1989999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.1059999999999999,"15min":2.0259999999999998},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.0090000000000003,"5min":13.602,"15min":13.602},"last":1.5129999999999999},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.712,"15min":1.4319999999999999},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.311},"max":{"1min":8.6440000000000001,"5min":10.773,"15min":10.773},"last":2.113}]},{"osd":5,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":2.351,"15min":2.1890000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.2750000000000004,"5min":13.666,"15min":13.666},"last":1.4870000000000001},{"interface":"front","average":{"1min":2.9239999999999999,"5min":1.9890000000000001,"15min":1.73},"min":{"1min":0.39100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.177,"5min":10.847,"15min":10.847},"last":2.0699999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":11.52,"15min":15.238},"min":{"1min":0.503,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.1029999999999998,"5min":209.221,"15min":209.221},"last":1.5369999999999999},{"interface":"front","average":{"1min":2.48,"5min":11.207000000000001,"15min":15.079000000000001},"min":{"1min":0.40000000000000002,"5min":0.377,"15min":0.377},"max":{"1min":8.1310000000000002,"5min":208.43600000000001,"15min":208.43600000000001},"last":1.4550000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9950000000000001,"5min":8.3640000000000008,"15min":11.084},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.3559999999999999,"5min":165.59399999999999,"15min":165.59399999999999},"last":2.2349999999999999},{"interface":"front","average":{"1min":3.3719999999999999,"5min":8.5670000000000002,"15min":11.308},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":10.565,"5min":165.833,"15min":165.833},"last":7.8280000000000003}]}]},{"osd":0,"up_from":94,"seq":403726926039,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":546408,"kb_used_data":197020,"kb_used_omap":5,"kb_used_meta":349370,"kb_avail":93198744,"statfs":{"total":95995035648,"available":95435513856,"internally_reserved":0,"allocated":201748480,"data_stored":199675406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":357755133},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:36:38 2023","interfaces":[{"interface":"back","average":{"1min":4.1920000000000002,"5min":3.0630000000000002,"15min":3.0190000000000001},"min":{"1min":0.24099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":15.401,"5min":17.966000000000001,"15min":17.966000000000001},"last":1.7549999999999999},{"interface":"front","average":{"1min":6.3339999999999996,"5min":7.7549999999999999,"15min":8.8680000000000003},"min":{"1min":0.31,"5min":0.22,"15min":0.22},"max":{"1min":25.869,"5min":138.92400000000001,"15min":138.92400000000001},"last":0.99099999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.1299999999999999,"15min":1.744},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.887,"5min":11.218999999999999,"15min":55.823999999999998},"last":0.747},{"interface":"front","average":{"1min":3.2080000000000002,"5min":2.5350000000000001,"15min":1.9319999999999999},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":10.763,"5min":40.128999999999998,"15min":55.790999999999997},"last":0.72099999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":8.5350000000000001,"15min":4.2089999999999996},"min":{"1min":0.30599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":16.271000000000001,"5min":218.06999999999999,"15min":218.06999999999999},"last":4.5339999999999998},{"interface":"front","average":{"1min":2.8199999999999998,"5min":6.2930000000000001,"15min":3.4159999999999999},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":14.987,"5min":218.63200000000001,"15min":218.63200000000001},"last":1.778}]},{"osd":4,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.6599999999999999,"15min":1.679},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.29099999999999998},"max":{"1min":10.614000000000001,"5min":10.614000000000001,"15min":121.486},"last":0.629},{"interface":"front","average":{"1min":3.3780000000000001,"5min":3.863,"15min":2.649},"min":{"1min":0.39900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":17.044,"5min":136.262,"15min":136.262},"last":4.569}]},{"osd":5,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":4.1710000000000003,"15min":3.0630000000000002},"min":{"1min":0.439,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.995999999999999,"5min":136.34,"15min":136.34},"last":4.4690000000000003},{"interface":"front","average":{"1min":2.4129999999999998,"5min":1.9419999999999999,"15min":2.2839999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":11.991,"5min":12.717000000000001,"15min":135.381},"last":1.6930000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":7.1440000000000001,"15min":3.2400000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":12.021000000000001,"5min":214.785,"15min":214.785},"last":1.649},{"interface":"front","average":{"1min":2.8999999999999999,"5min":7.2439999999999998,"15min":3.1850000000000001},"min":{"1min":0.46300000000000002,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":10.695,"5min":214.93899999999999,"15min":214.93899999999999},"last":0.66800000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":2.9870000000000001,"15min":3.1720000000000002},"min":{"1min":0.42999999999999999,"5min":0.38300000000000001,"15min":0.373},"max":{"1min":12.048,"5min":97.382000000000005,"15min":150.96899999999999},"last":1.7210000000000001},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.944,"15min":3.0150000000000001},"min":{"1min":0.43099999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.153,"5min":97.015000000000001,"15min":150.328},"last":0.69599999999999995}]}]},{"osd":2,"up_from":104,"seq":446676598980,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":348048,"kb_used_data":37640,"kb_used_omap":5,"kb_used_meta":310394,"kb_avail":93397104,"statfs":{"total":95995035648,"available":95638634496,"internally_reserved":0,"allocated":38543360,"data_stored":36666065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":317844109},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":4.1130000000000004,"5min":3.4630000000000001,"15min":2.1680000000000001},"min":{"1min":0.24199999999999999,"5min":0.187,"15min":0.187},"max":{"1min":15.063000000000001,"5min":77.379000000000005,"15min":77.379000000000005},"last":1.6379999999999999},{"interface":"front","average":{"1min":2.7509999999999999,"5min":3.2709999999999999,"15min":1.9670000000000001},"min":{"1min":0.31900000000000001,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":15.151,"5min":77.066999999999993,"15min":77.066999999999993},"last":1.387}]},{"osd":1,"last update":"Sat Sep 30 09:36:36 2023","interfaces":[{"interface":"back","average":{"1min":8.9789999999999992,"5min":4.8339999999999996,"15min":3.992},"min":{"1min":0.36399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":64.412999999999997,"5min":64.412999999999997,"15min":64.412999999999997},"last":1.177},{"interface":"front","average":{"1min":7.6420000000000003,"5min":4.1260000000000003,"15min":3.4129999999999998},"min":{"1min":0.33700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":63.247999999999998,"5min":63.247999999999998,"15min":63.247999999999998},"last":3.1680000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:36:51 2023","interfaces":[{"interface":"back","average":{"1min":15.153,"5min":7.4130000000000003,"15min":3.4689999999999999},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":134.078,"5min":134.078,"15min":134.078},"last":3.9889999999999999},{"interface":"front","average":{"1min":15.754,"5min":7.0709999999999997,"15min":3.5920000000000001},"min":{"1min":0.38900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":135.73400000000001,"5min":135.73400000000001,"15min":135.73400000000001},"last":0.89400000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":2.036,"15min":1.3380000000000001},"min":{"1min":0.39000000000000001,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":8.1240000000000006,"5min":35.972000000000001,"15min":35.972000000000001},"last":1.4510000000000001},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.931,"15min":1.597},"min":{"1min":0.42899999999999999,"5min":0.33800000000000002,"15min":0.32700000000000001},"max":{"1min":13.224,"5min":36.518000000000001,"15min":36.518000000000001},"last":0.78500000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.8610000000000002,"5min":2.024,"15min":2.0880000000000001},"min":{"1min":0.39400000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.41,"5min":12.651999999999999,"15min":155.477},"last":1.075},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.591,"15min":1.952},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":8.0839999999999996,"5min":8.0839999999999996,"15min":155.46000000000001},"last":1.421}]},{"osd":6,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":6.6180000000000003,"15min":3.2879999999999998},"min":{"1min":0.48199999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.273,"5min":154.417,"15min":154.417},"last":0.86399999999999999},{"interface":"front","average":{"1min":2.8359999999999999,"5min":7.2729999999999997,"15min":3.3399999999999999},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":10.382,"5min":154.04400000000001,"15min":154.04400000000001},"last":1.111}]},{"osd":7,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":3.8639999999999999,"15min":3.1480000000000001},"min":{"1min":0.52800000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":13.304,"5min":123.294,"15min":213.404},"last":0.83199999999999996},{"interface":"front","average":{"1min":2.9390000000000001,"5min":4.1269999999999998,"15min":3.1699999999999999},"min":{"1min":0.45100000000000001,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":10.35,"5min":123.16,"15min":213.27199999999999},"last":1.147}]}]},{"osd":3,"up_from":109,"seq":468151435453,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1299840,"kb_used_data":656432,"kb_used_omap":2,"kb_used_meta":643325,"kb_avail":92445312,"statfs":{"total":95995035648,"available":94663999488,"internally_reserved":0,"allocated":672186368,"data_stored":670163239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":658765159},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":5.4980000000000002,"5min":4.9889999999999999,"15min":3.097},"min":{"1min":0.23499999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":51.835000000000001,"5min":102.57299999999999,"15min":130.946},"last":1.0960000000000001},{"interface":"front","average":{"1min":5.6929999999999996,"5min":4.899,"15min":3.0579999999999998},"min":{"1min":0.317,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":52.093000000000004,"5min":102.745,"15min":130.96899999999999},"last":2.2370000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5489999999999999,"5min":14.395,"15min":18.975999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":9.3919999999999995,"5min":273.99599999999998,"15min":273.99599999999998},"last":2.5419999999999998},{"interface":"front","average":{"1min":4.4950000000000001,"5min":13.355,"15min":17.132999999999999},"min":{"1min":0.34899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.6910000000000007,"5min":274.21600000000001,"15min":274.21600000000001},"last":3.2959999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8360000000000003,"5min":4.5800000000000001,"15min":3.3759999999999999},"min":{"1min":0.53600000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":62.289000000000001,"5min":76.974000000000004,"15min":145.75899999999999},"last":1.139},{"interface":"front","average":{"1min":6.0650000000000004,"5min":4.3339999999999996,"15min":3.2170000000000001},"min":{"1min":0.33400000000000002,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":65.415999999999997,"5min":77.593999999999994,"15min":150.358},"last":1.8919999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":2.419,"5min":2.702,"15min":1.6899999999999999},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":10.805,"5min":60.637,"15min":60.637},"last":1.6180000000000001},{"interface":"front","average":{"1min":3.008,"5min":2.3100000000000001,"15min":1.627},"min":{"1min":0.40100000000000002,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":8.8049999999999997,"5min":10.731,"15min":15.638},"last":2.2890000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":2.9940000000000002,"15min":1.9390000000000001},"min":{"1min":0.375,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":8.8309999999999995,"5min":37.485999999999997,"15min":37.485999999999997},"last":3.9820000000000002},{"interface":"front","average":{"1min":2.9199999999999999,"5min":3.2930000000000001,"15min":2.113},"min":{"1min":0.437,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":7.7439999999999998,"5min":37.448,"15min":37.448},"last":0.72099999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":2.9740000000000002,"15min":1.8240000000000001},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.891999999999999,"5min":60.698,"15min":60.698},"last":0.65300000000000002},{"interface":"front","average":{"1min":3.895,"5min":2.7629999999999999,"15min":1.8999999999999999},"min":{"1min":0.61299999999999999,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":12.121,"5min":12.121,"15min":14.69},"last":2.8690000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":3.2469999999999999,"15min":2.7440000000000002},"min":{"1min":0.44500000000000001,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":7.7709999999999999,"5min":26.469000000000001,"15min":226.03999999999999},"last":3.3380000000000001},{"interface":"front","average":{"1min":2.8370000000000002,"5min":3.1989999999999998,"15min":2.6019999999999999},"min":{"1min":0.41799999999999998,"5min":0.38800000000000001,"15min":0.34799999999999998},"max":{"1min":10.855,"5min":60.670000000000002,"15min":226.06100000000001},"last":2.7629999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019216,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":485024,"kb_used_data":227928,"kb_used_omap":0,"kb_used_meta":257088,"kb_avail":93260128,"statfs":{"total":95995035648,"available":95498371072,"internally_reserved":0,"allocated":233398272,"data_stored":232208213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":263258112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":1.4399999999999999,"15min":2.8780000000000001},"min":{"1min":0.33200000000000002,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":4.5940000000000003,"5min":14.353,"15min":128.79300000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.873,"15min":2.8730000000000002},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.248},"max":{"1min":6.9939999999999998,"5min":10.618,"15min":128.38800000000001},"last":2.5800000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4169999999999998,"5min":6.8659999999999997,"15min":8.6240000000000006},"min":{"1min":0.26600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.915,"5min":160.69999999999999,"15min":160.69999999999999},"last":7.577},{"interface":"front","average":{"1min":3.2629999999999999,"5min":6.7960000000000003,"15min":8.6419999999999995},"min":{"1min":0.28399999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.7029999999999994,"5min":160.50700000000001,"15min":160.50700000000001},"last":0.314}]},{"osd":2,"last update":"Sat Sep 30 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.9850000000000001,"15min":2.9399999999999999},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9790000000000001,"5min":19.391999999999999,"15min":140.601},"last":1.71},{"interface":"front","average":{"1min":2.3399999999999999,"5min":2.0169999999999999,"15min":2.9300000000000002},"min":{"1min":0.39300000000000002,"5min":0.309,"15min":0.309},"max":{"1min":4.9900000000000002,"5min":19.422999999999998,"15min":140.54900000000001},"last":1.6919999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:36:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3010000000000002,"5min":2.964,"15min":3.6030000000000002},"min":{"1min":0.42799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.2650000000000006,"5min":52.762,"15min":281.42200000000003},"last":4.6929999999999996},{"interface":"front","average":{"1min":3.4359999999999999,"5min":2.992,"15min":3.5720000000000001},"min":{"1min":0.49099999999999999,"5min":0.312,"15min":0.312},"max":{"1min":9.2810000000000006,"5min":52.790999999999997,"15min":281.44999999999999},"last":4.702}]},{"osd":5,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.94399999999999995,"15min":1.7130000000000001},"min":{"1min":0.309,"5min":0.219,"15min":0.219},"max":{"1min":4.6120000000000001,"5min":8.6509999999999998,"15min":134.30600000000001},"last":2.5230000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1,"15min":1.772},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":2.2309999999999999,"5min":6.3929999999999998,"15min":134.35300000000001},"last":0.29199999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":3.0299999999999998,"15min":1.593},"min":{"1min":0.32800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.1079999999999997,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":2.927,"15min":1.597},"min":{"1min":0.34599999999999997,"5min":0.249,"15min":0.249},"max":{"1min":2.6459999999999999,"5min":202.42400000000001,"15min":202.42400000000001},"last":2.556}]},{"osd":7,"last update":"Sat Sep 30 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":4.4320000000000004,"15min":3.004},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":3.8330000000000002,"5min":230.90100000000001,"15min":230.90100000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":4.1289999999999996,"15min":2.5579999999999998},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":2.6779999999999999,"5min":229.97,"15min":229.97},"last":2.363}]}]},{"osd":5,"up_from":38,"seq":163208757580,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505312,"kb_used_data":231640,"kb_used_omap":0,"kb_used_meta":273664,"kb_avail":93239840,"statfs":{"total":95995035648,"available":95477596160,"internally_reserved":0,"allocated":237199360,"data_stored":236005849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":280231936},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.2069999999999999,"15min":2.4540000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.22},"max":{"1min":4.6459999999999999,"5min":56.563000000000002,"15min":148.79900000000001},"last":1.8},{"interface":"front","average":{"1min":1.738,"5min":2.3199999999999998,"15min":2.7589999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":6.2969999999999997,"5min":57.241999999999997,"15min":148.779},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:36:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":2.6619999999999999,"15min":2.1120000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.237,"5min":60.799999999999997,"15min":60.799999999999997},"last":0.54100000000000004},{"interface":"front","average":{"1min":3.012,"5min":3.581,"15min":3.3010000000000002},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.2539999999999996,"5min":63.798999999999999,"15min":63.798999999999999},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.8879999999999999,"15min":3.1150000000000002},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.71,"5min":30.613,"15min":165.768},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.0249999999999999,"15min":3.6030000000000002},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.1989999999999998,"5min":36.488,"15min":166.24000000000001},"last":2.0680000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:36:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":5.9180000000000001,"15min":4.0069999999999997},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5380000000000003,"5min":172.976,"15min":226.34700000000001},"last":1.877},{"interface":"front","average":{"1min":2.4689999999999999,"5min":5.8799999999999999,"15min":3.8300000000000001},"min":{"1min":0.376,"5min":0.36599999999999999,"15min":0.313},"max":{"1min":8.2240000000000002,"5min":173.01599999999999,"15min":226.36000000000001},"last":1.913}]},{"osd":4,"last update":"Sat Sep 30 09:36:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.76500000000000001,"15min":0.84899999999999998},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":1.0760000000000001,"5min":5.7160000000000002,"15min":8.5359999999999996},"last":0.434},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.96999999999999997,"15min":0.94199999999999995},"min":{"1min":0.29599999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.1230000000000002,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":3.1749999999999998,"15min":1.7390000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":4.6639999999999997,"5min":155.602,"15min":155.602},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.014,"5min":3.2309999999999999,"15min":1.8680000000000001},"min":{"1min":0.36499999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.1600000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":2.5630000000000002,"15min":1.639},"min":{"1min":0.33000000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":3.2000000000000002,"5min":133.40700000000001,"15min":133.40700000000001},"last":5.883},{"interface":"front","average":{"1min":0.56200000000000006,"5min":2.6549999999999998,"15min":1.5609999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":2.274,"5min":132.797,"15min":132.797},"last":5.7880000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":187305984,"data_stored":187301912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":91951104,"data_stored":91947032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":32899072,"data_stored":32899072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":662634496,"data_stored":662634496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":227737600,"data_stored":227737600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":231211008,"data_stored":231211008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":510332928,"data_stored":510328856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":773455872,"data_stored":773455872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:37:11.720 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:37:11.799 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:11.829 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:11.872 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:11.874 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:11 smithi167 ceph-mon[192880]: from='client.35712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:11.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:11.923 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:11.975 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:12.009 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:11 smithi019 ceph-mon[224043]: from='client.35712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:12.011 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:12.012 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:11 smithi019 ceph-mon[216815]: from='client.35712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:12.073 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:12.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:12.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:12.177 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:12.208 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:12.377 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:12.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:12.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:12.487 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:12.611 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:12.651 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:12 smithi019 ceph-mon[216815]: pgmap v891: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 895 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 7663/42813 objects degraded (17.899%); 11944/42813 objects misplaced (27.898%); 16 MiB/s, 331 objects/s recovering 2023-09-30T09:37:12.873 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:12.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:12.924 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:37:12.925 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:37:12.948 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:12 smithi167 ceph-mon[192880]: pgmap v891: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 895 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 7663/42813 objects degraded (17.899%); 11944/42813 objects misplaced (27.898%); 16 MiB/s, 331 objects/s recovering 2023-09-30T09:37:12.968 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:12 smithi019 ceph-mon[224043]: pgmap v891: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 895 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 7663/42813 objects degraded (17.899%); 11944/42813 objects misplaced (27.898%); 16 MiB/s, 331 objects/s recovering 2023-09-30T09:37:12.981 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:13.078 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:13.181 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:13.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:13.285 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:13.312 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:13.378 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:13.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:13.492 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:13.592 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:13.684 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:37:13.743 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:13.747 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:37:13.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:37:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:37:13] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-09-30T09:37:13.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3982920559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:37:13.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7663/42813 objects degraded (17.899%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:13.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3982920559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:37:13.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7663/42813 objects degraded (17.899%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:13.978 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:13.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3982920559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:37:14.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7663/42813 objects degraded (17.899%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:14.087 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:14.118 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:14.165 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:14.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:14.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:14.195 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:14.270 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:14.355 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:14.653 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[216815]: pgmap v892: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 828 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 6936/39597 objects degraded (17.516%); 10929/39597 objects misplaced (27.601%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:37:14.653 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:14.653 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3982920559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:37:14.654 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[216815]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.654 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[216815]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.654 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[216815]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.654 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[216815]: osdmap e345: 8 total, 8 up, 7 in 2023-09-30T09:37:14.659 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:14.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:14.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:14 smithi167 ceph-mon[192880]: pgmap v892: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 828 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 6936/39597 objects degraded (17.516%); 10929/39597 objects misplaced (27.601%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:37:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:14 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3982920559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:37:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:14 smithi167 ceph-mon[192880]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:14 smithi167 ceph-mon[192880]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:14 smithi167 ceph-mon[192880]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:14 smithi167 ceph-mon[192880]: osdmap e345: 8 total, 8 up, 7 in 2023-09-30T09:37:14.774 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:14.817 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:14.879 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:14.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:14.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[224043]: pgmap v892: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 828 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 6936/39597 objects degraded (17.516%); 10929/39597 objects misplaced (27.601%); 16 MiB/s, 254 objects/s recovering 2023-09-30T09:37:14.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:14.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3982920559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:37:14.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[224043]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[224043]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[224043]: failed to encode map e345 with expected crc 2023-09-30T09:37:14.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:14 smithi019 ceph-mon[224043]: osdmap e345: 8 total, 8 up, 7 in 2023-09-30T09:37:14.983 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:15.018 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:15.069 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:15.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:15.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:15.173 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:15.208 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:15.246 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:37:15.268 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:37:15.291 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:37:15.316 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:15.317 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:37:15.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:15.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:15.337 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:37:15.379 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:37:15.379 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:37:15.394 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:37:15.395 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:37:15.395 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:37:15.395 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:37:15.395 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:37:15.395 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:37:15.395 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:37:15.396 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:37:15.396 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:37:15.421 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:15.447 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:15.491 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:15.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:15.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:15.594 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:15.620 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: osdmap e346: 8 total, 8 up, 7 in 2023-09-30T09:37:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: failed to encode map e346 with expected crc 2023-09-30T09:37:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: failed to encode map e346 with expected crc 2023-09-30T09:37:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: failed to encode map e346 with expected crc 2023-09-30T09:37:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: failed to encode map e346 with expected crc 2023-09-30T09:37:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: failed to encode map e346 with expected crc 2023-09-30T09:37:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: failed to encode map e346 with expected crc 2023-09-30T09:37:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: failed to encode map e346 with expected crc 2023-09-30T09:37:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: failed to encode map e346 with expected crc 2023-09-30T09:37:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:37:15.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1476263374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:15.876 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:15.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:15.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:15.979 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:16.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: osdmap e346: 8 total, 8 up, 7 in 2023-09-30T09:37:16.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1476263374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: osdmap e346: 8 total, 8 up, 7 in 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: failed to encode map e346 with expected crc 2023-09-30T09:37:16.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:37:16.057 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1476263374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:16.073 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:16.187 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:16.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:16.288 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:16.341 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:16.456 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:37:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:37:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:37:16.456 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:37:16.457 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:37:16.689 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:37:16.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[216815]: pgmap v895: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 744 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 6381/35568 objects degraded (17.940%); 9686/35568 objects misplaced (27.232%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:37:16.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[216815]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[216815]: osdmap e347: 8 total, 8 up, 7 in 2023-09-30T09:37:16.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[216815]: 7.2 continuing backfill to osd.2 from (314'9050,345'13591] MIN to 345'13591 2023-09-30T09:37:16.692 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[216815]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[216815]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[216815]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[224043]: pgmap v895: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 744 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 6381/35568 objects degraded (17.940%); 9686/35568 objects misplaced (27.232%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[224043]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[224043]: osdmap e347: 8 total, 8 up, 7 in 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[224043]: 7.2 continuing backfill to osd.2 from (314'9050,345'13591] MIN to 345'13591 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[224043]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[224043]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[224043]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.694 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:16 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:37:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:16 smithi167 ceph-mon[192880]: pgmap v895: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 744 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 6381/35568 objects degraded (17.940%); 9686/35568 objects misplaced (27.232%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:37:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:16 smithi167 ceph-mon[192880]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:16 smithi167 ceph-mon[192880]: osdmap e347: 8 total, 8 up, 7 in 2023-09-30T09:37:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:16 smithi167 ceph-mon[192880]: 7.2 continuing backfill to osd.2 from (314'9050,345'13591] MIN to 345'13591 2023-09-30T09:37:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:16 smithi167 ceph-mon[192880]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:16 smithi167 ceph-mon[192880]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:16 smithi167 ceph-mon[192880]: failed to encode map e347 with expected crc 2023-09-30T09:37:16.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:16 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:37:16.892 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:16.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:16.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:17.016 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:17.052 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:17.130 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:17.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:17.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:17.240 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:17.338 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:17.467 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:17.470 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:37:17.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:17.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:17.575 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:17.580 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:17.668 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:17.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:17 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:37:17.785 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:17.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:17.897 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:17.974 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:17.994 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:17 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:37:17.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:17 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:37:18.022 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:18.089 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:37:18.089 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5922/32949 objects degraded (17.973%), 9 pgs degraded, 2 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1176,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":347,"num_osds":8,"num_up_osds":8,"osd_up_since":1696066411,"num_in_osds":7,"osd_in_since":1696066446,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":104},{"state_name":"active+recovery_wait+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+clean+scrubbing","count":1}],"num_pgs":119,"num_pools":5,"num_objects":10983,"data_bytes":722811748,"bytes_used":4176592896,"bytes_avail":667788656640,"bytes_total":671965249536,"degraded_objects":5922,"degraded_total":32949,"degraded_ratio":0.17973231357552583,"misplaced_objects":9642,"misplaced_total":32949,"misplaced_ratio":0.2926340708367477,"recovering_objects_per_sec":305,"recovering_bytes_per_sec":19993309,"recovering_keys_per_sec":0,"num_objects_recovered":1831,"num_bytes_recovered":119996416,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":547},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-30T09:34:31.415542+0000","services":{}},"progress_events":{"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"bf6cdc82-6db1-4694-b9dc-04effc0cdb2e":{"message":"Global Recovery Event (4m)\n [========================....] (remaining: 36s)","progress":0.88235294818878174,"add_to_ceph_s":true}}} 2023-09-30T09:37:18.169 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:18.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:18.282 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:18.368 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:19.018 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:37:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:18 smithi167 ceph-mon[192880]: pgmap v897: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 689 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 5922/32949 objects degraded (17.973%); 9642/32949 objects misplaced (29.263%); 19 MiB/s, 305 objects/s recovering 2023-09-30T09:37:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2508917875' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:37:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5922/32949 objects degraded (17.973%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:19.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:18 smithi019 ceph-mon[224043]: pgmap v897: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 689 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 5922/32949 objects degraded (17.973%); 9642/32949 objects misplaced (29.263%); 19 MiB/s, 305 objects/s recovering 2023-09-30T09:37:19.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:19.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2508917875' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:37:19.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5922/32949 objects degraded (17.973%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:19.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:18 smithi019 ceph-mon[216815]: pgmap v897: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 689 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 5922/32949 objects degraded (17.973%); 9642/32949 objects misplaced (29.263%); 19 MiB/s, 305 objects/s recovering 2023-09-30T09:37:19.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:19.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2508917875' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:37:19.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5922/32949 objects degraded (17.973%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:19.600 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:19.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:19.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:19.715 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:19.785 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:20.048 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:37:20.048 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:37:20.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:37:20.048 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:37:20.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.048 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55758 7.4 7:237201bd:::benchmark_data_smithi019_404601_object55757:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:37:20.048 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.487485+0000", 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 284.55522220799998, 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.018118769, 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55758 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487485+0000", 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487485+0000", 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487481+0000", 2023-09-30T09:37:20.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999952 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487521+0000", 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0756000000000001e-05 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487533+0000", 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1997000000000001e-05 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487540+0000", 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5139999999999998e-06 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487641+0000", 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100797 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487647+0000", 2023-09-30T09:37:20.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3230000000000003e-06 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.513556+0000", 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.025908515999999999 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.513562+0000", 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8390000000000001e-06 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.504438+0000", 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99087605700000003 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505604+0000", 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001165814 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55762 7.4 7:2a4e1ba2:::benchmark_data_smithi019_404601_object55761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.487546+0000", 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 284.55516126499998, 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.018159992, 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55762 2023-09-30T09:37:20.055 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487546+0000", 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487546+0000", 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487546+0000", 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487580+0000", 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4254000000000001e-05 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487587+0000", 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.007e-06 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487592+0000", 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4460000000000003e-06 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487691+0000", 2023-09-30T09:37:20.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8904e-05 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487696+0000", 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0710000000000002e-06 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.513582+0000", 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.025886560999999999 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.513587+0000", 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9699999999999998e-06 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505634+0000", 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99204647499999998 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505706+0000", 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2795999999999999e-05 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55764 7.4 7:2f6ef7de:::benchmark_data_smithi019_404601_object55763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.487596+0000", 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 284.55511123799999, 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0181986119999999, 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:37:20.061 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55764 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487596+0000", 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487596+0000", 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487596+0000", 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487629+0000", 2023-09-30T09:37:20.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3365000000000003e-05 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487636+0000", 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1720000000000004e-06 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487639+0000", 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.941e-06 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487712+0000", 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2329999999999994e-05 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487717+0000", 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6789999999999996e-06 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.514179+0000", 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.026462697 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.514187+0000", 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9799999999999998e-06 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505731+0000", 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99154362600000001 2023-09-30T09:37:20.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505795+0000", 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4257000000000005e-05 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55779 7.b 7:df81075e:::benchmark_data_smithi019_404601_object55778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.518341+0000", 2023-09-30T09:37:20.067 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 284.52436656100002, 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99813708499999998, 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55779 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.518341+0000", 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.518341+0000", 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.518341+0000", 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:37:20.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.518442+0000", 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101326 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.518449+0000", 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6560000000000003e-06 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.518454+0000", 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8250000000000004e-06 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.518506+0000", 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2299000000000002e-05 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.518514+0000", 2023-09-30T09:37:20.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2800000000000003e-06 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505039+0000", 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98652516000000001 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.506486+0000", 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001446815 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 4", 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.506655+0000", 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000168789 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.512762+0000", 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0061075410000000002 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.516341+0000", 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035785579999999999 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.516396+0000", 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4960000000000002e-05 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.516478+0000", 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2280999999999994e-05 2023-09-30T09:37:20.074 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55781 7.d 7:bd940ec6:::benchmark_data_smithi019_404601_object55780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.521034+0000", 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 284.52167384699999, 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98475530099999997, 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.075 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55781 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.521034+0000", 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.521034+0000", 2023-09-30T09:37:20.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.521033+0000", 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.521083+0000", 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9818000000000003e-05 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.521094+0000", 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1357e-05 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.521099+0000", 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5689999999999998e-06 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.521145+0000", 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6573000000000003e-05 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.521155+0000", 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6630000000000005e-06 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.492029+0000", 2023-09-30T09:37:20.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97087457499999996 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.492168+0000", 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013902000000000001 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0", 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.492298+0000", 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012919599999999999 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.502805+0000", 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01050735 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505660+0000", 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028553150000000002 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505708+0000", 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7806000000000003e-05 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.505789+0000", 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1087000000000005e-05 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145313 7.9 7:9885adcf:::benchmark_data_smithi019_404601_object32035:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.813845+0000", 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.22886282600001, 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.185121933, 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145313 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.083 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.813845+0000", 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.813845+0000", 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.084 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.813844+0000", 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.813847+0000", 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.385e-06 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.813853+0000", 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9039999999999997e-06 2023-09-30T09:37:20.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.813856+0000", 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0309999999999999e-06 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.813898+0000", 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2101000000000002e-05 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.814016+0000", 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011804899999999999 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2,7", 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.814151+0000", 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013473399999999999 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.816922+0000", 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027711789999999999 2023-09-30T09:37:20.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.816986+0000", 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4374000000000004e-05 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.843410+0000", 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.026423861 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.843422+0000", 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1646e-05 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.964830+0000", 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1214083130000001 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for missing object", 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.964989+0000", 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000158994 2023-09-30T09:37:20.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.998872+0000", 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.033882552000000003 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.998967+0000", 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5199999999999997e-05 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.090 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145314 7.9 7:9de1ac9b:::benchmark_data_smithi019_404601_object32036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.821073+0000", 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.22163482100001, 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2022353690000001, 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.091 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145314 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821073+0000", 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821073+0000", 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821071+0000", 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821078+0000", 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1210000000000001e-06 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.093 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821093+0000", 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4936000000000001e-05 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821100+0000", 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5270000000000004e-06 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821148+0000", 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8303000000000002e-05 2023-09-30T09:37:20.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821305+0000", 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015641599999999999 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2,7", 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.821472+0000", 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000167057 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.824807+0000", 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033357320000000001 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.826708+0000", 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019003189999999999 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.845057+0000", 2023-09-30T09:37:20.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.018349805 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.845066+0000", 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1140000000000006e-06 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.999010+0000", 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.153944399 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for missing object", 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.999171+0000", 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016092 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.023229+0000", 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.024057621000000001 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.023308+0000", 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9695000000000002e-05 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145332 7.4 7:2897ae8c:::benchmark_data_smithi019_404601_object32054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.865724+0000", 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.17698362799999, 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3332199330000001, 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.099 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145332 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.865724+0000", 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.100 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.865724+0000", 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.865723+0000", 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.865729+0000", 2023-09-30T09:37:20.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5809999999999996e-06 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.865741+0000", 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2655e-05 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.865746+0000", 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.023e-06 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.102 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.865789+0000", 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2472000000000003e-05 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.865798+0000", 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4339999999999998e-06 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.833712+0000", 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96791422999999999 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.833724+0000", 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2081e-05 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.177015+0000", 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3432901749999999 2023-09-30T09:37:20.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.177157+0000", 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014199199999999999 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,5", 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.177322+0000", 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000164943 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.105 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.185921+0000", 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0085997039999999997 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.185956+0000", 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4656000000000002e-05 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.185991+0000", 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5494000000000003e-05 2023-09-30T09:37:20.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.196299+0000", 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010307482999999999 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.196352+0000", 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3467999999999997e-05 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.198944+0000", 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002591604 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145340 7.4 7:2170588f:::benchmark_data_smithi019_404601_object32062:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.880798+0000", 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.161909665, 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3326320360000001, 2023-09-30T09:37:20.108 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145340 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.880798+0000", 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.109 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.880798+0000", 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.880797+0000", 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.110 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.880802+0000", 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2449999999999998e-06 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.880812+0000", 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.024e-05 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.880818+0000", 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1710000000000001e-06 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.880850+0000", 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2537999999999999e-05 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.880856+0000", 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.032e-06 2023-09-30T09:37:20.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.834949+0000", 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95409323800000001 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.834955+0000", 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0140000000000004e-06 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.182804+0000", 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.347848473 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.182961+0000", 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015765500000000001 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,5", 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.183127+0000", 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000165258 2023-09-30T09:37:20.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.196222+0000", 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013095035 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.196286+0000", 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4583000000000002e-05 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.115 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.198958+0000", 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026716880000000002 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.210326+0000", 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011367866000000001 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.210385+0000", 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8885000000000003e-05 2023-09-30T09:37:20.116 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.213430+0000", 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030452719999999999 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145356 7.9 7:9759da3e:::benchmark_data_smithi019_404601_object32078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.914641+0000", 2023-09-30T09:37:20.117 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.12806693600001, 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1053800359999999, 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145356 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.118 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.914641+0000", 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.914641+0000", 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.914640+0000", 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.119 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.914643+0000", 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.402e-06 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.914649+0000", 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4699999999999999e-06 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.120 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.914653+0000", 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4829999999999999e-06 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.914700+0000", 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7194000000000002e-05 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.914708+0000", 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0609999999999996e-06 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.121 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.999212+0000", 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.084504213 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.999345+0000", 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013250600000000001 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,7", 2023-09-30T09:37:20.122 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.999505+0000", 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000160443 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.003910+0000", 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044048000000000004 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.003978+0000", 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8183999999999996e-05 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.123 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008351+0000", 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004372997 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.017681+0000", 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0093297870000000008 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.124 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.017751+0000", 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9826999999999994e-05 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.020021+0000", 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022699740000000001 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.125 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145359 7.9 7:9a756b04:::benchmark_data_smithi019_404601_object32081:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.915511+0000", 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.12719656199999, 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1093094269999999, 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145359 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.126 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.915511+0000", 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.915511+0000", 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.915511+0000", 2023-09-30T09:37:20.127 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.915513+0000", 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3910000000000001e-06 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.915520+0000", 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4760000000000003e-06 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.128 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.915523+0000", 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8829999999999999e-06 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.915565+0000", 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1338000000000003e-05 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.129 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.915570+0000", 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3210000000000004e-06 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.001711+0000", 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.086141061 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.001807+0000", 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5488999999999993e-05 2023-09-30T09:37:20.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,7", 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.001900+0000", 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3561999999999996e-05 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.003967+0000", 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002066404 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008281+0000", 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043144129999999996 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008447+0000", 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016627999999999999 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.023353+0000", 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014905183000000001 2023-09-30T09:37:20.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.023426+0000", 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3876999999999998e-05 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.024821+0000", 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001394146 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.133 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145370 7.9 7:95d127f7:::benchmark_data_smithi019_404601_object32092:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.935617+0000", 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.107091083, 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0908896589999999, 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.134 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145370 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.935617+0000", 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.935617+0000", 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.935616+0000", 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.935620+0000", 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.578e-06 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.935632+0000", 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1639e-05 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.935637+0000", 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0479999999999998e-06 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.935677+0000", 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0176999999999998e-05 2023-09-30T09:37:20.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.935683+0000", 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8039999999999998e-06 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.004023+0000", 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.068339484 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.004164+0000", 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014140600000000001 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,7", 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.004329+0000", 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016474400000000001 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008412+0000", 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0040834820000000003 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008421+0000", 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9779999999999994e-06 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.012158+0000", 2023-09-30T09:37:20.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037366869999999998 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.024888+0000", 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01272979 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.024950+0000", 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2535999999999999e-05 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.141 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.026506+0000", 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015560019999999999 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145377 7.9 7:9291cbe7:::benchmark_data_smithi019_404601_object32099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.951191+0000", 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.09151642500001, 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.076470034, 2023-09-30T09:37:20.142 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145377 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.951191+0000", 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.143 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.951191+0000", 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.951190+0000", 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.951196+0000", 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.609e-06 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.951208+0000", 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2595000000000001e-05 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.951214+0000", 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4299999999999997e-06 2023-09-30T09:37:20.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.951265+0000", 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0954e-05 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.951275+0000", 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0727000000000001e-05 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.006079+0000", 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.054804002 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.006161+0000", 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2101000000000006e-05 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,7", 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.006247+0000", 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5477999999999997e-05 2023-09-30T09:37:20.147 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008491+0000", 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002243742 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008501+0000", 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0091e-05 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.012232+0000", 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037316509999999999 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.026571+0000", 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014338877999999999 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.026628+0000", 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6934000000000001e-05 2023-09-30T09:37:20.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.027661+0000", 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010330879999999999 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145385 7.9 7:9926bc57:::benchmark_data_smithi019_404601_object32107:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:36.971958+0000", 2023-09-30T09:37:20.150 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.070749409, 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0574806210000001, 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145385 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.971958+0000", 2023-09-30T09:37:20.151 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.971958+0000", 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.971957+0000", 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.971963+0000", 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1680000000000001e-06 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.971977+0000", 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3953e-05 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.971982+0000", 2023-09-30T09:37:20.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2499999999999997e-06 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.972025+0000", 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2276e-05 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:36.972032+0000", 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8789999999999997e-06 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.154 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008527+0000", 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0364950070000001 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008623+0000", 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6812999999999996e-05 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,7", 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.008713+0000", 2023-09-30T09:37:20.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9975999999999994e-05 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.012190+0000", 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034766459999999999 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.012198+0000", 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9799999999999998e-06 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.013434+0000", 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012359929999999999 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.027734+0000", 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014300294 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.027787+0000", 2023-09-30T09:37:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2620000000000001e-05 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.029439+0000", 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016520230000000001 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145397 7.4 7:2de57559:::benchmark_data_smithi019_404601_object32119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.158 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:37.027046+0000", 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 163.01566144, 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2007267850000001, 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145397 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.159 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.027046+0000", 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.027046+0000", 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.027045+0000", 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:20.160 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.027049+0000", 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9219999999999999e-06 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.027068+0000", 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9242999999999999e-05 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.161 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.027073+0000", 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3830000000000002e-06 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.027122+0000", 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9617999999999998e-05 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.027129+0000", 2023-09-30T09:37:20.162 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0010000000000004e-06 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.834970+0000", 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.80784105399999995 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.834974+0000", 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9940000000000002e-06 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.186014+0000", 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.351039181 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.186122+0000", 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010804899999999999 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,5", 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.186249+0000", 2023-09-30T09:37:20.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000127446 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.198966+0000", 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012717031 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.199023+0000", 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7445999999999999e-05 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.199063+0000", 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9647999999999997e-05 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.224737+0000", 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.025673404 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.166 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.224797+0000", 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0145000000000001e-05 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.227773+0000", 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029763569999999998 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.167 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145407 7.4 7:228f67aa:::benchmark_data_smithi019_404601_object32129:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:37.096212+0000", 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 162.94649557599999, 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.134114726, 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145407 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.168 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.096212+0000", 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.096212+0000", 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.169 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.096211+0000", 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.096217+0000", 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1140000000000002e-06 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.096230+0000", 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3419e-05 2023-09-30T09:37:20.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.096236+0000", 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2170000000000002e-06 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.096285+0000", 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9345999999999998e-05 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.096294+0000", 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4069999999999999e-06 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.834986+0000", 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.73869147400000001 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.172 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:37.834990+0000", 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.924e-06 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.188933+0000", 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3539434130000001 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.174 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.189067+0000", 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000134017 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,5", 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.189214+0000", 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014652 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.198974+0000", 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097601249999999997 2023-09-30T09:37:20.175 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.199097+0000", 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000123373 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.199131+0000", 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3442999999999998e-05 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.176 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.227826+0000", 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.028695670999999999 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.227888+0000", 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1634e-05 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.230327+0000", 2023-09-30T09:37:20.177 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024388320000000002 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145426 7.4 7:254b5760:::benchmark_data_smithi019_404601_object32148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:38.047605+0000", 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 161.99510234100001, 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.188721809, 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.178 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145426 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.047605+0000", 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.179 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.047605+0000", 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.047604+0000", 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.047611+0000", 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6409999999999996e-06 2023-09-30T09:37:20.180 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.047624+0000", 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3307e-05 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.047629+0000", 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7369999999999997e-06 2023-09-30T09:37:20.181 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.047687+0000", 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8270999999999998e-05 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.047700+0000", 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3118e-05 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.182 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.193182+0000", 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.145481969 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.193316+0000", 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013384399999999999 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,5", 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.193463+0000", 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000147412 2023-09-30T09:37:20.183 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.199161+0000", 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0056978699999999998 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.204905+0000", 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057442109999999999 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.184 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.213519+0000", 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0086133670000000002 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.233394+0000", 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.019875440000000001 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.233451+0000", 2023-09-30T09:37:20.185 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7232999999999999e-05 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.236327+0000", 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028757129999999998 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145435 7.4 7:207f1f1e:::benchmark_data_smithi019_404601_object32157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-30T09:37:20.186 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:38.084204+0000", 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 161.95850354500001, 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1548918189999999, 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145435 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.187 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.084204+0000", 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.084204+0000", 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.188 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.084203+0000", 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.084208+0000", 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4059999999999998e-06 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.084215+0000", 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3159999999999999e-06 2023-09-30T09:37:20.189 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.084220+0000", 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0189999999999997e-06 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.084284+0000", 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4268000000000002e-05 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.190 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.084296+0000", 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1892000000000001e-05 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.199186+0000", 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.114889671 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.191 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.199344+0000", 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000157863 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,5", 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.199495+0000", 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015106700000000001 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.192 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.213505+0000", 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014010425999999999 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.213584+0000", 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9276000000000005e-05 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.213681+0000", 2023-09-30T09:37:20.193 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6696999999999999e-05 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.236400+0000", 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.022718938000000001 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.236456+0000", 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6271999999999999e-05 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.194 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.239096+0000", 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026397159999999999 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:145455 7.4 7:2210dbf2:::benchmark_data_smithi019_404601_object32177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:34:38.147433+0000", 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 161.89527439299999, 2023-09-30T09:37:20.195 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.095874682, 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 145455 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.196 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.147433+0000", 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.147433+0000", 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.147432+0000", 2023-09-30T09:37:20.197 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.147438+0000", 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2829999999999998e-06 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.147452+0000", 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3453e-05 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.198 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.147457+0000", 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8000000000000004e-06 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.147513+0000", 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5952999999999998e-05 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.199 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:38.147525+0000", 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1735e-05 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.202043+0000", 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.05451768 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.202191+0000", 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014818000000000001 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.200 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,4,5", 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.202301+0000", 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109753 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.213553+0000", 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011251908999999999 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.201 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.213673+0000", 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000120938 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.213687+0000", 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3416e-05 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.240436+0000", 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.026749585999999999 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.202 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.240484+0000", 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7559999999999999e-05 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:34:39.243308+0000", 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002823945 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.203 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:184385 7.d 7:bae3696f:::benchmark_data_smithi019_404601_object71107:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:36:03.472770+0000", 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 76.569937296999996, 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1003519420000001, 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:20.204 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 184385 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.472770+0000", 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.472770+0000", 2023-09-30T09:37:20.205 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.472769+0000", 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.472775+0000", 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9000000000000003e-06 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.206 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.472788+0000", 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2357999999999999e-05 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.472795+0000", 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1350000000000003e-06 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.207 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.472834+0000", 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8945000000000001e-05 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.472928+0000", 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3900000000000006e-05 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1", 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.473019+0000", 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0854999999999997e-05 2023-09-30T09:37:20.208 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.475241+0000", 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022228560000000001 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.484760+0000", 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0095189920000000004 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.209 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.490246+0000", 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0054858830000000004 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:03.490258+0000", 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1482e-05 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:20.210 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:04.573067+0000", 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0828090880000001 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:36:04.573122+0000", 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5507999999999998e-05 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:20.211 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:20.212 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:37:20.212 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:20.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:20.213 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:37:20.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:37:20.313 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:20.403 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:20.426 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:20.676 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:20.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:20.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:20.779 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:20.899 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:20.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:20 smithi019 ceph-mon[224043]: pgmap v898: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 622 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 5010/29730 objects degraded (16.852%); 8569/29730 objects misplaced (28.823%); 22 MiB/s, 345 objects/s recovering 2023-09-30T09:37:20.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:20 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:37:20.988 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:20 smithi019 ceph-mon[216815]: pgmap v898: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 622 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 5010/29730 objects degraded (16.852%); 8569/29730 objects misplaced (28.823%); 22 MiB/s, 345 objects/s recovering 2023-09-30T09:37:20.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:20 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:37:21.010 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:20 smithi167 ceph-mon[192880]: pgmap v898: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 622 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 5010/29730 objects degraded (16.852%); 8569/29730 objects misplaced (28.823%); 22 MiB/s, 345 objects/s recovering 2023-09-30T09:37:21.011 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:20 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:37:21.031 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:37:21.036 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:21.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:21.143 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:21.263 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:21.452 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:21.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:21.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:21.566 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:21.606 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:21.615 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:21.693 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:21.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:21.750 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:37:21.799 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:21.874 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:21 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:37:21.875 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:21 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:37:21.875 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2360298680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:21.875 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:21 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:21.875 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:21 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:37:21.875 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:21 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:37:21.875 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2360298680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:21.875 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:21 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:21.888 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:22.005 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:22.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:22.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:22.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:21 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:37:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:21 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:37:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2360298680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:21 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:22.113 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:22.143 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:22.186 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:22.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:22.290 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:22.328 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:22.703 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:22.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:22.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: pgmap v899: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 614 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 5010/29319 objects degraded (17.088%); 8569/29319 objects misplaced (29.227%); 16 MiB/s, 263 objects/s recovering 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: osdmap e348: 8 total, 8 up, 7 in 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: failed to encode map e348 with expected crc 2023-09-30T09:37:22.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:22 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:37:22.815 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:22.843 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:23.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: pgmap v899: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 614 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 5010/29319 objects degraded (17.088%); 8569/29319 objects misplaced (29.227%); 16 MiB/s, 263 objects/s recovering 2023-09-30T09:37:23.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:37:23.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:23.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: osdmap e348: 8 total, 8 up, 7 in 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:37:23.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: pgmap v899: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 614 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 5010/29319 objects degraded (17.088%); 8569/29319 objects misplaced (29.227%); 16 MiB/s, 263 objects/s recovering 2023-09-30T09:37:23.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:37:23.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:23.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: osdmap e348: 8 total, 8 up, 7 in 2023-09-30T09:37:23.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: failed to encode map e348 with expected crc 2023-09-30T09:37:23.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:22 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:37:23.282 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:23.393 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:23.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:23.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:23.513 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:23.601 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:23.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:23 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:37:23.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5010/29319 objects degraded (17.088%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:23.810 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:37:23.810 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:37:23.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:219450 7.5 7:ae03b686:::benchmark_data_smithi019_404601_object106172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:37:23.802522+0000", 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0073550509999999996, 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007378089, 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:37:23.811 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 219450 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:23.802522+0000", 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:37:23.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:23.802522+0000", 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:23.802520+0000", 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:23.802528+0000", 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7670000000000003e-06 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:23.802542+0000", 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3654e-05 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:23.802549+0000", 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1729999999999998e-06 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:23.802612+0000", 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2881999999999999e-05 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:37:23.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:37:23.803671+0000", 2023-09-30T09:37:23.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001059445 2023-09-30T09:37:23.816 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:23.816 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:37:23.816 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:23.816 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:37:23.816 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:37:23.816 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 1 2023-09-30T09:37:23.816 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:37:23.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:23 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:37:23.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5010/29319 objects degraded (17.088%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:23.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:23 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:37:23.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5010/29319 objects degraded (17.088%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:23.840 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:37:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:37:23] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-30T09:37:23.849 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:23.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:23.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:23.957 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:24.010 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:24.093 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:24.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:24.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:24.199 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:24.251 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:24.311 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:24.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:24.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:24.416 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:24.446 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:24.493 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:24.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:24.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:24.561 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:37:24.561 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:37:24.561 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:37:24.582 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:37:24.597 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:24.619 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:37:24.629 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:24.668 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:37:24.711 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:37:24.748 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:37:24.767 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:37:24.768 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:37:24.768 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:37:24.768 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:37:24.768 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:37:24.768 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:37:24.768 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:37:24.768 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:37:24.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:24 smithi019 ceph-mon[216815]: pgmap v901: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 548 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 4580/26163 objects degraded (17.506%); 7655/26163 objects misplaced (29.259%); 15 MiB/s, 236 objects/s recovering 2023-09-30T09:37:24.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/485535322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:24 smithi019 ceph-mon[224043]: pgmap v901: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 548 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 4580/26163 objects degraded (17.506%); 7655/26163 objects misplaced (29.259%); 15 MiB/s, 236 objects/s recovering 2023-09-30T09:37:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/485535322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:24.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:24.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:24.952 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:37:24.987 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:24.994 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:24 smithi167 ceph-mon[192880]: pgmap v901: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 104 active+clean; 548 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 4580/26163 objects degraded (17.506%); 7655/26163 objects misplaced (29.259%); 15 MiB/s, 236 objects/s recovering 2023-09-30T09:37:24.994 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/485535322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:25.036 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:25.137 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:25.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:25.252 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:25.301 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:25.365 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:37:25.378 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:25.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:25.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:25.489 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:25.510 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:25.545 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:25.635 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:25.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:25.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:25.743 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:25.832 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:26.027 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:26.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:26.136 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:26.233 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:26.425 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:26.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:26.519 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:26.541 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:26.568 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:26.611 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:26.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:26.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:26.714 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:26.753 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:26.769 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:37:26.778 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:37:26.820 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:26.821 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:26 smithi167 ceph-mon[192880]: pgmap v902: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 463 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 3698/22074 objects degraded (16.753%); 6421/22074 objects misplaced (29.089%); 18 MiB/s, 287 objects/s recovering 2023-09-30T09:37:26.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:26.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:26.925 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:26.962 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:27.034 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:27.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:27.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:26 smithi019 ceph-mon[224043]: pgmap v902: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 463 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 3698/22074 objects degraded (16.753%); 6421/22074 objects misplaced (29.089%); 18 MiB/s, 287 objects/s recovering 2023-09-30T09:37:27.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:26 smithi019 ceph-mon[216815]: pgmap v902: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 463 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 3698/22074 objects degraded (16.753%); 6421/22074 objects misplaced (29.089%); 18 MiB/s, 287 objects/s recovering 2023-09-30T09:37:27.138 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:27.197 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:27.285 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:27.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:27.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:27.390 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:27.453 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:27.682 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:37:27.682 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:37:27.786 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:27 smithi019 ceph-mon[216815]: from='client.35733 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:27.786 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:27 smithi019 ceph-mon[216815]: 1.a deep-scrub starts 2023-09-30T09:37:28.049 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:37:28.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:27 smithi019 ceph-mon[224043]: from='client.35733 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:28.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:27 smithi019 ceph-mon[224043]: 1.a deep-scrub starts 2023-09-30T09:37:28.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:27 smithi167 ceph-mon[192880]: from='client.35733 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:27 smithi167 ceph-mon[192880]: 1.a deep-scrub starts 2023-09-30T09:37:28.571 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":902,"stamp":"2023-09-30T09:37:25.559674+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":485243748,"num_objects":7358,"num_object_clones":0,"num_object_copies":22074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3698,"num_objects_misplaced":6421,"num_objects_unfound":0,"num_objects_dirty":7358,"num_whiteouts":0,"num_read":6114,"num_read_kb":12467,"num_write":220211,"num_write_kb":7263166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42980,"num_bytes_recovered":2653069784,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88509,"ondisk_log_size":88509,"up":357,"acting":355,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":396,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3648816,"kb_used_data":1084280,"kb_used_omap":23,"kb_used_meta":2564264,"kb_avail":652567248,"statfs":{"total":671965249536,"available":668228861952,"internally_reserved":0,"allocated":1110302720,"data_stored":1098616284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2625806595},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,10,1,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-294780928,"num_objects":-4498,"num_object_clones":0,"num_object_copies":-13494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2683,"num_objects_misplaced":-3265,"num_objects_unfound":0,"num_objects_dirty":-4498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4498,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2841,"num_bytes_recovered":182648832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.883170"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":812,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690048+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:37:15.690048+0000","last_peered":"2023-09-30T09:37:15.690048+0000","last_clean":"2023-09-30T09:37:15.690048+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:37:15.690048+0000","last_undegraded":"2023-09-30T09:37:15.690048+0000","last_fullsized":"2023-09-30T09:37:15.690048+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":366,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731005+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:37:15.731005+0000","last_peered":"2023-09-30T09:37:15.731005+0000","last_clean":"2023-09-30T09:37:15.731005+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:37:15.731005+0000","last_undegraded":"2023-09-30T09:37:15.731005+0000","last_fullsized":"2023-09-30T09:37:15.731005+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":329,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.685114+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:37:15.685114+0000","last_peered":"2023-09-30T09:37:15.685114+0000","last_clean":"2023-09-30T09:37:15.685114+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:37:15.685114+0000","last_undegraded":"2023-09-30T09:37:15.685114+0000","last_fullsized":"2023-09-30T09:37:15.685114+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":792,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690330+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:37:15.690330+0000","last_peered":"2023-09-30T09:37:15.690330+0000","last_clean":"2023-09-30T09:37:15.690330+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:37:15.690330+0000","last_undegraded":"2023-09-30T09:37:15.690330+0000","last_fullsized":"2023-09-30T09:37:15.690330+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":221,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683896+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:37:15.683896+0000","last_peered":"2023-09-30T09:37:15.683896+0000","last_clean":"2023-09-30T09:37:15.683896+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:37:15.683896+0000","last_undegraded":"2023-09-30T09:37:15.683896+0000","last_fullsized":"2023-09-30T09:37:15.683896+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":253,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679379+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:37:15.679379+0000","last_peered":"2023-09-30T09:37:15.679379+0000","last_clean":"2023-09-30T09:37:15.679379+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:37:15.679379+0000","last_undegraded":"2023-09-30T09:37:15.679379+0000","last_fullsized":"2023-09-30T09:37:15.679379+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":561,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.730970+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:37:15.730970+0000","last_peered":"2023-09-30T09:37:15.730970+0000","last_clean":"2023-09-30T09:37:15.730970+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:37:15.730970+0000","last_undegraded":"2023-09-30T09:37:15.730970+0000","last_fullsized":"2023-09-30T09:37:15.730970+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":395,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683721+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:37:15.683721+0000","last_peered":"2023-09-30T09:37:15.683721+0000","last_clean":"2023-09-30T09:37:15.683721+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:37:15.683721+0000","last_undegraded":"2023-09-30T09:37:15.683721+0000","last_fullsized":"2023-09-30T09:37:15.683721+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":324,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689763+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:37:15.689763+0000","last_peered":"2023-09-30T09:37:15.689763+0000","last_clean":"2023-09-30T09:37:15.689763+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:37:15.689763+0000","last_undegraded":"2023-09-30T09:37:15.689763+0000","last_fullsized":"2023-09-30T09:37:15.689763+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":772,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.684761+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:37:15.684761+0000","last_peered":"2023-09-30T09:37:15.684761+0000","last_clean":"2023-09-30T09:37:15.684761+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:37:15.684761+0000","last_undegraded":"2023-09-30T09:37:15.684761+0000","last_fullsized":"2023-09-30T09:37:15.684761+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":243,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679564+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:37:15.679564+0000","last_peered":"2023-09-30T09:37:15.679564+0000","last_clean":"2023-09-30T09:37:15.679564+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:37:15.679564+0000","last_undegraded":"2023-09-30T09:37:15.679564+0000","last_fullsized":"2023-09-30T09:37:15.679564+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":329,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683742+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:37:15.683742+0000","last_peered":"2023-09-30T09:37:15.683742+0000","last_clean":"2023-09-30T09:37:15.683742+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:37:15.683742+0000","last_undegraded":"2023-09-30T09:37:15.683742+0000","last_fullsized":"2023-09-30T09:37:15.683742+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":511,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.680777+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:37:15.680777+0000","last_peered":"2023-09-30T09:37:15.680777+0000","last_clean":"2023-09-30T09:37:15.680777+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:37:15.680777+0000","last_undegraded":"2023-09-30T09:37:15.680777+0000","last_fullsized":"2023-09-30T09:37:15.680777+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":226,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690858+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:37:15.690858+0000","last_peered":"2023-09-30T09:37:15.690858+0000","last_clean":"2023-09-30T09:37:15.690858+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:37:15.690858+0000","last_undegraded":"2023-09-30T09:37:15.690858+0000","last_fullsized":"2023-09-30T09:37:15.690858+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":275,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683851+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:37:15.683851+0000","last_peered":"2023-09-30T09:37:15.683851+0000","last_clean":"2023-09-30T09:37:15.683851+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:37:15.683851+0000","last_undegraded":"2023-09-30T09:37:15.683851+0000","last_fullsized":"2023-09-30T09:37:15.683851+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":514,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.691081+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:37:15.691081+0000","last_peered":"2023-09-30T09:37:15.691081+0000","last_clean":"2023-09-30T09:37:15.691081+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:37:15.691081+0000","last_undegraded":"2023-09-30T09:37:15.691081+0000","last_fullsized":"2023-09-30T09:37:15.691081+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":70,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.685179+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:37:15.685179+0000","last_peered":"2023-09-30T09:37:15.685179+0000","last_clean":"2023-09-30T09:37:15.685179+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:37:15.685179+0000","last_undegraded":"2023-09-30T09:37:15.685179+0000","last_fullsized":"2023-09-30T09:37:15.685179+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":261,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.688810+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:37:15.688810+0000","last_peered":"2023-09-30T09:37:15.688810+0000","last_clean":"2023-09-30T09:37:15.688810+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:37:15.688810+0000","last_undegraded":"2023-09-30T09:37:15.688810+0000","last_fullsized":"2023-09-30T09:37:15.688810+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":508,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679537+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:37:15.679537+0000","last_peered":"2023-09-30T09:37:15.679537+0000","last_clean":"2023-09-30T09:37:15.679537+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:37:15.679537+0000","last_undegraded":"2023-09-30T09:37:15.679537+0000","last_fullsized":"2023-09-30T09:37:15.679537+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":332,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.730989+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:37:15.730989+0000","last_peered":"2023-09-30T09:37:15.730989+0000","last_clean":"2023-09-30T09:37:15.730989+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:37:15.730989+0000","last_undegraded":"2023-09-30T09:37:15.730989+0000","last_fullsized":"2023-09-30T09:37:15.730989+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":343,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731415+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:37:15.731415+0000","last_peered":"2023-09-30T09:37:15.731415+0000","last_clean":"2023-09-30T09:37:15.731415+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:37:15.731415+0000","last_undegraded":"2023-09-30T09:37:15.731415+0000","last_fullsized":"2023-09-30T09:37:15.731415+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":625,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679683+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:37:15.679683+0000","last_peered":"2023-09-30T09:37:15.679683+0000","last_clean":"2023-09-30T09:37:15.679683+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:37:15.679683+0000","last_undegraded":"2023-09-30T09:37:15.679683+0000","last_fullsized":"2023-09-30T09:37:15.679683+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":70,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731342+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:37:15.731342+0000","last_peered":"2023-09-30T09:37:15.731342+0000","last_clean":"2023-09-30T09:37:15.731342+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:37:15.731342+0000","last_undegraded":"2023-09-30T09:37:15.731342+0000","last_fullsized":"2023-09-30T09:37:15.731342+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":359,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.684798+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:37:15.684798+0000","last_peered":"2023-09-30T09:37:15.684798+0000","last_clean":"2023-09-30T09:37:15.684798+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:37:15.684798+0000","last_undegraded":"2023-09-30T09:37:15.684798+0000","last_fullsized":"2023-09-30T09:37:15.684798+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":694,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690131+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:37:15.690131+0000","last_peered":"2023-09-30T09:37:15.690131+0000","last_clean":"2023-09-30T09:37:15.690131+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:37:15.690131+0000","last_undegraded":"2023-09-30T09:37:15.690131+0000","last_fullsized":"2023-09-30T09:37:15.690131+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":252,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.678678+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:37:15.678678+0000","last_peered":"2023-09-30T09:37:15.678678+0000","last_clean":"2023-09-30T09:37:15.678678+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:37:15.678678+0000","last_undegraded":"2023-09-30T09:37:15.678678+0000","last_fullsized":"2023-09-30T09:37:15.678678+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":37,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690509+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:37:15.690509+0000","last_peered":"2023-09-30T09:37:15.690509+0000","last_clean":"2023-09-30T09:37:15.690509+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:37:15.690509+0000","last_undegraded":"2023-09-30T09:37:15.690509+0000","last_fullsized":"2023-09-30T09:37:15.690509+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":787,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.764215+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:37:21.764215+0000","last_peered":"2023-09-30T09:37:21.764215+0000","last_clean":"2023-09-30T09:37:21.764215+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:37:21.764215+0000","last_undegraded":"2023-09-30T09:37:21.764215+0000","last_fullsized":"2023-09-30T09:37:21.764215+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731261+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:37:15.731261+0000","last_peered":"2023-09-30T09:37:15.731261+0000","last_clean":"2023-09-30T09:37:15.731261+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:37:15.731261+0000","last_undegraded":"2023-09-30T09:37:15.731261+0000","last_fullsized":"2023-09-30T09:37:15.731261+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":251,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679462+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:37:15.679462+0000","last_peered":"2023-09-30T09:37:15.679462+0000","last_clean":"2023-09-30T09:37:15.679462+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:37:15.679462+0000","last_undegraded":"2023-09-30T09:37:15.679462+0000","last_fullsized":"2023-09-30T09:37:15.679462+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":813,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690662+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:37:15.690662+0000","last_peered":"2023-09-30T09:37:15.690662+0000","last_clean":"2023-09-30T09:37:15.690662+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:37:15.690662+0000","last_undegraded":"2023-09-30T09:37:15.690662+0000","last_fullsized":"2023-09-30T09:37:15.690662+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":507,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.688088+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:37:15.688088+0000","last_peered":"2023-09-30T09:37:15.688088+0000","last_clean":"2023-09-30T09:37:15.688088+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:37:15.688088+0000","last_undegraded":"2023-09-30T09:37:15.688088+0000","last_fullsized":"2023-09-30T09:37:15.688088+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":357,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.685749+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:37:15.685749+0000","last_peered":"2023-09-30T09:37:15.685749+0000","last_clean":"2023-09-30T09:37:15.685749+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:37:15.685749+0000","last_undegraded":"2023-09-30T09:37:15.685749+0000","last_fullsized":"2023-09-30T09:37:15.685749+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"348'13444","reported_seq":19744,"reported_epoch":348,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:24.093416+0000","last_change":"2023-09-30T09:34:48.517696+0000","last_active":"2023-09-30T09:37:24.093416+0000","last_peered":"2023-09-30T09:37:24.093416+0000","last_clean":"2023-09-30T09:32:35.457744+0000","last_became_active":"2023-09-30T09:34:48.499533+0000","last_became_peered":"2023-09-30T09:34:48.499533+0000","last_unstale":"2023-09-30T09:37:24.093416+0000","last_undegraded":"2023-09-30T09:34:48.469688+0000","last_fullsized":"2023-09-30T09:37:24.093416+0000","mapping_epoch":321,"log_start":"312'8621","ondisk_log_start":"312'8621","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4823,"log_dups_size":0,"ondisk_log_size":4823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:05:42.164341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29753344,"num_objects":454,"num_object_clones":0,"num_object_copies":1362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":452,"num_objects_misplaced":454,"num_objects_unfound":0,"num_objects_dirty":454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13444,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3090,"num_bytes_recovered":202506240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[6,1,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"1,4,6","objects":2},{"shards":"4,6","objects":452}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":839,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:23.002270+0000","last_change":"2023-09-30T09:37:23.002270+0000","last_active":"2023-09-30T09:37:23.002270+0000","last_peered":"2023-09-30T09:37:23.002270+0000","last_clean":"2023-09-30T09:37:23.002270+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:37:23.002270+0000","last_undegraded":"2023-09-30T09:37:23.002270+0000","last_fullsized":"2023-09-30T09:37:23.002270+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":82,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731518+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:37:15.731518+0000","last_peered":"2023-09-30T09:37:15.731518+0000","last_clean":"2023-09-30T09:37:15.731518+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:37:15.731518+0000","last_undegraded":"2023-09-30T09:37:15.731518+0000","last_fullsized":"2023-09-30T09:37:15.731518+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":37,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690010+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:37:15.690010+0000","last_peered":"2023-09-30T09:37:15.690010+0000","last_clean":"2023-09-30T09:37:15.690010+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:37:15.690010+0000","last_undegraded":"2023-09-30T09:37:15.690010+0000","last_fullsized":"2023-09-30T09:37:15.690010+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"348'13888","reported_seq":21316,"reported_epoch":348,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:22.270415+0000","last_change":"2023-09-30T09:35:10.766663+0000","last_active":"2023-09-30T09:37:22.270415+0000","last_peered":"2023-09-30T09:37:22.270415+0000","last_clean":"2023-09-30T09:32:35.454324+0000","last_became_active":"2023-09-30T09:33:33.826036+0000","last_became_peered":"2023-09-30T09:33:33.826036+0000","last_unstale":"2023-09-30T09:37:22.270415+0000","last_undegraded":"2023-09-30T09:33:33.732722+0000","last_fullsized":"2023-09-30T09:37:22.270415+0000","mapping_epoch":302,"log_start":"319'9399","ondisk_log_start":"319'9399","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4489,"log_dups_size":0,"ondisk_log_size":4489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:42.597216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33554432,"num_objects":512,"num_object_clones":0,"num_object_copies":1536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":477,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13888,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3661,"num_bytes_recovered":239927296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,5,1],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1},{"shards":"2,3,5","objects":35},{"shards":"3,5","objects":476}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":798,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.684713+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:37:15.684713+0000","last_peered":"2023-09-30T09:37:15.684713+0000","last_clean":"2023-09-30T09:37:15.684713+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:37:15.684713+0000","last_undegraded":"2023-09-30T09:37:15.684713+0000","last_fullsized":"2023-09-30T09:37:15.684713+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":60,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.730996+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:37:15.730996+0000","last_peered":"2023-09-30T09:37:15.730996+0000","last_clean":"2023-09-30T09:37:15.730996+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:37:15.730996+0000","last_undegraded":"2023-09-30T09:37:15.730996+0000","last_fullsized":"2023-09-30T09:37:15.730996+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":340,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731166+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:37:15.731166+0000","last_peered":"2023-09-30T09:37:15.731166+0000","last_clean":"2023-09-30T09:37:15.731166+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:37:15.731166+0000","last_undegraded":"2023-09-30T09:37:15.731166+0000","last_fullsized":"2023-09-30T09:37:15.731166+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"348'13530","reported_seq":19975,"reported_epoch":348,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:23.999129+0000","last_change":"2023-09-30T09:35:10.879711+0000","last_active":"2023-09-30T09:37:23.999129+0000","last_peered":"2023-09-30T09:37:23.999129+0000","last_clean":"2023-09-30T09:32:35.467048+0000","last_became_active":"2023-09-30T09:35:10.845361+0000","last_became_peered":"2023-09-30T09:35:10.845361+0000","last_unstale":"2023-09-30T09:37:23.999129+0000","last_undegraded":"2023-09-30T09:35:10.791338+0000","last_fullsized":"2023-09-30T09:37:23.999129+0000","mapping_epoch":327,"log_start":"313'8676","ondisk_log_start":"313'8676","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4854,"log_dups_size":0,"ondisk_log_size":4854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:41.089234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29229056,"num_objects":446,"num_object_clones":0,"num_object_copies":1338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":446,"num_objects_misplaced":892,"num_objects_unfound":0,"num_objects_dirty":446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13530,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3162,"num_bytes_recovered":207093760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[6,1,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":446}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":631,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689671+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:37:15.689671+0000","last_peered":"2023-09-30T09:37:15.689671+0000","last_clean":"2023-09-30T09:37:15.689671+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:37:15.689671+0000","last_undegraded":"2023-09-30T09:37:15.689671+0000","last_fullsized":"2023-09-30T09:37:15.689671+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":41,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690428+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:37:15.690428+0000","last_peered":"2023-09-30T09:37:15.690428+0000","last_clean":"2023-09-30T09:37:15.690428+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:37:15.690428+0000","last_undegraded":"2023-09-30T09:37:15.690428+0000","last_fullsized":"2023-09-30T09:37:15.690428+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":327,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.688720+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:37:15.688720+0000","last_peered":"2023-09-30T09:37:15.688720+0000","last_clean":"2023-09-30T09:37:15.688720+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:37:15.688720+0000","last_undegraded":"2023-09-30T09:37:15.688720+0000","last_fullsized":"2023-09-30T09:37:15.688720+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"348'13798","reported_seq":13888,"reported_epoch":348,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:22.260547+0000","last_change":"2023-09-30T09:34:08.527633+0000","last_active":"2023-09-30T09:37:22.260547+0000","last_peered":"2023-09-30T09:37:22.260547+0000","last_clean":"2023-09-30T09:33:09.318417+0000","last_became_active":"2023-09-30T09:34:08.510746+0000","last_became_peered":"2023-09-30T09:34:08.510746+0000","last_unstale":"2023-09-30T09:37:22.260547+0000","last_undegraded":"2023-09-30T09:37:22.260547+0000","last_fullsized":"2023-09-30T09:37:22.260547+0000","mapping_epoch":308,"log_start":"319'9276","ondisk_log_start":"319'9276","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4522,"log_dups_size":0,"ondisk_log_size":4522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:07.374969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32768000,"num_objects":500,"num_object_clones":0,"num_object_copies":1500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":500,"num_objects_unfound":0,"num_objects_dirty":500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13798,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":500}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":510,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:22.012364+0000","last_change":"2023-09-30T09:37:22.012364+0000","last_active":"2023-09-30T09:37:22.012364+0000","last_peered":"2023-09-30T09:37:22.012364+0000","last_clean":"2023-09-30T09:37:22.012364+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:37:22.012364+0000","last_undegraded":"2023-09-30T09:37:22.012364+0000","last_fullsized":"2023-09-30T09:37:22.012364+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":307,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690797+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:37:15.690797+0000","last_peered":"2023-09-30T09:37:15.690797+0000","last_clean":"2023-09-30T09:37:15.690797+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:37:15.690797+0000","last_undegraded":"2023-09-30T09:37:15.690797+0000","last_fullsized":"2023-09-30T09:37:15.690797+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":254,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679194+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:37:15.679194+0000","last_peered":"2023-09-30T09:37:15.679194+0000","last_clean":"2023-09-30T09:37:15.679194+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:37:15.679194+0000","last_undegraded":"2023-09-30T09:37:15.679194+0000","last_fullsized":"2023-09-30T09:37:15.679194+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"348'13544","reported_seq":13647,"reported_epoch":348,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:22.051791+0000","last_change":"2023-09-30T09:35:10.779076+0000","last_active":"2023-09-30T09:37:22.051791+0000","last_peered":"2023-09-30T09:37:22.051791+0000","last_clean":"2023-09-30T09:34:06.422096+0000","last_became_active":"2023-09-30T09:35:10.747716+0000","last_became_peered":"2023-09-30T09:35:10.747716+0000","last_unstale":"2023-09-30T09:37:22.051791+0000","last_undegraded":"2023-09-30T09:37:22.051791+0000","last_fullsized":"2023-09-30T09:37:22.051791+0000","mapping_epoch":327,"log_start":"314'9020","ondisk_log_start":"314'9020","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4524,"log_dups_size":0,"ondisk_log_size":4524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:09.191024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32899072,"num_objects":502,"num_object_clones":0,"num_object_copies":1506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":502,"num_objects_unfound":0,"num_objects_dirty":502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13544,"num_write_kb":449472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":502}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":694,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683698+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:37:15.683698+0000","last_peered":"2023-09-30T09:37:15.683698+0000","last_clean":"2023-09-30T09:37:15.683698+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:37:15.683698+0000","last_undegraded":"2023-09-30T09:37:15.683698+0000","last_fullsized":"2023-09-30T09:37:15.683698+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":376,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.686938+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:37:15.686938+0000","last_peered":"2023-09-30T09:37:15.686938+0000","last_clean":"2023-09-30T09:37:15.686938+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:37:15.686938+0000","last_undegraded":"2023-09-30T09:37:15.686938+0000","last_fullsized":"2023-09-30T09:37:15.686938+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":306,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731057+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:37:15.731057+0000","last_peered":"2023-09-30T09:37:15.731057+0000","last_clean":"2023-09-30T09:37:15.731057+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:37:15.731057+0000","last_undegraded":"2023-09-30T09:37:15.731057+0000","last_fullsized":"2023-09-30T09:37:15.731057+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"348'13763","reported_seq":20204,"reported_epoch":348,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:22.271804+0000","last_change":"2023-09-30T09:35:10.871065+0000","last_active":"2023-09-30T09:37:22.271804+0000","last_peered":"2023-09-30T09:37:22.271804+0000","last_clean":"2023-09-30T09:32:35.449714+0000","last_became_active":"2023-09-30T09:35:10.838732+0000","last_became_peered":"2023-09-30T09:35:10.838732+0000","last_unstale":"2023-09-30T09:37:22.271804+0000","last_undegraded":"2023-09-30T09:35:10.818868+0000","last_fullsized":"2023-09-30T09:37:22.271804+0000","mapping_epoch":327,"log_start":"319'9206","ondisk_log_start":"319'9206","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4557,"log_dups_size":0,"ondisk_log_size":4557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:44.318381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32047104,"num_objects":489,"num_object_clones":0,"num_object_copies":1467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":486,"num_objects_misplaced":981,"num_objects_unfound":0,"num_objects_dirty":489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13763,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3162,"num_bytes_recovered":207224832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[3,7,1],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"1,3,7","objects":3},{"shards":"3,7","objects":486}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":769,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.680094+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:37:15.680094+0000","last_peered":"2023-09-30T09:37:15.680094+0000","last_clean":"2023-09-30T09:37:15.680094+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:37:15.680094+0000","last_undegraded":"2023-09-30T09:37:15.680094+0000","last_fullsized":"2023-09-30T09:37:15.680094+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":325,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.688551+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:37:15.688551+0000","last_peered":"2023-09-30T09:37:15.688551+0000","last_clean":"2023-09-30T09:37:15.688551+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:37:15.688551+0000","last_undegraded":"2023-09-30T09:37:15.688551+0000","last_fullsized":"2023-09-30T09:37:15.688551+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":42,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.730876+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:37:15.730876+0000","last_peered":"2023-09-30T09:37:15.730876+0000","last_clean":"2023-09-30T09:37:15.730876+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:37:15.730876+0000","last_undegraded":"2023-09-30T09:37:15.730876+0000","last_fullsized":"2023-09-30T09:37:15.730876+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"348'13597","reported_seq":20830,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:24.059919+0000","last_change":"2023-09-30T09:36:04.573419+0000","last_active":"2023-09-30T09:37:24.059919+0000","last_peered":"2023-09-30T09:37:24.059919+0000","last_clean":"2023-09-30T09:37:24.059919+0000","last_became_active":"2023-09-30T09:36:04.549763+0000","last_became_peered":"2023-09-30T09:36:04.549763+0000","last_unstale":"2023-09-30T09:37:24.059919+0000","last_undegraded":"2023-09-30T09:37:24.059919+0000","last_fullsized":"2023-09-30T09:37:24.059919+0000","mapping_epoch":337,"log_start":"313'8821","ondisk_log_start":"313'8821","created":279,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4776,"log_dups_size":0,"ondisk_log_size":4776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:27:18.185457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29229080,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13597,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3562,"num_bytes_recovered":233439232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":784,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.678988+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:37:15.678988+0000","last_peered":"2023-09-30T09:37:15.678988+0000","last_clean":"2023-09-30T09:37:15.678988+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:37:15.678988+0000","last_undegraded":"2023-09-30T09:37:15.678988+0000","last_fullsized":"2023-09-30T09:37:15.678988+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":344,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731083+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:37:15.731083+0000","last_peered":"2023-09-30T09:37:15.731083+0000","last_clean":"2023-09-30T09:37:15.731083+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:37:15.731083+0000","last_undegraded":"2023-09-30T09:37:15.731083+0000","last_fullsized":"2023-09-30T09:37:15.731083+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":323,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731427+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:37:15.731427+0000","last_peered":"2023-09-30T09:37:15.731427+0000","last_clean":"2023-09-30T09:37:15.731427+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:37:15.731427+0000","last_undegraded":"2023-09-30T09:37:15.731427+0000","last_fullsized":"2023-09-30T09:37:15.731427+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"348'13750","reported_seq":13844,"reported_epoch":348,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:23.995914+0000","last_change":"2023-09-30T09:34:47.630136+0000","last_active":"2023-09-30T09:37:23.995914+0000","last_peered":"2023-09-30T09:37:23.995914+0000","last_clean":"2023-09-30T09:34:06.422409+0000","last_became_active":"2023-09-30T09:34:08.626702+0000","last_became_peered":"2023-09-30T09:34:08.626702+0000","last_unstale":"2023-09-30T09:37:23.995914+0000","last_undegraded":"2023-09-30T09:37:23.995914+0000","last_fullsized":"2023-09-30T09:37:23.995914+0000","mapping_epoch":308,"log_start":"313'8906","ondisk_log_start":"313'8906","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4844,"log_dups_size":0,"ondisk_log_size":4844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:53.753813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27918336,"num_objects":426,"num_object_clones":0,"num_object_copies":1278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":426,"num_objects_unfound":0,"num_objects_dirty":426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13750,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":426}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":496,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:20.751860+0000","last_change":"2023-09-30T09:37:21.738073+0000","last_active":"2023-09-30T09:37:20.751860+0000","last_peered":"2023-09-30T09:37:20.751860+0000","last_clean":"2023-09-30T09:37:20.751860+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:37:20.751860+0000","last_undegraded":"2023-09-30T09:37:20.751860+0000","last_fullsized":"2023-09-30T09:37:20.751860+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":339,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683624+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:37:15.683624+0000","last_peered":"2023-09-30T09:37:15.683624+0000","last_clean":"2023-09-30T09:37:15.683624+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:37:15.683624+0000","last_undegraded":"2023-09-30T09:37:15.683624+0000","last_fullsized":"2023-09-30T09:37:15.683624+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":116,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.685654+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:37:15.685654+0000","last_peered":"2023-09-30T09:37:15.685654+0000","last_clean":"2023-09-30T09:37:15.685654+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:37:15.685654+0000","last_undegraded":"2023-09-30T09:37:15.685654+0000","last_fullsized":"2023-09-30T09:37:15.685654+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"348'13507","reported_seq":19590,"reported_epoch":348,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:22.282942+0000","last_change":"2023-09-30T09:34:39.166112+0000","last_active":"2023-09-30T09:37:22.282942+0000","last_peered":"2023-09-30T09:37:22.282942+0000","last_clean":"2023-09-30T09:32:35.465008+0000","last_became_active":"2023-09-30T09:34:39.150257+0000","last_became_peered":"2023-09-30T09:34:39.150257+0000","last_unstale":"2023-09-30T09:37:22.282942+0000","last_undegraded":"2023-09-30T09:34:38.934740+0000","last_fullsized":"2023-09-30T09:37:22.282942+0000","mapping_epoch":316,"log_start":"314'8978","ondisk_log_start":"314'8978","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4529,"log_dups_size":0,"ondisk_log_size":4529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:50.805799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31391744,"num_objects":479,"num_object_clones":0,"num_object_copies":1437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":478,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13507,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":195952640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6,1],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1},{"shards":"3,6","objects":478}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":642,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689851+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:37:15.689851+0000","last_peered":"2023-09-30T09:37:15.689851+0000","last_clean":"2023-09-30T09:37:15.689851+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:37:15.689851+0000","last_undegraded":"2023-09-30T09:37:15.689851+0000","last_fullsized":"2023-09-30T09:37:15.689851+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":540,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.678971+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:37:15.678971+0000","last_peered":"2023-09-30T09:37:15.678971+0000","last_clean":"2023-09-30T09:37:15.678971+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:37:15.678971+0000","last_undegraded":"2023-09-30T09:37:15.678971+0000","last_fullsized":"2023-09-30T09:37:15.678971+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":344,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690647+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:37:15.690647+0000","last_peered":"2023-09-30T09:37:15.690647+0000","last_clean":"2023-09-30T09:37:15.690647+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:37:15.690647+0000","last_undegraded":"2023-09-30T09:37:15.690647+0000","last_fullsized":"2023-09-30T09:37:15.690647+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":37,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690355+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:37:15.690355+0000","last_peered":"2023-09-30T09:37:15.690355+0000","last_clean":"2023-09-30T09:37:15.690355+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:37:15.690355+0000","last_undegraded":"2023-09-30T09:37:15.690355+0000","last_fullsized":"2023-09-30T09:37:15.690355+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"348'13814","reported_seq":13891,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:24.060423+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:37:24.060423+0000","last_peered":"2023-09-30T09:37:24.060423+0000","last_clean":"2023-09-30T09:37:24.060423+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:37:24.060423+0000","last_undegraded":"2023-09-30T09:37:24.060423+0000","last_fullsized":"2023-09-30T09:37:24.060423+0000","mapping_epoch":279,"log_start":"313'8957","ondisk_log_start":"313'8957","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4857,"log_dups_size":0,"ondisk_log_size":4857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28966912,"num_objects":442,"num_object_clones":0,"num_object_copies":1326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13814,"num_write_kb":456192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":810,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690362+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:37:15.690362+0000","last_peered":"2023-09-30T09:37:15.690362+0000","last_clean":"2023-09-30T09:37:15.690362+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:37:15.690362+0000","last_undegraded":"2023-09-30T09:37:15.690362+0000","last_fullsized":"2023-09-30T09:37:15.690362+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":339,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.688639+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:37:15.688639+0000","last_peered":"2023-09-30T09:37:15.688639+0000","last_clean":"2023-09-30T09:37:15.688639+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:37:15.688639+0000","last_undegraded":"2023-09-30T09:37:15.688639+0000","last_fullsized":"2023-09-30T09:37:15.688639+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":327,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731465+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:37:15.731465+0000","last_peered":"2023-09-30T09:37:15.731465+0000","last_clean":"2023-09-30T09:37:15.731465+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:37:15.731465+0000","last_undegraded":"2023-09-30T09:37:15.731465+0000","last_fullsized":"2023-09-30T09:37:15.731465+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:20.389320+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:37:20.389320+0000","last_peered":"2023-09-30T09:37:20.389320+0000","last_clean":"2023-09-30T09:37:20.389320+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:37:20.389320+0000","last_undegraded":"2023-09-30T09:37:20.389320+0000","last_fullsized":"2023-09-30T09:37:20.389320+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"348'13897","reported_seq":20536,"reported_epoch":348,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:24.064518+0000","last_change":"2023-09-30T09:35:10.808625+0000","last_active":"2023-09-30T09:37:24.064518+0000","last_peered":"2023-09-30T09:37:24.064518+0000","last_clean":"2023-09-30T09:32:35.460821+0000","last_became_active":"2023-09-30T09:35:10.778683+0000","last_became_peered":"2023-09-30T09:35:10.778683+0000","last_unstale":"2023-09-30T09:37:24.064518+0000","last_undegraded":"2023-09-30T09:35:10.766189+0000","last_fullsized":"2023-09-30T09:37:24.064518+0000","mapping_epoch":327,"log_start":"314'9133","ondisk_log_start":"314'9133","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4764,"log_dups_size":0,"ondisk_log_size":4764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:45.905717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":455,"num_objects_misplaced":910,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13897,"num_write_kb":459264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3261,"num_bytes_recovered":213712896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[6,1,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":455}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":708,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.678771+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:37:15.678771+0000","last_peered":"2023-09-30T09:37:15.678771+0000","last_clean":"2023-09-30T09:37:15.678771+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:37:15.678771+0000","last_undegraded":"2023-09-30T09:37:15.678771+0000","last_fullsized":"2023-09-30T09:37:15.678771+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":527,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:20.389244+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:37:20.389244+0000","last_peered":"2023-09-30T09:37:20.389244+0000","last_clean":"2023-09-30T09:37:20.389244+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:37:20.389244+0000","last_undegraded":"2023-09-30T09:37:20.389244+0000","last_fullsized":"2023-09-30T09:37:20.389244+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":74,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679961+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:37:15.679961+0000","last_peered":"2023-09-30T09:37:15.679961+0000","last_clean":"2023-09-30T09:37:15.679961+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:37:15.679961+0000","last_undegraded":"2023-09-30T09:37:15.679961+0000","last_fullsized":"2023-09-30T09:37:15.679961+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":267,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.691230+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:37:15.691230+0000","last_peered":"2023-09-30T09:37:15.691230+0000","last_clean":"2023-09-30T09:37:15.691230+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:37:15.691230+0000","last_undegraded":"2023-09-30T09:37:15.691230+0000","last_fullsized":"2023-09-30T09:37:15.691230+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"348'13636","reported_seq":27769,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:25.213676+0000","last_change":"2023-09-30T09:37:06.584279+0000","last_active":"2023-09-30T09:37:25.213676+0000","last_peered":"2023-09-30T09:37:25.213676+0000","last_clean":"2023-09-30T09:37:25.213676+0000","last_became_active":"2023-09-30T09:37:06.541995+0000","last_became_peered":"2023-09-30T09:37:06.541995+0000","last_unstale":"2023-09-30T09:37:25.213676+0000","last_undegraded":"2023-09-30T09:37:25.213676+0000","last_fullsized":"2023-09-30T09:37:25.213676+0000","mapping_epoch":343,"log_start":"304'7350","ondisk_log_start":"304'7350","created":279,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":6286,"log_dups_size":0,"ondisk_log_size":6286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:09:29.630262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25559040,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13636,"num_write_kb":448832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7014,"num_bytes_recovered":304349184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":920,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690434+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:37:15.690434+0000","last_peered":"2023-09-30T09:37:15.690434+0000","last_clean":"2023-09-30T09:37:15.690434+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:37:15.690434+0000","last_undegraded":"2023-09-30T09:37:15.690434+0000","last_fullsized":"2023-09-30T09:37:15.690434+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":613,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.678641+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:37:15.678641+0000","last_peered":"2023-09-30T09:37:15.678641+0000","last_clean":"2023-09-30T09:37:15.678641+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:37:15.678641+0000","last_undegraded":"2023-09-30T09:37:15.678641+0000","last_fullsized":"2023-09-30T09:37:15.678641+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":267,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683916+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:37:15.683916+0000","last_peered":"2023-09-30T09:37:15.683916+0000","last_clean":"2023-09-30T09:37:15.683916+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:37:15.683916+0000","last_undegraded":"2023-09-30T09:37:15.683916+0000","last_fullsized":"2023-09-30T09:37:15.683916+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":300,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689572+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:37:15.689572+0000","last_peered":"2023-09-30T09:37:15.689572+0000","last_clean":"2023-09-30T09:37:15.689572+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:37:15.689572+0000","last_undegraded":"2023-09-30T09:37:15.689572+0000","last_fullsized":"2023-09-30T09:37:15.689572+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"348'13764","reported_seq":26436,"reported_epoch":348,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:24.435424+0000","last_change":"2023-09-30T09:35:10.837720+0000","last_active":"2023-09-30T09:37:24.435424+0000","last_peered":"2023-09-30T09:37:24.435424+0000","last_clean":"2023-09-30T09:32:35.481650+0000","last_became_active":"2023-09-30T09:35:10.788829+0000","last_became_peered":"2023-09-30T09:35:10.788829+0000","last_unstale":"2023-09-30T09:37:24.435424+0000","last_undegraded":"2023-09-30T09:35:10.757584+0000","last_fullsized":"2023-09-30T09:37:24.435424+0000","mapping_epoch":327,"log_start":"289'3700","ondisk_log_start":"289'3700","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":10064,"log_dups_size":0,"ondisk_log_size":10064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:05.384343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25821184,"num_objects":394,"num_object_clones":0,"num_object_copies":1182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":215,"num_objects_misplaced":786,"num_objects_unfound":0,"num_objects_dirty":394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13764,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6272,"num_bytes_recovered":411041792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[7,1,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":2},{"shards":"2,6,7","objects":179},{"shards":"6,7","objects":213}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":493,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689028+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:37:15.689028+0000","last_peered":"2023-09-30T09:37:15.689028+0000","last_clean":"2023-09-30T09:37:15.689028+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:37:15.689028+0000","last_undegraded":"2023-09-30T09:37:15.689028+0000","last_fullsized":"2023-09-30T09:37:15.689028+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":724,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.678726+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:37:15.678726+0000","last_peered":"2023-09-30T09:37:15.678726+0000","last_clean":"2023-09-30T09:37:15.678726+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:37:15.678726+0000","last_undegraded":"2023-09-30T09:37:15.678726+0000","last_fullsized":"2023-09-30T09:37:15.678726+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":276,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679602+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:37:15.679602+0000","last_peered":"2023-09-30T09:37:15.679602+0000","last_clean":"2023-09-30T09:37:15.679602+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:37:15.679602+0000","last_undegraded":"2023-09-30T09:37:15.679602+0000","last_fullsized":"2023-09-30T09:37:15.679602+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":33,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:20.248851+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:37:20.248851+0000","last_peered":"2023-09-30T09:37:20.248851+0000","last_clean":"2023-09-30T09:37:20.248851+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:37:20.248851+0000","last_undegraded":"2023-09-30T09:37:20.248851+0000","last_fullsized":"2023-09-30T09:37:20.248851+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"348'13734","reported_seq":19719,"reported_epoch":348,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:37:24.290247+0000","last_change":"2023-09-30T09:37:14.249676+0000","last_active":"2023-09-30T09:37:24.290247+0000","last_peered":"2023-09-30T09:37:24.290247+0000","last_clean":"2023-09-30T09:32:35.469075+0000","last_became_active":"2023-09-30T09:33:33.792569+0000","last_became_peered":"2023-09-30T09:33:33.792569+0000","last_unstale":"2023-09-30T09:37:24.290247+0000","last_undegraded":"2023-09-30T09:33:33.610726+0000","last_fullsized":"2023-09-30T09:33:33.587755+0000","mapping_epoch":302,"log_start":"310'8245","ondisk_log_start":"310'8245","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":5489,"log_dups_size":0,"ondisk_log_size":5489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:16.121133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27656192,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":675,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13734,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2943,"num_bytes_recovered":189333504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-253}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":622,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689593+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:37:15.689593+0000","last_peered":"2023-09-30T09:37:15.689593+0000","last_clean":"2023-09-30T09:37:15.689593+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:37:15.689593+0000","last_undegraded":"2023-09-30T09:37:15.689593+0000","last_fullsized":"2023-09-30T09:37:15.689593+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":856,"reported_epoch":347,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:37:15.684774+0000","last_change":"2023-09-30T09:35:11.685841+0000","last_active":"2023-09-30T09:37:15.684774+0000","last_peered":"2023-09-30T09:37:15.684774+0000","last_clean":"2023-09-30T09:26:47.308941+0000","last_became_active":"2023-09-30T09:35:11.685274+0000","last_became_peered":"2023-09-30T09:35:11.685274+0000","last_unstale":"2023-09-30T09:37:15.684774+0000","last_undegraded":"2023-09-30T09:35:11.661789+0000","last_fullsized":"2023-09-30T09:35:11.661213+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:38.516737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[1,4],"avail_no_missing":["1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":74,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:20.389289+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:37:20.389289+0000","last_peered":"2023-09-30T09:37:20.389289+0000","last_clean":"2023-09-30T09:37:20.389289+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:37:20.389289+0000","last_undegraded":"2023-09-30T09:37:20.389289+0000","last_fullsized":"2023-09-30T09:37:20.389289+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":248,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679313+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:37:15.679313+0000","last_peered":"2023-09-30T09:37:15.679313+0000","last_clean":"2023-09-30T09:37:15.679313+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:37:15.679313+0000","last_undegraded":"2023-09-30T09:37:15.679313+0000","last_fullsized":"2023-09-30T09:37:15.679313+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"348'13765","reported_seq":13944,"reported_epoch":348,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:21.985477+0000","last_change":"2023-09-30T09:33:32.864041+0000","last_active":"2023-09-30T09:37:21.985477+0000","last_peered":"2023-09-30T09:37:21.985477+0000","last_clean":"2023-09-30T09:33:09.329441+0000","last_became_active":"2023-09-30T09:33:11.506402+0000","last_became_peered":"2023-09-30T09:33:11.506402+0000","last_unstale":"2023-09-30T09:37:21.985477+0000","last_undegraded":"2023-09-30T09:37:21.985477+0000","last_fullsized":"2023-09-30T09:37:21.985477+0000","mapping_epoch":295,"log_start":"319'9198","ondisk_log_start":"319'9198","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4567,"log_dups_size":0,"ondisk_log_size":4567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:51:06.485777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31391744,"num_objects":479,"num_object_clones":0,"num_object_copies":1437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":477,"num_objects_unfound":0,"num_objects_dirty":479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13765,"num_write_kb":455808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":479}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":820,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731071+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:37:15.731071+0000","last_peered":"2023-09-30T09:37:15.731071+0000","last_clean":"2023-09-30T09:37:15.731071+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:37:15.731071+0000","last_undegraded":"2023-09-30T09:37:15.731071+0000","last_fullsized":"2023-09-30T09:37:15.731071+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":349,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.685705+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:37:15.685705+0000","last_peered":"2023-09-30T09:37:15.685705+0000","last_clean":"2023-09-30T09:37:15.685705+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:37:15.685705+0000","last_undegraded":"2023-09-30T09:37:15.685705+0000","last_fullsized":"2023-09-30T09:37:15.685705+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1114,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689410+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:37:15.689410+0000","last_peered":"2023-09-30T09:37:15.689410+0000","last_clean":"2023-09-30T09:37:15.689410+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:37:15.689410+0000","last_undegraded":"2023-09-30T09:37:15.689410+0000","last_fullsized":"2023-09-30T09:37:15.689410+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":365,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683638+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:37:15.683638+0000","last_peered":"2023-09-30T09:37:15.683638+0000","last_clean":"2023-09-30T09:37:15.683638+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:37:15.683638+0000","last_undegraded":"2023-09-30T09:37:15.683638+0000","last_fullsized":"2023-09-30T09:37:15.683638+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"348'13792","reported_seq":21105,"reported_epoch":348,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:22.003565+0000","last_change":"2023-09-30T09:37:15.721158+0000","last_active":"2023-09-30T09:37:22.003565+0000","last_peered":"2023-09-30T09:37:22.003565+0000","last_clean":"2023-09-30T09:32:35.469896+0000","last_became_active":"2023-09-30T09:37:15.688891+0000","last_became_peered":"2023-09-30T09:37:15.688891+0000","last_unstale":"2023-09-30T09:37:22.003565+0000","last_undegraded":"2023-09-30T09:37:22.003565+0000","last_fullsized":"2023-09-30T09:37:22.003565+0000","mapping_epoch":346,"log_start":"319'9250","ondisk_log_start":"319'9250","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4542,"log_dups_size":0,"ondisk_log_size":4542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:53:44.682029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32243712,"num_objects":492,"num_object_clones":0,"num_object_copies":1476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":492,"num_objects_unfound":0,"num_objects_dirty":492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13792,"num_write_kb":457088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3599,"num_bytes_recovered":235864064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":492}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":814,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.685388+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:37:15.685388+0000","last_peered":"2023-09-30T09:37:15.685388+0000","last_clean":"2023-09-30T09:37:15.685388+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:37:15.685388+0000","last_undegraded":"2023-09-30T09:37:15.685388+0000","last_fullsized":"2023-09-30T09:37:15.685388+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":673,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689994+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:37:15.689994+0000","last_peered":"2023-09-30T09:37:15.689994+0000","last_clean":"2023-09-30T09:37:15.689994+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:37:15.689994+0000","last_undegraded":"2023-09-30T09:37:15.689994+0000","last_fullsized":"2023-09-30T09:37:15.689994+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":36,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:20.248873+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:37:20.248873+0000","last_peered":"2023-09-30T09:37:20.248873+0000","last_clean":"2023-09-30T09:37:20.248873+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:37:20.248873+0000","last_undegraded":"2023-09-30T09:37:20.248873+0000","last_fullsized":"2023-09-30T09:37:20.248873+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":289,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731567+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:37:15.731567+0000","last_peered":"2023-09-30T09:37:15.731567+0000","last_clean":"2023-09-30T09:37:15.731567+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:37:15.731567+0000","last_undegraded":"2023-09-30T09:37:15.731567+0000","last_fullsized":"2023-09-30T09:37:15.731567+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":762,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683611+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:37:15.683611+0000","last_peered":"2023-09-30T09:37:15.683611+0000","last_clean":"2023-09-30T09:37:15.683611+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:37:15.683611+0000","last_undegraded":"2023-09-30T09:37:15.683611+0000","last_fullsized":"2023-09-30T09:37:15.683611+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":31,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679299+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:37:15.679299+0000","last_peered":"2023-09-30T09:37:15.679299+0000","last_clean":"2023-09-30T09:37:15.679299+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:37:15.679299+0000","last_undegraded":"2023-09-30T09:37:15.679299+0000","last_fullsized":"2023-09-30T09:37:15.679299+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":25,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.680563+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:37:15.680563+0000","last_peered":"2023-09-30T09:37:15.680563+0000","last_clean":"2023-09-30T09:37:15.680563+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:37:15.680563+0000","last_undegraded":"2023-09-30T09:37:15.680563+0000","last_fullsized":"2023-09-30T09:37:15.680563+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":533,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.685062+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:37:15.685062+0000","last_peered":"2023-09-30T09:37:15.685062+0000","last_clean":"2023-09-30T09:37:15.685062+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:37:15.685062+0000","last_undegraded":"2023-09-30T09:37:15.685062+0000","last_fullsized":"2023-09-30T09:37:15.685062+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":326,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690764+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:37:15.690764+0000","last_peered":"2023-09-30T09:37:15.690764+0000","last_clean":"2023-09-30T09:37:15.690764+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:37:15.690764+0000","last_undegraded":"2023-09-30T09:37:15.690764+0000","last_fullsized":"2023-09-30T09:37:15.690764+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":42,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.730902+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:37:15.730902+0000","last_peered":"2023-09-30T09:37:15.730902+0000","last_clean":"2023-09-30T09:37:15.730902+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:37:15.730902+0000","last_undegraded":"2023-09-30T09:37:15.730902+0000","last_fullsized":"2023-09-30T09:37:15.730902+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":292,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.690592+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:37:15.690592+0000","last_peered":"2023-09-30T09:37:15.690592+0000","last_clean":"2023-09-30T09:37:15.690592+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:37:15.690592+0000","last_undegraded":"2023-09-30T09:37:15.690592+0000","last_fullsized":"2023-09-30T09:37:15.690592+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":42,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.731116+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:37:15.731116+0000","last_peered":"2023-09-30T09:37:15.731116+0000","last_clean":"2023-09-30T09:37:15.731116+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:37:15.731116+0000","last_undegraded":"2023-09-30T09:37:15.731116+0000","last_fullsized":"2023-09-30T09:37:15.731116+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":476,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.689417+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:37:15.689417+0000","last_peered":"2023-09-30T09:37:15.689417+0000","last_clean":"2023-09-30T09:37:15.689417+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:37:15.689417+0000","last_undegraded":"2023-09-30T09:37:15.689417+0000","last_fullsized":"2023-09-30T09:37:15.689417+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":344,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.686543+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:37:15.686543+0000","last_peered":"2023-09-30T09:37:15.686543+0000","last_clean":"2023-09-30T09:37:15.686543+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:37:15.686543+0000","last_undegraded":"2023-09-30T09:37:15.686543+0000","last_fullsized":"2023-09-30T09:37:15.686543+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":369,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.683810+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:37:15.683810+0000","last_peered":"2023-09-30T09:37:15.683810+0000","last_clean":"2023-09-30T09:37:15.683810+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:37:15.683810+0000","last_undegraded":"2023-09-30T09:37:15.683810+0000","last_fullsized":"2023-09-30T09:37:15.683810+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":796,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.684858+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:37:15.684858+0000","last_peered":"2023-09-30T09:37:15.684858+0000","last_clean":"2023-09-30T09:37:15.684858+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:37:15.684858+0000","last_undegraded":"2023-09-30T09:37:15.684858+0000","last_fullsized":"2023-09-30T09:37:15.684858+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":52,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.679770+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:37:15.679770+0000","last_peered":"2023-09-30T09:37:15.679770+0000","last_clean":"2023-09-30T09:37:15.679770+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:37:15.679770+0000","last_undegraded":"2023-09-30T09:37:15.679770+0000","last_fullsized":"2023-09-30T09:37:15.679770+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":323,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-09-30T09:37:15.730785+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:37:15.730785+0000","last_peered":"2023-09-30T09:37:15.730785+0000","last_clean":"2023-09-30T09:37:15.730785+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:37:15.730785+0000","last_undegraded":"2023-09-30T09:37:15.730785+0000","last_fullsized":"2023-09-30T09:37:15.730785+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":480247832,"num_objects":7329,"num_object_clones":0,"num_object_copies":21987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3684,"num_objects_misplaced":6421,"num_objects_unfound":0,"num_objects_dirty":7329,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":219223,"num_write_kb":7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42761,"num_bytes_recovered":2643263488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1596534784,"data_stored":1596522568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82487,"ondisk_log_size":82487,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234309,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495946,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":667288,"kb_used_data":266256,"kb_used_omap":0,"kb_used_meta":401024,"kb_avail":93077864,"statfs":{"total":95995035648,"available":95311732736,"internally_reserved":0,"allocated":272646144,"data_stored":271433342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":410648576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9380000000000002,"5min":2.2330000000000001,"15min":1.802},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":44.113,"5min":44.113,"15min":45.927999999999997},"last":0.91100000000000003},{"interface":"front","average":{"1min":4.6150000000000002,"5min":2.3639999999999999,"15min":2.0449999999999999},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":44.164000000000001,"5min":44.164000000000001,"15min":45.316000000000003},"last":0.48499999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.0419999999999998,"15min":1.8819999999999999},"min":{"1min":0.39100000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.343999999999999,"5min":13.343999999999999,"15min":13.343999999999999},"last":0.96799999999999997},{"interface":"front","average":{"1min":3.169,"5min":2.3300000000000001,"15min":2.1520000000000001},"min":{"1min":0.35999999999999999,"5min":0.313,"15min":0.313},"max":{"1min":18.425999999999998,"5min":18.425999999999998,"15min":18.425999999999998},"last":0.52900000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:36:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2210000000000001,"5min":2.226,"15min":1.905},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":17.126000000000001,"5min":22.158000000000001,"15min":37.978000000000002},"last":0.33400000000000002},{"interface":"front","average":{"1min":3.7589999999999999,"5min":2.5110000000000001,"15min":1.851},"min":{"1min":0.32500000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":25.164999999999999,"5min":25.164999999999999,"15min":37.759999999999998},"last":0.94399999999999995}]},{"osd":3,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7839999999999998,"5min":7.1130000000000004,"15min":3.532},"min":{"1min":0.45300000000000001,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":90.147999999999996,"5min":212.44800000000001,"15min":212.44800000000001},"last":1.0900000000000001},{"interface":"front","average":{"1min":8.9450000000000003,"5min":7.8339999999999996,"15min":3.9199999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":89.859999999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.0640000000000001,"15min":1.593},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23400000000000001},"max":{"1min":3.004,"5min":7.4359999999999999,"15min":73.656000000000006},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.0209999999999999,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.23000000000000001},"max":{"1min":1.8089999999999999,"5min":9.6590000000000007,"15min":73.882999999999996},"last":1.232}]},{"osd":5,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.1930000000000001,"15min":1.819},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":1.024,"5min":8.2040000000000006,"15min":76.613},"last":11.428000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.1910000000000001,"15min":1.786},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.0979999999999999,"5min":9.7750000000000004,"15min":76.329999999999998},"last":7.7220000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":1.458,"15min":2.1419999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":3.9870000000000001,"5min":8.7400000000000002,"15min":91.120999999999995},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.1679999999999999,"15min":2.0910000000000002},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.4870000000000001,"5min":9.9489999999999998,"15min":91.733999999999995},"last":1.2609999999999999}]}]},{"osd":1,"up_from":301,"seq":1292785156143,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":388124,"kb_used_data":100436,"kb_used_omap":9,"kb_used_meta":287670,"kb_avail":93357028,"statfs":{"total":95995035648,"available":95597596672,"internally_reserved":0,"allocated":102846464,"data_stored":100873126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":294574098},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":9.7590000000000003,"5min":4.0979999999999999,"15min":3.0600000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":138.43799999999999,"5min":138.43799999999999,"15min":138.43799999999999},"last":0.19400000000000001},{"interface":"front","average":{"1min":9.4320000000000004,"5min":4.5140000000000002,"15min":3.52},"min":{"1min":0.27800000000000002,"5min":0.25,"15min":0.25},"max":{"1min":138.482,"5min":138.482,"15min":138.482},"last":2.4620000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.8100000000000001,"15min":2.7999999999999998},"min":{"1min":0.254,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":13.656000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":3.9609999999999999,"5min":3.0899999999999999,"15min":2.8340000000000001},"min":{"1min":0.309,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":18.978999999999999,"5min":18.978999999999999,"15min":18.978999999999999},"last":2.4359999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":6.8879999999999999,"5min":7.9370000000000003,"15min":8.9429999999999996},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":40.606000000000002,"5min":77.959000000000003,"15min":77.959000000000003},"last":2.1640000000000001},{"interface":"front","average":{"1min":7.5449999999999999,"5min":8.2690000000000001,"15min":9.5410000000000004},"min":{"1min":0.439,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":41.247,"5min":77.828999999999994,"15min":77.828999999999994},"last":2.7330000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.1059999999999999,"15min":2.0259999999999998},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.0090000000000003,"5min":13.602,"15min":13.602},"last":2.3969999999999998},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.712,"15min":1.4319999999999999},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.311},"max":{"1min":8.6440000000000001,"5min":10.773,"15min":10.773},"last":2.198}]},{"osd":5,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":2.351,"15min":2.1890000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.2750000000000004,"5min":13.666,"15min":13.666},"last":2.3660000000000001},{"interface":"front","average":{"1min":2.9239999999999999,"5min":1.9890000000000001,"15min":1.73},"min":{"1min":0.39100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.177,"5min":10.847,"15min":10.847},"last":2.2250000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":11.52,"15min":15.238},"min":{"1min":0.503,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.1029999999999998,"5min":209.221,"15min":209.221},"last":2.323},{"interface":"front","average":{"1min":2.48,"5min":11.207000000000001,"15min":15.079000000000001},"min":{"1min":0.40000000000000002,"5min":0.377,"15min":0.377},"max":{"1min":8.1310000000000002,"5min":208.43600000000001,"15min":208.43600000000001},"last":2.4159999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9950000000000001,"5min":8.3640000000000008,"15min":11.084},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.3559999999999999,"5min":165.59399999999999,"15min":165.59399999999999},"last":2.7749999999999999},{"interface":"front","average":{"1min":3.3719999999999999,"5min":8.5670000000000002,"15min":11.308},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":10.565,"5min":165.833,"15min":165.833},"last":0.46000000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926043,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":440324,"kb_used_data":90940,"kb_used_omap":5,"kb_used_meta":349370,"kb_avail":93304828,"statfs":{"total":95995035648,"available":95544143872,"internally_reserved":0,"allocated":93122560,"data_stored":91004230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":357755133},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:36:38 2023","interfaces":[{"interface":"back","average":{"1min":4.1920000000000002,"5min":3.0630000000000002,"15min":3.0190000000000001},"min":{"1min":0.24099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":15.401,"5min":17.966000000000001,"15min":17.966000000000001},"last":2.1360000000000001},{"interface":"front","average":{"1min":6.3339999999999996,"5min":7.7549999999999999,"15min":8.8680000000000003},"min":{"1min":0.31,"5min":0.22,"15min":0.22},"max":{"1min":25.869,"5min":138.92400000000001,"15min":138.92400000000001},"last":2.9849999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.1299999999999999,"15min":1.744},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.887,"5min":11.218999999999999,"15min":55.823999999999998},"last":1.044},{"interface":"front","average":{"1min":3.2080000000000002,"5min":2.5350000000000001,"15min":1.9319999999999999},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":10.763,"5min":40.128999999999998,"15min":55.790999999999997},"last":2.4990000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":8.5350000000000001,"15min":4.2089999999999996},"min":{"1min":0.30599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":16.271000000000001,"5min":218.06999999999999,"15min":218.06999999999999},"last":3.3809999999999998},{"interface":"front","average":{"1min":2.8199999999999998,"5min":6.2930000000000001,"15min":3.4159999999999999},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":14.987,"5min":218.63200000000001,"15min":218.63200000000001},"last":2.3929999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.6599999999999999,"15min":1.679},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.29099999999999998},"max":{"1min":10.614000000000001,"5min":10.614000000000001,"15min":121.486},"last":1.129},{"interface":"front","average":{"1min":3.3780000000000001,"5min":3.863,"15min":2.649},"min":{"1min":0.39900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":17.044,"5min":136.262,"15min":136.262},"last":3.4670000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":4.1710000000000003,"15min":3.0630000000000002},"min":{"1min":0.439,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.995999999999999,"5min":136.34,"15min":136.34},"last":3.427},{"interface":"front","average":{"1min":2.4129999999999998,"5min":1.9419999999999999,"15min":2.2839999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":11.991,"5min":12.717000000000001,"15min":135.381},"last":2.2320000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":7.1440000000000001,"15min":3.2400000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":12.021000000000001,"5min":214.785,"15min":214.785},"last":2.1890000000000001},{"interface":"front","average":{"1min":2.8999999999999999,"5min":7.2439999999999998,"15min":3.1850000000000001},"min":{"1min":0.46300000000000002,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":10.695,"5min":214.93899999999999,"15min":214.93899999999999},"last":1.095}]},{"osd":7,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":2.9870000000000001,"15min":3.1720000000000002},"min":{"1min":0.42999999999999999,"5min":0.38300000000000001,"15min":0.373},"max":{"1min":12.048,"5min":97.382000000000005,"15min":150.96899999999999},"last":2.262},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.944,"15min":3.0150000000000001},"min":{"1min":0.43099999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.153,"5min":97.015000000000001,"15min":150.328},"last":1.1639999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598984,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":324208,"kb_used_data":17960,"kb_used_omap":5,"kb_used_meta":306234,"kb_avail":93420944,"statfs":{"total":95995035648,"available":95663046656,"internally_reserved":0,"allocated":18391040,"data_stored":16468489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":313584269},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":4.1130000000000004,"5min":3.4630000000000001,"15min":2.1680000000000001},"min":{"1min":0.24199999999999999,"5min":0.187,"15min":0.187},"max":{"1min":15.063000000000001,"5min":77.379000000000005,"15min":77.379000000000005},"last":0.20499999999999999},{"interface":"front","average":{"1min":2.7509999999999999,"5min":3.2709999999999999,"15min":1.9670000000000001},"min":{"1min":0.31900000000000001,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":15.151,"5min":77.066999999999993,"15min":77.066999999999993},"last":3.3279999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:36:36 2023","interfaces":[{"interface":"back","average":{"1min":8.9789999999999992,"5min":4.8339999999999996,"15min":3.992},"min":{"1min":0.36399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":64.412999999999997,"5min":64.412999999999997,"15min":64.412999999999997},"last":0.373},{"interface":"front","average":{"1min":7.6420000000000003,"5min":4.1260000000000003,"15min":3.4129999999999998},"min":{"1min":0.33700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":63.247999999999998,"5min":63.247999999999998,"15min":63.247999999999998},"last":3.3769999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:36:51 2023","interfaces":[{"interface":"back","average":{"1min":15.153,"5min":7.4130000000000003,"15min":3.4689999999999999},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":134.078,"5min":134.078,"15min":134.078},"last":0.45100000000000001},{"interface":"front","average":{"1min":15.754,"5min":7.0709999999999997,"15min":3.5920000000000001},"min":{"1min":0.38900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":135.73400000000001,"5min":135.73400000000001,"15min":135.73400000000001},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":2.036,"15min":1.3380000000000001},"min":{"1min":0.39000000000000001,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":8.1240000000000006,"5min":35.972000000000001,"15min":35.972000000000001},"last":6.5979999999999999},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.931,"15min":1.597},"min":{"1min":0.42899999999999999,"5min":0.33800000000000002,"15min":0.32700000000000001},"max":{"1min":13.224,"5min":36.518000000000001,"15min":36.518000000000001},"last":2.3759999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.8610000000000002,"5min":2.024,"15min":2.0880000000000001},"min":{"1min":0.39400000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.41,"5min":12.651999999999999,"15min":155.477},"last":6.7359999999999998},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.591,"15min":1.952},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":8.0839999999999996,"5min":8.0839999999999996,"15min":155.46000000000001},"last":2.536}]},{"osd":6,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":6.6180000000000003,"15min":3.2879999999999998},"min":{"1min":0.48199999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.273,"5min":154.417,"15min":154.417},"last":4.5780000000000003},{"interface":"front","average":{"1min":2.8359999999999999,"5min":7.2729999999999997,"15min":3.3399999999999999},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":10.382,"5min":154.04400000000001,"15min":154.04400000000001},"last":1.4750000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:36:29 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":3.8639999999999999,"15min":3.1480000000000001},"min":{"1min":0.52800000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":13.304,"5min":123.294,"15min":213.404},"last":0.71999999999999997},{"interface":"front","average":{"1min":2.9390000000000001,"5min":4.1269999999999998,"15min":3.1699999999999999},"min":{"1min":0.45100000000000001,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":10.35,"5min":123.16,"15min":213.27199999999999},"last":1.282}]}]},{"osd":3,"up_from":109,"seq":468151435456,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1032424,"kb_used_data":375136,"kb_used_omap":2,"kb_used_meta":657213,"kb_avail":92712728,"statfs":{"total":95995035648,"available":94937833472,"internally_reserved":0,"allocated":384139264,"data_stored":382093515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":672986471},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":5.4980000000000002,"5min":4.9889999999999999,"15min":3.097},"min":{"1min":0.23499999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":51.835000000000001,"5min":102.57299999999999,"15min":130.946},"last":5.9450000000000003},{"interface":"front","average":{"1min":5.6929999999999996,"5min":4.899,"15min":3.0579999999999998},"min":{"1min":0.317,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":52.093000000000004,"5min":102.745,"15min":130.96899999999999},"last":1.204}]},{"osd":1,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5489999999999999,"5min":14.395,"15min":18.975999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":9.3919999999999995,"5min":273.99599999999998,"15min":273.99599999999998},"last":2.004},{"interface":"front","average":{"1min":4.4950000000000001,"5min":13.355,"15min":17.132999999999999},"min":{"1min":0.34899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.6910000000000007,"5min":274.21600000000001,"15min":274.21600000000001},"last":2.6579999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8360000000000003,"5min":4.5800000000000001,"15min":3.3759999999999999},"min":{"1min":0.53600000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":62.289000000000001,"5min":76.974000000000004,"15min":145.75899999999999},"last":2.6859999999999999},{"interface":"front","average":{"1min":6.0650000000000004,"5min":4.3339999999999996,"15min":3.2170000000000001},"min":{"1min":0.33400000000000002,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":65.415999999999997,"5min":77.593999999999994,"15min":150.358},"last":5.9109999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":2.419,"5min":2.702,"15min":1.6899999999999999},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":10.805,"5min":60.637,"15min":60.637},"last":2.5030000000000001},{"interface":"front","average":{"1min":3.008,"5min":2.3100000000000001,"15min":1.627},"min":{"1min":0.40100000000000002,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":8.8049999999999997,"5min":10.731,"15min":15.638},"last":1.1339999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":2.9940000000000002,"15min":1.9390000000000001},"min":{"1min":0.375,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":8.8309999999999995,"5min":37.485999999999997,"15min":37.485999999999997},"last":2.04},{"interface":"front","average":{"1min":2.9199999999999999,"5min":3.2930000000000001,"15min":2.113},"min":{"1min":0.437,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":7.7439999999999998,"5min":37.448,"15min":37.448},"last":2.7090000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":2.9740000000000002,"15min":1.8240000000000001},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.891999999999999,"5min":60.698,"15min":60.698},"last":2.5830000000000002},{"interface":"front","average":{"1min":3.895,"5min":2.7629999999999999,"15min":1.8999999999999999},"min":{"1min":0.61299999999999999,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":12.121,"5min":12.121,"15min":14.69},"last":1.619}]},{"osd":7,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":3.2469999999999999,"15min":2.7440000000000002},"min":{"1min":0.44500000000000001,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":7.7709999999999999,"5min":26.469000000000001,"15min":226.03999999999999},"last":2.734},{"interface":"front","average":{"1min":2.8370000000000002,"5min":3.1989999999999998,"15min":2.6019999999999999},"min":{"1min":0.41799999999999998,"5min":0.38800000000000001,"15min":0.34799999999999998},"max":{"1min":10.855,"5min":60.670000000000002,"15min":226.06100000000001},"last":2.621}]}]},{"osd":4,"up_from":30,"seq":128849019219,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":391120,"kb_used_data":117704,"kb_used_omap":0,"kb_used_meta":273408,"kb_avail":93354032,"statfs":{"total":95995035648,"available":95594528768,"internally_reserved":0,"allocated":120528896,"data_stored":119323789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":279969792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":1.4399999999999999,"15min":2.8780000000000001},"min":{"1min":0.33200000000000002,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":4.5940000000000003,"5min":14.353,"15min":128.79300000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.873,"15min":2.8730000000000002},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.248},"max":{"1min":6.9939999999999998,"5min":10.618,"15min":128.38800000000001},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4169999999999998,"5min":6.8659999999999997,"15min":8.6240000000000006},"min":{"1min":0.26600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.915,"5min":160.69999999999999,"15min":160.69999999999999},"last":25.663},{"interface":"front","average":{"1min":3.2629999999999999,"5min":6.7960000000000003,"15min":8.6419999999999995},"min":{"1min":0.28399999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.7029999999999994,"5min":160.50700000000001,"15min":160.50700000000001},"last":25.367999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.9850000000000001,"15min":2.9399999999999999},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9790000000000001,"5min":19.391999999999999,"15min":140.601},"last":0.376},{"interface":"front","average":{"1min":2.3399999999999999,"5min":2.0169999999999999,"15min":2.9300000000000002},"min":{"1min":0.39300000000000002,"5min":0.309,"15min":0.309},"max":{"1min":4.9900000000000002,"5min":19.422999999999998,"15min":140.54900000000001},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 30 09:36:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3010000000000002,"5min":2.964,"15min":3.6030000000000002},"min":{"1min":0.42799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.2650000000000006,"5min":52.762,"15min":281.42200000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":3.4359999999999999,"5min":2.992,"15min":3.5720000000000001},"min":{"1min":0.49099999999999999,"5min":0.312,"15min":0.312},"max":{"1min":9.2810000000000006,"5min":52.790999999999997,"15min":281.44999999999999},"last":0.39300000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.94399999999999995,"15min":1.7130000000000001},"min":{"1min":0.309,"5min":0.219,"15min":0.219},"max":{"1min":4.6120000000000001,"5min":8.6509999999999998,"15min":134.30600000000001},"last":0.309},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1,"15min":1.772},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":2.2309999999999999,"5min":6.3929999999999998,"15min":134.35300000000001},"last":0.33700000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":3.0299999999999998,"15min":1.593},"min":{"1min":0.32800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.1079999999999997,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.91700000000000004,"5min":2.927,"15min":1.597},"min":{"1min":0.34599999999999997,"5min":0.249,"15min":0.249},"max":{"1min":2.6459999999999999,"5min":202.42400000000001,"15min":202.42400000000001},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":4.4320000000000004,"15min":3.004},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":3.8330000000000002,"5min":230.90100000000001,"15min":230.90100000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.73199999999999998,"5min":4.1289999999999996,"15min":2.5579999999999998},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":2.6779999999999999,"5min":229.97,"15min":229.97},"last":0.497}]}]},{"osd":5,"up_from":38,"seq":163208757583,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":405328,"kb_used_data":115848,"kb_used_omap":0,"kb_used_meta":289344,"kb_avail":93339824,"statfs":{"total":95995035648,"available":95579979776,"internally_reserved":0,"allocated":118628352,"data_stored":117419793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":296288256},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.2069999999999999,"15min":2.4540000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.22},"max":{"1min":4.6459999999999999,"5min":56.563000000000002,"15min":148.79900000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.738,"5min":2.3199999999999998,"15min":2.7589999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":6.2969999999999997,"5min":57.241999999999997,"15min":148.779},"last":3.2440000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:36:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":2.6619999999999999,"15min":2.1120000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.237,"5min":60.799999999999997,"15min":60.799999999999997},"last":2.0289999999999999},{"interface":"front","average":{"1min":3.012,"5min":3.581,"15min":3.3010000000000002},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.2539999999999996,"5min":63.798999999999999,"15min":63.798999999999999},"last":2.7890000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.8879999999999999,"15min":3.1150000000000002},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.71,"5min":30.613,"15min":165.768},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.0249999999999999,"15min":3.6030000000000002},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.1989999999999998,"5min":36.488,"15min":166.24000000000001},"last":0.54800000000000004}]},{"osd":3,"last update":"Sat Sep 30 09:36:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":5.9180000000000001,"15min":4.0069999999999997},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5380000000000003,"5min":172.976,"15min":226.34700000000001},"last":4.0419999999999998},{"interface":"front","average":{"1min":2.4689999999999999,"5min":5.8799999999999999,"15min":3.8300000000000001},"min":{"1min":0.376,"5min":0.36599999999999999,"15min":0.313},"max":{"1min":8.2240000000000002,"5min":173.01599999999999,"15min":226.36000000000001},"last":4.0540000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:36:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.76500000000000001,"15min":0.84899999999999998},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":1.0760000000000001,"5min":5.7160000000000002,"15min":8.5359999999999996},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.96999999999999997,"15min":0.94199999999999995},"min":{"1min":0.29599999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.1230000000000002,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":2.9980000000000002,"15min":1.7509999999999999},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":3.2069999999999999,"5min":155.602,"15min":155.602},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":3.012,"15min":1.867},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.9490000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.45700000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":2.5590000000000002,"15min":1.659},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.883,"5min":133.40700000000001,"15min":133.40700000000001},"last":3.0710000000000002},{"interface":"front","average":{"1min":0.98599999999999999,"5min":2.7229999999999999,"15min":1.5840000000000001},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":5.7880000000000003,"5min":132.797,"15min":132.797},"last":0.68500000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":78909440,"data_stored":78905368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":93065216,"data_stored":93061144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12582912,"data_stored":12582912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":355008512,"data_stored":355008512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":113836032,"data_stored":113836032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":113508352,"data_stored":113508352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":255397888,"data_stored":255393816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":574226432,"data_stored":574226432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:37:28.678 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:28.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:28.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:28.709 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:37:28.791 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:28.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:28 smithi019 ceph-mon[216815]: pgmap v903: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 388 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 3207/18468 objects degraded (17.365%); 5382/18468 objects misplaced (29.142%); 18 MiB/s, 287 objects/s recovering 2023-09-30T09:37:28.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 3207/18468 objects degraded (17.365%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:28.918 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:28.991 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:28 smithi167 ceph-mon[192880]: pgmap v903: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 388 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 3207/18468 objects degraded (17.365%); 5382/18468 objects misplaced (29.142%); 18 MiB/s, 287 objects/s recovering 2023-09-30T09:37:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 3207/18468 objects degraded (17.365%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:29.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:28 smithi019 ceph-mon[224043]: pgmap v903: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 388 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 3207/18468 objects degraded (17.365%); 5382/18468 objects misplaced (29.142%); 18 MiB/s, 287 objects/s recovering 2023-09-30T09:37:29.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 3207/18468 objects degraded (17.365%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:29.095 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:29.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:29.191 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:37:29.206 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:29.263 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:29.345 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:29.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:29.448 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:29.562 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:29.685 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:29.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:29.793 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:29.870 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:29 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:37:30.043 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:30.054 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:29 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:37:30.054 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:29 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:37:30.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:30.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:30.161 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:30.243 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:30.302 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:30.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:30.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:30.406 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:30.433 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:30.478 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:30.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:30.582 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:30.626 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:30.723 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:30.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:30.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:30.834 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:30.857 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:37:30.927 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:30.978 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:30 smithi167 ceph-mon[192880]: pgmap v904: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 304 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 2250/14436 objects degraded (15.586%); 4037/14436 objects misplaced (27.965%); 16 MiB/s, 292 objects/s recovering 2023-09-30T09:37:30.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:30 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:37:30.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2483363309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:37:30.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:37:31.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:30 smithi019 ceph-mon[216815]: pgmap v904: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 304 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 2250/14436 objects degraded (15.586%); 4037/14436 objects misplaced (27.965%); 16 MiB/s, 292 objects/s recovering 2023-09-30T09:37:31.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:30 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:37:31.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2483363309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:37:31.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:37:31.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:30 smithi019 ceph-mon[224043]: pgmap v904: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 304 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 2250/14436 objects degraded (15.586%); 4037/14436 objects misplaced (27.965%); 16 MiB/s, 292 objects/s recovering 2023-09-30T09:37:31.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:30 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:37:31.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2483363309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:37:31.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:37:31.156 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:31.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:31.238 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:37:31.238 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:37:31.266 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:31.315 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:31.488 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:31.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:31.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:31.582 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:37:31.592 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:31.628 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:31.671 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:31.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:31.774 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:31.801 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:31.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:31.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:37:31.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[216815]: osdmap e349: 8 total, 8 up, 7 in 2023-09-30T09:37:31.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[216815]: failed to encode map e349 with expected crc 2023-09-30T09:37:31.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[216815]: failed to encode map e349 with expected crc 2023-09-30T09:37:31.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:31.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:37:31.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[224043]: osdmap e349: 8 total, 8 up, 7 in 2023-09-30T09:37:31.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[224043]: failed to encode map e349 with expected crc 2023-09-30T09:37:31.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:31 smithi019 ceph-mon[224043]: failed to encode map e349 with expected crc 2023-09-30T09:37:32.008 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:31 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:37:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:31 smithi167 ceph-mon[192880]: osdmap e349: 8 total, 8 up, 7 in 2023-09-30T09:37:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:31 smithi167 ceph-mon[192880]: failed to encode map e349 with expected crc 2023-09-30T09:37:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:31 smithi167 ceph-mon[192880]: failed to encode map e349 with expected crc 2023-09-30T09:37:32.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:32.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:32.124 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:32.167 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:32.199 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:37:32.235 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:32.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:32.345 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:32.377 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:32.386 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:37:32.420 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:32.436 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:32.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:32.444 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:37:32.507 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:37:32.541 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:32.584 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:37:32.621 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:32.622 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:37:32.663 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:37:32.710 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:37:32.763 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:37:32.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:37:32.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:37:32.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:37:32.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:37:32.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:37:32.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:37:32.787 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:37:32.787 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:37:32.832 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:32.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:32.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:32.875 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:32 smithi019 ceph-mon[216815]: pgmap v906: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 293 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 2250/13947 objects degraded (16.133%); 4037/13947 objects misplaced (28.945%); 16 MiB/s, 297 objects/s recovering 2023-09-30T09:37:32.875 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4017384199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:32.875 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:32 smithi019 ceph-mon[224043]: pgmap v906: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 293 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 2250/13947 objects degraded (16.133%); 4037/13947 objects misplaced (28.945%); 16 MiB/s, 297 objects/s recovering 2023-09-30T09:37:32.875 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4017384199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:32.951 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:33.073 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:32 smithi167 ceph-mon[192880]: pgmap v906: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 293 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 2250/13947 objects degraded (16.133%); 4037/13947 objects misplaced (28.945%); 16 MiB/s, 297 objects/s recovering 2023-09-30T09:37:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4017384199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:33.712 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:33.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:33.817 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:33.846 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:33.855 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:37:33.919 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:33.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:34.027 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:37:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:37:33] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-30T09:37:34.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:34.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 2250/13947 objects degraded (16.133%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:34.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:34.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 2250/13947 objects degraded (16.133%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:34.145 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 2250/13947 objects degraded (16.133%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:34.437 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:34.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:34.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:34.546 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:34.575 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:34.982 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:34.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:34.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:35.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:34 smithi019 ceph-mon[224043]: pgmap v907: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 226 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 1780/10686 objects degraded (16.657%); 3122/10686 objects misplaced (29.216%); 16 MiB/s, 296 objects/s recovering 2023-09-30T09:37:35.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:34 smithi019 ceph-mon[224043]: osdmap e350: 8 total, 8 up, 7 in 2023-09-30T09:37:35.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:34 smithi019 ceph-mon[224043]: failed to encode map e350 with expected crc 2023-09-30T09:37:35.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:34 smithi019 ceph-mon[216815]: pgmap v907: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 226 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 1780/10686 objects degraded (16.657%); 3122/10686 objects misplaced (29.216%); 16 MiB/s, 296 objects/s recovering 2023-09-30T09:37:35.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:34 smithi019 ceph-mon[216815]: osdmap e350: 8 total, 8 up, 7 in 2023-09-30T09:37:35.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:34 smithi019 ceph-mon[216815]: failed to encode map e350 with expected crc 2023-09-30T09:37:35.087 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:35.109 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:35.145 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:35.236 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:35.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:35.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:34 smithi167 ceph-mon[192880]: pgmap v907: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 105 active+clean; 226 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 1780/10686 objects degraded (16.657%); 3122/10686 objects misplaced (29.216%); 16 MiB/s, 296 objects/s recovering 2023-09-30T09:37:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:34 smithi167 ceph-mon[192880]: osdmap e350: 8 total, 8 up, 7 in 2023-09-30T09:37:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:34 smithi167 ceph-mon[192880]: failed to encode map e350 with expected crc 2023-09-30T09:37:35.341 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:35.378 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:35.379 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:35.668 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:35.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:35.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:35.782 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:35.875 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:35.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[224043]: failed to encode map e350 with expected crc 2023-09-30T09:37:35.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[224043]: failed to encode map e350 with expected crc 2023-09-30T09:37:35.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[224043]: osdmap e351: 8 total, 8 up, 7 in 2023-09-30T09:37:35.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[224043]: failed to encode map e351 with expected crc 2023-09-30T09:37:35.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[224043]: failed to encode map e351 with expected crc 2023-09-30T09:37:35.941 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[216815]: failed to encode map e350 with expected crc 2023-09-30T09:37:35.941 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[216815]: failed to encode map e350 with expected crc 2023-09-30T09:37:35.941 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[216815]: osdmap e351: 8 total, 8 up, 7 in 2023-09-30T09:37:35.941 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[216815]: failed to encode map e351 with expected crc 2023-09-30T09:37:35.941 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:35 smithi019 ceph-mon[216815]: failed to encode map e351 with expected crc 2023-09-30T09:37:36.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:35 smithi167 ceph-mon[192880]: failed to encode map e350 with expected crc 2023-09-30T09:37:36.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:35 smithi167 ceph-mon[192880]: failed to encode map e350 with expected crc 2023-09-30T09:37:36.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:35 smithi167 ceph-mon[192880]: osdmap e351: 8 total, 8 up, 7 in 2023-09-30T09:37:36.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:35 smithi167 ceph-mon[192880]: failed to encode map e351 with expected crc 2023-09-30T09:37:36.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:35 smithi167 ceph-mon[192880]: failed to encode map e351 with expected crc 2023-09-30T09:37:36.312 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:36.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:36.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:36.358 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:37:36.360 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:37:36.429 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:36.457 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:36.506 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:36.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:36.624 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:36.706 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:36.828 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:37:37.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[216815]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[216815]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[216815]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[216815]: pgmap v910: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 105 active+clean; 138 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 869/6495 objects degraded (13.380%); 1863/6495 objects misplaced (28.684%); 12 MiB/s, 242 objects/s recovering 2023-09-30T09:37:37.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[216815]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[216815]: osdmap e352: 8 total, 8 up, 7 in 2023-09-30T09:37:37.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[216815]: failed to encode map e352 with expected crc 2023-09-30T09:37:37.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[224043]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[224043]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[224043]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[224043]: pgmap v910: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 105 active+clean; 138 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 869/6495 objects degraded (13.380%); 1863/6495 objects misplaced (28.684%); 12 MiB/s, 242 objects/s recovering 2023-09-30T09:37:37.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[224043]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[224043]: osdmap e352: 8 total, 8 up, 7 in 2023-09-30T09:37:37.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:36 smithi019 ceph-mon[224043]: failed to encode map e352 with expected crc 2023-09-30T09:37:37.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:36 smithi167 ceph-mon[192880]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:36 smithi167 ceph-mon[192880]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:36 smithi167 ceph-mon[192880]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:36 smithi167 ceph-mon[192880]: pgmap v910: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 105 active+clean; 138 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 869/6495 objects degraded (13.380%); 1863/6495 objects misplaced (28.684%); 12 MiB/s, 242 objects/s recovering 2023-09-30T09:37:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:36 smithi167 ceph-mon[192880]: failed to encode map e351 with expected crc 2023-09-30T09:37:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:36 smithi167 ceph-mon[192880]: osdmap e352: 8 total, 8 up, 7 in 2023-09-30T09:37:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:36 smithi167 ceph-mon[192880]: failed to encode map e352 with expected crc 2023-09-30T09:37:37.450 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:37.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:37.563 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:37.654 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:37.745 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Removed 113275 objects 2023-09-30T09:37:37.745 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Clean up completed and total clean up time :243.298 2023-09-30T09:37:38.114 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":910,"stamp":"2023-09-30T09:37:35.563438+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":144915300,"num_objects":2165,"num_object_clones":0,"num_object_copies":6495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":869,"num_objects_misplaced":1863,"num_objects_unfound":0,"num_objects_dirty":2165,"num_whiteouts":0,"num_read":6114,"num_read_kb":12467,"num_write":225404,"num_write_kb":7263166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45970,"num_bytes_recovered":2804785624,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88502,"ondisk_log_size":88502,"up":357,"acting":355,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":396,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2966332,"kb_used_data":367740,"kb_used_omap":23,"kb_used_meta":2598440,"kb_avail":653249732,"statfs":{"total":671965249536,"available":668927725568,"internally_reserved":0,"allocated":376565760,"data_stored":364822711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2660802819},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-173473792,"num_objects":-2647,"num_object_clones":0,"num_object_copies":-7941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1381,"num_objects_misplaced":-2174,"num_objects_unfound":0,"num_objects_dirty":-2647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2647,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1457,"num_bytes_recovered":74055680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002174"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":813,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750338+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:37:21.750338+0000","last_peered":"2023-09-30T09:37:21.750338+0000","last_clean":"2023-09-30T09:37:21.750338+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:37:21.750338+0000","last_undegraded":"2023-09-30T09:37:21.750338+0000","last_fullsized":"2023-09-30T09:37:21.750338+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":367,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.762949+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:37:21.762949+0000","last_peered":"2023-09-30T09:37:21.762949+0000","last_clean":"2023-09-30T09:37:21.762949+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:37:21.762949+0000","last_undegraded":"2023-09-30T09:37:21.762949+0000","last_fullsized":"2023-09-30T09:37:21.762949+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":330,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.756392+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:37:21.756392+0000","last_peered":"2023-09-30T09:37:21.756392+0000","last_clean":"2023-09-30T09:37:21.756392+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:37:21.756392+0000","last_undegraded":"2023-09-30T09:37:21.756392+0000","last_fullsized":"2023-09-30T09:37:21.756392+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":793,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.751424+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:37:21.751424+0000","last_peered":"2023-09-30T09:37:21.751424+0000","last_clean":"2023-09-30T09:37:21.751424+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:37:21.751424+0000","last_undegraded":"2023-09-30T09:37:21.751424+0000","last_fullsized":"2023-09-30T09:37:21.751424+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":223,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.840259+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:37:30.840259+0000","last_peered":"2023-09-30T09:37:30.840259+0000","last_clean":"2023-09-30T09:37:30.840259+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:37:30.840259+0000","last_undegraded":"2023-09-30T09:37:30.840259+0000","last_fullsized":"2023-09-30T09:37:30.840259+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":255,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845770+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:37:30.845770+0000","last_peered":"2023-09-30T09:37:30.845770+0000","last_clean":"2023-09-30T09:37:30.845770+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:37:30.845770+0000","last_undegraded":"2023-09-30T09:37:30.845770+0000","last_fullsized":"2023-09-30T09:37:30.845770+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":562,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.762861+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:37:21.762861+0000","last_peered":"2023-09-30T09:37:21.762861+0000","last_clean":"2023-09-30T09:37:21.762861+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:37:21.762861+0000","last_undegraded":"2023-09-30T09:37:21.762861+0000","last_fullsized":"2023-09-30T09:37:21.762861+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":396,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.759591+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:37:21.759591+0000","last_peered":"2023-09-30T09:37:21.759591+0000","last_clean":"2023-09-30T09:37:21.759591+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:37:21.759591+0000","last_undegraded":"2023-09-30T09:37:21.759591+0000","last_fullsized":"2023-09-30T09:37:21.759591+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":325,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.795730+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:37:21.795730+0000","last_peered":"2023-09-30T09:37:21.795730+0000","last_clean":"2023-09-30T09:37:21.795730+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:37:21.795730+0000","last_undegraded":"2023-09-30T09:37:21.795730+0000","last_fullsized":"2023-09-30T09:37:21.795730+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":773,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.756197+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:37:21.756197+0000","last_peered":"2023-09-30T09:37:21.756197+0000","last_clean":"2023-09-30T09:37:21.756197+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:37:21.756197+0000","last_undegraded":"2023-09-30T09:37:21.756197+0000","last_fullsized":"2023-09-30T09:37:21.756197+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":245,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845895+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:37:30.845895+0000","last_peered":"2023-09-30T09:37:30.845895+0000","last_clean":"2023-09-30T09:37:30.845895+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:37:30.845895+0000","last_undegraded":"2023-09-30T09:37:30.845895+0000","last_fullsized":"2023-09-30T09:37:30.845895+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":330,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.759659+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:37:21.759659+0000","last_peered":"2023-09-30T09:37:21.759659+0000","last_clean":"2023-09-30T09:37:21.759659+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:37:21.759659+0000","last_undegraded":"2023-09-30T09:37:21.759659+0000","last_fullsized":"2023-09-30T09:37:21.759659+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":513,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.842150+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:37:30.842150+0000","last_peered":"2023-09-30T09:37:30.842150+0000","last_clean":"2023-09-30T09:37:30.842150+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:37:30.842150+0000","last_undegraded":"2023-09-30T09:37:30.842150+0000","last_fullsized":"2023-09-30T09:37:30.842150+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":227,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.751071+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:37:21.751071+0000","last_peered":"2023-09-30T09:37:21.751071+0000","last_clean":"2023-09-30T09:37:21.751071+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:37:21.751071+0000","last_undegraded":"2023-09-30T09:37:21.751071+0000","last_fullsized":"2023-09-30T09:37:21.751071+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":276,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.765372+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:37:21.765372+0000","last_peered":"2023-09-30T09:37:21.765372+0000","last_clean":"2023-09-30T09:37:21.765372+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:37:21.765372+0000","last_undegraded":"2023-09-30T09:37:21.765372+0000","last_fullsized":"2023-09-30T09:37:21.765372+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":515,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.751256+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:37:21.751256+0000","last_peered":"2023-09-30T09:37:21.751256+0000","last_clean":"2023-09-30T09:37:21.751256+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:37:21.751256+0000","last_undegraded":"2023-09-30T09:37:21.751256+0000","last_fullsized":"2023-09-30T09:37:21.751256+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":71,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.756476+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:37:21.756476+0000","last_peered":"2023-09-30T09:37:21.756476+0000","last_clean":"2023-09-30T09:37:21.756476+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:37:21.756476+0000","last_undegraded":"2023-09-30T09:37:21.756476+0000","last_fullsized":"2023-09-30T09:37:21.756476+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":262,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.795259+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:37:21.795259+0000","last_peered":"2023-09-30T09:37:21.795259+0000","last_clean":"2023-09-30T09:37:21.795259+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:37:21.795259+0000","last_undegraded":"2023-09-30T09:37:21.795259+0000","last_fullsized":"2023-09-30T09:37:21.795259+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":510,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845924+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:37:30.845924+0000","last_peered":"2023-09-30T09:37:30.845924+0000","last_clean":"2023-09-30T09:37:30.845924+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:37:30.845924+0000","last_undegraded":"2023-09-30T09:37:30.845924+0000","last_fullsized":"2023-09-30T09:37:30.845924+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":333,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.791944+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:37:21.791944+0000","last_peered":"2023-09-30T09:37:21.791944+0000","last_clean":"2023-09-30T09:37:21.791944+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:37:21.791944+0000","last_undegraded":"2023-09-30T09:37:21.791944+0000","last_fullsized":"2023-09-30T09:37:21.791944+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":344,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.795681+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:37:21.795681+0000","last_peered":"2023-09-30T09:37:21.795681+0000","last_clean":"2023-09-30T09:37:21.795681+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:37:21.795681+0000","last_undegraded":"2023-09-30T09:37:21.795681+0000","last_fullsized":"2023-09-30T09:37:21.795681+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":628,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.844797+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:37:30.844797+0000","last_peered":"2023-09-30T09:37:30.844797+0000","last_clean":"2023-09-30T09:37:30.844797+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:37:30.844797+0000","last_undegraded":"2023-09-30T09:37:30.844797+0000","last_fullsized":"2023-09-30T09:37:30.844797+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":71,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.792192+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:37:21.792192+0000","last_peered":"2023-09-30T09:37:21.792192+0000","last_clean":"2023-09-30T09:37:21.792192+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:37:21.792192+0000","last_undegraded":"2023-09-30T09:37:21.792192+0000","last_fullsized":"2023-09-30T09:37:21.792192+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":360,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.755812+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:37:21.755812+0000","last_peered":"2023-09-30T09:37:21.755812+0000","last_clean":"2023-09-30T09:37:21.755812+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:37:21.755812+0000","last_undegraded":"2023-09-30T09:37:21.755812+0000","last_fullsized":"2023-09-30T09:37:21.755812+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":695,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750871+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:37:21.750871+0000","last_peered":"2023-09-30T09:37:21.750871+0000","last_clean":"2023-09-30T09:37:21.750871+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:37:21.750871+0000","last_undegraded":"2023-09-30T09:37:21.750871+0000","last_fullsized":"2023-09-30T09:37:21.750871+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":254,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.844668+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:37:30.844668+0000","last_peered":"2023-09-30T09:37:30.844668+0000","last_clean":"2023-09-30T09:37:30.844668+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:37:30.844668+0000","last_undegraded":"2023-09-30T09:37:30.844668+0000","last_fullsized":"2023-09-30T09:37:30.844668+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":38,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750568+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:37:21.750568+0000","last_peered":"2023-09-30T09:37:21.750568+0000","last_clean":"2023-09-30T09:37:21.750568+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:37:21.750568+0000","last_undegraded":"2023-09-30T09:37:21.750568+0000","last_fullsized":"2023-09-30T09:37:21.750568+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":788,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:22.287712+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:37:22.287712+0000","last_peered":"2023-09-30T09:37:22.287712+0000","last_clean":"2023-09-30T09:37:22.287712+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:37:22.287712+0000","last_undegraded":"2023-09-30T09:37:22.287712+0000","last_fullsized":"2023-09-30T09:37:22.287712+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763365+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:37:21.763365+0000","last_peered":"2023-09-30T09:37:21.763365+0000","last_clean":"2023-09-30T09:37:21.763365+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:37:21.763365+0000","last_undegraded":"2023-09-30T09:37:21.763365+0000","last_fullsized":"2023-09-30T09:37:21.763365+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":253,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845804+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:37:30.845804+0000","last_peered":"2023-09-30T09:37:30.845804+0000","last_clean":"2023-09-30T09:37:30.845804+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:37:30.845804+0000","last_undegraded":"2023-09-30T09:37:30.845804+0000","last_fullsized":"2023-09-30T09:37:30.845804+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":814,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.751631+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:37:21.751631+0000","last_peered":"2023-09-30T09:37:21.751631+0000","last_clean":"2023-09-30T09:37:21.751631+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:37:21.751631+0000","last_undegraded":"2023-09-30T09:37:21.751631+0000","last_fullsized":"2023-09-30T09:37:21.751631+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":508,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.794903+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:37:21.794903+0000","last_peered":"2023-09-30T09:37:21.794903+0000","last_clean":"2023-09-30T09:37:21.794903+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:37:21.794903+0000","last_undegraded":"2023-09-30T09:37:21.794903+0000","last_fullsized":"2023-09-30T09:37:21.794903+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":358,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.757487+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:37:21.757487+0000","last_peered":"2023-09-30T09:37:21.757487+0000","last_clean":"2023-09-30T09:37:21.757487+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:37:21.757487+0000","last_undegraded":"2023-09-30T09:37:21.757487+0000","last_fullsized":"2023-09-30T09:37:21.757487+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"349'13782","reported_seq":20815,"reported_epoch":349,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-30T09:37:34.104381+0000","last_change":"2023-09-30T09:37:33.542827+0000","last_active":"2023-09-30T09:37:34.104381+0000","last_peered":"2023-09-30T09:37:34.104381+0000","last_clean":"2023-09-30T09:32:35.457744+0000","last_became_active":"2023-09-30T09:34:48.499533+0000","last_became_peered":"2023-09-30T09:34:48.499533+0000","last_unstale":"2023-09-30T09:37:34.104381+0000","last_undegraded":"2023-09-30T09:34:48.469688+0000","last_fullsized":"2023-09-30T09:37:34.104381+0000","mapping_epoch":321,"log_start":"317'8921","ondisk_log_start":"317'8921","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4861,"log_dups_size":0,"ondisk_log_size":4861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:05:42.164341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7602176,"num_objects":116,"num_object_clones":0,"num_object_copies":348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":87,"num_objects_misplaced":87,"num_objects_unfound":0,"num_objects_dirty":116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13782,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3455,"num_bytes_recovered":226426880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[6,1,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"1,4,6","objects":29},{"shards":"4,6","objects":87}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":843,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:29.893973+0000","last_change":"2023-09-30T09:37:29.893943+0000","last_active":"2023-09-30T09:37:29.893973+0000","last_peered":"2023-09-30T09:37:29.893973+0000","last_clean":"2023-09-30T09:37:29.893973+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:37:29.893973+0000","last_undegraded":"2023-09-30T09:37:29.893973+0000","last_fullsized":"2023-09-30T09:37:29.893973+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":83,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.791783+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:37:21.791783+0000","last_peered":"2023-09-30T09:37:21.791783+0000","last_clean":"2023-09-30T09:37:21.791783+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:37:21.791783+0000","last_undegraded":"2023-09-30T09:37:21.791783+0000","last_fullsized":"2023-09-30T09:37:21.791783+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":38,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750276+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:37:21.750276+0000","last_peered":"2023-09-30T09:37:21.750276+0000","last_clean":"2023-09-30T09:37:21.750276+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:37:21.750276+0000","last_undegraded":"2023-09-30T09:37:21.750276+0000","last_fullsized":"2023-09-30T09:37:21.750276+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"349'14220","reported_seq":22311,"reported_epoch":349,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:32.215643+0000","last_change":"2023-09-30T09:35:10.766663+0000","last_active":"2023-09-30T09:37:32.215643+0000","last_peered":"2023-09-30T09:37:32.215643+0000","last_clean":"2023-09-30T09:32:35.454324+0000","last_became_active":"2023-09-30T09:33:33.826036+0000","last_became_peered":"2023-09-30T09:33:33.826036+0000","last_unstale":"2023-09-30T09:37:32.215643+0000","last_undegraded":"2023-09-30T09:33:33.732722+0000","last_fullsized":"2023-09-30T09:37:32.215643+0000","mapping_epoch":302,"log_start":"324'9699","ondisk_log_start":"324'9699","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4521,"log_dups_size":0,"ondisk_log_size":4521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:42.597216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11796480,"num_objects":180,"num_object_clones":0,"num_object_copies":540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":169,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14220,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3992,"num_bytes_recovered":261619712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,5,1],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":11},{"shards":"3,5","objects":169}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":799,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.755724+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:37:21.755724+0000","last_peered":"2023-09-30T09:37:21.755724+0000","last_clean":"2023-09-30T09:37:21.755724+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:37:21.755724+0000","last_undegraded":"2023-09-30T09:37:21.755724+0000","last_fullsized":"2023-09-30T09:37:21.755724+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":61,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763720+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:37:21.763720+0000","last_peered":"2023-09-30T09:37:21.763720+0000","last_clean":"2023-09-30T09:37:21.763720+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:37:21.763720+0000","last_undegraded":"2023-09-30T09:37:21.763720+0000","last_fullsized":"2023-09-30T09:37:21.763720+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":341,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763336+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:37:21.763336+0000","last_peered":"2023-09-30T09:37:21.763336+0000","last_clean":"2023-09-30T09:37:21.763336+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:37:21.763336+0000","last_undegraded":"2023-09-30T09:37:21.763336+0000","last_fullsized":"2023-09-30T09:37:21.763336+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"349'13854","reported_seq":20948,"reported_epoch":349,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:34.084000+0000","last_change":"2023-09-30T09:35:10.879711+0000","last_active":"2023-09-30T09:37:34.084000+0000","last_peered":"2023-09-30T09:37:34.084000+0000","last_clean":"2023-09-30T09:32:35.467048+0000","last_became_active":"2023-09-30T09:35:10.845361+0000","last_became_peered":"2023-09-30T09:35:10.845361+0000","last_unstale":"2023-09-30T09:37:34.084000+0000","last_undegraded":"2023-09-30T09:35:10.791338+0000","last_fullsized":"2023-09-30T09:37:34.084000+0000","mapping_epoch":327,"log_start":"319'9076","ondisk_log_start":"319'9076","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4778,"log_dups_size":0,"ondisk_log_size":4778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:41.089234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7995392,"num_objects":122,"num_object_clones":0,"num_object_copies":366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":122,"num_objects_misplaced":244,"num_objects_unfound":0,"num_objects_dirty":122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13854,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3486,"num_bytes_recovered":228327424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[6,1,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":122}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":632,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.795659+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:37:21.795659+0000","last_peered":"2023-09-30T09:37:21.795659+0000","last_clean":"2023-09-30T09:37:21.795659+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:37:21.795659+0000","last_undegraded":"2023-09-30T09:37:21.795659+0000","last_fullsized":"2023-09-30T09:37:21.795659+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":42,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750493+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:37:21.750493+0000","last_peered":"2023-09-30T09:37:21.750493+0000","last_clean":"2023-09-30T09:37:21.750493+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:37:21.750493+0000","last_undegraded":"2023-09-30T09:37:21.750493+0000","last_fullsized":"2023-09-30T09:37:21.750493+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":328,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.795194+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:37:21.795194+0000","last_peered":"2023-09-30T09:37:21.795194+0000","last_clean":"2023-09-30T09:37:21.795194+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:37:21.795194+0000","last_undegraded":"2023-09-30T09:37:21.795194+0000","last_fullsized":"2023-09-30T09:37:21.795194+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"349'14116","reported_seq":14207,"reported_epoch":349,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:32.287157+0000","last_change":"2023-09-30T09:34:08.527633+0000","last_active":"2023-09-30T09:37:32.287157+0000","last_peered":"2023-09-30T09:37:32.287157+0000","last_clean":"2023-09-30T09:33:09.318417+0000","last_became_active":"2023-09-30T09:34:08.510746+0000","last_became_peered":"2023-09-30T09:34:08.510746+0000","last_unstale":"2023-09-30T09:37:32.287157+0000","last_undegraded":"2023-09-30T09:37:32.287157+0000","last_fullsized":"2023-09-30T09:37:32.287157+0000","mapping_epoch":308,"log_start":"324'9576","ondisk_log_start":"324'9576","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4540,"log_dups_size":0,"ondisk_log_size":4540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:07.374969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":182,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14116,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":182}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":514,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:28.926431+0000","last_change":"2023-09-30T09:37:28.926398+0000","last_active":"2023-09-30T09:37:28.926431+0000","last_peered":"2023-09-30T09:37:28.926431+0000","last_clean":"2023-09-30T09:37:28.926431+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:37:28.926431+0000","last_undegraded":"2023-09-30T09:37:28.926431+0000","last_fullsized":"2023-09-30T09:37:28.926431+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":308,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750787+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:37:21.750787+0000","last_peered":"2023-09-30T09:37:21.750787+0000","last_clean":"2023-09-30T09:37:21.750787+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:37:21.750787+0000","last_undegraded":"2023-09-30T09:37:21.750787+0000","last_fullsized":"2023-09-30T09:37:21.750787+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":256,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845499+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:37:30.845499+0000","last_peered":"2023-09-30T09:37:30.845499+0000","last_clean":"2023-09-30T09:37:30.845499+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:37:30.845499+0000","last_undegraded":"2023-09-30T09:37:30.845499+0000","last_fullsized":"2023-09-30T09:37:30.845499+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"349'13894","reported_seq":13998,"reported_epoch":349,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:32.247434+0000","last_change":"2023-09-30T09:35:10.779076+0000","last_active":"2023-09-30T09:37:32.247434+0000","last_peered":"2023-09-30T09:37:32.247434+0000","last_clean":"2023-09-30T09:34:06.422096+0000","last_became_active":"2023-09-30T09:35:10.747716+0000","last_became_peered":"2023-09-30T09:35:10.747716+0000","last_unstale":"2023-09-30T09:37:32.247434+0000","last_undegraded":"2023-09-30T09:37:32.247434+0000","last_fullsized":"2023-09-30T09:37:32.247434+0000","mapping_epoch":327,"log_start":"322'9320","ondisk_log_start":"322'9320","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4574,"log_dups_size":0,"ondisk_log_size":4574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:09.191024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9961472,"num_objects":152,"num_object_clones":0,"num_object_copies":456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":152,"num_objects_unfound":0,"num_objects_dirty":152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13894,"num_write_kb":449472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":152}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":695,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.757769+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:37:21.757769+0000","last_peered":"2023-09-30T09:37:21.757769+0000","last_clean":"2023-09-30T09:37:21.757769+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:37:21.757769+0000","last_undegraded":"2023-09-30T09:37:21.757769+0000","last_fullsized":"2023-09-30T09:37:21.757769+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":377,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.764606+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:37:21.764606+0000","last_peered":"2023-09-30T09:37:21.764606+0000","last_clean":"2023-09-30T09:37:21.764606+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:37:21.764606+0000","last_undegraded":"2023-09-30T09:37:21.764606+0000","last_fullsized":"2023-09-30T09:37:21.764606+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":307,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.792030+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:37:21.792030+0000","last_peered":"2023-09-30T09:37:21.792030+0000","last_clean":"2023-09-30T09:37:21.792030+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:37:21.792030+0000","last_undegraded":"2023-09-30T09:37:21.792030+0000","last_fullsized":"2023-09-30T09:37:21.792030+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"349'14089","reported_seq":21179,"reported_epoch":349,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:32.295396+0000","last_change":"2023-09-30T09:35:10.871065+0000","last_active":"2023-09-30T09:37:32.295396+0000","last_peered":"2023-09-30T09:37:32.295396+0000","last_clean":"2023-09-30T09:32:35.449714+0000","last_became_active":"2023-09-30T09:35:10.838732+0000","last_became_peered":"2023-09-30T09:35:10.838732+0000","last_unstale":"2023-09-30T09:37:32.295396+0000","last_undegraded":"2023-09-30T09:35:10.818868+0000","last_fullsized":"2023-09-30T09:37:32.295396+0000","mapping_epoch":327,"log_start":"324'9606","ondisk_log_start":"324'9606","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4483,"log_dups_size":0,"ondisk_log_size":4483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:00:44.318381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10682368,"num_objects":163,"num_object_clones":0,"num_object_copies":489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":162,"num_objects_misplaced":327,"num_objects_unfound":0,"num_objects_dirty":163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14089,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3486,"num_bytes_recovered":228458496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[3,7,1],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"1,3,7","objects":1},{"shards":"3,7","objects":162}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":772,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.841262+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:37:30.841262+0000","last_peered":"2023-09-30T09:37:30.841262+0000","last_clean":"2023-09-30T09:37:30.841262+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:37:30.841262+0000","last_undegraded":"2023-09-30T09:37:30.841262+0000","last_fullsized":"2023-09-30T09:37:30.841262+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":326,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.795042+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:37:21.795042+0000","last_peered":"2023-09-30T09:37:21.795042+0000","last_clean":"2023-09-30T09:37:21.795042+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:37:21.795042+0000","last_undegraded":"2023-09-30T09:37:21.795042+0000","last_fullsized":"2023-09-30T09:37:21.795042+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":43,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763152+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:37:21.763152+0000","last_peered":"2023-09-30T09:37:21.763152+0000","last_clean":"2023-09-30T09:37:21.763152+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:37:21.763152+0000","last_undegraded":"2023-09-30T09:37:21.763152+0000","last_fullsized":"2023-09-30T09:37:21.763152+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"349'13921","reported_seq":21155,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:34.061468+0000","last_change":"2023-09-30T09:36:04.573419+0000","last_active":"2023-09-30T09:37:34.061468+0000","last_peered":"2023-09-30T09:37:34.061468+0000","last_clean":"2023-09-30T09:37:34.061468+0000","last_became_active":"2023-09-30T09:36:04.549763+0000","last_became_peered":"2023-09-30T09:36:04.549763+0000","last_unstale":"2023-09-30T09:37:34.061468+0000","last_undegraded":"2023-09-30T09:37:34.061468+0000","last_fullsized":"2023-09-30T09:37:34.061468+0000","mapping_epoch":337,"log_start":"319'9121","ondisk_log_start":"319'9121","created":279,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4800,"log_dups_size":0,"ondisk_log_size":4800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:27:18.185457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7995416,"num_objects":123,"num_object_clones":0,"num_object_copies":369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":123,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13921,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3562,"num_bytes_recovered":233439232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":786,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845287+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:37:30.845287+0000","last_peered":"2023-09-30T09:37:30.845287+0000","last_clean":"2023-09-30T09:37:30.845287+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:37:30.845287+0000","last_undegraded":"2023-09-30T09:37:30.845287+0000","last_fullsized":"2023-09-30T09:37:30.845287+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":345,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763263+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:37:21.763263+0000","last_peered":"2023-09-30T09:37:21.763263+0000","last_clean":"2023-09-30T09:37:21.763263+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:37:21.763263+0000","last_undegraded":"2023-09-30T09:37:21.763263+0000","last_fullsized":"2023-09-30T09:37:21.763263+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":324,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763967+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:37:21.763967+0000","last_peered":"2023-09-30T09:37:21.763967+0000","last_clean":"2023-09-30T09:37:21.763967+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:37:21.763967+0000","last_undegraded":"2023-09-30T09:37:21.763967+0000","last_fullsized":"2023-09-30T09:37:21.763967+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"349'14057","reported_seq":14151,"reported_epoch":349,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:34.082710+0000","last_change":"2023-09-30T09:34:47.630136+0000","last_active":"2023-09-30T09:37:34.082710+0000","last_peered":"2023-09-30T09:37:34.082710+0000","last_clean":"2023-09-30T09:34:06.422409+0000","last_became_active":"2023-09-30T09:34:08.626702+0000","last_became_peered":"2023-09-30T09:34:08.626702+0000","last_unstale":"2023-09-30T09:37:34.082710+0000","last_undegraded":"2023-09-30T09:37:34.082710+0000","last_fullsized":"2023-09-30T09:37:34.082710+0000","mapping_epoch":308,"log_start":"318'9206","ondisk_log_start":"318'9206","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4851,"log_dups_size":0,"ondisk_log_size":4851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:53.753813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7798784,"num_objects":119,"num_object_clones":0,"num_object_copies":357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":119,"num_objects_unfound":0,"num_objects_dirty":119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14057,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":119}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":500,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.841652+0000","last_change":"2023-09-30T09:37:30.390725+0000","last_active":"2023-09-30T09:37:30.841652+0000","last_peered":"2023-09-30T09:37:30.841652+0000","last_clean":"2023-09-30T09:37:30.841652+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:37:30.841652+0000","last_undegraded":"2023-09-30T09:37:30.841652+0000","last_fullsized":"2023-09-30T09:37:30.841652+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":340,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.759448+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:37:21.759448+0000","last_peered":"2023-09-30T09:37:21.759448+0000","last_clean":"2023-09-30T09:37:21.759448+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:37:21.759448+0000","last_undegraded":"2023-09-30T09:37:21.759448+0000","last_fullsized":"2023-09-30T09:37:21.759448+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":117,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.757311+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:37:21.757311+0000","last_peered":"2023-09-30T09:37:21.757311+0000","last_clean":"2023-09-30T09:37:21.757311+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:37:21.757311+0000","last_undegraded":"2023-09-30T09:37:21.757311+0000","last_fullsized":"2023-09-30T09:37:21.757311+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"349'13836","reported_seq":20578,"reported_epoch":349,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:32.291773+0000","last_change":"2023-09-30T09:34:39.166112+0000","last_active":"2023-09-30T09:37:32.291773+0000","last_peered":"2023-09-30T09:37:32.291773+0000","last_clean":"2023-09-30T09:32:35.465008+0000","last_became_active":"2023-09-30T09:34:39.150257+0000","last_became_peered":"2023-09-30T09:34:39.150257+0000","last_unstale":"2023-09-30T09:37:32.291773+0000","last_undegraded":"2023-09-30T09:34:38.934740+0000","last_fullsized":"2023-09-30T09:37:32.291773+0000","mapping_epoch":316,"log_start":"319'9278","ondisk_log_start":"319'9278","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4558,"log_dups_size":0,"ondisk_log_size":4558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:55:50.805799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9830400,"num_objects":150,"num_object_clones":0,"num_object_copies":450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":149,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13836,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3320,"num_bytes_recovered":217513984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6,1],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1},{"shards":"3,6","objects":149}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":643,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.796538+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:37:21.796538+0000","last_peered":"2023-09-30T09:37:21.796538+0000","last_clean":"2023-09-30T09:37:21.796538+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:37:21.796538+0000","last_undegraded":"2023-09-30T09:37:21.796538+0000","last_fullsized":"2023-09-30T09:37:21.796538+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":542,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.846290+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:37:30.846290+0000","last_peered":"2023-09-30T09:37:30.846290+0000","last_clean":"2023-09-30T09:37:30.846290+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:37:30.846290+0000","last_undegraded":"2023-09-30T09:37:30.846290+0000","last_fullsized":"2023-09-30T09:37:30.846290+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":345,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750798+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:37:21.750798+0000","last_peered":"2023-09-30T09:37:21.750798+0000","last_clean":"2023-09-30T09:37:21.750798+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:37:21.750798+0000","last_undegraded":"2023-09-30T09:37:21.750798+0000","last_fullsized":"2023-09-30T09:37:21.750798+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":38,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.751464+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:37:21.751464+0000","last_peered":"2023-09-30T09:37:21.751464+0000","last_clean":"2023-09-30T09:37:21.751464+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:37:21.751464+0000","last_undegraded":"2023-09-30T09:37:21.751464+0000","last_fullsized":"2023-09-30T09:37:21.751464+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"349'14142","reported_seq":14220,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:34.082606+0000","last_change":"2023-09-30T09:31:55.924897+0000","last_active":"2023-09-30T09:37:34.082606+0000","last_peered":"2023-09-30T09:37:34.082606+0000","last_clean":"2023-09-30T09:37:34.082606+0000","last_became_active":"2023-09-30T09:31:55.924632+0000","last_became_peered":"2023-09-30T09:31:55.924632+0000","last_unstale":"2023-09-30T09:37:34.082606+0000","last_undegraded":"2023-09-30T09:37:34.082606+0000","last_fullsized":"2023-09-30T09:37:34.082606+0000","mapping_epoch":279,"log_start":"319'9357","ondisk_log_start":"319'9357","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4785,"log_dups_size":0,"ondisk_log_size":4785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:21.044695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7471104,"num_objects":114,"num_object_clones":0,"num_object_copies":342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14142,"num_write_kb":456192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":811,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.751350+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:37:21.751350+0000","last_peered":"2023-09-30T09:37:21.751350+0000","last_clean":"2023-09-30T09:37:21.751350+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:37:21.751350+0000","last_undegraded":"2023-09-30T09:37:21.751350+0000","last_fullsized":"2023-09-30T09:37:21.751350+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":340,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.795120+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:37:21.795120+0000","last_peered":"2023-09-30T09:37:21.795120+0000","last_clean":"2023-09-30T09:37:21.795120+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:37:21.795120+0000","last_undegraded":"2023-09-30T09:37:21.795120+0000","last_fullsized":"2023-09-30T09:37:21.795120+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":328,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.792840+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:37:21.792840+0000","last_peered":"2023-09-30T09:37:21.792840+0000","last_clean":"2023-09-30T09:37:21.792840+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:37:21.792840+0000","last_undegraded":"2023-09-30T09:37:21.792840+0000","last_fullsized":"2023-09-30T09:37:21.792840+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":68,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.842253+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:37:30.842253+0000","last_peered":"2023-09-30T09:37:30.842253+0000","last_clean":"2023-09-30T09:37:30.842253+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:37:30.842253+0000","last_undegraded":"2023-09-30T09:37:30.842253+0000","last_fullsized":"2023-09-30T09:37:30.842253+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"349'14230","reported_seq":21536,"reported_epoch":349,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:34.103401+0000","last_change":"2023-09-30T09:35:10.808625+0000","last_active":"2023-09-30T09:37:34.103401+0000","last_peered":"2023-09-30T09:37:34.103401+0000","last_clean":"2023-09-30T09:32:35.460821+0000","last_became_active":"2023-09-30T09:35:10.778683+0000","last_became_peered":"2023-09-30T09:35:10.778683+0000","last_unstale":"2023-09-30T09:37:34.103401+0000","last_undegraded":"2023-09-30T09:35:10.766189+0000","last_fullsized":"2023-09-30T09:37:34.103401+0000","mapping_epoch":327,"log_start":"323'9433","ondisk_log_start":"323'9433","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4797,"log_dups_size":0,"ondisk_log_size":4797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:45.905717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7995392,"num_objects":122,"num_object_clones":0,"num_object_copies":366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":121,"num_objects_misplaced":244,"num_objects_unfound":0,"num_objects_dirty":122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14230,"num_write_kb":459264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3595,"num_bytes_recovered":235601920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[6,1,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"1,5,6","objects":1},{"shards":"5,6","objects":121}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":710,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845116+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:37:30.845116+0000","last_peered":"2023-09-30T09:37:30.845116+0000","last_clean":"2023-09-30T09:37:30.845116+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:37:30.845116+0000","last_undegraded":"2023-09-30T09:37:30.845116+0000","last_fullsized":"2023-09-30T09:37:30.845116+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":529,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.840789+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:37:30.840789+0000","last_peered":"2023-09-30T09:37:30.840789+0000","last_clean":"2023-09-30T09:37:30.840789+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:37:30.840789+0000","last_undegraded":"2023-09-30T09:37:30.840789+0000","last_fullsized":"2023-09-30T09:37:30.840789+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":76,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.841170+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:37:30.841170+0000","last_peered":"2023-09-30T09:37:30.841170+0000","last_clean":"2023-09-30T09:37:30.841170+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:37:30.841170+0000","last_undegraded":"2023-09-30T09:37:30.841170+0000","last_fullsized":"2023-09-30T09:37:30.841170+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":268,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.751384+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:37:21.751384+0000","last_peered":"2023-09-30T09:37:21.751384+0000","last_clean":"2023-09-30T09:37:21.751384+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:37:21.751384+0000","last_undegraded":"2023-09-30T09:37:21.751384+0000","last_fullsized":"2023-09-30T09:37:21.751384+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"350'13946","reported_seq":28081,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-30T09:37:35.207520+0000","last_change":"2023-09-30T09:37:06.584279+0000","last_active":"2023-09-30T09:37:35.207520+0000","last_peered":"2023-09-30T09:37:35.207520+0000","last_clean":"2023-09-30T09:37:35.207520+0000","last_became_active":"2023-09-30T09:37:06.541995+0000","last_became_peered":"2023-09-30T09:37:06.541995+0000","last_unstale":"2023-09-30T09:37:35.207520+0000","last_undegraded":"2023-09-30T09:37:35.207520+0000","last_fullsized":"2023-09-30T09:37:35.207520+0000","mapping_epoch":343,"log_start":"305'7650","ondisk_log_start":"305'7650","created":279,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:09:29.630262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5242880,"num_objects":80,"num_object_clones":0,"num_object_copies":240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13946,"num_write_kb":448832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7014,"num_bytes_recovered":304349184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":921,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.751476+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:37:21.751476+0000","last_peered":"2023-09-30T09:37:21.751476+0000","last_clean":"2023-09-30T09:37:21.751476+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:37:21.751476+0000","last_undegraded":"2023-09-30T09:37:21.751476+0000","last_fullsized":"2023-09-30T09:37:21.751476+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":615,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.835000+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:37:30.835000+0000","last_peered":"2023-09-30T09:37:30.835000+0000","last_clean":"2023-09-30T09:37:30.835000+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:37:30.835000+0000","last_undegraded":"2023-09-30T09:37:30.835000+0000","last_fullsized":"2023-09-30T09:37:30.835000+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":268,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.757024+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:37:21.757024+0000","last_peered":"2023-09-30T09:37:21.757024+0000","last_clean":"2023-09-30T09:37:21.757024+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:37:21.757024+0000","last_undegraded":"2023-09-30T09:37:21.757024+0000","last_fullsized":"2023-09-30T09:37:21.757024+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":301,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.803624+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:37:21.803624+0000","last_peered":"2023-09-30T09:37:21.803624+0000","last_clean":"2023-09-30T09:37:21.803624+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:37:21.803624+0000","last_undegraded":"2023-09-30T09:37:21.803624+0000","last_fullsized":"2023-09-30T09:37:21.803624+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"349'14072","reported_seq":27362,"reported_epoch":349,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:37:34.440066+0000","last_change":"2023-09-30T09:35:10.837720+0000","last_active":"2023-09-30T09:37:34.440066+0000","last_peered":"2023-09-30T09:37:34.440066+0000","last_clean":"2023-09-30T09:32:35.481650+0000","last_became_active":"2023-09-30T09:35:10.788829+0000","last_became_peered":"2023-09-30T09:35:10.788829+0000","last_unstale":"2023-09-30T09:37:34.440066+0000","last_undegraded":"2023-09-30T09:35:10.757584+0000","last_fullsized":"2023-09-30T09:37:34.440066+0000","mapping_epoch":327,"log_start":"289'4000","ondisk_log_start":"289'4000","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":10072,"log_dups_size":0,"ondisk_log_size":10072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:05.384343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5636096,"num_objects":86,"num_object_clones":0,"num_object_copies":258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":45,"num_objects_misplaced":170,"num_objects_unfound":0,"num_objects_dirty":86,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14072,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6580,"num_bytes_recovered":431226880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[7,1,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":2},{"shards":"2,6,7","objects":41},{"shards":"6,7","objects":43}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":494,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.797820+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:37:21.797820+0000","last_peered":"2023-09-30T09:37:21.797820+0000","last_clean":"2023-09-30T09:37:21.797820+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:37:21.797820+0000","last_undegraded":"2023-09-30T09:37:21.797820+0000","last_fullsized":"2023-09-30T09:37:21.797820+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":726,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.835047+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:37:30.835047+0000","last_peered":"2023-09-30T09:37:30.835047+0000","last_clean":"2023-09-30T09:37:30.835047+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:37:30.835047+0000","last_undegraded":"2023-09-30T09:37:30.835047+0000","last_fullsized":"2023-09-30T09:37:30.835047+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":278,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.844717+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:37:30.844717+0000","last_peered":"2023-09-30T09:37:30.844717+0000","last_clean":"2023-09-30T09:37:30.844717+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:37:30.844717+0000","last_undegraded":"2023-09-30T09:37:30.844717+0000","last_fullsized":"2023-09-30T09:37:30.844717+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":35,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845370+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:37:30.845370+0000","last_peered":"2023-09-30T09:37:30.845370+0000","last_clean":"2023-09-30T09:37:30.845370+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:37:30.845370+0000","last_undegraded":"2023-09-30T09:37:30.845370+0000","last_fullsized":"2023-09-30T09:37:30.845370+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"349'14066","reported_seq":21404,"reported_epoch":349,"state":"active+undersized+remapped","last_fresh":"2023-09-30T09:37:34.308839+0000","last_change":"2023-09-30T09:37:33.541688+0000","last_active":"2023-09-30T09:37:34.308839+0000","last_peered":"2023-09-30T09:37:34.308839+0000","last_clean":"2023-09-30T09:32:35.469075+0000","last_became_active":"2023-09-30T09:33:33.792569+0000","last_became_peered":"2023-09-30T09:33:33.792569+0000","last_unstale":"2023-09-30T09:37:34.308839+0000","last_undegraded":"2023-09-30T09:37:34.308839+0000","last_fullsized":"2023-09-30T09:33:33.587755+0000","mapping_epoch":302,"log_start":"312'8545","ondisk_log_start":"312'8545","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:16.121133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5898240,"num_objects":90,"num_object_clones":0,"num_object_copies":270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14066,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3618,"num_bytes_recovered":189333504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"3,5","objects":90}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":623,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.797604+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:37:21.797604+0000","last_peered":"2023-09-30T09:37:21.797604+0000","last_clean":"2023-09-30T09:37:21.797604+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:37:21.797604+0000","last_undegraded":"2023-09-30T09:37:21.797604+0000","last_fullsized":"2023-09-30T09:37:21.797604+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":858,"reported_epoch":348,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:37:21.746824+0000","last_change":"2023-09-30T09:35:11.685841+0000","last_active":"2023-09-30T09:37:21.746824+0000","last_peered":"2023-09-30T09:37:21.746824+0000","last_clean":"2023-09-30T09:26:47.308941+0000","last_became_active":"2023-09-30T09:35:11.685274+0000","last_became_peered":"2023-09-30T09:35:11.685274+0000","last_unstale":"2023-09-30T09:37:21.746824+0000","last_undegraded":"2023-09-30T09:35:11.661789+0000","last_fullsized":"2023-09-30T09:35:11.661213+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:38.516737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[1,4],"avail_no_missing":["1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":76,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.842061+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:37:30.842061+0000","last_peered":"2023-09-30T09:37:30.842061+0000","last_clean":"2023-09-30T09:37:30.842061+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:37:30.842061+0000","last_undegraded":"2023-09-30T09:37:30.842061+0000","last_fullsized":"2023-09-30T09:37:30.842061+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":250,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845679+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:37:30.845679+0000","last_peered":"2023-09-30T09:37:30.845679+0000","last_clean":"2023-09-30T09:37:30.845679+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:37:30.845679+0000","last_undegraded":"2023-09-30T09:37:30.845679+0000","last_fullsized":"2023-09-30T09:37:30.845679+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"349'14085","reported_seq":14265,"reported_epoch":349,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:32.260226+0000","last_change":"2023-09-30T09:33:32.864041+0000","last_active":"2023-09-30T09:37:32.260226+0000","last_peered":"2023-09-30T09:37:32.260226+0000","last_clean":"2023-09-30T09:33:09.329441+0000","last_became_active":"2023-09-30T09:33:11.506402+0000","last_became_peered":"2023-09-30T09:33:11.506402+0000","last_unstale":"2023-09-30T09:37:32.260226+0000","last_undegraded":"2023-09-30T09:37:32.260226+0000","last_fullsized":"2023-09-30T09:37:32.260226+0000","mapping_epoch":295,"log_start":"324'9598","ondisk_log_start":"324'9598","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:51:06.485777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10420224,"num_objects":159,"num_object_clones":0,"num_object_copies":477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":159,"num_objects_unfound":0,"num_objects_dirty":159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14085,"num_write_kb":455808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":159}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":821,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763362+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:37:21.763362+0000","last_peered":"2023-09-30T09:37:21.763362+0000","last_clean":"2023-09-30T09:37:21.763362+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:37:21.763362+0000","last_undegraded":"2023-09-30T09:37:21.763362+0000","last_fullsized":"2023-09-30T09:37:21.763362+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":350,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.757401+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:37:21.757401+0000","last_peered":"2023-09-30T09:37:21.757401+0000","last_clean":"2023-09-30T09:37:21.757401+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:37:21.757401+0000","last_undegraded":"2023-09-30T09:37:21.757401+0000","last_fullsized":"2023-09-30T09:37:21.757401+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1115,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.797417+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:37:21.797417+0000","last_peered":"2023-09-30T09:37:21.797417+0000","last_clean":"2023-09-30T09:37:21.797417+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:37:21.797417+0000","last_undegraded":"2023-09-30T09:37:21.797417+0000","last_fullsized":"2023-09-30T09:37:21.797417+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":367,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.836067+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:37:30.836067+0000","last_peered":"2023-09-30T09:37:30.836067+0000","last_clean":"2023-09-30T09:37:30.836067+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:37:30.836067+0000","last_undegraded":"2023-09-30T09:37:30.836067+0000","last_fullsized":"2023-09-30T09:37:30.836067+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"349'14106","reported_seq":21420,"reported_epoch":349,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:37:32.175952+0000","last_change":"2023-09-30T09:37:15.721158+0000","last_active":"2023-09-30T09:37:32.175952+0000","last_peered":"2023-09-30T09:37:32.175952+0000","last_clean":"2023-09-30T09:32:35.469896+0000","last_became_active":"2023-09-30T09:37:15.688891+0000","last_became_peered":"2023-09-30T09:37:15.688891+0000","last_unstale":"2023-09-30T09:37:32.175952+0000","last_undegraded":"2023-09-30T09:37:32.175952+0000","last_fullsized":"2023-09-30T09:37:32.175952+0000","mapping_epoch":346,"log_start":"324'9550","ondisk_log_start":"324'9550","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:31:54.899919+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.899919+0000","objects_scrubbed":0,"log_size":4556,"log_dups_size":0,"ondisk_log_size":4556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:53:44.682029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11665408,"num_objects":178,"num_object_clones":0,"num_object_copies":534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":178,"num_objects_unfound":0,"num_objects_dirty":178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14106,"num_write_kb":457088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3599,"num_bytes_recovered":235864064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":178}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":815,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.756099+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:37:21.756099+0000","last_peered":"2023-09-30T09:37:21.756099+0000","last_clean":"2023-09-30T09:37:21.756099+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:37:21.756099+0000","last_undegraded":"2023-09-30T09:37:21.756099+0000","last_fullsized":"2023-09-30T09:37:21.756099+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":674,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.796675+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:37:21.796675+0000","last_peered":"2023-09-30T09:37:21.796675+0000","last_clean":"2023-09-30T09:37:21.796675+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:37:21.796675+0000","last_undegraded":"2023-09-30T09:37:21.796675+0000","last_fullsized":"2023-09-30T09:37:21.796675+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":38,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845468+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:37:30.845468+0000","last_peered":"2023-09-30T09:37:30.845468+0000","last_clean":"2023-09-30T09:37:30.845468+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:37:30.845468+0000","last_undegraded":"2023-09-30T09:37:30.845468+0000","last_fullsized":"2023-09-30T09:37:30.845468+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":290,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763285+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:37:21.763285+0000","last_peered":"2023-09-30T09:37:21.763285+0000","last_clean":"2023-09-30T09:37:21.763285+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:37:21.763285+0000","last_undegraded":"2023-09-30T09:37:21.763285+0000","last_fullsized":"2023-09-30T09:37:21.763285+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":763,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.757686+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:37:21.757686+0000","last_peered":"2023-09-30T09:37:21.757686+0000","last_clean":"2023-09-30T09:37:21.757686+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:37:21.757686+0000","last_undegraded":"2023-09-30T09:37:21.757686+0000","last_fullsized":"2023-09-30T09:37:21.757686+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":33,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.845591+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:37:30.845591+0000","last_peered":"2023-09-30T09:37:30.845591+0000","last_clean":"2023-09-30T09:37:30.845591+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:37:30.845591+0000","last_undegraded":"2023-09-30T09:37:30.845591+0000","last_fullsized":"2023-09-30T09:37:30.845591+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":27,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.841974+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:37:30.841974+0000","last_peered":"2023-09-30T09:37:30.841974+0000","last_clean":"2023-09-30T09:37:30.841974+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:37:30.841974+0000","last_undegraded":"2023-09-30T09:37:30.841974+0000","last_fullsized":"2023-09-30T09:37:30.841974+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":534,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.755817+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:37:21.755817+0000","last_peered":"2023-09-30T09:37:21.755817+0000","last_clean":"2023-09-30T09:37:21.755817+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:37:21.755817+0000","last_undegraded":"2023-09-30T09:37:21.755817+0000","last_fullsized":"2023-09-30T09:37:21.755817+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":327,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750703+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:37:21.750703+0000","last_peered":"2023-09-30T09:37:21.750703+0000","last_clean":"2023-09-30T09:37:21.750703+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:37:21.750703+0000","last_undegraded":"2023-09-30T09:37:21.750703+0000","last_fullsized":"2023-09-30T09:37:21.750703+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":43,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.791739+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:37:21.791739+0000","last_peered":"2023-09-30T09:37:21.791739+0000","last_clean":"2023-09-30T09:37:21.791739+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:37:21.791739+0000","last_undegraded":"2023-09-30T09:37:21.791739+0000","last_fullsized":"2023-09-30T09:37:21.791739+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":293,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.750726+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:37:21.750726+0000","last_peered":"2023-09-30T09:37:21.750726+0000","last_clean":"2023-09-30T09:37:21.750726+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:37:21.750726+0000","last_undegraded":"2023-09-30T09:37:21.750726+0000","last_fullsized":"2023-09-30T09:37:21.750726+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":43,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.763016+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:37:21.763016+0000","last_peered":"2023-09-30T09:37:21.763016+0000","last_clean":"2023-09-30T09:37:21.763016+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:37:21.763016+0000","last_undegraded":"2023-09-30T09:37:21.763016+0000","last_fullsized":"2023-09-30T09:37:21.763016+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":477,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.795739+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:37:21.795739+0000","last_peered":"2023-09-30T09:37:21.795739+0000","last_clean":"2023-09-30T09:37:21.795739+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:37:21.795739+0000","last_undegraded":"2023-09-30T09:37:21.795739+0000","last_fullsized":"2023-09-30T09:37:21.795739+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":346,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.837097+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:37:30.837097+0000","last_peered":"2023-09-30T09:37:30.837097+0000","last_clean":"2023-09-30T09:37:30.837097+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:37:30.837097+0000","last_undegraded":"2023-09-30T09:37:30.837097+0000","last_fullsized":"2023-09-30T09:37:30.837097+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":370,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.765293+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:37:21.765293+0000","last_peered":"2023-09-30T09:37:21.765293+0000","last_clean":"2023-09-30T09:37:21.765293+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:37:21.765293+0000","last_undegraded":"2023-09-30T09:37:21.765293+0000","last_fullsized":"2023-09-30T09:37:21.765293+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":797,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.756275+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:37:21.756275+0000","last_peered":"2023-09-30T09:37:21.756275+0000","last_clean":"2023-09-30T09:37:21.756275+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:37:21.756275+0000","last_undegraded":"2023-09-30T09:37:21.756275+0000","last_fullsized":"2023-09-30T09:37:21.756275+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":54,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-09-30T09:37:30.846073+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:37:30.846073+0000","last_peered":"2023-09-30T09:37:30.846073+0000","last_clean":"2023-09-30T09:37:30.846073+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:37:30.846073+0000","last_undegraded":"2023-09-30T09:37:30.846073+0000","last_fullsized":"2023-09-30T09:37:30.846073+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":324,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-09-30T09:37:21.791895+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:37:21.791895+0000","last_peered":"2023-09-30T09:37:21.791895+0000","last_clean":"2023-09-30T09:37:21.791895+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:37:21.791895+0000","last_undegraded":"2023-09-30T09:37:21.791895+0000","last_fullsized":"2023-09-30T09:37:21.791895+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":139919384,"num_objects":2136,"num_object_clones":0,"num_object_copies":6408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":855,"num_objects_misplaced":1863,"num_objects_unfound":0,"num_objects_dirty":2136,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":224416,"num_write_kb":7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45751,"num_bytes_recovered":2794979328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":727592960,"data_stored":727580744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82480,"ondisk_log_size":82480,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234311,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495948,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":471012,"kb_used_data":74332,"kb_used_omap":0,"kb_used_meta":396608,"kb_avail":93274140,"statfs":{"total":95995035648,"available":95512719360,"internally_reserved":0,"allocated":76115968,"data_stored":74899405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":406126592},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9380000000000002,"5min":2.2330000000000001,"15min":1.802},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":44.113,"5min":44.113,"15min":45.927999999999997},"last":3.2400000000000002},{"interface":"front","average":{"1min":4.6150000000000002,"5min":2.3639999999999999,"15min":2.0449999999999999},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":44.164000000000001,"5min":44.164000000000001,"15min":45.316000000000003},"last":2.8479999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.0419999999999998,"15min":1.8819999999999999},"min":{"1min":0.39100000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.343999999999999,"5min":13.343999999999999,"15min":13.343999999999999},"last":3.0299999999999998},{"interface":"front","average":{"1min":3.169,"5min":2.3300000000000001,"15min":2.1520000000000001},"min":{"1min":0.35999999999999999,"5min":0.313,"15min":0.313},"max":{"1min":18.425999999999998,"5min":18.425999999999998,"15min":18.425999999999998},"last":3.0470000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:37:31 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":2.2040000000000002,"15min":1.9910000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.1200000000000001,"5min":17.126000000000001,"15min":37.978000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.272,"5min":2.3399999999999999,"15min":1.8959999999999999},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2720000000000002,"5min":25.164999999999999,"15min":37.759999999999998},"last":2.3069999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7839999999999998,"5min":7.1130000000000004,"15min":3.532},"min":{"1min":0.45300000000000001,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":90.147999999999996,"5min":212.44800000000001,"15min":212.44800000000001},"last":3.4689999999999999},{"interface":"front","average":{"1min":8.9450000000000003,"5min":7.8339999999999996,"15min":3.9199999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":89.859999999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":3.3969999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.0640000000000001,"15min":1.593},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23400000000000001},"max":{"1min":3.004,"5min":7.4359999999999999,"15min":73.656000000000006},"last":2.5699999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.0209999999999999,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.23000000000000001},"max":{"1min":1.8089999999999999,"5min":9.6590000000000007,"15min":73.882999999999996},"last":0.35099999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.1930000000000001,"15min":1.819},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":1.024,"5min":8.2040000000000006,"15min":76.613},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.1910000000000001,"15min":1.786},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.0979999999999999,"5min":9.7750000000000004,"15min":76.329999999999998},"last":3.8799999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:37:31 2023","interfaces":[{"interface":"back","average":{"1min":8.1950000000000003,"5min":2.7679999999999998,"15min":2.6499999999999999},"min":{"1min":0.371,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":138.191,"5min":138.191,"15min":138.191},"last":2.649},{"interface":"front","average":{"1min":8.0210000000000008,"5min":2.5619999999999998,"15min":2.5830000000000002},"min":{"1min":0.35099999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":140.33500000000001,"5min":140.33500000000001,"15min":140.33500000000001},"last":0.38400000000000001}]}]},{"osd":1,"up_from":301,"seq":1292785156145,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":340140,"kb_used_data":52452,"kb_used_omap":9,"kb_used_meta":287670,"kb_avail":93405012,"statfs":{"total":95995035648,"available":95646732288,"internally_reserved":0,"allocated":53710848,"data_stored":51729955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":294574098},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":9.7590000000000003,"5min":4.0979999999999999,"15min":3.0600000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":138.43799999999999,"5min":138.43799999999999,"15min":138.43799999999999},"last":3.5659999999999998},{"interface":"front","average":{"1min":9.4320000000000004,"5min":4.5140000000000002,"15min":3.52},"min":{"1min":0.27800000000000002,"5min":0.25,"15min":0.25},"max":{"1min":138.482,"5min":138.482,"15min":138.482},"last":3.2490000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.8100000000000001,"15min":2.7999999999999998},"min":{"1min":0.254,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":13.656000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":2.927},{"interface":"front","average":{"1min":3.9609999999999999,"5min":3.0899999999999999,"15min":2.8340000000000001},"min":{"1min":0.309,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":18.978999999999999,"5min":18.978999999999999,"15min":18.978999999999999},"last":0.21299999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":6.8879999999999999,"5min":7.9370000000000003,"15min":8.9429999999999996},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":40.606000000000002,"5min":77.959000000000003,"15min":77.959000000000003},"last":3.7890000000000001},{"interface":"front","average":{"1min":7.5449999999999999,"5min":8.2690000000000001,"15min":9.5410000000000004},"min":{"1min":0.439,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":41.247,"5min":77.828999999999994,"15min":77.828999999999994},"last":12.608000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.1059999999999999,"15min":2.0259999999999998},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.0090000000000003,"5min":13.602,"15min":13.602},"last":3.3570000000000002},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.712,"15min":1.4319999999999999},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.311},"max":{"1min":8.6440000000000001,"5min":10.773,"15min":10.773},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":2.351,"15min":2.1890000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.2750000000000004,"5min":13.666,"15min":13.666},"last":3.3159999999999998},{"interface":"front","average":{"1min":2.9239999999999999,"5min":1.9890000000000001,"15min":1.73},"min":{"1min":0.39100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.177,"5min":10.847,"15min":10.847},"last":3.7490000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":11.52,"15min":15.238},"min":{"1min":0.503,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.1029999999999998,"5min":209.221,"15min":209.221},"last":3.387},{"interface":"front","average":{"1min":2.48,"5min":11.207000000000001,"15min":15.079000000000001},"min":{"1min":0.40000000000000002,"5min":0.377,"15min":0.377},"max":{"1min":8.1310000000000002,"5min":208.43600000000001,"15min":208.43600000000001},"last":3.2839999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9950000000000001,"5min":8.3640000000000008,"15min":11.084},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.3559999999999999,"5min":165.59399999999999,"15min":165.59399999999999},"last":2.02},{"interface":"front","average":{"1min":3.3719999999999999,"5min":8.5670000000000002,"15min":11.308},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":10.565,"5min":165.833,"15min":165.833},"last":2.056}]}]},{"osd":0,"up_from":94,"seq":403726926045,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":392020,"kb_used_data":29132,"kb_used_omap":5,"kb_used_meta":362874,"kb_avail":93353132,"statfs":{"total":95995035648,"available":95593607168,"internally_reserved":0,"allocated":29831168,"data_stored":27705283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":371583229},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:36:38 2023","interfaces":[{"interface":"back","average":{"1min":4.1920000000000002,"5min":3.0630000000000002,"15min":3.0190000000000001},"min":{"1min":0.24099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":15.401,"5min":17.966000000000001,"15min":17.966000000000001},"last":4.4569999999999999},{"interface":"front","average":{"1min":6.3339999999999996,"5min":7.7549999999999999,"15min":8.8680000000000003},"min":{"1min":0.31,"5min":0.22,"15min":0.22},"max":{"1min":25.869,"5min":138.92400000000001,"15min":138.92400000000001},"last":4.0700000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:37:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9750000000000001,"5min":2.5209999999999999,"15min":1.9179999999999999},"min":{"1min":0.26600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":15.324,"5min":15.324,"15min":55.823999999999998},"last":4.3609999999999998},{"interface":"front","average":{"1min":2.8799999999999999,"5min":2.976,"15min":2.0950000000000002},"min":{"1min":0.36199999999999999,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":14.848000000000001,"5min":40.128999999999998,"15min":55.790999999999997},"last":4.1189999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":8.5350000000000001,"15min":4.2089999999999996},"min":{"1min":0.30599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":16.271000000000001,"5min":218.06999999999999,"15min":218.06999999999999},"last":3.3170000000000002},{"interface":"front","average":{"1min":2.8199999999999998,"5min":6.2930000000000001,"15min":3.4159999999999999},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":14.987,"5min":218.63200000000001,"15min":218.63200000000001},"last":3.9750000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.6599999999999999,"15min":1.679},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.29099999999999998},"max":{"1min":10.614000000000001,"5min":10.614000000000001,"15min":121.486},"last":5.5060000000000002},{"interface":"front","average":{"1min":3.3780000000000001,"5min":3.863,"15min":2.649},"min":{"1min":0.39900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":17.044,"5min":136.262,"15min":136.262},"last":3.3839999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":4.1710000000000003,"15min":3.0630000000000002},"min":{"1min":0.439,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.995999999999999,"5min":136.34,"15min":136.34},"last":3.3540000000000001},{"interface":"front","average":{"1min":2.4129999999999998,"5min":1.9419999999999999,"15min":2.2839999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":11.991,"5min":12.717000000000001,"15min":135.381},"last":4.016}]},{"osd":6,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":7.1440000000000001,"15min":3.2400000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":12.021000000000001,"5min":214.785,"15min":214.785},"last":5.835},{"interface":"front","average":{"1min":2.8999999999999999,"5min":7.2439999999999998,"15min":3.1850000000000001},"min":{"1min":0.46300000000000002,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":10.695,"5min":214.93899999999999,"15min":214.93899999999999},"last":7.8319999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":2.9870000000000001,"15min":3.1720000000000002},"min":{"1min":0.42999999999999999,"5min":0.38300000000000001,"15min":0.373},"max":{"1min":12.048,"5min":97.382000000000005,"15min":150.96899999999999},"last":7.7889999999999997},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.944,"15min":3.0150000000000001},"min":{"1min":0.43099999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.153,"5min":97.015000000000001,"15min":150.328},"last":10.135999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598986,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":340372,"kb_used_data":7752,"kb_used_omap":5,"kb_used_meta":332602,"kb_avail":93404780,"statfs":{"total":95995035648,"available":95646494720,"internally_reserved":0,"allocated":7938048,"data_stored":6000387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":340585101},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.8420000000000001,"15min":2.242},"min":{"1min":0.20499999999999999,"5min":0.187,"15min":0.187},"max":{"1min":6.9480000000000004,"5min":77.379000000000005,"15min":77.379000000000005},"last":0.23300000000000001},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.8010000000000002,"15min":2.0750000000000002},"min":{"1min":0.29499999999999998,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":10.590999999999999,"5min":76.796999999999997,"15min":77.066999999999993},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:36:36 2023","interfaces":[{"interface":"back","average":{"1min":8.9789999999999992,"5min":4.8339999999999996,"15min":3.992},"min":{"1min":0.36399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":64.412999999999997,"5min":64.412999999999997,"15min":64.412999999999997},"last":1.274},{"interface":"front","average":{"1min":7.6420000000000003,"5min":4.1260000000000003,"15min":3.4129999999999998},"min":{"1min":0.33700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":63.247999999999998,"5min":63.247999999999998,"15min":63.247999999999998},"last":0.61699999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:36:51 2023","interfaces":[{"interface":"back","average":{"1min":15.153,"5min":7.4130000000000003,"15min":3.4689999999999999},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":134.078,"5min":134.078,"15min":134.078},"last":0.30599999999999999},{"interface":"front","average":{"1min":15.754,"5min":7.0709999999999997,"15min":3.5920000000000001},"min":{"1min":0.38900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":135.73400000000001,"5min":135.73400000000001,"15min":135.73400000000001},"last":0.374}]},{"osd":4,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":2.1909999999999998,"15min":1.403},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":6.5979999999999999,"5min":35.972000000000001,"15min":35.972000000000001},"last":2.5390000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.917,"15min":1.6279999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":2.3759999999999999,"5min":36.518000000000001,"15min":36.518000000000001},"last":1.1899999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":2.2989999999999999,"15min":2.2200000000000002},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.6280000000000001,"5min":12.651999999999999,"15min":155.477},"last":4.5730000000000004},{"interface":"front","average":{"1min":1.585,"5min":1.6850000000000001,"15min":2.0139999999999998},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":4.9450000000000003,"5min":8.0839999999999996,"15min":155.46000000000001},"last":1.7509999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":5.5700000000000003,"15min":3.3690000000000002},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.8040000000000003,"5min":154.417,"15min":154.417},"last":5.8040000000000003},{"interface":"front","average":{"1min":2.3809999999999998,"5min":6.3570000000000002,"15min":3.4540000000000002},"min":{"1min":0.39000000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":7.6799999999999997,"5min":154.04400000000001,"15min":154.04400000000001},"last":2.419}]},{"osd":7,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":3.7839999999999998,"15min":3.1890000000000001},"min":{"1min":0.45000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0229999999999997,"5min":123.294,"15min":213.404},"last":1.897},{"interface":"front","average":{"1min":2.2970000000000002,"5min":4.3479999999999999,"15min":3.2749999999999999},"min":{"1min":0.48099999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.71,"5min":123.16,"15min":213.27199999999999},"last":1.792}]}]},{"osd":3,"up_from":109,"seq":468151435458,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":802700,"kb_used_data":131584,"kb_used_omap":2,"kb_used_meta":671101,"kb_avail":92942452,"statfs":{"total":95995035648,"available":95173070848,"internally_reserved":0,"allocated":134742016,"data_stored":132681157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":687207783},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":5.4980000000000002,"5min":4.9889999999999999,"15min":3.097},"min":{"1min":0.23499999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":51.835000000000001,"5min":102.57299999999999,"15min":130.946},"last":5.9699999999999998},{"interface":"front","average":{"1min":5.6929999999999996,"5min":4.899,"15min":3.0579999999999998},"min":{"1min":0.317,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":52.093000000000004,"5min":102.745,"15min":130.96899999999999},"last":0.46200000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:36:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5489999999999999,"5min":14.395,"15min":18.975999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":9.3919999999999995,"5min":273.99599999999998,"15min":273.99599999999998},"last":6.4509999999999996},{"interface":"front","average":{"1min":4.4950000000000001,"5min":13.355,"15min":17.132999999999999},"min":{"1min":0.34899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.6910000000000007,"5min":274.21600000000001,"15min":274.21600000000001},"last":8.7850000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8360000000000003,"5min":4.5800000000000001,"15min":3.3759999999999999},"min":{"1min":0.53600000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":62.289000000000001,"5min":76.974000000000004,"15min":145.75899999999999},"last":5.1360000000000001},{"interface":"front","average":{"1min":6.0650000000000004,"5min":4.3339999999999996,"15min":3.2170000000000001},"min":{"1min":0.33400000000000002,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":65.415999999999997,"5min":77.593999999999994,"15min":150.358},"last":5.9400000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":2.419,"5min":2.702,"15min":1.6899999999999999},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":10.805,"5min":60.637,"15min":60.637},"last":5.9109999999999996},{"interface":"front","average":{"1min":3.008,"5min":2.3100000000000001,"15min":1.627},"min":{"1min":0.40100000000000002,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":8.8049999999999997,"5min":10.731,"15min":15.638},"last":0.503}]},{"osd":5,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":2.9940000000000002,"15min":1.9390000000000001},"min":{"1min":0.375,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":8.8309999999999995,"5min":37.485999999999997,"15min":37.485999999999997},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.9199999999999999,"5min":3.2930000000000001,"15min":2.113},"min":{"1min":0.437,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":7.7439999999999998,"5min":37.448,"15min":37.448},"last":5.008}]},{"osd":6,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":2.9740000000000002,"15min":1.8240000000000001},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.891999999999999,"5min":60.698,"15min":60.698},"last":5.8780000000000001},{"interface":"front","average":{"1min":3.895,"5min":2.7629999999999999,"15min":1.8999999999999999},"min":{"1min":0.61299999999999999,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":12.121,"5min":12.121,"15min":14.69},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":3.2469999999999999,"15min":2.7440000000000002},"min":{"1min":0.44500000000000001,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":7.7709999999999999,"5min":26.469000000000001,"15min":226.03999999999999},"last":5.0819999999999999},{"interface":"front","average":{"1min":2.8370000000000002,"5min":3.1989999999999998,"15min":2.6019999999999999},"min":{"1min":0.41799999999999998,"5min":0.38800000000000001,"15min":0.34799999999999998},"max":{"1min":10.855,"5min":60.670000000000002,"15min":226.06100000000001},"last":5.8369999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019221,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":301980,"kb_used_data":33684,"kb_used_omap":0,"kb_used_meta":268288,"kb_avail":93443172,"statfs":{"total":95995035648,"available":95685808128,"internally_reserved":0,"allocated":34492416,"data_stored":33283548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":274726912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":1.4399999999999999,"15min":2.8780000000000001},"min":{"1min":0.33200000000000002,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":4.5940000000000003,"5min":14.353,"15min":128.79300000000001},"last":10.452},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.873,"15min":2.8730000000000002},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.248},"max":{"1min":6.9939999999999998,"5min":10.618,"15min":128.38800000000001},"last":2.0569999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4169999999999998,"5min":6.8659999999999997,"15min":8.6240000000000006},"min":{"1min":0.26600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.915,"5min":160.69999999999999,"15min":160.69999999999999},"last":3.5800000000000001},{"interface":"front","average":{"1min":3.2629999999999999,"5min":6.7960000000000003,"15min":8.6419999999999995},"min":{"1min":0.28399999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.7029999999999994,"5min":160.50700000000001,"15min":160.50700000000001},"last":0.28100000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.9850000000000001,"15min":2.9399999999999999},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9790000000000001,"5min":19.391999999999999,"15min":140.601},"last":1.944},{"interface":"front","average":{"1min":2.3399999999999999,"5min":2.0169999999999999,"15min":2.9300000000000002},"min":{"1min":0.39300000000000002,"5min":0.309,"15min":0.309},"max":{"1min":4.9900000000000002,"5min":19.422999999999998,"15min":140.54900000000001},"last":1.962}]},{"osd":3,"last update":"Sat Sep 30 09:36:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3010000000000002,"5min":2.964,"15min":3.6030000000000002},"min":{"1min":0.42799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.2650000000000006,"5min":52.762,"15min":281.42200000000003},"last":4.6779999999999999},{"interface":"front","average":{"1min":3.4359999999999999,"5min":2.992,"15min":3.5720000000000001},"min":{"1min":0.49099999999999999,"5min":0.312,"15min":0.312},"max":{"1min":9.2810000000000006,"5min":52.790999999999997,"15min":281.44999999999999},"last":4.6890000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.94399999999999995,"15min":1.7130000000000001},"min":{"1min":0.309,"5min":0.219,"15min":0.219},"max":{"1min":4.6120000000000001,"5min":8.6509999999999998,"15min":134.30600000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1,"15min":1.772},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":2.2309999999999999,"5min":6.3929999999999998,"15min":134.35300000000001},"last":0.376}]},{"osd":6,"last update":"Sat Sep 30 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":3.0299999999999998,"15min":1.593},"min":{"1min":0.32800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.1079999999999997,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.91700000000000004,"5min":2.927,"15min":1.597},"min":{"1min":0.34599999999999997,"5min":0.249,"15min":0.249},"max":{"1min":2.6459999999999999,"5min":202.42400000000001,"15min":202.42400000000001},"last":0.34799999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":4.4320000000000004,"15min":3.004},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":3.8330000000000002,"5min":230.90100000000001,"15min":230.90100000000001},"last":2.3010000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":4.1289999999999996,"15min":2.5579999999999998},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":2.6779999999999999,"5min":229.97,"15min":229.97},"last":0.432}]}]},{"osd":5,"up_from":38,"seq":163208757585,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318108,"kb_used_data":38804,"kb_used_omap":0,"kb_used_meta":279296,"kb_avail":93427044,"statfs":{"total":95995035648,"available":95669293056,"internally_reserved":0,"allocated":39735296,"data_stored":38522976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285999104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.2069999999999999,"15min":2.4540000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.22},"max":{"1min":4.6459999999999999,"5min":56.563000000000002,"15min":148.79900000000001},"last":2.5649999999999999},{"interface":"front","average":{"1min":1.738,"5min":2.3199999999999998,"15min":2.7589999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":6.2969999999999997,"5min":57.241999999999997,"15min":148.779},"last":1.8320000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:36:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":2.6619999999999999,"15min":2.1120000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.237,"5min":60.799999999999997,"15min":60.799999999999997},"last":6.5620000000000003},{"interface":"front","average":{"1min":3.012,"5min":3.581,"15min":3.3010000000000002},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.2539999999999996,"5min":63.798999999999999,"15min":63.798999999999999},"last":2.7000000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:37:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9199999999999999,"15min":3.149},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.9180000000000001,"5min":30.613,"15min":165.768},"last":2.544},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.327,"15min":3.714},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.5439999999999996,"5min":36.488,"15min":166.24000000000001},"last":7.5949999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:36:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":5.9180000000000001,"15min":4.0069999999999997},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5380000000000003,"5min":172.976,"15min":226.34700000000001},"last":7.351},{"interface":"front","average":{"1min":2.4689999999999999,"5min":5.8799999999999999,"15min":3.8300000000000001},"min":{"1min":0.376,"5min":0.36599999999999999,"15min":0.313},"max":{"1min":8.2240000000000002,"5min":173.01599999999999,"15min":226.36000000000001},"last":7.3769999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:36:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.76500000000000001,"15min":0.84899999999999998},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":1.0760000000000001,"5min":5.7160000000000002,"15min":8.5359999999999996},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.96999999999999997,"15min":0.94199999999999995},"min":{"1min":0.29599999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.1230000000000002,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.28599999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":2.9980000000000002,"15min":1.7509999999999999},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":3.2069999999999999,"5min":155.602,"15min":155.602},"last":0.308},{"interface":"front","average":{"1min":0.67400000000000004,"5min":3.012,"15min":1.867},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.9490000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.38700000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":2.5590000000000002,"15min":1.659},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.883,"5min":133.40700000000001,"15min":133.40700000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.98599999999999999,"5min":2.7229999999999999,"15min":1.5840000000000001},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":5.7880000000000003,"5min":132.797,"15min":132.797},"last":0.39700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16388096,"data_stored":16384024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":38604800,"data_stored":38600728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2686976,"data_stored":2686976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":119078912,"data_stored":119078912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":27787264,"data_stored":27787264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28049408,"data_stored":28049408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":66064384,"data_stored":66060312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":428933120,"data_stored":428933120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:37:38.124 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:37:38.128 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:37 smithi019 ceph-mon[216815]: from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:38.128 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:37 smithi019 ceph-mon[216815]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.128 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:37 smithi019 ceph-mon[216815]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.128 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:37 smithi019 ceph-mon[216815]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.128 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:37 smithi019 ceph-mon[224043]: from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:38.128 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:37 smithi019 ceph-mon[224043]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.128 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:37 smithi019 ceph-mon[224043]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.128 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:37 smithi019 ceph-mon[224043]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.223 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:38.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:37 smithi167 ceph-mon[192880]: from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:37 smithi167 ceph-mon[192880]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:37 smithi167 ceph-mon[192880]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:37 smithi167 ceph-mon[192880]: failed to encode map e352 with expected crc 2023-09-30T09:37:38.327 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:38.358 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:38.405 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:38.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:38.450 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:38.510 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:38.513 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:37:38.513 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:37:38.513 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 0 2023-09-30T09:37:38.513 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:37:38.538 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:38.564 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-30T09:37:38.564 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-30T09:37:38.581 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:38.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:38.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:38.686 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:38.715 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: pgmap v912: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 2 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 72 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 416/3321 objects degraded (12.526%); 982/3321 objects misplaced (29.569%); 17 MiB/s, 319 objects/s recovering 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: osdmap e353: 8 total, 8 up, 7 in 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 416/3321 objects degraded (12.526%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: pgmap v912: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 2 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 72 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 416/3321 objects degraded (12.526%); 982/3321 objects misplaced (29.569%); 17 MiB/s, 319 objects/s recovering 2023-09-30T09:37:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: osdmap e353: 8 total, 8 up, 7 in 2023-09-30T09:37:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 416/3321 objects degraded (12.526%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: pgmap v912: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 2 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 72 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 416/3321 objects degraded (12.526%); 982/3321 objects misplaced (29.569%); 17 MiB/s, 319 objects/s recovering 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: osdmap e353: 8 total, 8 up, 7 in 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: failed to encode map e353 with expected crc 2023-09-30T09:37:39.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 416/3321 objects degraded (12.526%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:37:39.593 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:39.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:39.697 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:39.732 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:40.088 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:40.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:40.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:40.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: osdmap e354: 8 total, 8 up, 7 in 2023-09-30T09:37:40.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: osdmap e354: 8 total, 8 up, 7 in 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:39 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.193 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:40.223 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: osdmap e354: 8 total, 8 up, 7 in 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:39 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:40.483 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:40.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:40.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:40.588 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:40.616 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:40.924 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:37:40.967 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:40.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:41.071 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:41.104 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:41.143 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:40 smithi019 ceph-mon[224043]: failed to encode map e354 with expected crc 2023-09-30T09:37:41.143 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:40 smithi019 ceph-mon[224043]: pgmap v915: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 31 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 284/1341 objects degraded (21.178%); 594/1341 objects misplaced (44.295%); 9.8 MiB/s, 157 objects/s recovering 2023-09-30T09:37:41.143 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:40 smithi019 ceph-mon[216815]: failed to encode map e354 with expected crc 2023-09-30T09:37:41.143 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:40 smithi019 ceph-mon[216815]: pgmap v915: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 31 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 284/1341 objects degraded (21.178%); 594/1341 objects misplaced (44.295%); 9.8 MiB/s, 157 objects/s recovering 2023-09-30T09:37:41.169 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:41.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:40 smithi167 ceph-mon[192880]: failed to encode map e354 with expected crc 2023-09-30T09:37:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:40 smithi167 ceph-mon[192880]: pgmap v915: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 31 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 284/1341 objects degraded (21.178%); 594/1341 objects misplaced (44.295%); 9.8 MiB/s, 157 objects/s recovering 2023-09-30T09:37:41.277 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:41.309 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:41.312 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:41.358 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:41.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:41.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:41.466 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:41.494 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:41.538 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:41.544 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:41.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:41.646 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:41.649 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:37:41.673 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:37:41.673 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:41.683 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:41.704 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:37:41.729 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:37:41.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:37:41.770 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:37:41.790 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:37:41.804 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:41.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:41.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:41.812 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:37:41.813 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:37:41.813 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:37:41.814 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:37:41.814 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:37:41.814 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:37:41.814 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:37:41.814 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:37:41.814 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:37:41.896 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:37:41.909 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:41.937 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1972546481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:41.937 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1079599371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:41.938 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1972546481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:41.938 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1079599371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:41.941 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:42.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1972546481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:37:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1079599371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:43.004 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:43.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:43.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:43.109 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:43.139 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:42 smithi167 ceph-mon[192880]: pgmap v916: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped, 1 active+recovering+degraded+remapped, 115 active+clean; 7.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 8/228 objects degraded (3.509%); 36/228 objects misplaced (15.789%); 9.3 MiB/s, 151 objects/s recovering 2023-09-30T09:37:43.140 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:42 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:43.140 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1972546481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:43.140 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:42 smithi167 ceph-mon[192880]: osdmap e355: 8 total, 8 up, 7 in 2023-09-30T09:37:43.140 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:42 smithi167 ceph-mon[192880]: failed to encode map e355 with expected crc 2023-09-30T09:37:43.142 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:43.192 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:43.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:43.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[216815]: pgmap v916: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped, 1 active+recovering+degraded+remapped, 115 active+clean; 7.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 8/228 objects degraded (3.509%); 36/228 objects misplaced (15.789%); 9.3 MiB/s, 151 objects/s recovering 2023-09-30T09:37:43.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:43.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1972546481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:43.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[216815]: osdmap e355: 8 total, 8 up, 7 in 2023-09-30T09:37:43.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[216815]: failed to encode map e355 with expected crc 2023-09-30T09:37:43.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[224043]: pgmap v916: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped, 1 active+recovering+degraded+remapped, 115 active+clean; 7.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 8/228 objects degraded (3.509%); 36/228 objects misplaced (15.789%); 9.3 MiB/s, 151 objects/s recovering 2023-09-30T09:37:43.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:43.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1972546481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:37:43.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[224043]: osdmap e355: 8 total, 8 up, 7 in 2023-09-30T09:37:43.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:42 smithi019 ceph-mon[224043]: failed to encode map e355 with expected crc 2023-09-30T09:37:43.297 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:43.328 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:43.573 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:43.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:43.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:43.677 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:43.708 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:43.758 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:43.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:43.863 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:43.892 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:43.936 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:43.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:43.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:44.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:43 smithi167 ceph-mon[192880]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:43 smithi167 ceph-mon[192880]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:43 smithi167 ceph-mon[192880]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 8/228 objects degraded (3.509%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:37:44.044 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:44.072 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:37:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:37:43] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-30T09:37:44.072 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:43 smithi019 ceph-mon[216815]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.072 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:43 smithi019 ceph-mon[216815]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.072 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:43 smithi019 ceph-mon[216815]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.073 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 8/228 objects degraded (3.509%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:37:44.073 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:43 smithi019 ceph-mon[224043]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.073 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:43 smithi019 ceph-mon[224043]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.073 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:43 smithi019 ceph-mon[224043]: failed to encode map e355 with expected crc 2023-09-30T09:37:44.073 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 8/228 objects degraded (3.509%), 1 pg degraded (PG_DEGRADED) 2023-09-30T09:37:44.075 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:45.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:44 smithi019 ceph-mon[216815]: pgmap v918: 119 pgs: 1 active+remapped, 118 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 4.6 MiB/s, 76 objects/s recovering 2023-09-30T09:37:45.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:44 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/228 objects degraded (3.509%), 1 pg degraded) 2023-09-30T09:37:45.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:44 smithi019 ceph-mon[224043]: pgmap v918: 119 pgs: 1 active+remapped, 118 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 4.6 MiB/s, 76 objects/s recovering 2023-09-30T09:37:45.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:44 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/228 objects degraded (3.509%), 1 pg degraded) 2023-09-30T09:37:45.271 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:37:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:44 smithi167 ceph-mon[192880]: pgmap v918: 119 pgs: 1 active+remapped, 118 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 4.6 MiB/s, 76 objects/s recovering 2023-09-30T09:37:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:44 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/228 objects degraded (3.509%), 1 pg degraded) 2023-09-30T09:37:45.483 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:45.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:45.588 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:45.632 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:45.679 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:45.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:45.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:45.786 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:45.827 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:45.934 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:45.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:46.010 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:46.024 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:37:46.024 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:37:46.038 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:46.068 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:46.114 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:46.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:46.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:46.219 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:46.250 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:46.348 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:46.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:46.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:46.452 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:46.480 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:46.525 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:46.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:46.633 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:46.660 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:46.703 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:46.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:46.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:46.807 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:46.836 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:46.949 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:46.949 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_2' removed 2023-09-30T09:37:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:46.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:46.979 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:37:47.081 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:47.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:46 smithi019 ceph-mon[216815]: pgmap v919: 119 pgs: 1 active+remapped, 118 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 3.6 MiB/s, 59 objects/s recovering 2023-09-30T09:37:47.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3559590126' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:37:47.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:46 smithi019 ceph-mon[224043]: pgmap v919: 119 pgs: 1 active+remapped, 118 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 3.6 MiB/s, 59 objects/s recovering 2023-09-30T09:37:47.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3559590126' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:37:47.147 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:47.158 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:37:47.158 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1205,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":356,"num_osds":8,"num_up_osds":8,"osd_up_since":1696066411,"num_in_osds":7,"osd_in_since":1696066446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":118},{"state_name":"active+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":29,"data_bytes":4995916,"bytes_used":2773647360,"bytes_avail":669191602176,"bytes_total":671965249536,"recovering_objects_per_sec":59,"recovering_bytes_per_sec":3771287,"recovering_keys_per_sec":0,"num_objects_recovered":457,"num_bytes_recovered":29032448,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":140},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-30T09:37:27.560882+0000","services":{}},"progress_events":{"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"bf6cdc82-6db1-4694-b9dc-04effc0cdb2e":{"message":"Global Recovery Event (5m)\n [===========================.] (remaining: 10s)","progress":0.96638655662536621,"add_to_ceph_s":true}}} 2023-09-30T09:37:47.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:46 smithi167 ceph-mon[192880]: pgmap v919: 119 pgs: 1 active+remapped, 118 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 3.6 MiB/s, 59 objects/s recovering 2023-09-30T09:37:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3559590126' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:37:47.313 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:47.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:47.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:47.392 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:37:47.418 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:47.463 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:47.905 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:47.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:47.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:48.012 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:48.018 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-30T09:37:48.018 INFO:tasks.radosbench:Beginning radosbench... 2023-09-30T09:37:48.019 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-30T09:37:48.019 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-30T09:37:48.044 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:48.094 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:48.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:48.200 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:48.225 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3559590126' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:37:48.225 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[224043]: osdmap e356: 8 total, 8 up, 7 in 2023-09-30T09:37:48.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[224043]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[224043]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[224043]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[224043]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1362105211' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:37:48.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3559590126' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:37:48.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[216815]: osdmap e356: 8 total, 8 up, 7 in 2023-09-30T09:37:48.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[216815]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[216815]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[216815]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[216815]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1362105211' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:37:48.229 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3559590126' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:37:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:47 smithi167 ceph-mon[192880]: osdmap e356: 8 total, 8 up, 7 in 2023-09-30T09:37:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:47 smithi167 ceph-mon[192880]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:47 smithi167 ceph-mon[192880]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:47 smithi167 ceph-mon[192880]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:47 smithi167 ceph-mon[192880]: failed to encode map e356 with expected crc 2023-09-30T09:37:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1362105211' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:37:48.480 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:48.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:48.585 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:48.616 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:48.652 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:48.666 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:37:48.667 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:37:48.667 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:48.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:48.771 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:48.801 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:48.848 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:48.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:48.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:48.955 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:48.984 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:49.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:48 smithi019 ceph-mon[216815]: pgmap v921: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:49.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:49.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:48 smithi019 ceph-mon[224043]: pgmap v921: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:49.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:48 smithi167 ceph-mon[192880]: pgmap v921: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:37:49.409 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:49.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:49.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:49.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:49.544 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:49.593 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:49.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:49.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:49.665 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:49.698 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:49.729 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:49.826 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:49.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:49.930 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:49.958 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:50.259 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:37:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:50.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:37:50.356 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:50.357 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:50.363 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:50.401 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:51.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:50 smithi167 ceph-mon[192880]: pgmap v922: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:51.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:50 smithi019 ceph-mon[216815]: pgmap v922: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:51.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:50 smithi019 ceph-mon[224043]: pgmap v922: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:51.410 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:51.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:51.517 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:51.555 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:51.601 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:51.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:51.707 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:51.746 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:51.984 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:37:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1607493740' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:37:52.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1607493740' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:37:52.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1607493740' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:37:52.564 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:37:52.564 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:37:52.750 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:52.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:52.854 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:52.884 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:53.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[224043]: pgmap v923: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:53.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:53.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1607493740' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:37:53.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[224043]: failed to encode map e357 with expected crc 2023-09-30T09:37:53.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[224043]: osdmap e357: 8 total, 8 up, 7 in 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[224043]: failed to encode map e357 with expected crc 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[224043]: 1.a scrub starts 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[224043]: 1.a scrub ok 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[216815]: pgmap v923: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1607493740' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[216815]: failed to encode map e357 with expected crc 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[216815]: osdmap e357: 8 total, 8 up, 7 in 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[216815]: failed to encode map e357 with expected crc 2023-09-30T09:37:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[216815]: 1.a scrub starts 2023-09-30T09:37:53.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:52 smithi019 ceph-mon[216815]: 1.a scrub ok 2023-09-30T09:37:53.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:52 smithi167 ceph-mon[192880]: pgmap v923: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:53.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:52 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:37:53.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1607493740' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:37:53.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:52 smithi167 ceph-mon[192880]: failed to encode map e357 with expected crc 2023-09-30T09:37:53.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:52 smithi167 ceph-mon[192880]: osdmap e357: 8 total, 8 up, 7 in 2023-09-30T09:37:53.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:52 smithi167 ceph-mon[192880]: failed to encode map e357 with expected crc 2023-09-30T09:37:53.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:52 smithi167 ceph-mon[192880]: 1.a scrub starts 2023-09-30T09:37:53.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:52 smithi167 ceph-mon[192880]: 1.a scrub ok 2023-09-30T09:37:53.482 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:53.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:53.587 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:53.619 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:53.668 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:53.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:53.773 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:53.802 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:53.846 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:53.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:53.955 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:37:53.978 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:53 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:37:53.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:53 smithi167 ceph-mon[192880]: 1.d scrub ok 2023-09-30T09:37:53.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:53 smithi167 ceph-mon[192880]: 1.3 scrub starts 2023-09-30T09:37:53.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:53 smithi167 ceph-mon[192880]: 1.3 scrub ok 2023-09-30T09:37:53.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:53 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:53.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:53 smithi167 ceph-mon[192880]: 1.14 scrub starts 2023-09-30T09:37:53.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:53 smithi167 ceph-mon[192880]: 1.14 scrub ok 2023-09-30T09:37:53.981 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:54.001 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:37:54.023 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:37:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:54.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:37:54.039 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:37:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:37:53] "GET /metrics HTTP/1.1" 200 34008 "" "Prometheus/2.43.0" 2023-09-30T09:37:54.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:37:54.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[216815]: 1.d scrub ok 2023-09-30T09:37:54.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[216815]: 1.3 scrub starts 2023-09-30T09:37:54.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[216815]: 1.3 scrub ok 2023-09-30T09:37:54.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:54.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[216815]: 1.14 scrub starts 2023-09-30T09:37:54.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[216815]: 1.14 scrub ok 2023-09-30T09:37:54.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:37:54.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[224043]: 1.d scrub ok 2023-09-30T09:37:54.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[224043]: 1.3 scrub starts 2023-09-30T09:37:54.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[224043]: 1.3 scrub ok 2023-09-30T09:37:54.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:37:54.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[224043]: 1.14 scrub starts 2023-09-30T09:37:54.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:53 smithi019 ceph-mon[224043]: 1.14 scrub ok 2023-09-30T09:37:54.131 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:54.159 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:54.710 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:54.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:54.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:54.817 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:37:54.846 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:54.849 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:54.955 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:37:54.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:54.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:37:55.062 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:37:55.092 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:55.137 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:37:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:55.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:37:55.242 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:55.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:55 smithi019 ceph-mon[224043]: pgmap v925: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:55.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:55 smithi019 ceph-mon[224043]: 1.e scrub starts 2023-09-30T09:37:55.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:55 smithi019 ceph-mon[224043]: 1.e scrub ok 2023-09-30T09:37:55.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:55 smithi019 ceph-mon[216815]: pgmap v925: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:55.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:55 smithi019 ceph-mon[216815]: 1.e scrub starts 2023-09-30T09:37:55.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:55 smithi019 ceph-mon[216815]: 1.e scrub ok 2023-09-30T09:37:55.269 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:55.312 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:55.317 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:55 smithi167 ceph-mon[192880]: pgmap v925: 103 pgs: 103 active+clean; 4.8 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:55.317 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:55 smithi167 ceph-mon[192880]: 1.e scrub starts 2023-09-30T09:37:55.317 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:55 smithi167 ceph-mon[192880]: 1.e scrub ok 2023-09-30T09:37:55.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:55.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:55.397 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:37:55.403 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:37:55.418 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:55.452 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:55.532 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_3' created 2023-09-30T09:37:55.670 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:37:55.671 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:37:55.678 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:37:55.697 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:37:55.716 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:37:55.735 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:37:55.754 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:37:55.774 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:37:55.775 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:37:55.775 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:37:55.775 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:37:55.775 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:37:55.775 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:37:55.775 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:37:55.775 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:37:55.775 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:37:55.962 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:37:56.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:37:56.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[216815]: 1.1c scrub ok 2023-09-30T09:37:56.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3045501001' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-30T09:37:56.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3045501001' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-30T09:37:56.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[216815]: failed to encode map e358 with expected crc 2023-09-30T09:37:56.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[216815]: osdmap e358: 8 total, 8 up, 7 in 2023-09-30T09:37:56.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[216815]: failed to encode map e358 with expected crc 2023-09-30T09:37:56.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2048052342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:56.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:37:56.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[224043]: 1.1c scrub ok 2023-09-30T09:37:56.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3045501001' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-30T09:37:56.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3045501001' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-30T09:37:56.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[224043]: failed to encode map e358 with expected crc 2023-09-30T09:37:56.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[224043]: osdmap e358: 8 total, 8 up, 7 in 2023-09-30T09:37:56.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[224043]: failed to encode map e358 with expected crc 2023-09-30T09:37:56.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2048052342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:56 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:37:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:56 smithi167 ceph-mon[192880]: 1.1c scrub ok 2023-09-30T09:37:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3045501001' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-30T09:37:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3045501001' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-30T09:37:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:56 smithi167 ceph-mon[192880]: failed to encode map e358 with expected crc 2023-09-30T09:37:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:56 smithi167 ceph-mon[192880]: osdmap e358: 8 total, 8 up, 7 in 2023-09-30T09:37:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:56 smithi167 ceph-mon[192880]: failed to encode map e358 with expected crc 2023-09-30T09:37:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2048052342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:37:57.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[216815]: from='client.35787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:57.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[216815]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[216815]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[216815]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[216815]: pgmap v927: 119 pgs: 16 unknown, 103 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:57.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[216815]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:37:57.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[216815]: osdmap e359: 8 total, 8 up, 7 in 2023-09-30T09:37:57.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[216815]: failed to encode map e359 with expected crc 2023-09-30T09:37:57.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[224043]: from='client.35787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:57.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[224043]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[224043]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[224043]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[224043]: pgmap v927: 119 pgs: 16 unknown, 103 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:57.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[224043]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:37:57.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[224043]: osdmap e359: 8 total, 8 up, 7 in 2023-09-30T09:37:57.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:57 smithi019 ceph-mon[224043]: failed to encode map e359 with expected crc 2023-09-30T09:37:57.139 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-30T09:37:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:57 smithi167 ceph-mon[192880]: from='client.35787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:37:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:57 smithi167 ceph-mon[192880]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:57 smithi167 ceph-mon[192880]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:57 smithi167 ceph-mon[192880]: failed to encode map e358 with expected crc 2023-09-30T09:37:57.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:57 smithi167 ceph-mon[192880]: pgmap v927: 119 pgs: 16 unknown, 103 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:57.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:57 smithi167 ceph-mon[192880]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:37:57.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:57 smithi167 ceph-mon[192880]: osdmap e359: 8 total, 8 up, 7 in 2023-09-30T09:37:57.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:57 smithi167 ceph-mon[192880]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.115 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":925,"stamp":"2023-09-30T09:37:53.568995+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4995916,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":6113,"num_read_kb":12466,"num_write":988,"num_write_kb":13565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":233,"num_bytes_recovered":9806296,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2695760,"kb_used_data":44360,"kb_used_omap":23,"kb_used_meta":2651304,"kb_avail":653520304,"statfs":{"total":671965249536,"available":669204791296,"internally_reserved":0,"allocated":45424640,"data_stored":33493035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2714935555},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":101,"apply_latency_ms":101,"commit_latency_ns":101000000,"apply_latency_ns":101000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-226552,"num_write_kb":-7249601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-46657,"num_bytes_recovered":-2854354944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002112"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":820,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407336+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:37:42.407336+0000","last_peered":"2023-09-30T09:37:42.407336+0000","last_clean":"2023-09-30T09:37:42.407336+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:37:42.407336+0000","last_undegraded":"2023-09-30T09:37:42.407336+0000","last_fullsized":"2023-09-30T09:37:42.407336+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":374,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406597+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:37:42.406597+0000","last_peered":"2023-09-30T09:37:42.406597+0000","last_clean":"2023-09-30T09:37:42.406597+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:37:42.406597+0000","last_undegraded":"2023-09-30T09:37:42.406597+0000","last_fullsized":"2023-09-30T09:37:42.406597+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":337,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.894107+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:37:41.894107+0000","last_peered":"2023-09-30T09:37:41.894107+0000","last_clean":"2023-09-30T09:37:41.894107+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:37:41.894107+0000","last_undegraded":"2023-09-30T09:37:41.894107+0000","last_fullsized":"2023-09-30T09:37:41.894107+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":800,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406073+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:37:42.406073+0000","last_peered":"2023-09-30T09:37:42.406073+0000","last_clean":"2023-09-30T09:37:42.406073+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:37:42.406073+0000","last_undegraded":"2023-09-30T09:37:42.406073+0000","last_fullsized":"2023-09-30T09:37:42.406073+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":230,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.894026+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:37:41.894026+0000","last_peered":"2023-09-30T09:37:41.894026+0000","last_clean":"2023-09-30T09:37:41.894026+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:37:41.894026+0000","last_undegraded":"2023-09-30T09:37:41.894026+0000","last_fullsized":"2023-09-30T09:37:41.894026+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":261,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.892168+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:37:41.892168+0000","last_peered":"2023-09-30T09:37:41.892168+0000","last_clean":"2023-09-30T09:37:41.892168+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:37:41.892168+0000","last_undegraded":"2023-09-30T09:37:41.892168+0000","last_fullsized":"2023-09-30T09:37:41.892168+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":569,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406474+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:37:42.406474+0000","last_peered":"2023-09-30T09:37:42.406474+0000","last_clean":"2023-09-30T09:37:42.406474+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:37:42.406474+0000","last_undegraded":"2023-09-30T09:37:42.406474+0000","last_fullsized":"2023-09-30T09:37:42.406474+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":403,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893512+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:37:41.893512+0000","last_peered":"2023-09-30T09:37:41.893512+0000","last_clean":"2023-09-30T09:37:41.893512+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:37:41.893512+0000","last_undegraded":"2023-09-30T09:37:41.893512+0000","last_fullsized":"2023-09-30T09:37:41.893512+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":332,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407736+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:37:42.407736+0000","last_peered":"2023-09-30T09:37:42.407736+0000","last_clean":"2023-09-30T09:37:42.407736+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:37:42.407736+0000","last_undegraded":"2023-09-30T09:37:42.407736+0000","last_fullsized":"2023-09-30T09:37:42.407736+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":780,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893380+0000","last_change":"2023-09-30T09:34:10.503067+0000","last_active":"2023-09-30T09:37:41.893380+0000","last_peered":"2023-09-30T09:37:41.893380+0000","last_clean":"2023-09-30T09:37:41.893380+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:37:41.893380+0000","last_undegraded":"2023-09-30T09:37:41.893380+0000","last_fullsized":"2023-09-30T09:37:41.893380+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.060845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.060845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":252,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:45.255980+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:37:45.255980+0000","last_peered":"2023-09-30T09:37:45.255980+0000","last_clean":"2023-09-30T09:37:45.255980+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:37:45.255980+0000","last_undegraded":"2023-09-30T09:37:45.255980+0000","last_fullsized":"2023-09-30T09:37:45.255980+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":338,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.313054+0000","last_change":"2023-09-30T09:33:32.614697+0000","last_active":"2023-09-30T09:37:42.313054+0000","last_peered":"2023-09-30T09:37:42.313054+0000","last_clean":"2023-09-30T09:37:42.313054+0000","last_became_active":"2023-09-30T09:33:32.614208+0000","last_became_peered":"2023-09-30T09:33:32.614208+0000","last_unstale":"2023-09-30T09:37:42.313054+0000","last_undegraded":"2023-09-30T09:37:42.313054+0000","last_fullsized":"2023-09-30T09:37:42.313054+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:14:40.141016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":520,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-30T09:37:46.908643+0000","last_change":"2023-09-30T09:35:23.848516+0000","last_active":"2023-09-30T09:37:46.908643+0000","last_peered":"2023-09-30T09:37:46.908643+0000","last_clean":"2023-09-30T09:37:46.908643+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:37:46.908643+0000","last_undegraded":"2023-09-30T09:37:46.908643+0000","last_fullsized":"2023-09-30T09:37:46.908643+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.000956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.000956+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":234,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407430+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:37:42.407430+0000","last_peered":"2023-09-30T09:37:42.407430+0000","last_clean":"2023-09-30T09:37:42.407430+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:37:42.407430+0000","last_undegraded":"2023-09-30T09:37:42.407430+0000","last_fullsized":"2023-09-30T09:37:42.407430+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":284,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-30T09:37:46.905266+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:37:46.905266+0000","last_peered":"2023-09-30T09:37:46.905266+0000","last_clean":"2023-09-30T09:37:46.905266+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:37:46.905266+0000","last_undegraded":"2023-09-30T09:37:46.905266+0000","last_fullsized":"2023-09-30T09:37:46.905266+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":522,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406827+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:37:42.406827+0000","last_peered":"2023-09-30T09:37:42.406827+0000","last_clean":"2023-09-30T09:37:42.406827+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:37:42.406827+0000","last_undegraded":"2023-09-30T09:37:42.406827+0000","last_fullsized":"2023-09-30T09:37:42.406827+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":78,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.894200+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:37:41.894200+0000","last_peered":"2023-09-30T09:37:41.894200+0000","last_clean":"2023-09-30T09:37:41.894200+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:37:41.894200+0000","last_undegraded":"2023-09-30T09:37:41.894200+0000","last_fullsized":"2023-09-30T09:37:41.894200+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":269,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408344+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:37:42.408344+0000","last_peered":"2023-09-30T09:37:42.408344+0000","last_clean":"2023-09-30T09:37:42.408344+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:37:42.408344+0000","last_undegraded":"2023-09-30T09:37:42.408344+0000","last_fullsized":"2023-09-30T09:37:42.408344+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":516,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.892510+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:37:41.892510+0000","last_peered":"2023-09-30T09:37:41.892510+0000","last_clean":"2023-09-30T09:37:41.892510+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:37:41.892510+0000","last_undegraded":"2023-09-30T09:37:41.892510+0000","last_fullsized":"2023-09-30T09:37:41.892510+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":340,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406347+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:37:42.406347+0000","last_peered":"2023-09-30T09:37:42.406347+0000","last_clean":"2023-09-30T09:37:42.406347+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:37:42.406347+0000","last_undegraded":"2023-09-30T09:37:42.406347+0000","last_fullsized":"2023-09-30T09:37:42.406347+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":351,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408182+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:37:42.408182+0000","last_peered":"2023-09-30T09:37:42.408182+0000","last_clean":"2023-09-30T09:37:42.408182+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:37:42.408182+0000","last_undegraded":"2023-09-30T09:37:42.408182+0000","last_fullsized":"2023-09-30T09:37:42.408182+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":634,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.890048+0000","last_change":"2023-09-30T09:35:13.717514+0000","last_active":"2023-09-30T09:37:41.890048+0000","last_peered":"2023-09-30T09:37:41.890048+0000","last_clean":"2023-09-30T09:37:41.890048+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:37:41.890048+0000","last_undegraded":"2023-09-30T09:37:41.890048+0000","last_fullsized":"2023-09-30T09:37:41.890048+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.652131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.652131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":78,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408099+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:37:42.408099+0000","last_peered":"2023-09-30T09:37:42.408099+0000","last_clean":"2023-09-30T09:37:42.408099+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:37:42.408099+0000","last_undegraded":"2023-09-30T09:37:42.408099+0000","last_fullsized":"2023-09-30T09:37:42.408099+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":368,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-30T09:37:46.908196+0000","last_change":"2023-09-30T09:34:07.478739+0000","last_active":"2023-09-30T09:37:46.908196+0000","last_peered":"2023-09-30T09:37:46.908196+0000","last_clean":"2023-09-30T09:37:46.908196+0000","last_became_active":"2023-09-30T09:34:07.477388+0000","last_became_peered":"2023-09-30T09:34:07.477388+0000","last_unstale":"2023-09-30T09:37:46.908196+0000","last_undegraded":"2023-09-30T09:37:46.908196+0000","last_fullsized":"2023-09-30T09:37:46.908196+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:44.050540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":702,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407536+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:37:42.407536+0000","last_peered":"2023-09-30T09:37:42.407536+0000","last_clean":"2023-09-30T09:37:42.407536+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:37:42.407536+0000","last_undegraded":"2023-09-30T09:37:42.407536+0000","last_fullsized":"2023-09-30T09:37:42.407536+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":260,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889544+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:37:41.889544+0000","last_peered":"2023-09-30T09:37:41.889544+0000","last_clean":"2023-09-30T09:37:41.889544+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:37:41.889544+0000","last_undegraded":"2023-09-30T09:37:41.889544+0000","last_fullsized":"2023-09-30T09:37:41.889544+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":45,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407279+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:37:42.407279+0000","last_peered":"2023-09-30T09:37:42.407279+0000","last_clean":"2023-09-30T09:37:42.407279+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:37:42.407279+0000","last_undegraded":"2023-09-30T09:37:42.407279+0000","last_fullsized":"2023-09-30T09:37:42.407279+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":795,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.894349+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:37:41.894349+0000","last_peered":"2023-09-30T09:37:41.894349+0000","last_clean":"2023-09-30T09:37:41.894349+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:37:41.894349+0000","last_undegraded":"2023-09-30T09:37:41.894349+0000","last_fullsized":"2023-09-30T09:37:41.894349+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":61,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408158+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:37:42.408158+0000","last_peered":"2023-09-30T09:37:42.408158+0000","last_clean":"2023-09-30T09:37:42.408158+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:37:42.408158+0000","last_undegraded":"2023-09-30T09:37:42.408158+0000","last_fullsized":"2023-09-30T09:37:42.408158+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":260,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.892353+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:37:41.892353+0000","last_peered":"2023-09-30T09:37:41.892353+0000","last_clean":"2023-09-30T09:37:41.892353+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:37:41.892353+0000","last_undegraded":"2023-09-30T09:37:41.892353+0000","last_fullsized":"2023-09-30T09:37:41.892353+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":821,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406129+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:37:42.406129+0000","last_peered":"2023-09-30T09:37:42.406129+0000","last_clean":"2023-09-30T09:37:42.406129+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:37:42.406129+0000","last_undegraded":"2023-09-30T09:37:42.406129+0000","last_fullsized":"2023-09-30T09:37:42.406129+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":515,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406343+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:37:42.406343+0000","last_peered":"2023-09-30T09:37:42.406343+0000","last_clean":"2023-09-30T09:37:42.406343+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:37:42.406343+0000","last_undegraded":"2023-09-30T09:37:42.406343+0000","last_fullsized":"2023-09-30T09:37:42.406343+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":365,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893893+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:37:41.893893+0000","last_peered":"2023-09-30T09:37:41.893893+0000","last_clean":"2023-09-30T09:37:41.893893+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:37:41.893893+0000","last_undegraded":"2023-09-30T09:37:41.893893+0000","last_fullsized":"2023-09-30T09:37:41.893893+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":850,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406877+0000","last_change":"2023-09-30T09:37:29.893943+0000","last_active":"2023-09-30T09:37:42.406877+0000","last_peered":"2023-09-30T09:37:42.406877+0000","last_clean":"2023-09-30T09:37:42.406877+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:37:42.406877+0000","last_undegraded":"2023-09-30T09:37:42.406877+0000","last_fullsized":"2023-09-30T09:37:42.406877+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:32:08.030550+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":90,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406681+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:37:42.406681+0000","last_peered":"2023-09-30T09:37:42.406681+0000","last_clean":"2023-09-30T09:37:42.406681+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:37:42.406681+0000","last_undegraded":"2023-09-30T09:37:42.406681+0000","last_fullsized":"2023-09-30T09:37:42.406681+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407255+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:37:42.407255+0000","last_peered":"2023-09-30T09:37:42.407255+0000","last_clean":"2023-09-30T09:37:42.407255+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:37:42.407255+0000","last_undegraded":"2023-09-30T09:37:42.407255+0000","last_fullsized":"2023-09-30T09:37:42.407255+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":806,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.894718+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:37:41.894718+0000","last_peered":"2023-09-30T09:37:41.894718+0000","last_clean":"2023-09-30T09:37:41.894718+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:37:41.894718+0000","last_undegraded":"2023-09-30T09:37:41.894718+0000","last_fullsized":"2023-09-30T09:37:41.894718+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":68,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406470+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:37:42.406470+0000","last_peered":"2023-09-30T09:37:42.406470+0000","last_clean":"2023-09-30T09:37:42.406470+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:37:42.406470+0000","last_undegraded":"2023-09-30T09:37:42.406470+0000","last_fullsized":"2023-09-30T09:37:42.406470+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":348,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408073+0000","last_change":"2023-09-30T09:33:32.650019+0000","last_active":"2023-09-30T09:37:42.408073+0000","last_peered":"2023-09-30T09:37:42.408073+0000","last_clean":"2023-09-30T09:37:42.408073+0000","last_became_active":"2023-09-30T09:33:32.649704+0000","last_became_peered":"2023-09-30T09:33:32.649704+0000","last_unstale":"2023-09-30T09:37:42.408073+0000","last_undegraded":"2023-09-30T09:37:42.408073+0000","last_fullsized":"2023-09-30T09:37:42.408073+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:17.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":639,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407656+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:37:42.407656+0000","last_peered":"2023-09-30T09:37:42.407656+0000","last_clean":"2023-09-30T09:37:42.407656+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:37:42.407656+0000","last_undegraded":"2023-09-30T09:37:42.407656+0000","last_fullsized":"2023-09-30T09:37:42.407656+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":49,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407200+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:37:42.407200+0000","last_peered":"2023-09-30T09:37:42.407200+0000","last_clean":"2023-09-30T09:37:42.407200+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:37:42.407200+0000","last_undegraded":"2023-09-30T09:37:42.407200+0000","last_fullsized":"2023-09-30T09:37:42.407200+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":335,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408189+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:37:42.408189+0000","last_peered":"2023-09-30T09:37:42.408189+0000","last_clean":"2023-09-30T09:37:42.408189+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:37:42.408189+0000","last_undegraded":"2023-09-30T09:37:42.408189+0000","last_fullsized":"2023-09-30T09:37:42.408189+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":521,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406583+0000","last_change":"2023-09-30T09:37:28.926398+0000","last_active":"2023-09-30T09:37:42.406583+0000","last_peered":"2023-09-30T09:37:42.406583+0000","last_clean":"2023-09-30T09:37:42.406583+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:37:42.406583+0000","last_undegraded":"2023-09-30T09:37:42.406583+0000","last_fullsized":"2023-09-30T09:37:42.406583+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:06.043188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:06.043188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":315,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407761+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:37:42.407761+0000","last_peered":"2023-09-30T09:37:42.407761+0000","last_clean":"2023-09-30T09:37:42.407761+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:37:42.407761+0000","last_undegraded":"2023-09-30T09:37:42.407761+0000","last_fullsized":"2023-09-30T09:37:42.407761+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":262,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.890725+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:37:41.890725+0000","last_peered":"2023-09-30T09:37:41.890725+0000","last_clean":"2023-09-30T09:37:41.890725+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:37:41.890725+0000","last_undegraded":"2023-09-30T09:37:41.890725+0000","last_fullsized":"2023-09-30T09:37:41.890725+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":702,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893718+0000","last_change":"2023-09-30T09:34:10.500608+0000","last_active":"2023-09-30T09:37:41.893718+0000","last_peered":"2023-09-30T09:37:41.893718+0000","last_clean":"2023-09-30T09:37:41.893718+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:37:41.893718+0000","last_undegraded":"2023-09-30T09:37:41.893718+0000","last_fullsized":"2023-09-30T09:37:41.893718+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.035086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.035086+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":384,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.894568+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:37:41.894568+0000","last_peered":"2023-09-30T09:37:41.894568+0000","last_clean":"2023-09-30T09:37:41.894568+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:37:41.894568+0000","last_undegraded":"2023-09-30T09:37:41.894568+0000","last_fullsized":"2023-09-30T09:37:41.894568+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":314,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406447+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:37:42.406447+0000","last_peered":"2023-09-30T09:37:42.406447+0000","last_clean":"2023-09-30T09:37:42.406447+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:37:42.406447+0000","last_undegraded":"2023-09-30T09:37:42.406447+0000","last_fullsized":"2023-09-30T09:37:42.406447+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":778,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889309+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:37:41.889309+0000","last_peered":"2023-09-30T09:37:41.889309+0000","last_clean":"2023-09-30T09:37:41.889309+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:37:41.889309+0000","last_undegraded":"2023-09-30T09:37:41.889309+0000","last_fullsized":"2023-09-30T09:37:41.889309+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":333,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408086+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:37:42.408086+0000","last_peered":"2023-09-30T09:37:42.408086+0000","last_clean":"2023-09-30T09:37:42.408086+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:37:42.408086+0000","last_undegraded":"2023-09-30T09:37:42.408086+0000","last_fullsized":"2023-09-30T09:37:42.408086+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":50,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406712+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:37:42.406712+0000","last_peered":"2023-09-30T09:37:42.406712+0000","last_clean":"2023-09-30T09:37:42.406712+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:37:42.406712+0000","last_undegraded":"2023-09-30T09:37:42.406712+0000","last_fullsized":"2023-09-30T09:37:42.406712+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":792,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.890534+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:37:41.890534+0000","last_peered":"2023-09-30T09:37:41.890534+0000","last_clean":"2023-09-30T09:37:41.890534+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:37:41.890534+0000","last_undegraded":"2023-09-30T09:37:41.890534+0000","last_fullsized":"2023-09-30T09:37:41.890534+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":352,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407913+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:37:42.407913+0000","last_peered":"2023-09-30T09:37:42.407913+0000","last_clean":"2023-09-30T09:37:42.407913+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:37:42.407913+0000","last_undegraded":"2023-09-30T09:37:42.407913+0000","last_fullsized":"2023-09-30T09:37:42.407913+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":331,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406705+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:37:42.406705+0000","last_peered":"2023-09-30T09:37:42.406705+0000","last_clean":"2023-09-30T09:37:42.406705+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:37:42.406705+0000","last_undegraded":"2023-09-30T09:37:42.406705+0000","last_fullsized":"2023-09-30T09:37:42.406705+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":506,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889597+0000","last_change":"2023-09-30T09:37:30.390725+0000","last_active":"2023-09-30T09:37:41.889597+0000","last_peered":"2023-09-30T09:37:41.889597+0000","last_clean":"2023-09-30T09:37:41.889597+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:37:41.889597+0000","last_undegraded":"2023-09-30T09:37:41.889597+0000","last_fullsized":"2023-09-30T09:37:41.889597+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.381424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.381424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":347,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893326+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:37:41.893326+0000","last_peered":"2023-09-30T09:37:41.893326+0000","last_clean":"2023-09-30T09:37:41.893326+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:37:41.893326+0000","last_undegraded":"2023-09-30T09:37:41.893326+0000","last_fullsized":"2023-09-30T09:37:41.893326+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":124,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893714+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:37:41.893714+0000","last_peered":"2023-09-30T09:37:41.893714+0000","last_clean":"2023-09-30T09:37:41.893714+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:37:41.893714+0000","last_undegraded":"2023-09-30T09:37:41.893714+0000","last_fullsized":"2023-09-30T09:37:41.893714+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":650,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407819+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:37:42.407819+0000","last_peered":"2023-09-30T09:37:42.407819+0000","last_clean":"2023-09-30T09:37:42.407819+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:37:42.407819+0000","last_undegraded":"2023-09-30T09:37:42.407819+0000","last_fullsized":"2023-09-30T09:37:42.407819+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":548,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.891356+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:37:41.891356+0000","last_peered":"2023-09-30T09:37:41.891356+0000","last_clean":"2023-09-30T09:37:41.891356+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:37:41.891356+0000","last_undegraded":"2023-09-30T09:37:41.891356+0000","last_fullsized":"2023-09-30T09:37:41.891356+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":352,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.405739+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:37:42.405739+0000","last_peered":"2023-09-30T09:37:42.405739+0000","last_clean":"2023-09-30T09:37:42.405739+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:37:42.405739+0000","last_undegraded":"2023-09-30T09:37:42.405739+0000","last_fullsized":"2023-09-30T09:37:42.405739+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":45,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.405914+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:37:42.405914+0000","last_peered":"2023-09-30T09:37:42.405914+0000","last_clean":"2023-09-30T09:37:42.405914+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:37:42.405914+0000","last_undegraded":"2023-09-30T09:37:42.405914+0000","last_fullsized":"2023-09-30T09:37:42.405914+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":818,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406315+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:37:42.406315+0000","last_peered":"2023-09-30T09:37:42.406315+0000","last_clean":"2023-09-30T09:37:42.406315+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:37:42.406315+0000","last_undegraded":"2023-09-30T09:37:42.406315+0000","last_fullsized":"2023-09-30T09:37:42.406315+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":347,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408141+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:37:42.408141+0000","last_peered":"2023-09-30T09:37:42.408141+0000","last_clean":"2023-09-30T09:37:42.408141+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:37:42.408141+0000","last_undegraded":"2023-09-30T09:37:42.408141+0000","last_fullsized":"2023-09-30T09:37:42.408141+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":335,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408264+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:37:42.408264+0000","last_peered":"2023-09-30T09:37:42.408264+0000","last_clean":"2023-09-30T09:37:42.408264+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:37:42.408264+0000","last_undegraded":"2023-09-30T09:37:42.408264+0000","last_fullsized":"2023-09-30T09:37:42.408264+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":74,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.890015+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:37:41.890015+0000","last_peered":"2023-09-30T09:37:41.890015+0000","last_clean":"2023-09-30T09:37:41.890015+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:37:41.890015+0000","last_undegraded":"2023-09-30T09:37:41.890015+0000","last_fullsized":"2023-09-30T09:37:41.890015+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":716,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.890267+0000","last_change":"2023-09-30T09:34:40.869212+0000","last_active":"2023-09-30T09:37:41.890267+0000","last_peered":"2023-09-30T09:37:41.890267+0000","last_clean":"2023-09-30T09:37:41.890267+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:37:41.890267+0000","last_undegraded":"2023-09-30T09:37:41.890267+0000","last_fullsized":"2023-09-30T09:37:41.890267+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:52.056816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:52.056816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":535,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889659+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:37:41.889659+0000","last_peered":"2023-09-30T09:37:41.889659+0000","last_clean":"2023-09-30T09:37:41.889659+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:37:41.889659+0000","last_undegraded":"2023-09-30T09:37:41.889659+0000","last_fullsized":"2023-09-30T09:37:41.889659+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":82,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889220+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:37:41.889220+0000","last_peered":"2023-09-30T09:37:41.889220+0000","last_clean":"2023-09-30T09:37:41.889220+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:37:41.889220+0000","last_undegraded":"2023-09-30T09:37:41.889220+0000","last_fullsized":"2023-09-30T09:37:41.889220+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":275,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407093+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:37:42.407093+0000","last_peered":"2023-09-30T09:37:42.407093+0000","last_clean":"2023-09-30T09:37:42.407093+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:37:42.407093+0000","last_undegraded":"2023-09-30T09:37:42.407093+0000","last_fullsized":"2023-09-30T09:37:42.407093+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":928,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406155+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:37:42.406155+0000","last_peered":"2023-09-30T09:37:42.406155+0000","last_clean":"2023-09-30T09:37:42.406155+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:37:42.406155+0000","last_undegraded":"2023-09-30T09:37:42.406155+0000","last_fullsized":"2023-09-30T09:37:42.406155+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":621,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889737+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:37:41.889737+0000","last_peered":"2023-09-30T09:37:41.889737+0000","last_clean":"2023-09-30T09:37:41.889737+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:37:41.889737+0000","last_undegraded":"2023-09-30T09:37:41.889737+0000","last_fullsized":"2023-09-30T09:37:41.889737+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":275,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.895090+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:37:41.895090+0000","last_peered":"2023-09-30T09:37:41.895090+0000","last_clean":"2023-09-30T09:37:41.895090+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:37:41.895090+0000","last_undegraded":"2023-09-30T09:37:41.895090+0000","last_fullsized":"2023-09-30T09:37:41.895090+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":308,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407145+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:37:42.407145+0000","last_peered":"2023-09-30T09:37:42.407145+0000","last_clean":"2023-09-30T09:37:42.407145+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:37:42.407145+0000","last_undegraded":"2023-09-30T09:37:42.407145+0000","last_fullsized":"2023-09-30T09:37:42.407145+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":501,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407522+0000","last_change":"2023-09-30T09:33:32.748871+0000","last_active":"2023-09-30T09:37:42.407522+0000","last_peered":"2023-09-30T09:37:42.407522+0000","last_clean":"2023-09-30T09:37:42.407522+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:37:42.407522+0000","last_undegraded":"2023-09-30T09:37:42.407522+0000","last_fullsized":"2023-09-30T09:37:42.407522+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.412569+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.412569+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":732,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889817+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:37:41.889817+0000","last_peered":"2023-09-30T09:37:41.889817+0000","last_clean":"2023-09-30T09:37:41.889817+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:37:41.889817+0000","last_undegraded":"2023-09-30T09:37:41.889817+0000","last_fullsized":"2023-09-30T09:37:41.889817+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":285,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:45.255786+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:37:45.255786+0000","last_peered":"2023-09-30T09:37:45.255786+0000","last_clean":"2023-09-30T09:37:45.255786+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:37:45.255786+0000","last_undegraded":"2023-09-30T09:37:45.255786+0000","last_fullsized":"2023-09-30T09:37:45.255786+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":41,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.891913+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:37:41.891913+0000","last_peered":"2023-09-30T09:37:41.891913+0000","last_clean":"2023-09-30T09:37:41.891913+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:37:41.891913+0000","last_undegraded":"2023-09-30T09:37:41.891913+0000","last_fullsized":"2023-09-30T09:37:41.891913+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":630,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408305+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:37:42.408305+0000","last_peered":"2023-09-30T09:37:42.408305+0000","last_clean":"2023-09-30T09:37:42.408305+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:37:42.408305+0000","last_undegraded":"2023-09-30T09:37:42.408305+0000","last_fullsized":"2023-09-30T09:37:42.408305+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":847,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.891032+0000","last_change":"2023-09-30T09:37:36.568108+0000","last_active":"2023-09-30T09:37:41.891032+0000","last_peered":"2023-09-30T09:37:41.891032+0000","last_clean":"2023-09-30T09:37:41.891032+0000","last_became_active":"2023-09-30T09:37:36.567318+0000","last_became_peered":"2023-09-30T09:37:36.567318+0000","last_unstale":"2023-09-30T09:37:41.891032+0000","last_undegraded":"2023-09-30T09:37:41.891032+0000","last_fullsized":"2023-09-30T09:37:41.891032+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:25:45.846255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":82,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889856+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:37:41.889856+0000","last_peered":"2023-09-30T09:37:41.889856+0000","last_clean":"2023-09-30T09:37:41.889856+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:37:41.889856+0000","last_undegraded":"2023-09-30T09:37:41.889856+0000","last_fullsized":"2023-09-30T09:37:41.889856+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":257,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-30T09:37:46.908639+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:37:46.908639+0000","last_peered":"2023-09-30T09:37:46.908639+0000","last_clean":"2023-09-30T09:37:46.908639+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:37:46.908639+0000","last_undegraded":"2023-09-30T09:37:46.908639+0000","last_fullsized":"2023-09-30T09:37:46.908639+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":828,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406913+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:37:42.406913+0000","last_peered":"2023-09-30T09:37:42.406913+0000","last_clean":"2023-09-30T09:37:42.406913+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:37:42.406913+0000","last_undegraded":"2023-09-30T09:37:42.406913+0000","last_fullsized":"2023-09-30T09:37:42.406913+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":357,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893807+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:37:41.893807+0000","last_peered":"2023-09-30T09:37:41.893807+0000","last_clean":"2023-09-30T09:37:41.893807+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:37:41.893807+0000","last_undegraded":"2023-09-30T09:37:41.893807+0000","last_fullsized":"2023-09-30T09:37:41.893807+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1122,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.408065+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:37:42.408065+0000","last_peered":"2023-09-30T09:37:42.408065+0000","last_clean":"2023-09-30T09:37:42.408065+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:37:42.408065+0000","last_undegraded":"2023-09-30T09:37:42.408065+0000","last_fullsized":"2023-09-30T09:37:42.408065+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":374,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893422+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:37:41.893422+0000","last_peered":"2023-09-30T09:37:41.893422+0000","last_clean":"2023-09-30T09:37:41.893422+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:37:41.893422+0000","last_undegraded":"2023-09-30T09:37:41.893422+0000","last_fullsized":"2023-09-30T09:37:41.893422+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":822,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.894902+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:37:41.894902+0000","last_peered":"2023-09-30T09:37:41.894902+0000","last_clean":"2023-09-30T09:37:41.894902+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:37:41.894902+0000","last_undegraded":"2023-09-30T09:37:41.894902+0000","last_fullsized":"2023-09-30T09:37:41.894902+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":681,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407967+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:37:42.407967+0000","last_peered":"2023-09-30T09:37:42.407967+0000","last_clean":"2023-09-30T09:37:42.407967+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:37:42.407967+0000","last_undegraded":"2023-09-30T09:37:42.407967+0000","last_fullsized":"2023-09-30T09:37:42.407967+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":44,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.892010+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:37:41.892010+0000","last_peered":"2023-09-30T09:37:41.892010+0000","last_clean":"2023-09-30T09:37:41.892010+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:37:41.892010+0000","last_undegraded":"2023-09-30T09:37:41.892010+0000","last_fullsized":"2023-09-30T09:37:41.892010+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":297,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407995+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:37:42.407995+0000","last_peered":"2023-09-30T09:37:42.407995+0000","last_clean":"2023-09-30T09:37:42.407995+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:37:42.407995+0000","last_undegraded":"2023-09-30T09:37:42.407995+0000","last_fullsized":"2023-09-30T09:37:42.407995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":770,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893626+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:37:41.893626+0000","last_peered":"2023-09-30T09:37:41.893626+0000","last_clean":"2023-09-30T09:37:41.893626+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:37:41.893626+0000","last_undegraded":"2023-09-30T09:37:41.893626+0000","last_fullsized":"2023-09-30T09:37:41.893626+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.890809+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:37:41.890809+0000","last_peered":"2023-09-30T09:37:41.890809+0000","last_clean":"2023-09-30T09:37:41.890809+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:37:41.890809+0000","last_undegraded":"2023-09-30T09:37:41.890809+0000","last_fullsized":"2023-09-30T09:37:41.890809+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":33,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.889765+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:37:41.889765+0000","last_peered":"2023-09-30T09:37:41.889765+0000","last_clean":"2023-09-30T09:37:41.889765+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:37:41.889765+0000","last_undegraded":"2023-09-30T09:37:41.889765+0000","last_fullsized":"2023-09-30T09:37:41.889765+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":541,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893527+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:37:41.893527+0000","last_peered":"2023-09-30T09:37:41.893527+0000","last_clean":"2023-09-30T09:37:41.893527+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:37:41.893527+0000","last_undegraded":"2023-09-30T09:37:41.893527+0000","last_fullsized":"2023-09-30T09:37:41.893527+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":334,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407702+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:37:42.407702+0000","last_peered":"2023-09-30T09:37:42.407702+0000","last_clean":"2023-09-30T09:37:42.407702+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:37:42.407702+0000","last_undegraded":"2023-09-30T09:37:42.407702+0000","last_fullsized":"2023-09-30T09:37:42.407702+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":50,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407754+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:37:42.407754+0000","last_peered":"2023-09-30T09:37:42.407754+0000","last_clean":"2023-09-30T09:37:42.407754+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:37:42.407754+0000","last_undegraded":"2023-09-30T09:37:42.407754+0000","last_fullsized":"2023-09-30T09:37:42.407754+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":300,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407432+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:37:42.407432+0000","last_peered":"2023-09-30T09:37:42.407432+0000","last_clean":"2023-09-30T09:37:42.407432+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:37:42.407432+0000","last_undegraded":"2023-09-30T09:37:42.407432+0000","last_fullsized":"2023-09-30T09:37:42.407432+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":50,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.406699+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:37:42.406699+0000","last_peered":"2023-09-30T09:37:42.406699+0000","last_clean":"2023-09-30T09:37:42.406699+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:37:42.406699+0000","last_undegraded":"2023-09-30T09:37:42.406699+0000","last_fullsized":"2023-09-30T09:37:42.406699+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":484,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407103+0000","last_change":"2023-09-30T09:34:10.498047+0000","last_active":"2023-09-30T09:37:42.407103+0000","last_peered":"2023-09-30T09:37:42.407103+0000","last_clean":"2023-09-30T09:37:42.407103+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:37:42.407103+0000","last_undegraded":"2023-09-30T09:37:42.407103+0000","last_fullsized":"2023-09-30T09:37:42.407103+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:00.045701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:00.045701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":353,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893894+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:37:41.893894+0000","last_peered":"2023-09-30T09:37:41.893894+0000","last_clean":"2023-09-30T09:37:41.893894+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:37:41.893894+0000","last_undegraded":"2023-09-30T09:37:41.893894+0000","last_fullsized":"2023-09-30T09:37:41.893894+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":377,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.893860+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:37:41.893860+0000","last_peered":"2023-09-30T09:37:41.893860+0000","last_clean":"2023-09-30T09:37:41.893860+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:37:41.893860+0000","last_undegraded":"2023-09-30T09:37:41.893860+0000","last_fullsized":"2023-09-30T09:37:41.893860+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":805,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.313232+0000","last_change":"2023-09-30T09:33:32.662406+0000","last_active":"2023-09-30T09:37:42.313232+0000","last_peered":"2023-09-30T09:37:42.313232+0000","last_clean":"2023-09-30T09:37:42.313232+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:37:42.313232+0000","last_undegraded":"2023-09-30T09:37:42.313232+0000","last_fullsized":"2023-09-30T09:37:42.313232+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.346428+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.346428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":60,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:41.892574+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:37:41.892574+0000","last_peered":"2023-09-30T09:37:41.892574+0000","last_clean":"2023-09-30T09:37:41.892574+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:37:41.892574+0000","last_undegraded":"2023-09-30T09:37:41.892574+0000","last_fullsized":"2023-09-30T09:37:41.892574+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":331,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-30T09:37:42.407390+0000","last_change":"2023-09-30T09:33:32.613706+0000","last_active":"2023-09-30T09:37:42.407390+0000","last_peered":"2023-09-30T09:37:42.407390+0000","last_clean":"2023-09-30T09:37:42.407390+0000","last_became_active":"2023-09-30T09:33:32.613407+0000","last_became_peered":"2023-09-30T09:33:32.613407+0000","last_unstale":"2023-09-30T09:37:42.407390+0000","last_undegraded":"2023-09-30T09:37:42.407390+0000","last_fullsized":"2023-09-30T09:37:42.407390+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:50.315323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":796101,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":171,"num_read_kb":320,"num_write":81,"num_write_kb":840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1410926,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":162,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234314,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495951,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408484,"kb_used_data":4124,"kb_used_omap":0,"kb_used_meta":404352,"kb_avail":93336668,"statfs":{"total":95995035648,"available":95576748032,"internally_reserved":0,"allocated":4222976,"data_stored":2998126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":414056448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9380000000000002,"5min":2.2330000000000001,"15min":1.802},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":44.113,"5min":44.113,"15min":45.927999999999997},"last":0.53400000000000003},{"interface":"front","average":{"1min":4.6150000000000002,"5min":2.3639999999999999,"15min":2.0449999999999999},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":44.164000000000001,"5min":44.164000000000001,"15min":45.316000000000003},"last":0.34799999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":2.5910000000000002,"15min":2.0649999999999999},"min":{"1min":0.316,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":52.893000000000001,"5min":52.893000000000001,"15min":52.893000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":4.673,"5min":2.8519999999999999,"15min":2.3260000000000001},"min":{"1min":0.47799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":52.905999999999999,"5min":52.905999999999999,"15min":52.905999999999999},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:37:31 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":2.2040000000000002,"15min":1.9910000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.1200000000000001,"5min":17.126000000000001,"15min":37.978000000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.272,"5min":2.3399999999999999,"15min":1.8959999999999999},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2720000000000002,"5min":25.164999999999999,"15min":37.759999999999998},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7839999999999998,"5min":7.1130000000000004,"15min":3.532},"min":{"1min":0.45300000000000001,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":90.147999999999996,"5min":212.44800000000001,"15min":212.44800000000001},"last":0.44},{"interface":"front","average":{"1min":8.9450000000000003,"5min":7.8339999999999996,"15min":3.9199999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":89.859999999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.0640000000000001,"15min":1.593},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23400000000000001},"max":{"1min":3.004,"5min":7.4359999999999999,"15min":73.656000000000006},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.0209999999999999,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.23000000000000001},"max":{"1min":1.8089999999999999,"5min":9.6590000000000007,"15min":73.882999999999996},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.1930000000000001,"15min":1.819},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":1.024,"5min":8.2040000000000006,"15min":76.613},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.1910000000000001,"15min":1.786},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.0979999999999999,"5min":9.7750000000000004,"15min":76.329999999999998},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:37:31 2023","interfaces":[{"interface":"back","average":{"1min":8.1950000000000003,"5min":2.7679999999999998,"15min":2.6499999999999999},"min":{"1min":0.371,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":138.191,"5min":138.191,"15min":138.191},"last":0.42599999999999999},{"interface":"front","average":{"1min":8.0210000000000008,"5min":2.5619999999999998,"15min":2.5830000000000002},"min":{"1min":0.35099999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":140.33500000000001,"5min":140.33500000000001,"15min":140.33500000000001},"last":0.51200000000000001}]}]},{"osd":1,"up_from":301,"seq":1292785156149,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328664,"kb_used_data":9036,"kb_used_omap":9,"kb_used_meta":319606,"kb_avail":93416488,"statfs":{"total":95995035648,"available":95658483712,"internally_reserved":0,"allocated":9252864,"data_stored":7233986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":327276562},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":5.5,"5min":4.6900000000000004,"15min":3.258},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":59.115000000000002,"5min":138.43799999999999,"15min":138.43799999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":3.347,"5min":4.5780000000000003,"15min":3.5419999999999998},"min":{"1min":0.32600000000000001,"5min":0.25,"15min":0.25},"max":{"1min":12.332000000000001,"5min":138.482,"15min":138.482},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":2.895,"15min":2.8290000000000002},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":11.595000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":5.5129999999999999,"5min":3.6520000000000001,"15min":3.0209999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":59.078000000000003,"5min":59.078000000000003,"15min":59.078000000000003},"last":0.54500000000000004}]},{"osd":3,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":14.836,"5min":9.0150000000000006,"15min":9.3030000000000008},"min":{"1min":0.32000000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":191.07300000000001,"5min":191.07300000000001,"15min":191.07300000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":13.679,"5min":8.9689999999999994,"15min":9.7739999999999991},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":191.11500000000001,"5min":191.11500000000001,"15min":191.11500000000001},"last":0.48399999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.0569999999999999,"15min":2.0099999999999998},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.1099999999999994,"5min":13.602,"15min":13.602},"last":0.64300000000000002},{"interface":"front","average":{"1min":4.8929999999999998,"5min":2.4319999999999999,"15min":1.6719999999999999},"min":{"1min":0.35099999999999998,"5min":0.311,"15min":0.311},"max":{"1min":58.939999999999998,"5min":58.939999999999998,"15min":58.939999999999998},"last":0.69899999999999995}]},{"osd":5,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":3.0640000000000001,"5min":2.5419999999999998,"15min":2.2530000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":24.25,"5min":24.25,"15min":24.25},"last":0.73599999999999999},{"interface":"front","average":{"1min":6.4749999999999996,"5min":2.9630000000000001,"15min":2.0550000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":58.999000000000002,"5min":58.999000000000002,"15min":58.999000000000002},"last":0.65300000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":6.0739999999999998,"5min":9.3149999999999995,"15min":14.503},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":58.683999999999997,"5min":209.221,"15min":209.221},"last":0.71099999999999997},{"interface":"front","average":{"1min":6.4400000000000004,"5min":9.0920000000000005,"15min":14.374000000000001},"min":{"1min":0.40899999999999997,"5min":0.377,"15min":0.377},"max":{"1min":59.143000000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":0.67500000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":3.222,"5min":6.5190000000000001,"15min":10.468999999999999},"min":{"1min":0.495,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":11.73,"5min":165.59399999999999,"15min":165.59399999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":2.9969999999999999,"5min":6.6310000000000002,"15min":10.663},"min":{"1min":0.46000000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":11.692,"5min":165.833,"15min":165.833},"last":0.77500000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926048,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":376312,"kb_used_data":9260,"kb_used_omap":5,"kb_used_meta":367034,"kb_avail":93368840,"statfs":{"total":95995035648,"available":95609692160,"internally_reserved":0,"allocated":9482240,"data_stored":7318366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":375843069},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":19,"apply_latency_ms":19,"commit_latency_ns":19000000,"apply_latency_ns":19000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":2.835,"15min":2.9430000000000001},"min":{"1min":0.20799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":7.4820000000000002,"5min":17.966000000000001,"15min":17.966000000000001},"last":0.314},{"interface":"front","average":{"1min":2.552,"5min":6.3799999999999999,"15min":8.4100000000000001},"min":{"1min":0.35199999999999998,"5min":0.22,"15min":0.22},"max":{"1min":7.9210000000000003,"5min":138.92400000000001,"15min":138.92400000000001},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:37:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9750000000000001,"5min":2.5209999999999999,"15min":1.9179999999999999},"min":{"1min":0.26600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":15.324,"5min":15.324,"15min":55.823999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.8799999999999999,"5min":2.976,"15min":2.0950000000000002},"min":{"1min":0.36199999999999999,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":14.848000000000001,"5min":40.128999999999998,"15min":55.790999999999997},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":8.5350000000000001,"15min":4.2089999999999996},"min":{"1min":0.30599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":16.271000000000001,"5min":218.06999999999999,"15min":218.06999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.8199999999999998,"5min":6.2930000000000001,"15min":3.4159999999999999},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":14.987,"5min":218.63200000000001,"15min":218.63200000000001},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.6599999999999999,"15min":1.679},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.29099999999999998},"max":{"1min":10.614000000000001,"5min":10.614000000000001,"15min":121.486},"last":0.59499999999999997},{"interface":"front","average":{"1min":3.3780000000000001,"5min":3.863,"15min":2.649},"min":{"1min":0.39900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":17.044,"5min":136.262,"15min":136.262},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":4.1710000000000003,"15min":3.0630000000000002},"min":{"1min":0.439,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.995999999999999,"5min":136.34,"15min":136.34},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.4129999999999998,"5min":1.9419999999999999,"15min":2.2839999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":11.991,"5min":12.717000000000001,"15min":135.381},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":7.1440000000000001,"15min":3.2400000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":12.021000000000001,"5min":214.785,"15min":214.785},"last":0.502},{"interface":"front","average":{"1min":2.8999999999999999,"5min":7.2439999999999998,"15min":3.1850000000000001},"min":{"1min":0.46300000000000002,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":10.695,"5min":214.93899999999999,"15min":214.93899999999999},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 30 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":2.9870000000000001,"15min":3.1720000000000002},"min":{"1min":0.42999999999999999,"5min":0.38300000000000001,"15min":0.373},"max":{"1min":12.048,"5min":97.382000000000005,"15min":150.96899999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.944,"15min":3.0150000000000001},"min":{"1min":0.43099999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.153,"5min":97.015000000000001,"15min":150.328},"last":0.52700000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598989,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325032,"kb_used_data":4892,"kb_used_omap":5,"kb_used_meta":320122,"kb_avail":93420120,"statfs":{"total":95995035648,"available":95662202880,"internally_reserved":0,"allocated":5009408,"data_stored":3037241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":327805581},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.8420000000000001,"15min":2.242},"min":{"1min":0.20499999999999999,"5min":0.187,"15min":0.187},"max":{"1min":6.9480000000000004,"5min":77.379000000000005,"15min":77.379000000000005},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.8010000000000002,"15min":2.0750000000000002},"min":{"1min":0.29499999999999998,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":10.590999999999999,"5min":76.796999999999997,"15min":77.066999999999993},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":4.2720000000000002,"5min":4.9740000000000002,"15min":4.0389999999999997},"min":{"1min":0.35799999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":37.701999999999998,"5min":64.412999999999997,"15min":64.412999999999997},"last":0.33200000000000002},{"interface":"front","average":{"1min":3.9129999999999998,"5min":4.2969999999999997,"15min":3.4700000000000002},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":37.728999999999999,"5min":63.247999999999998,"15min":63.247999999999998},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:36:51 2023","interfaces":[{"interface":"back","average":{"1min":15.153,"5min":7.4130000000000003,"15min":3.4689999999999999},"min":{"1min":0.46300000000000002,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":134.078,"5min":134.078,"15min":134.078},"last":0.64800000000000002},{"interface":"front","average":{"1min":15.754,"5min":7.0709999999999997,"15min":3.5920000000000001},"min":{"1min":0.38900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":135.73400000000001,"5min":135.73400000000001,"15min":135.73400000000001},"last":0.56599999999999995}]},{"osd":4,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":2.1909999999999998,"15min":1.403},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":6.5979999999999999,"5min":35.972000000000001,"15min":35.972000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.099,"5min":1.917,"15min":1.6279999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":2.3759999999999999,"5min":36.518000000000001,"15min":36.518000000000001},"last":0.443}]},{"osd":5,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":2.2989999999999999,"15min":2.2200000000000002},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.6280000000000001,"5min":12.651999999999999,"15min":155.477},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.585,"5min":1.6850000000000001,"15min":2.0139999999999998},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":4.9450000000000003,"5min":8.0839999999999996,"15min":155.46000000000001},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":5.5700000000000003,"15min":3.3690000000000002},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.8040000000000003,"5min":154.417,"15min":154.417},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.3809999999999998,"5min":6.3570000000000002,"15min":3.4540000000000002},"min":{"1min":0.39000000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":7.6799999999999997,"5min":154.04400000000001,"15min":154.04400000000001},"last":0.502}]},{"osd":7,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":3.7839999999999998,"15min":3.1890000000000001},"min":{"1min":0.45000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0229999999999997,"5min":123.294,"15min":213.404},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.2970000000000002,"5min":4.3479999999999999,"15min":3.2749999999999999},"min":{"1min":0.48099999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.71,"5min":123.16,"15min":213.27199999999999},"last":0.628}]}]},{"osd":3,"up_from":109,"seq":468151435462,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":688624,"kb_used_data":9188,"kb_used_omap":2,"kb_used_meta":679421,"kb_avail":93056528,"statfs":{"total":95995035648,"available":95289884672,"internally_reserved":0,"allocated":9408512,"data_stored":7305592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":695727463},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":5.4980000000000002,"5min":4.9889999999999999,"15min":3.097},"min":{"1min":0.23499999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":51.835000000000001,"5min":102.57299999999999,"15min":130.946},"last":0.439},{"interface":"front","average":{"1min":5.6929999999999996,"5min":4.899,"15min":3.0579999999999998},"min":{"1min":0.317,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":52.093000000000004,"5min":102.745,"15min":130.96899999999999},"last":0.38400000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":8.1340000000000003,"5min":11.768000000000001,"15min":18.100000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":98.826999999999998,"5min":273.99599999999998,"15min":273.99599999999998},"last":0.41899999999999998},{"interface":"front","average":{"1min":7.7400000000000002,"5min":11.098000000000001,"15min":16.379999999999999},"min":{"1min":0.28899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":98.918000000000006,"5min":274.21600000000001,"15min":274.21600000000001},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8360000000000003,"5min":4.5800000000000001,"15min":3.3759999999999999},"min":{"1min":0.53600000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":62.289000000000001,"5min":76.974000000000004,"15min":145.75899999999999},"last":0.31},{"interface":"front","average":{"1min":6.0650000000000004,"5min":4.3339999999999996,"15min":3.2170000000000001},"min":{"1min":0.33400000000000002,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":65.415999999999997,"5min":77.593999999999994,"15min":150.358},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":2.419,"5min":2.702,"15min":1.6899999999999999},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":10.805,"5min":60.637,"15min":60.637},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.008,"5min":2.3100000000000001,"15min":1.627},"min":{"1min":0.40100000000000002,"5min":0.35999999999999999,"15min":0.27400000000000002},"max":{"1min":8.8049999999999997,"5min":10.731,"15min":15.638},"last":0.72599999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":2.9940000000000002,"15min":1.9390000000000001},"min":{"1min":0.375,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":8.8309999999999995,"5min":37.485999999999997,"15min":37.485999999999997},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.9199999999999999,"5min":3.2930000000000001,"15min":2.113},"min":{"1min":0.437,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":7.7439999999999998,"5min":37.448,"15min":37.448},"last":0.53500000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":2.9740000000000002,"15min":1.8240000000000001},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.891999999999999,"5min":60.698,"15min":60.698},"last":0.503},{"interface":"front","average":{"1min":3.895,"5min":2.7629999999999999,"15min":1.8999999999999999},"min":{"1min":0.61299999999999999,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":12.121,"5min":12.121,"15min":14.69},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:37:02 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":3.2469999999999999,"15min":2.7440000000000002},"min":{"1min":0.44500000000000001,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":7.7709999999999999,"5min":26.469000000000001,"15min":226.03999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.8370000000000002,"5min":3.1989999999999998,"15min":2.6019999999999999},"min":{"1min":0.41799999999999998,"5min":0.38800000000000001,"15min":0.34799999999999998},"max":{"1min":10.855,"5min":60.670000000000002,"15min":226.06100000000001},"last":0.59099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019224,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278692,"kb_used_data":3804,"kb_used_omap":0,"kb_used_meta":274880,"kb_avail":93466460,"statfs":{"total":95995035648,"available":95709655040,"internally_reserved":0,"allocated":3895296,"data_stored":2670527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":281477120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:42 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":1.958,"15min":3.0649999999999999},"min":{"1min":0.314,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":26.309999999999999,"5min":26.309999999999999,"15min":128.79300000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.6230000000000002,"5min":2.2389999999999999,"15min":3.0129999999999999},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.248},"max":{"1min":26.433,"5min":26.433,"15min":128.38800000000001},"last":0.47799999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:37:42 2023","interfaces":[{"interface":"back","average":{"1min":4.1150000000000002,"5min":5.7889999999999997,"15min":8.2650000000000006},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":25.663,"5min":160.69999999999999,"15min":160.69999999999999},"last":0.71699999999999997},{"interface":"front","average":{"1min":2.8370000000000002,"5min":5.4500000000000002,"15min":8.1929999999999996},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":25.367999999999999,"5min":160.50700000000001,"15min":160.50700000000001},"last":0.68500000000000005}]},{"osd":2,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":2.089,"15min":2.9980000000000002},"min":{"1min":0.309,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.984,"5min":19.391999999999999,"15min":140.601},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.2080000000000002,"15min":3.0230000000000001},"min":{"1min":0.36899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":7.085,"5min":19.422999999999998,"15min":140.54900000000001},"last":0.77700000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:37:48 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":3.399,"15min":3.802},"min":{"1min":0.40500000000000003,"5min":0.315,"15min":0.26400000000000001},"max":{"1min":28.527000000000001,"5min":52.762,"15min":281.42200000000003},"last":0.752},{"interface":"front","average":{"1min":3.4390000000000001,"5min":3.4220000000000002,"15min":3.7599999999999998},"min":{"1min":0.39100000000000001,"5min":0.312,"15min":0.312},"max":{"1min":28.486000000000001,"5min":52.790999999999997,"15min":281.44999999999999},"last":0.57599999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:37:42 2023","interfaces":[{"interface":"back","average":{"1min":10.028,"5min":2.766,"15min":2.3500000000000001},"min":{"1min":0.30099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":172.97800000000001,"5min":172.97800000000001,"15min":172.97800000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":9.8870000000000005,"5min":2.8039999999999998,"15min":2.395},"min":{"1min":0.29199999999999998,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":172.92400000000001,"5min":172.92400000000001,"15min":172.92400000000001},"last":0.623}]},{"osd":6,"last update":"Sat Sep 30 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":3.0299999999999998,"15min":1.593},"min":{"1min":0.32800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.1079999999999997,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.91700000000000004,"5min":2.927,"15min":1.597},"min":{"1min":0.34599999999999997,"5min":0.249,"15min":0.249},"max":{"1min":2.6459999999999999,"5min":202.42400000000001,"15min":202.42400000000001},"last":0.64200000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.0910000000000002,"15min":3.0619999999999998},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.30299999999999999},"max":{"1min":5.4059999999999997,"5min":85.114000000000004,"15min":230.90100000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.22,"5min":1.772,"15min":2.5960000000000001},"min":{"1min":0.33200000000000002,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":7.1550000000000002,"5min":84.697999999999993,"15min":229.97},"last":0.82599999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757588,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":289952,"kb_used_data":4056,"kb_used_omap":0,"kb_used_meta":285888,"kb_avail":93455200,"statfs":{"total":95995035648,"available":95698124800,"internally_reserved":0,"allocated":4153344,"data_stored":2929197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":292749312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.2069999999999999,"15min":2.4540000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.22},"max":{"1min":4.6459999999999999,"5min":56.563000000000002,"15min":148.79900000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.738,"5min":2.3199999999999998,"15min":2.7589999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":6.2969999999999997,"5min":57.241999999999997,"15min":148.779},"last":0.48099999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:37:40 2023","interfaces":[{"interface":"back","average":{"1min":3.3079999999999998,"5min":2.956,"15min":2.21},"min":{"1min":0.27300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":13.363,"5min":60.799999999999997,"15min":60.799999999999997},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.5790000000000002,"5min":3.464,"15min":3.262},"min":{"1min":0.36199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":16.757000000000001,"5min":63.798999999999999,"15min":63.798999999999999},"last":0.75600000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:37:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9199999999999999,"15min":3.149},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.9180000000000001,"5min":30.613,"15min":165.768},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.327,"15min":3.714},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.5439999999999996,"5min":36.488,"15min":166.24000000000001},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:36:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":5.9180000000000001,"15min":4.0069999999999997},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5380000000000003,"5min":172.976,"15min":226.34700000000001},"last":0.79400000000000004},{"interface":"front","average":{"1min":2.4689999999999999,"5min":5.8799999999999999,"15min":3.8300000000000001},"min":{"1min":0.376,"5min":0.36599999999999999,"15min":0.313},"max":{"1min":8.2240000000000002,"5min":173.01599999999999,"15min":226.36000000000001},"last":0.53600000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.76900000000000002,"15min":0.85999999999999999},"min":{"1min":0.26400000000000001,"5min":0.25800000000000001,"15min":0.222},"max":{"1min":3.1360000000000001,"5min":5.7160000000000002,"15min":8.5359999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.94799999999999995,"15min":0.95499999999999996},"min":{"1min":0.28499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.4790000000000001,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.63300000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":2.9980000000000002,"15min":1.7509999999999999},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":3.2069999999999999,"5min":155.602,"15min":155.602},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":3.012,"15min":1.867},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.9490000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.82299999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":2.5590000000000002,"15min":1.659},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.883,"5min":133.40700000000001,"15min":133.40700000000001},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":2.7229999999999999,"15min":1.5840000000000001},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":5.7880000000000003,"5min":132.797,"15min":132.797},"last":0.71899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:37:58.138 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:58.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:58.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:58.190 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:58.225 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[224043]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[224043]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[224043]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[224043]: 1.1d scrub starts 2023-09-30T09:37:58.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[224043]: 1.1d scrub ok 2023-09-30T09:37:58.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[224043]: osdmap e360: 8 total, 8 up, 7 in 2023-09-30T09:37:58.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[216815]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[216815]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[216815]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[216815]: 1.1d scrub starts 2023-09-30T09:37:58.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[216815]: 1.1d scrub ok 2023-09-30T09:37:58.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:58 smithi019 ceph-mon[216815]: osdmap e360: 8 total, 8 up, 7 in 2023-09-30T09:37:58.242 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:37:58.250 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:37:58.282 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:58.331 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:37:58.337 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:58 smithi167 ceph-mon[192880]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.337 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:58 smithi167 ceph-mon[192880]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.337 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:58 smithi167 ceph-mon[192880]: failed to encode map e359 with expected crc 2023-09-30T09:37:58.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:58 smithi167 ceph-mon[192880]: 1.1d scrub starts 2023-09-30T09:37:58.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:58 smithi167 ceph-mon[192880]: 1.1d scrub ok 2023-09-30T09:37:58.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:58 smithi167 ceph-mon[192880]: osdmap e360: 8 total, 8 up, 7 in 2023-09-30T09:37:58.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:58.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:37:58.439 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:37:58.469 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:58.512 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:37:58.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:58.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:37:58.619 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:37:58.650 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:58.902 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:37:58.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:58.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:37:59.007 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:37:59.037 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:59 smithi019 ceph-mon[224043]: pgmap v930: 119 pgs: 12 unknown, 107 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:59.037 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:59 smithi019 ceph-mon[224043]: failed to encode map e360 with expected crc 2023-09-30T09:37:59.038 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:37:59 smithi019 ceph-mon[224043]: failed to encode map e360 with expected crc 2023-09-30T09:37:59.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:59 smithi019 ceph-mon[216815]: pgmap v930: 119 pgs: 12 unknown, 107 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:59.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:59 smithi019 ceph-mon[216815]: failed to encode map e360 with expected crc 2023-09-30T09:37:59.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:37:59 smithi019 ceph-mon[216815]: failed to encode map e360 with expected crc 2023-09-30T09:37:59.040 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:59.096 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:37:59.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:37:59.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:37:59.168 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:37:59.168 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:37:59.168 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 0 2023-09-30T09:37:59.168 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:37:59.200 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:37:59.230 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:37:59.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:37:59.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:59 smithi167 ceph-mon[192880]: pgmap v930: 119 pgs: 12 unknown, 107 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:37:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:59 smithi167 ceph-mon[192880]: failed to encode map e360 with expected crc 2023-09-30T09:37:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:37:59 smithi167 ceph-mon[192880]: failed to encode map e360 with expected crc 2023-09-30T09:38:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:00 smithi019 ceph-mon[216815]: failed to encode map e360 with expected crc 2023-09-30T09:38:00.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:00 smithi019 ceph-mon[216815]: failed to encode map e360 with expected crc 2023-09-30T09:38:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:00 smithi019 ceph-mon[224043]: failed to encode map e360 with expected crc 2023-09-30T09:38:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:00 smithi019 ceph-mon[224043]: failed to encode map e360 with expected crc 2023-09-30T09:38:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:00 smithi167 ceph-mon[192880]: failed to encode map e360 with expected crc 2023-09-30T09:38:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:00 smithi167 ceph-mon[192880]: failed to encode map e360 with expected crc 2023-09-30T09:38:00.733 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:00.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:00.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:00.760 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:38:00.840 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:00.865 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:00.889 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:01.051 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:01.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:01.125 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:38:01.156 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:01.200 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:01.314 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:01.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:01 smithi019 ceph-mon[224043]: pgmap v931: 119 pgs: 5 unknown, 114 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:01.317 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:01 smithi019 ceph-mon[216815]: pgmap v931: 119 pgs: 5 unknown, 114 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:01.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:01.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:01.419 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:01.448 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:01 smithi167 ceph-mon[192880]: pgmap v931: 119 pgs: 5 unknown, 114 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:01.451 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:01.501 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:01.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:01.607 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:01.638 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:01.791 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:01.792 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:01.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:01.894 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:01.923 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:01.968 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:01.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:01.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:02.057 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:38:02.063 INFO:teuthology.orchestra.run.smithi019.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-30T09:38:02.072 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:02.104 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:02.152 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:02.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:02.259 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:02.288 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:02.332 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[224043]: 1.2 scrub starts 2023-09-30T09:38:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[224043]: 1.2 scrub ok 2023-09-30T09:38:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/395490636' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3654833386' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:38:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:38:02.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[216815]: 1.2 scrub starts 2023-09-30T09:38:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[216815]: 1.2 scrub ok 2023-09-30T09:38:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/395490636' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3654833386' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:38:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:02 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:38:02.337 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:02 smithi167 ceph-mon[192880]: 1.2 scrub starts 2023-09-30T09:38:02.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:02 smithi167 ceph-mon[192880]: 1.2 scrub ok 2023-09-30T09:38:02.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:02 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/395490636' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:02.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:02 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3654833386' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:38:02.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:02 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:38:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:02.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:02.439 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:02.467 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:02.510 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:02.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:02.618 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:02.648 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:03.297 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:03.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:03.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[224043]: pgmap v932: 119 pgs: 119 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:38:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/395490636' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:38:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[224043]: osdmap e361: 8 total, 8 up, 7 in 2023-09-30T09:38:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[224043]: failed to encode map e361 with expected crc 2023-09-30T09:38:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[216815]: pgmap v932: 119 pgs: 119 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:38:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/395490636' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:38:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[216815]: osdmap e361: 8 total, 8 up, 7 in 2023-09-30T09:38:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[216815]: failed to encode map e361 with expected crc 2023-09-30T09:38:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:03.402 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:03.437 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:03.485 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:03.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:03.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:03.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:03 smithi167 ceph-mon[192880]: pgmap v932: 119 pgs: 119 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:03 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:38:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/395490636' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:38:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:03 smithi167 ceph-mon[192880]: osdmap e361: 8 total, 8 up, 7 in 2023-09-30T09:38:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:03 smithi167 ceph-mon[192880]: failed to encode map e361 with expected crc 2023-09-30T09:38:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:03.589 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:03.617 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:03.661 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:03.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:03.769 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:03.796 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:03.839 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:03.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:03.946 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:03.978 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:04.013 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:38:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:38:03] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.43.0" 2023-09-30T09:38:04.106 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:04.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:04.210 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:04.236 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:04.277 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:04.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:04.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:04.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[224043]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[224043]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[224043]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 0]}]: dispatch 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[224043]: pgmap v934: 119 pgs: 119 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[216815]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[216815]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[216815]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-09-30T09:38:04.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2023-09-30T09:38:04.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 0]}]: dispatch 2023-09-30T09:38:04.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-30T09:38:04.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:04 smithi019 ceph-mon[216815]: pgmap v934: 119 pgs: 119 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:04.382 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:04.405 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:04 smithi167 ceph-mon[192880]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.406 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:04 smithi167 ceph-mon[192880]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.406 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:04 smithi167 ceph-mon[192880]: failed to encode map e361 with expected crc 2023-09-30T09:38:04.406 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-09-30T09:38:04.406 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2023-09-30T09:38:04.406 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 0]}]: dispatch 2023-09-30T09:38:04.406 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-30T09:38:04.406 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:04 smithi167 ceph-mon[192880]: pgmap v934: 119 pgs: 119 active+clean; 4.7 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:04.408 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:04.449 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:04.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:04.517 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:38:04.538 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:38:04.555 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:04.558 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:38:04.580 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:38:04.585 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:04.600 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:38:04.622 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:38:04.632 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:04.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:04.641 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:38:04.661 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:38:04.662 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:38:04.662 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:38:04.662 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:38:04.662 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:38:04.662 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:38:04.663 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:38:04.663 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:38:04.663 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:38:04.736 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:04.763 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:04.807 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:04.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:04.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:04.911 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:04.942 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:05.186 DEBUG:teuthology.orchestra.run.smithi019:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-30T09:38:05.189 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-30T09:38:05.189 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-30T09:38:05.202 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:05.212 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:05.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:05.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:05.286 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:38:05.320 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]': finished 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 0]}]': finished 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: osdmap e362: 8 total, 8 up, 7 in 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: 1.11 scrub starts 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: 1.11 scrub ok 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2224645860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]': finished 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 0]}]': finished 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-30T09:38:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: osdmap e362: 8 total, 8 up, 7 in 2023-09-30T09:38:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: 1.11 scrub starts 2023-09-30T09:38:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: 1.11 scrub ok 2023-09-30T09:38:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2224645860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:38:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:05 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:38:05.349 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-09-30T09:38:05.349 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]': finished 2023-09-30T09:38:05.349 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 0]}]': finished 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: osdmap e362: 8 total, 8 up, 7 in 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: failed to encode map e362 with expected crc 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: 1.11 scrub starts 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: 1.11 scrub ok 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2224645860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:38:05.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:05 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:38:05.352 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:05.399 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:05.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:05.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:05.504 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:05.506 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:hints = 1 2023-09-30T09:38:05.506 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-30T09:38:05.507 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object prefix: benchmark_data_smithi019_478679 2023-09-30T09:38:05.507 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:38:05.507 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 0 0 0 0 0 0 - 0 2023-09-30T09:38:05.545 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:05.613 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:05.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:05.719 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:05.779 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:05.957 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:05.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:06.064 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:06.119 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:06.260 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:06.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:06.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: 1.15 deep-scrub starts 2023-09-30T09:38:06.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: 1.15 deep-scrub ok 2023-09-30T09:38:06.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: osdmap e363: 8 total, 8 up, 7 in 2023-09-30T09:38:06.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[224043]: pgmap v937: 119 pgs: 2 peering, 117 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: 1.15 deep-scrub starts 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: 1.15 deep-scrub ok 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: osdmap e363: 8 total, 8 up, 7 in 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:06 smithi019 ceph-mon[216815]: pgmap v937: 119 pgs: 2 peering, 117 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:06.365 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: 1.15 deep-scrub starts 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: 1.15 deep-scrub ok 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: osdmap e363: 8 total, 8 up, 7 in 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: failed to encode map e363 with expected crc 2023-09-30T09:38:06.398 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:06 smithi167 ceph-mon[192880]: pgmap v937: 119 pgs: 2 peering, 117 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-30T09:38:06.401 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:06.466 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:06.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:06.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:06.505 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 1 16 3045 3029 189.302 189.312 0.00266724 0.00526715 2023-09-30T09:38:06.526 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:06.559 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:38:06.559 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:38:06.574 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:06.605 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:06.667 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:06.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:06.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:06.774 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:06.838 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:07.505 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 2 16 6173 6157 192.388 195.5 0.0042266 0.00518457 2023-09-30T09:38:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:07 smithi167 ceph-mon[192880]: 1.9 deep-scrub starts 2023-09-30T09:38:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:07 smithi167 ceph-mon[192880]: 1.9 deep-scrub ok 2023-09-30T09:38:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:07 smithi167 ceph-mon[192880]: failed to encode map e363 with expected crc 2023-09-30T09:38:07.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:07 smithi019 ceph-mon[224043]: 1.9 deep-scrub starts 2023-09-30T09:38:07.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:07 smithi019 ceph-mon[224043]: 1.9 deep-scrub ok 2023-09-30T09:38:07.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:07 smithi019 ceph-mon[224043]: failed to encode map e363 with expected crc 2023-09-30T09:38:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:07 smithi019 ceph-mon[216815]: 1.9 deep-scrub starts 2023-09-30T09:38:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:07 smithi019 ceph-mon[216815]: 1.9 deep-scrub ok 2023-09-30T09:38:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:07 smithi019 ceph-mon[216815]: failed to encode map e363 with expected crc 2023-09-30T09:38:08.045 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:38:08.507 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 3 16 9018 9002 187.525 177.812 0.00415007 0.00532756 2023-09-30T09:38:08.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:08 smithi167 ceph-mon[192880]: pgmap v938: 119 pgs: 2 peering, 117 active+clean; 90 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 223 op/s 2023-09-30T09:38:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:08 smithi019 ceph-mon[216815]: pgmap v938: 119 pgs: 2 peering, 117 active+clean; 90 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 223 op/s 2023-09-30T09:38:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:08 smithi019 ceph-mon[224043]: pgmap v938: 119 pgs: 2 peering, 117 active+clean; 90 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 223 op/s 2023-09-30T09:38:09.303 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:09.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:09.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:09.410 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:09.460 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:09.461 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:09 smithi167 ceph-mon[192880]: 1.0 scrub starts 2023-09-30T09:38:09.461 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:09 smithi167 ceph-mon[192880]: 1.0 scrub ok 2023-09-30T09:38:09.461 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:38:09.461 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.461 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.461 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.506 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 4 16 11572 11556 180.545 159.625 0.00150322 0.00548476 2023-09-30T09:38:09.511 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:09.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[216815]: 1.0 scrub starts 2023-09-30T09:38:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[216815]: 1.0 scrub ok 2023-09-30T09:38:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:38:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[224043]: 1.0 scrub starts 2023-09-30T09:38:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[224043]: 1.0 scrub ok 2023-09-30T09:38:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:38:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:09.615 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:09.786 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:10.317 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:38:10.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:10 smithi019 ceph-mon[216815]: 1.b scrub starts 2023-09-30T09:38:10.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:10 smithi019 ceph-mon[216815]: 1.b scrub ok 2023-09-30T09:38:10.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:10 smithi019 ceph-mon[216815]: pgmap v939: 119 pgs: 119 active+clean; 386 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 51 MiB/s wr, 807 op/s 2023-09-30T09:38:10.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:10 smithi019 ceph-mon[224043]: 1.b scrub starts 2023-09-30T09:38:10.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:10 smithi019 ceph-mon[224043]: 1.b scrub ok 2023-09-30T09:38:10.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:10 smithi019 ceph-mon[224043]: pgmap v939: 119 pgs: 119 active+clean; 386 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 51 MiB/s wr, 807 op/s 2023-09-30T09:38:10.428 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:10.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:10.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:10.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 5 16 13094 13078 163.404 95.125 0.00362881 0.0061068 2023-09-30T09:38:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:10 smithi167 ceph-mon[192880]: 1.b scrub starts 2023-09-30T09:38:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:10 smithi167 ceph-mon[192880]: 1.b scrub ok 2023-09-30T09:38:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:10 smithi167 ceph-mon[192880]: pgmap v939: 119 pgs: 119 active+clean; 386 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 51 MiB/s wr, 807 op/s 2023-09-30T09:38:10.537 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:10.570 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:10.676 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:10.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:10.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:10.779 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:10.867 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:10.876 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:11.112 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:11.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:11.218 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:11.257 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:11.335 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:11.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:11.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:11.442 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:11.481 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:11.481 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:11.485 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:11.508 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 6 16 15064 15048 156.684 123.125 0.00786078 0.00637239 2023-09-30T09:38:11.558 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:38:11.565 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:38:11.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:11.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:11.599 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:11.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:11.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:11.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:11.764 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:12.508 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 7 16 16916 16900 150.835 115.75 0.00158089 0.00654832 2023-09-30T09:38:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:12 smithi167 ceph-mon[192880]: from='client.45752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:38:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:12 smithi167 ceph-mon[192880]: pgmap v940: 119 pgs: 119 active+clean; 634 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 79 MiB/s wr, 1.25k op/s 2023-09-30T09:38:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:12 smithi019 ceph-mon[224043]: from='client.45752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:38:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:12 smithi019 ceph-mon[224043]: pgmap v940: 119 pgs: 119 active+clean; 634 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 79 MiB/s wr, 1.25k op/s 2023-09-30T09:38:12.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:12 smithi019 ceph-mon[216815]: from='client.45752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:38:12.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:12 smithi019 ceph-mon[216815]: pgmap v940: 119 pgs: 119 active+clean; 634 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 79 MiB/s wr, 1.25k op/s 2023-09-30T09:38:13.356 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:13.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:13.461 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:13.491 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:13.508 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 8 16 18695 18679 145.879 111.188 0.00130581 0.006785 2023-09-30T09:38:13.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:13 smithi167 ceph-mon[192880]: 1.16 scrub starts 2023-09-30T09:38:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:13 smithi167 ceph-mon[192880]: 1.16 scrub ok 2023-09-30T09:38:13.809 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:13 smithi019 ceph-mon[224043]: 1.16 scrub starts 2023-09-30T09:38:13.809 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:13 smithi019 ceph-mon[224043]: 1.16 scrub ok 2023-09-30T09:38:13.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:13 smithi019 ceph-mon[216815]: 1.16 scrub starts 2023-09-30T09:38:13.810 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:13 smithi019 ceph-mon[216815]: 1.16 scrub ok 2023-09-30T09:38:14.018 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:14.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:14.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:38:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:38:13] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.43.0" 2023-09-30T09:38:14.125 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:14.164 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:14.189 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":939,"stamp":"2023-09-30T09:38:09.573338+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":404993251,"num_objects":6134,"num_object_clones":0,"num_object_copies":18402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6134,"num_whiteouts":0,"num_read":5975,"num_read_kb":12209,"num_write":7029,"num_write_kb":403673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":9095172,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12128,"ondisk_log_size":12128,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3657196,"kb_used_data":1006248,"kb_used_omap":23,"kb_used_meta":2650856,"kb_avail":652558868,"statfs":{"total":671965249536,"available":668220280832,"internally_reserved":0,"allocated":1030397952,"data_stored":1018373411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24317,"internal_metadata":2714476803},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":400082311,"num_objects":6105,"num_object_clones":0,"num_object_copies":18315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6105,"num_whiteouts":0,"num_read":-68,"num_read_kb":-126,"num_write":6074,"num_write_kb":390455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-456193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.522830"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":827,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.070685+0000","last_change":"2023-09-30T09:35:12.683213+0000","last_active":"2023-09-30T09:38:04.070685+0000","last_peered":"2023-09-30T09:38:04.070685+0000","last_clean":"2023-09-30T09:38:04.070685+0000","last_became_active":"2023-09-30T09:35:12.682797+0000","last_became_peered":"2023-09-30T09:35:12.682797+0000","last_unstale":"2023-09-30T09:38:04.070685+0000","last_undegraded":"2023-09-30T09:38:04.070685+0000","last_fullsized":"2023-09-30T09:38:04.070685+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:54.499835+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":381,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064791+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:38:04.064791+0000","last_peered":"2023-09-30T09:38:04.064791+0000","last_clean":"2023-09-30T09:38:04.064791+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:38:04.064791+0000","last_undegraded":"2023-09-30T09:38:04.064791+0000","last_fullsized":"2023-09-30T09:38:04.064791+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":344,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065339+0000","last_change":"2023-09-30T09:33:32.663081+0000","last_active":"2023-09-30T09:38:04.065339+0000","last_peered":"2023-09-30T09:38:04.065339+0000","last_clean":"2023-09-30T09:38:04.065339+0000","last_became_active":"2023-09-30T09:33:32.659841+0000","last_became_peered":"2023-09-30T09:33:32.659841+0000","last_unstale":"2023-09-30T09:38:04.065339+0000","last_undegraded":"2023-09-30T09:38:04.065339+0000","last_fullsized":"2023-09-30T09:38:04.065339+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:54.523951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":807,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071050+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:38:04.071050+0000","last_peered":"2023-09-30T09:38:04.071050+0000","last_clean":"2023-09-30T09:38:04.071050+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:38:04.071050+0000","last_undegraded":"2023-09-30T09:38:04.071050+0000","last_fullsized":"2023-09-30T09:38:04.071050+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":237,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065453+0000","last_change":"2023-09-30T09:34:37.840497+0000","last_active":"2023-09-30T09:38:04.065453+0000","last_peered":"2023-09-30T09:38:04.065453+0000","last_clean":"2023-09-30T09:38:04.065453+0000","last_became_active":"2023-09-30T09:34:37.840103+0000","last_became_peered":"2023-09-30T09:34:37.840103+0000","last_unstale":"2023-09-30T09:38:04.065453+0000","last_undegraded":"2023-09-30T09:38:04.065453+0000","last_fullsized":"2023-09-30T09:38:04.065453+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:23:21.260454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":268,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.063447+0000","last_change":"2023-09-30T09:34:07.475717+0000","last_active":"2023-09-30T09:38:04.063447+0000","last_peered":"2023-09-30T09:38:04.063447+0000","last_clean":"2023-09-30T09:38:04.063447+0000","last_became_active":"2023-09-30T09:34:07.474941+0000","last_became_peered":"2023-09-30T09:34:07.474941+0000","last_unstale":"2023-09-30T09:38:04.063447+0000","last_undegraded":"2023-09-30T09:38:04.063447+0000","last_fullsized":"2023-09-30T09:38:04.063447+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:49.770309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":576,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064966+0000","last_change":"2023-09-30T09:34:10.522404+0000","last_active":"2023-09-30T09:38:04.064966+0000","last_peered":"2023-09-30T09:38:04.064966+0000","last_clean":"2023-09-30T09:38:04.064966+0000","last_became_active":"2023-09-30T09:34:10.521992+0000","last_became_peered":"2023-09-30T09:34:10.521992+0000","last_unstale":"2023-09-30T09:38:04.064966+0000","last_undegraded":"2023-09-30T09:38:04.064966+0000","last_fullsized":"2023-09-30T09:38:04.064966+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:07.920688+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":410,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064700+0000","last_change":"2023-09-30T09:35:10.651069+0000","last_active":"2023-09-30T09:38:04.064700+0000","last_peered":"2023-09-30T09:38:04.064700+0000","last_clean":"2023-09-30T09:38:04.064700+0000","last_became_active":"2023-09-30T09:35:10.648038+0000","last_became_peered":"2023-09-30T09:35:10.648038+0000","last_unstale":"2023-09-30T09:38:04.064700+0000","last_undegraded":"2023-09-30T09:38:04.064700+0000","last_fullsized":"2023-09-30T09:38:04.064700+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:22.640376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":339,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.069707+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:38:04.069707+0000","last_peered":"2023-09-30T09:38:04.069707+0000","last_clean":"2023-09-30T09:38:04.069707+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:38:04.069707+0000","last_undegraded":"2023-09-30T09:38:04.069707+0000","last_fullsized":"2023-09-30T09:38:04.069707+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":795,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.164145+0000","last_change":"2023-09-30T09:38:04.164145+0000","last_active":"2023-09-30T09:38:04.164145+0000","last_peered":"2023-09-30T09:38:04.164145+0000","last_clean":"2023-09-30T09:38:04.164145+0000","last_became_active":"2023-09-30T09:34:10.502282+0000","last_became_peered":"2023-09-30T09:34:10.502282+0000","last_unstale":"2023-09-30T09:38:04.164145+0000","last_undegraded":"2023-09-30T09:38:04.164145+0000","last_fullsized":"2023-09-30T09:38:04.164145+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.164029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:36:08.710706+0000","scrub_duration":0.00154265,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":259,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062792+0000","last_change":"2023-09-30T09:23:08.025889+0000","last_active":"2023-09-30T09:38:04.062792+0000","last_peered":"2023-09-30T09:38:04.062792+0000","last_clean":"2023-09-30T09:38:04.062792+0000","last_became_active":"2023-09-30T09:23:08.025590+0000","last_became_peered":"2023-09-30T09:23:08.025590+0000","last_unstale":"2023-09-30T09:38:04.062792+0000","last_undegraded":"2023-09-30T09:38:04.062792+0000","last_fullsized":"2023-09-30T09:38:04.062792+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:56.066917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":353,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:05.412503+0000","last_change":"2023-09-30T09:38:05.412503+0000","last_active":"2023-09-30T09:38:05.412503+0000","last_peered":"2023-09-30T09:38:05.412503+0000","last_clean":"2023-09-30T09:38:05.412503+0000","last_became_active":"2023-09-30T09:38:05.412252+0000","last_became_peered":"2023-09-30T09:38:05.412252+0000","last_unstale":"2023-09-30T09:38:05.412503+0000","last_undegraded":"2023-09-30T09:38:05.412503+0000","last_fullsized":"2023-09-30T09:38:05.412503+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:20:21.936182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":535,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068884+0000","last_change":"2023-09-30T09:37:53.801871+0000","last_active":"2023-09-30T09:38:04.068884+0000","last_peered":"2023-09-30T09:38:04.068884+0000","last_clean":"2023-09-30T09:38:04.068884+0000","last_became_active":"2023-09-30T09:34:40.874145+0000","last_became_peered":"2023-09-30T09:34:40.874145+0000","last_unstale":"2023-09-30T09:38:04.068884+0000","last_undegraded":"2023-09-30T09:38:04.068884+0000","last_fullsized":"2023-09-30T09:38:04.068884+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:38.085701+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":241,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071120+0000","last_change":"2023-09-30T09:34:07.488621+0000","last_active":"2023-09-30T09:38:04.071120+0000","last_peered":"2023-09-30T09:38:04.071120+0000","last_clean":"2023-09-30T09:38:04.071120+0000","last_became_active":"2023-09-30T09:34:07.487793+0000","last_became_peered":"2023-09-30T09:34:07.487793+0000","last_unstale":"2023-09-30T09:38:04.071120+0000","last_undegraded":"2023-09-30T09:38:04.071120+0000","last_fullsized":"2023-09-30T09:38:04.071120+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:18.479324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":291,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065357+0000","last_change":"2023-09-30T09:34:07.478590+0000","last_active":"2023-09-30T09:38:04.065357+0000","last_peered":"2023-09-30T09:38:04.065357+0000","last_clean":"2023-09-30T09:38:04.065357+0000","last_became_active":"2023-09-30T09:34:07.478048+0000","last_became_peered":"2023-09-30T09:34:07.478048+0000","last_unstale":"2023-09-30T09:38:04.065357+0000","last_undegraded":"2023-09-30T09:38:04.065357+0000","last_fullsized":"2023-09-30T09:38:04.065357+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:26.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":529,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071058+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:38:04.071058+0000","last_peered":"2023-09-30T09:38:04.071058+0000","last_clean":"2023-09-30T09:38:04.071058+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:38:04.071058+0000","last_undegraded":"2023-09-30T09:38:04.071058+0000","last_fullsized":"2023-09-30T09:38:04.071058+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":85,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065429+0000","last_change":"2023-09-30T09:34:07.497116+0000","last_active":"2023-09-30T09:38:04.065429+0000","last_peered":"2023-09-30T09:38:04.065429+0000","last_clean":"2023-09-30T09:38:04.065429+0000","last_became_active":"2023-09-30T09:34:07.484719+0000","last_became_peered":"2023-09-30T09:34:07.484719+0000","last_unstale":"2023-09-30T09:38:04.065429+0000","last_undegraded":"2023-09-30T09:38:04.065429+0000","last_fullsized":"2023-09-30T09:38:04.065429+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:06.814003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":276,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068213+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:38:04.068213+0000","last_peered":"2023-09-30T09:38:04.068213+0000","last_clean":"2023-09-30T09:38:04.068213+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:38:04.068213+0000","last_undegraded":"2023-09-30T09:38:04.068213+0000","last_fullsized":"2023-09-30T09:38:04.068213+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":523,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.063587+0000","last_change":"2023-09-30T09:34:06.469105+0000","last_active":"2023-09-30T09:38:04.063587+0000","last_peered":"2023-09-30T09:38:04.063587+0000","last_clean":"2023-09-30T09:38:04.063587+0000","last_became_active":"2023-09-30T09:33:32.609263+0000","last_became_peered":"2023-09-30T09:33:32.609263+0000","last_unstale":"2023-09-30T09:38:04.063587+0000","last_undegraded":"2023-09-30T09:38:04.063587+0000","last_fullsized":"2023-09-30T09:38:04.063587+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:04.392210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:04.392210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":347,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065063+0000","last_change":"2023-09-30T09:34:07.517971+0000","last_active":"2023-09-30T09:38:04.065063+0000","last_peered":"2023-09-30T09:38:04.065063+0000","last_clean":"2023-09-30T09:38:04.065063+0000","last_became_active":"2023-09-30T09:34:07.485156+0000","last_became_peered":"2023-09-30T09:34:07.485156+0000","last_unstale":"2023-09-30T09:38:04.065063+0000","last_undegraded":"2023-09-30T09:38:04.065063+0000","last_fullsized":"2023-09-30T09:38:04.065063+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:29.009235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4722,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":358,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065178+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:38:04.065178+0000","last_peered":"2023-09-30T09:38:04.065178+0000","last_clean":"2023-09-30T09:38:04.065178+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:38:04.065178+0000","last_undegraded":"2023-09-30T09:38:04.065178+0000","last_fullsized":"2023-09-30T09:38:04.065178+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":649,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.211283+0000","last_change":"2023-09-30T09:38:04.211283+0000","last_active":"2023-09-30T09:38:04.211283+0000","last_peered":"2023-09-30T09:38:04.211283+0000","last_clean":"2023-09-30T09:38:04.211283+0000","last_became_active":"2023-09-30T09:35:13.717030+0000","last_became_peered":"2023-09-30T09:35:13.717030+0000","last_unstale":"2023-09-30T09:38:04.211283+0000","last_undegraded":"2023-09-30T09:38:04.211283+0000","last_fullsized":"2023-09-30T09:38:04.211283+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:16:00.435409+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":85,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065214+0000","last_change":"2023-09-30T09:34:07.541559+0000","last_active":"2023-09-30T09:38:04.065214+0000","last_peered":"2023-09-30T09:38:04.065214+0000","last_clean":"2023-09-30T09:38:04.065214+0000","last_became_active":"2023-09-30T09:34:07.491110+0000","last_became_peered":"2023-09-30T09:34:07.491110+0000","last_unstale":"2023-09-30T09:38:04.065214+0000","last_undegraded":"2023-09-30T09:38:04.065214+0000","last_fullsized":"2023-09-30T09:38:04.065214+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:10.141097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":383,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:05.151959+0000","last_change":"2023-09-30T09:38:05.151959+0000","last_active":"2023-09-30T09:38:05.151959+0000","last_peered":"2023-09-30T09:38:05.151959+0000","last_clean":"2023-09-30T09:38:05.151959+0000","last_became_active":"2023-09-30T09:38:05.151570+0000","last_became_peered":"2023-09-30T09:38:05.151570+0000","last_unstale":"2023-09-30T09:38:05.151959+0000","last_undegraded":"2023-09-30T09:38:05.151959+0000","last_fullsized":"2023-09-30T09:38:05.151959+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:26:38.881804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":709,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.070772+0000","last_change":"2023-09-30T09:34:10.522643+0000","last_active":"2023-09-30T09:38:04.070772+0000","last_peered":"2023-09-30T09:38:04.070772+0000","last_clean":"2023-09-30T09:38:04.070772+0000","last_became_active":"2023-09-30T09:34:10.522318+0000","last_became_peered":"2023-09-30T09:34:10.522318+0000","last_unstale":"2023-09-30T09:38:04.070772+0000","last_undegraded":"2023-09-30T09:38:04.070772+0000","last_fullsized":"2023-09-30T09:38:04.070772+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:16.321230+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":267,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062259+0000","last_change":"2023-09-30T09:34:07.474646+0000","last_active":"2023-09-30T09:38:04.062259+0000","last_peered":"2023-09-30T09:38:04.062259+0000","last_clean":"2023-09-30T09:38:04.062259+0000","last_became_active":"2023-09-30T09:34:07.473940+0000","last_became_peered":"2023-09-30T09:34:07.473940+0000","last_unstale":"2023-09-30T09:38:04.062259+0000","last_undegraded":"2023-09-30T09:38:04.062259+0000","last_fullsized":"2023-09-30T09:38:04.062259+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:23.500436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":52,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071130+0000","last_change":"2023-09-30T09:34:47.542202+0000","last_active":"2023-09-30T09:38:04.071130+0000","last_peered":"2023-09-30T09:38:04.071130+0000","last_clean":"2023-09-30T09:38:04.071130+0000","last_became_active":"2023-09-30T09:34:47.541975+0000","last_became_peered":"2023-09-30T09:34:47.541975+0000","last_unstale":"2023-09-30T09:38:04.071130+0000","last_undegraded":"2023-09-30T09:38:04.071130+0000","last_fullsized":"2023-09-30T09:38:04.071130+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:45.989960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":802,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064829+0000","last_change":"2023-09-30T09:34:40.898312+0000","last_active":"2023-09-30T09:38:04.064829+0000","last_peered":"2023-09-30T09:38:04.064829+0000","last_clean":"2023-09-30T09:38:04.064829+0000","last_became_active":"2023-09-30T09:34:40.897975+0000","last_became_peered":"2023-09-30T09:34:40.897975+0000","last_unstale":"2023-09-30T09:38:04.064829+0000","last_undegraded":"2023-09-30T09:38:04.064829+0000","last_fullsized":"2023-09-30T09:38:04.064829+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":68,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064789+0000","last_change":"2023-09-30T09:34:07.491676+0000","last_active":"2023-09-30T09:38:04.064789+0000","last_peered":"2023-09-30T09:38:04.064789+0000","last_clean":"2023-09-30T09:38:04.064789+0000","last_became_active":"2023-09-30T09:34:07.491332+0000","last_became_peered":"2023-09-30T09:34:07.491332+0000","last_unstale":"2023-09-30T09:38:04.064789+0000","last_undegraded":"2023-09-30T09:38:04.064789+0000","last_fullsized":"2023-09-30T09:38:04.064789+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:12.834112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":267,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.063480+0000","last_change":"2023-09-30T09:34:07.475908+0000","last_active":"2023-09-30T09:38:04.063480+0000","last_peered":"2023-09-30T09:38:04.063480+0000","last_clean":"2023-09-30T09:38:04.063480+0000","last_became_active":"2023-09-30T09:34:07.475392+0000","last_became_peered":"2023-09-30T09:34:07.475392+0000","last_unstale":"2023-09-30T09:38:04.063480+0000","last_undegraded":"2023-09-30T09:38:04.063480+0000","last_fullsized":"2023-09-30T09:38:04.063480+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:21:36.339923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":828,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.072015+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:38:04.072015+0000","last_peered":"2023-09-30T09:38:04.072015+0000","last_clean":"2023-09-30T09:38:04.072015+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:38:04.072015+0000","last_undegraded":"2023-09-30T09:38:04.072015+0000","last_fullsized":"2023-09-30T09:38:04.072015+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":522,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068524+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:38:04.068524+0000","last_peered":"2023-09-30T09:38:04.068524+0000","last_clean":"2023-09-30T09:38:04.068524+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:38:04.068524+0000","last_undegraded":"2023-09-30T09:38:04.068524+0000","last_fullsized":"2023-09-30T09:38:04.068524+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":372,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065899+0000","last_change":"2023-09-30T09:34:07.478900+0000","last_active":"2023-09-30T09:38:04.065899+0000","last_peered":"2023-09-30T09:38:04.065899+0000","last_clean":"2023-09-30T09:38:04.065899+0000","last_became_active":"2023-09-30T09:34:07.478559+0000","last_became_peered":"2023-09-30T09:34:07.478559+0000","last_unstale":"2023-09-30T09:38:04.065899+0000","last_undegraded":"2023-09-30T09:38:04.065899+0000","last_fullsized":"2023-09-30T09:38:04.065899+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:55:20.384058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"363'687","reported_seq":700,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:09.309054+0000","last_change":"2023-09-30T09:37:56.539717+0000","last_active":"2023-09-30T09:38:09.309054+0000","last_peered":"2023-09-30T09:38:09.309054+0000","last_clean":"2023-09-30T09:38:09.309054+0000","last_became_active":"2023-09-30T09:37:56.539331+0000","last_became_peered":"2023-09-30T09:37:56.539331+0000","last_unstale":"2023-09-30T09:38:09.309054+0000","last_undegraded":"2023-09-30T09:38:09.309054+0000","last_fullsized":"2023-09-30T09:38:09.309054+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":687,"log_dups_size":0,"ondisk_log_size":687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:15:01.898944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45023232,"num_objects":687,"num_object_clones":0,"num_object_copies":2061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":687,"num_write_kb":43968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":867,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065276+0000","last_change":"2023-09-30T09:37:52.740944+0000","last_active":"2023-09-30T09:38:04.065276+0000","last_peered":"2023-09-30T09:38:04.065276+0000","last_clean":"2023-09-30T09:38:04.065276+0000","last_became_active":"2023-09-30T09:34:10.522226+0000","last_became_peered":"2023-09-30T09:34:10.522226+0000","last_unstale":"2023-09-30T09:38:04.065276+0000","last_undegraded":"2023-09-30T09:38:04.065276+0000","last_fullsized":"2023-09-30T09:38:04.065276+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.740851+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.740851+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:24.266149+0000","scrub_duration":0.0033539569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":97,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065478+0000","last_change":"2023-09-30T09:35:10.652869+0000","last_active":"2023-09-30T09:38:04.065478+0000","last_peered":"2023-09-30T09:38:04.065478+0000","last_clean":"2023-09-30T09:38:04.065478+0000","last_became_active":"2023-09-30T09:35:10.646589+0000","last_became_peered":"2023-09-30T09:35:10.646589+0000","last_unstale":"2023-09-30T09:38:04.065478+0000","last_undegraded":"2023-09-30T09:38:04.065478+0000","last_fullsized":"2023-09-30T09:38:04.065478+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:02.914131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.070590+0000","last_change":"2023-09-30T09:34:47.538222+0000","last_active":"2023-09-30T09:38:04.070590+0000","last_peered":"2023-09-30T09:38:04.070590+0000","last_clean":"2023-09-30T09:38:04.070590+0000","last_became_active":"2023-09-30T09:34:47.537859+0000","last_became_peered":"2023-09-30T09:34:47.537859+0000","last_unstale":"2023-09-30T09:38:04.070590+0000","last_undegraded":"2023-09-30T09:38:04.070590+0000","last_fullsized":"2023-09-30T09:38:04.070590+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:36:13.874526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"363'636","reported_seq":649,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:09.101099+0000","last_change":"2023-09-30T09:37:56.534373+0000","last_active":"2023-09-30T09:38:09.101099+0000","last_peered":"2023-09-30T09:38:09.101099+0000","last_clean":"2023-09-30T09:38:09.101099+0000","last_became_active":"2023-09-30T09:37:56.533900+0000","last_became_peered":"2023-09-30T09:37:56.533900+0000","last_unstale":"2023-09-30T09:38:09.101099+0000","last_undegraded":"2023-09-30T09:38:09.101099+0000","last_fullsized":"2023-09-30T09:38:09.101099+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":636,"log_dups_size":0,"ondisk_log_size":636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:19:11.902983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41680896,"num_objects":636,"num_object_clones":0,"num_object_copies":1908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":636,"num_write_kb":40704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":813,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065108+0000","last_change":"2023-09-30T09:32:31.861949+0000","last_active":"2023-09-30T09:38:04.065108+0000","last_peered":"2023-09-30T09:38:04.065108+0000","last_clean":"2023-09-30T09:38:04.065108+0000","last_became_active":"2023-09-30T09:31:55.916681+0000","last_became_peered":"2023-09-30T09:31:55.916681+0000","last_unstale":"2023-09-30T09:38:04.065108+0000","last_undegraded":"2023-09-30T09:38:04.065108+0000","last_fullsized":"2023-09-30T09:38:04.065108+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:09.343756+0000","scrub_duration":0.0067749280000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":75,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065316+0000","last_change":"2023-09-30T09:35:10.762863+0000","last_active":"2023-09-30T09:38:04.065316+0000","last_peered":"2023-09-30T09:38:04.065316+0000","last_clean":"2023-09-30T09:38:04.065316+0000","last_became_active":"2023-09-30T09:35:10.650823+0000","last_became_peered":"2023-09-30T09:35:10.650823+0000","last_unstale":"2023-09-30T09:38:04.065316+0000","last_undegraded":"2023-09-30T09:38:04.065316+0000","last_fullsized":"2023-09-30T09:38:04.065316+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:31.531047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":365,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:05.152460+0000","last_change":"2023-09-30T09:38:05.152460+0000","last_active":"2023-09-30T09:38:05.152460+0000","last_peered":"2023-09-30T09:38:05.152460+0000","last_clean":"2023-09-30T09:38:05.152460+0000","last_became_active":"2023-09-30T09:38:05.152115+0000","last_became_peered":"2023-09-30T09:38:05.152115+0000","last_unstale":"2023-09-30T09:38:05.152460+0000","last_undegraded":"2023-09-30T09:38:05.152460+0000","last_fullsized":"2023-09-30T09:38:05.152460+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:00:01.393950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":16,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.063281+0000","last_change":"2023-09-30T09:37:56.537822+0000","last_active":"2023-09-30T09:38:04.063281+0000","last_peered":"2023-09-30T09:38:04.063281+0000","last_clean":"2023-09-30T09:38:04.063281+0000","last_became_active":"2023-09-30T09:37:56.537466+0000","last_became_peered":"2023-09-30T09:37:56.537466+0000","last_unstale":"2023-09-30T09:38:04.063281+0000","last_undegraded":"2023-09-30T09:38:04.063281+0000","last_fullsized":"2023-09-30T09:38:04.063281+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:18:24.275646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":646,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.069683+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:38:04.069683+0000","last_peered":"2023-09-30T09:38:04.069683+0000","last_clean":"2023-09-30T09:38:04.069683+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:38:04.069683+0000","last_undegraded":"2023-09-30T09:38:04.069683+0000","last_fullsized":"2023-09-30T09:38:04.069683+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":56,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071020+0000","last_change":"2023-09-30T09:34:47.553208+0000","last_active":"2023-09-30T09:38:04.071020+0000","last_peered":"2023-09-30T09:38:04.071020+0000","last_clean":"2023-09-30T09:38:04.071020+0000","last_became_active":"2023-09-30T09:34:47.542740+0000","last_became_peered":"2023-09-30T09:34:47.542740+0000","last_unstale":"2023-09-30T09:38:04.071020+0000","last_undegraded":"2023-09-30T09:38:04.071020+0000","last_fullsized":"2023-09-30T09:38:04.071020+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:00.136569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":342,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068151+0000","last_change":"2023-09-30T09:34:07.482526+0000","last_active":"2023-09-30T09:38:04.068151+0000","last_peered":"2023-09-30T09:38:04.068151+0000","last_clean":"2023-09-30T09:38:04.068151+0000","last_became_active":"2023-09-30T09:34:07.482220+0000","last_became_peered":"2023-09-30T09:34:07.482220+0000","last_unstale":"2023-09-30T09:38:04.068151+0000","last_undegraded":"2023-09-30T09:38:04.068151+0000","last_fullsized":"2023-09-30T09:38:04.068151+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:40.155251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"363'381","reported_seq":397,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:07.315791+0000","last_change":"2023-09-30T09:37:56.537414+0000","last_active":"2023-09-30T09:38:07.315791+0000","last_peered":"2023-09-30T09:38:07.315791+0000","last_clean":"2023-09-30T09:38:07.315791+0000","last_became_active":"2023-09-30T09:37:56.537069+0000","last_became_peered":"2023-09-30T09:37:56.537069+0000","last_unstale":"2023-09-30T09:38:07.315791+0000","last_undegraded":"2023-09-30T09:38:07.315791+0000","last_fullsized":"2023-09-30T09:38:07.315791+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":381,"log_dups_size":0,"ondisk_log_size":381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:39.717983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":381,"num_write_kb":24384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":538,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065520+0000","last_change":"2023-09-30T09:37:53.710570+0000","last_active":"2023-09-30T09:38:04.065520+0000","last_peered":"2023-09-30T09:38:04.065520+0000","last_clean":"2023-09-30T09:38:04.065520+0000","last_became_active":"2023-09-30T09:34:40.911971+0000","last_became_peered":"2023-09-30T09:34:40.911971+0000","last_unstale":"2023-09-30T09:38:04.065520+0000","last_undegraded":"2023-09-30T09:38:04.065520+0000","last_fullsized":"2023-09-30T09:38:04.065520+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:46.389016+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":322,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071060+0000","last_change":"2023-09-30T09:21:29.104438+0000","last_active":"2023-09-30T09:38:04.071060+0000","last_peered":"2023-09-30T09:38:04.071060+0000","last_clean":"2023-09-30T09:38:04.071060+0000","last_became_active":"2023-09-30T09:21:29.104089+0000","last_became_peered":"2023-09-30T09:21:29.104089+0000","last_unstale":"2023-09-30T09:38:04.071060+0000","last_undegraded":"2023-09-30T09:38:04.071060+0000","last_fullsized":"2023-09-30T09:38:04.071060+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:03.466805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":269,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062707+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:38:04.062707+0000","last_peered":"2023-09-30T09:38:04.062707+0000","last_clean":"2023-09-30T09:38:04.062707+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:38:04.062707+0000","last_undegraded":"2023-09-30T09:38:04.062707+0000","last_fullsized":"2023-09-30T09:38:04.062707+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"363'342","reported_seq":358,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:07.303056+0000","last_change":"2023-09-30T09:37:56.539803+0000","last_active":"2023-09-30T09:38:07.303056+0000","last_peered":"2023-09-30T09:38:07.303056+0000","last_clean":"2023-09-30T09:38:07.303056+0000","last_became_active":"2023-09-30T09:37:56.539499+0000","last_became_peered":"2023-09-30T09:37:56.539499+0000","last_unstale":"2023-09-30T09:38:07.303056+0000","last_undegraded":"2023-09-30T09:38:07.303056+0000","last_fullsized":"2023-09-30T09:38:07.303056+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":342,"log_dups_size":0,"ondisk_log_size":342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:46.239457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22413312,"num_objects":342,"num_object_clones":0,"num_object_copies":1026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":342,"num_write_kb":21888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":717,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:05.154770+0000","last_change":"2023-09-30T09:38:05.154857+0000","last_active":"2023-09-30T09:38:05.154770+0000","last_peered":"2023-09-30T09:38:05.154770+0000","last_clean":"2023-09-30T09:38:05.154770+0000","last_became_active":"2023-09-30T09:34:10.499491+0000","last_became_peered":"2023-09-30T09:34:10.499491+0000","last_unstale":"2023-09-30T09:38:05.154770+0000","last_undegraded":"2023-09-30T09:38:05.154770+0000","last_fullsized":"2023-09-30T09:38:05.154770+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:24:15.951217+0000","scrub_duration":0.005268366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":391,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065063+0000","last_change":"2023-09-30T09:35:10.683584+0000","last_active":"2023-09-30T09:38:04.065063+0000","last_peered":"2023-09-30T09:38:04.065063+0000","last_clean":"2023-09-30T09:38:04.065063+0000","last_became_active":"2023-09-30T09:35:10.648082+0000","last_became_peered":"2023-09-30T09:35:10.648082+0000","last_unstale":"2023-09-30T09:38:04.065063+0000","last_undegraded":"2023-09-30T09:38:04.065063+0000","last_fullsized":"2023-09-30T09:38:04.065063+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:58.621685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":321,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065112+0000","last_change":"2023-09-30T09:21:29.104691+0000","last_active":"2023-09-30T09:38:04.065112+0000","last_peered":"2023-09-30T09:38:04.065112+0000","last_clean":"2023-09-30T09:38:04.065112+0000","last_became_active":"2023-09-30T09:21:29.104264+0000","last_became_peered":"2023-09-30T09:21:29.104264+0000","last_unstale":"2023-09-30T09:38:04.065112+0000","last_undegraded":"2023-09-30T09:38:04.065112+0000","last_fullsized":"2023-09-30T09:38:04.065112+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:54:19.191803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":16,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062096+0000","last_change":"2023-09-30T09:37:56.535045+0000","last_active":"2023-09-30T09:38:04.062096+0000","last_peered":"2023-09-30T09:38:04.062096+0000","last_clean":"2023-09-30T09:38:04.062096+0000","last_became_active":"2023-09-30T09:37:56.534751+0000","last_became_peered":"2023-09-30T09:37:56.534751+0000","last_unstale":"2023-09-30T09:38:04.062096+0000","last_undegraded":"2023-09-30T09:38:04.062096+0000","last_fullsized":"2023-09-30T09:38:04.062096+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:27:36.529588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":785,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.067098+0000","last_change":"2023-09-30T09:34:50.450122+0000","last_active":"2023-09-30T09:38:04.067098+0000","last_peered":"2023-09-30T09:38:04.067098+0000","last_clean":"2023-09-30T09:38:04.067098+0000","last_became_active":"2023-09-30T09:34:50.449725+0000","last_became_peered":"2023-09-30T09:34:50.449725+0000","last_unstale":"2023-09-30T09:38:04.067098+0000","last_undegraded":"2023-09-30T09:38:04.067098+0000","last_fullsized":"2023-09-30T09:38:04.067098+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:31.660679+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":340,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068071+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:38:04.068071+0000","last_peered":"2023-09-30T09:38:04.068071+0000","last_clean":"2023-09-30T09:38:04.068071+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:38:04.068071+0000","last_undegraded":"2023-09-30T09:38:04.068071+0000","last_fullsized":"2023-09-30T09:38:04.068071+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":57,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065292+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:38:04.065292+0000","last_peered":"2023-09-30T09:38:04.065292+0000","last_clean":"2023-09-30T09:38:04.065292+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:38:04.065292+0000","last_undegraded":"2023-09-30T09:38:04.065292+0000","last_fullsized":"2023-09-30T09:38:04.065292+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"363'637","reported_seq":650,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:09.098236+0000","last_change":"2023-09-30T09:37:56.538248+0000","last_active":"2023-09-30T09:38:09.098236+0000","last_peered":"2023-09-30T09:38:09.098236+0000","last_clean":"2023-09-30T09:38:09.098236+0000","last_became_active":"2023-09-30T09:37:56.537940+0000","last_became_peered":"2023-09-30T09:37:56.537940+0000","last_unstale":"2023-09-30T09:38:09.098236+0000","last_undegraded":"2023-09-30T09:38:09.098236+0000","last_fullsized":"2023-09-30T09:38:09.098236+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":637,"log_dups_size":0,"ondisk_log_size":637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:56:16.621887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41746432,"num_objects":637,"num_object_clones":0,"num_object_copies":1911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":637,"num_write_kb":40768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":799,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062487+0000","last_change":"2023-09-30T09:34:07.455095+0000","last_active":"2023-09-30T09:38:04.062487+0000","last_peered":"2023-09-30T09:38:04.062487+0000","last_clean":"2023-09-30T09:38:04.062487+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:38:04.062487+0000","last_undegraded":"2023-09-30T09:38:04.062487+0000","last_fullsized":"2023-09-30T09:38:04.062487+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.322017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.322017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":359,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064692+0000","last_change":"2023-09-30T09:34:07.486156+0000","last_active":"2023-09-30T09:38:04.064692+0000","last_peered":"2023-09-30T09:38:04.064692+0000","last_clean":"2023-09-30T09:38:04.064692+0000","last_became_active":"2023-09-30T09:34:07.485965+0000","last_became_peered":"2023-09-30T09:34:07.485965+0000","last_unstale":"2023-09-30T09:38:04.064692+0000","last_undegraded":"2023-09-30T09:38:04.064692+0000","last_fullsized":"2023-09-30T09:38:04.064692+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:47.837443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":338,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065566+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:38:04.065566+0000","last_peered":"2023-09-30T09:38:04.065566+0000","last_clean":"2023-09-30T09:38:04.065566+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:38:04.065566+0000","last_undegraded":"2023-09-30T09:38:04.065566+0000","last_fullsized":"2023-09-30T09:38:04.065566+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"363'667","reported_seq":680,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:09.112207+0000","last_change":"2023-09-30T09:37:56.538633+0000","last_active":"2023-09-30T09:38:09.112207+0000","last_peered":"2023-09-30T09:38:09.112207+0000","last_clean":"2023-09-30T09:38:09.112207+0000","last_became_active":"2023-09-30T09:37:56.538313+0000","last_became_peered":"2023-09-30T09:37:56.538313+0000","last_unstale":"2023-09-30T09:38:09.112207+0000","last_undegraded":"2023-09-30T09:38:09.112207+0000","last_fullsized":"2023-09-30T09:38:09.112207+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":667,"log_dups_size":0,"ondisk_log_size":667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:12.090507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43712512,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":667,"num_write_kb":42688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":521,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068679+0000","last_change":"2023-09-30T09:37:52.780622+0000","last_active":"2023-09-30T09:38:04.068679+0000","last_peered":"2023-09-30T09:38:04.068679+0000","last_clean":"2023-09-30T09:38:04.068679+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:38:04.068679+0000","last_undegraded":"2023-09-30T09:38:04.068679+0000","last_fullsized":"2023-09-30T09:38:04.068679+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:16:18.525289+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":354,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064647+0000","last_change":"2023-09-30T09:34:07.484844+0000","last_active":"2023-09-30T09:38:04.064647+0000","last_peered":"2023-09-30T09:38:04.064647+0000","last_clean":"2023-09-30T09:38:04.064647+0000","last_became_active":"2023-09-30T09:34:07.484523+0000","last_became_peered":"2023-09-30T09:34:07.484523+0000","last_unstale":"2023-09-30T09:38:04.064647+0000","last_undegraded":"2023-09-30T09:38:04.064647+0000","last_fullsized":"2023-09-30T09:38:04.064647+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:15:33.742560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065678+0000","last_change":"2023-09-30T09:33:32.662230+0000","last_active":"2023-09-30T09:38:04.065678+0000","last_peered":"2023-09-30T09:38:04.065678+0000","last_clean":"2023-09-30T09:38:04.065678+0000","last_became_active":"2023-09-30T09:33:32.661526+0000","last_became_peered":"2023-09-30T09:33:32.661526+0000","last_unstale":"2023-09-30T09:38:04.065678+0000","last_undegraded":"2023-09-30T09:38:04.065678+0000","last_fullsized":"2023-09-30T09:38:04.065678+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:28.251364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":16,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.063553+0000","last_change":"2023-09-30T09:37:56.536019+0000","last_active":"2023-09-30T09:38:04.063553+0000","last_peered":"2023-09-30T09:38:04.063553+0000","last_clean":"2023-09-30T09:38:04.063553+0000","last_became_active":"2023-09-30T09:37:56.535667+0000","last_became_peered":"2023-09-30T09:37:56.535667+0000","last_unstale":"2023-09-30T09:38:04.063553+0000","last_undegraded":"2023-09-30T09:38:04.063553+0000","last_fullsized":"2023-09-30T09:38:04.063553+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:35:07.105737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":657,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.069774+0000","last_change":"2023-09-30T09:34:10.500130+0000","last_active":"2023-09-30T09:38:04.069774+0000","last_peered":"2023-09-30T09:38:04.069774+0000","last_clean":"2023-09-30T09:38:04.069774+0000","last_became_active":"2023-09-30T09:34:10.499770+0000","last_became_peered":"2023-09-30T09:34:10.499770+0000","last_unstale":"2023-09-30T09:38:04.069774+0000","last_undegraded":"2023-09-30T09:38:04.069774+0000","last_fullsized":"2023-09-30T09:38:04.069774+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:57:03.852067+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":555,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062535+0000","last_change":"2023-09-30T09:34:07.553285+0000","last_active":"2023-09-30T09:38:04.062535+0000","last_peered":"2023-09-30T09:38:04.062535+0000","last_clean":"2023-09-30T09:38:04.062535+0000","last_became_active":"2023-09-30T09:34:07.474030+0000","last_became_peered":"2023-09-30T09:34:07.474030+0000","last_unstale":"2023-09-30T09:38:04.062535+0000","last_undegraded":"2023-09-30T09:38:04.062535+0000","last_fullsized":"2023-09-30T09:38:04.062535+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:43.407021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":359,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071228+0000","last_change":"2023-09-30T09:33:10.370728+0000","last_active":"2023-09-30T09:38:04.071228+0000","last_peered":"2023-09-30T09:38:04.071228+0000","last_clean":"2023-09-30T09:38:04.071228+0000","last_became_active":"2023-09-30T09:33:10.370323+0000","last_became_peered":"2023-09-30T09:33:10.370323+0000","last_unstale":"2023-09-30T09:38:04.071228+0000","last_undegraded":"2023-09-30T09:38:04.071228+0000","last_fullsized":"2023-09-30T09:38:04.071228+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:32:25.178858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.070698+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:38:04.070698+0000","last_peered":"2023-09-30T09:38:04.070698+0000","last_clean":"2023-09-30T09:38:04.070698+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:38:04.070698+0000","last_undegraded":"2023-09-30T09:38:04.070698+0000","last_fullsized":"2023-09-30T09:38:04.070698+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"363'751","reported_seq":763,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:09.297428+0000","last_change":"2023-09-30T09:37:56.539999+0000","last_active":"2023-09-30T09:38:09.297428+0000","last_peered":"2023-09-30T09:38:09.297428+0000","last_clean":"2023-09-30T09:38:09.297428+0000","last_became_active":"2023-09-30T09:37:56.539603+0000","last_became_peered":"2023-09-30T09:37:56.539603+0000","last_unstale":"2023-09-30T09:38:09.297428+0000","last_undegraded":"2023-09-30T09:38:09.297428+0000","last_fullsized":"2023-09-30T09:38:09.297428+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":751,"log_dups_size":0,"ondisk_log_size":751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:34:41.424470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49217536,"num_objects":751,"num_object_clones":0,"num_object_copies":2253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":751,"num_write_kb":48064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":825,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.070816+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:38:04.070816+0000","last_peered":"2023-09-30T09:38:04.070816+0000","last_clean":"2023-09-30T09:38:04.070816+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:38:04.070816+0000","last_undegraded":"2023-09-30T09:38:04.070816+0000","last_fullsized":"2023-09-30T09:38:04.070816+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":354,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068140+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:38:04.068140+0000","last_peered":"2023-09-30T09:38:04.068140+0000","last_clean":"2023-09-30T09:38:04.068140+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:38:04.068140+0000","last_undegraded":"2023-09-30T09:38:04.068140+0000","last_fullsized":"2023-09-30T09:38:04.068140+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":342,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065285+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:38:04.065285+0000","last_peered":"2023-09-30T09:38:04.065285+0000","last_clean":"2023-09-30T09:38:04.065285+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:38:04.065285+0000","last_undegraded":"2023-09-30T09:38:04.065285+0000","last_fullsized":"2023-09-30T09:38:04.065285+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":81,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068931+0000","last_change":"2023-09-30T09:33:10.367229+0000","last_active":"2023-09-30T09:38:04.068931+0000","last_peered":"2023-09-30T09:38:04.068931+0000","last_clean":"2023-09-30T09:38:04.068931+0000","last_became_active":"2023-09-30T09:33:10.366858+0000","last_became_peered":"2023-09-30T09:33:10.366858+0000","last_unstale":"2023-09-30T09:38:04.068931+0000","last_undegraded":"2023-09-30T09:38:04.068931+0000","last_fullsized":"2023-09-30T09:38:04.068931+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:02:10.574573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"363'327","reported_seq":343,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:07.312338+0000","last_change":"2023-09-30T09:37:56.534927+0000","last_active":"2023-09-30T09:38:07.312338+0000","last_peered":"2023-09-30T09:38:07.312338+0000","last_clean":"2023-09-30T09:38:07.312338+0000","last_became_active":"2023-09-30T09:37:56.534621+0000","last_became_peered":"2023-09-30T09:37:56.534621+0000","last_unstale":"2023-09-30T09:38:07.312338+0000","last_undegraded":"2023-09-30T09:38:07.312338+0000","last_fullsized":"2023-09-30T09:38:07.312338+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":327,"log_dups_size":0,"ondisk_log_size":327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:26.982589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21430272,"num_objects":327,"num_object_clones":0,"num_object_copies":981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":327,"num_write_kb":20928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":731,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062250+0000","last_change":"2023-09-30T09:38:01.203788+0000","last_active":"2023-09-30T09:38:04.062250+0000","last_peered":"2023-09-30T09:38:04.062250+0000","last_clean":"2023-09-30T09:38:04.062250+0000","last_became_active":"2023-09-30T09:34:40.868878+0000","last_became_peered":"2023-09-30T09:34:40.868878+0000","last_unstale":"2023-09-30T09:38:04.062250+0000","last_undegraded":"2023-09-30T09:38:04.062250+0000","last_fullsized":"2023-09-30T09:38:04.062250+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:18:40.468133+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":542,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068121+0000","last_change":"2023-09-30T09:33:13.377587+0000","last_active":"2023-09-30T09:38:04.068121+0000","last_peered":"2023-09-30T09:38:04.068121+0000","last_clean":"2023-09-30T09:38:04.068121+0000","last_became_active":"2023-09-30T09:33:13.376995+0000","last_became_peered":"2023-09-30T09:33:13.376995+0000","last_unstale":"2023-09-30T09:38:04.068121+0000","last_undegraded":"2023-09-30T09:38:04.068121+0000","last_fullsized":"2023-09-30T09:38:04.068121+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:48:42.884644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.067008+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:38:04.067008+0000","last_peered":"2023-09-30T09:38:04.067008+0000","last_clean":"2023-09-30T09:38:04.067008+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:38:04.067008+0000","last_undegraded":"2023-09-30T09:38:04.067008+0000","last_fullsized":"2023-09-30T09:38:04.067008+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":282,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071350+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:38:04.071350+0000","last_peered":"2023-09-30T09:38:04.071350+0000","last_clean":"2023-09-30T09:38:04.071350+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:38:04.071350+0000","last_undegraded":"2023-09-30T09:38:04.071350+0000","last_fullsized":"2023-09-30T09:38:04.071350+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"363'654","reported_seq":667,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:09.108981+0000","last_change":"2023-09-30T09:37:56.537696+0000","last_active":"2023-09-30T09:38:09.108981+0000","last_peered":"2023-09-30T09:38:09.108981+0000","last_clean":"2023-09-30T09:38:09.108981+0000","last_became_active":"2023-09-30T09:37:56.537311+0000","last_became_peered":"2023-09-30T09:37:56.537311+0000","last_unstale":"2023-09-30T09:38:09.108981+0000","last_undegraded":"2023-09-30T09:38:09.108981+0000","last_fullsized":"2023-09-30T09:38:09.108981+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":654,"log_dups_size":0,"ondisk_log_size":654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:04:15.717625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42860544,"num_objects":654,"num_object_clones":0,"num_object_copies":1962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":654,"num_write_kb":41856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":935,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.070921+0000","last_change":"2023-09-30T09:34:49.429359+0000","last_active":"2023-09-30T09:38:04.070921+0000","last_peered":"2023-09-30T09:38:04.070921+0000","last_clean":"2023-09-30T09:38:04.070921+0000","last_became_active":"2023-09-30T09:34:49.428875+0000","last_became_peered":"2023-09-30T09:34:49.428875+0000","last_unstale":"2023-09-30T09:38:04.070921+0000","last_undegraded":"2023-09-30T09:38:04.070921+0000","last_fullsized":"2023-09-30T09:38:04.070921+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:55.638649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":628,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062143+0000","last_change":"2023-09-30T09:35:13.706062+0000","last_active":"2023-09-30T09:38:04.062143+0000","last_peered":"2023-09-30T09:38:04.062143+0000","last_clean":"2023-09-30T09:38:04.062143+0000","last_became_active":"2023-09-30T09:35:13.705670+0000","last_became_peered":"2023-09-30T09:35:13.705670+0000","last_unstale":"2023-09-30T09:38:04.062143+0000","last_undegraded":"2023-09-30T09:38:04.062143+0000","last_fullsized":"2023-09-30T09:38:04.062143+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:54.097534+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:54.097534+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":283,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:05.065946+0000","last_change":"2023-09-30T09:21:29.104912+0000","last_active":"2023-09-30T09:38:05.065946+0000","last_peered":"2023-09-30T09:38:05.065946+0000","last_clean":"2023-09-30T09:38:05.065946+0000","last_became_active":"2023-09-30T09:21:29.104022+0000","last_became_peered":"2023-09-30T09:21:29.104022+0000","last_unstale":"2023-09-30T09:38:05.065946+0000","last_undegraded":"2023-09-30T09:38:05.065946+0000","last_fullsized":"2023-09-30T09:38:05.065946+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:40.217511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":315,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.069525+0000","last_change":"2023-09-30T09:34:07.484187+0000","last_active":"2023-09-30T09:38:04.069525+0000","last_peered":"2023-09-30T09:38:04.069525+0000","last_clean":"2023-09-30T09:38:04.069525+0000","last_became_active":"2023-09-30T09:34:07.483587+0000","last_became_peered":"2023-09-30T09:34:07.483587+0000","last_unstale":"2023-09-30T09:38:04.069525+0000","last_undegraded":"2023-09-30T09:38:04.069525+0000","last_fullsized":"2023-09-30T09:38:04.069525+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:53:58.563223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":16,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068406+0000","last_change":"2023-09-30T09:37:56.539888+0000","last_active":"2023-09-30T09:38:04.068406+0000","last_peered":"2023-09-30T09:38:04.068406+0000","last_clean":"2023-09-30T09:38:04.068406+0000","last_became_active":"2023-09-30T09:37:56.539552+0000","last_became_peered":"2023-09-30T09:37:56.539552+0000","last_unstale":"2023-09-30T09:38:04.068406+0000","last_undegraded":"2023-09-30T09:38:04.068406+0000","last_fullsized":"2023-09-30T09:38:04.068406+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:00:59.888085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":518,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068069+0000","last_change":"2023-09-30T09:37:53.041461+0000","last_active":"2023-09-30T09:38:04.068069+0000","last_peered":"2023-09-30T09:38:04.068069+0000","last_clean":"2023-09-30T09:38:04.068069+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:38:04.068069+0000","last_undegraded":"2023-09-30T09:38:04.068069+0000","last_fullsized":"2023-09-30T09:38:04.068069+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:03:14.451639+0000","scrub_duration":0.0016789540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":739,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062201+0000","last_change":"2023-09-30T09:34:49.436256+0000","last_active":"2023-09-30T09:38:04.062201+0000","last_peered":"2023-09-30T09:38:04.062201+0000","last_clean":"2023-09-30T09:38:04.062201+0000","last_became_active":"2023-09-30T09:34:49.435423+0000","last_became_peered":"2023-09-30T09:34:49.435423+0000","last_unstale":"2023-09-30T09:38:04.062201+0000","last_undegraded":"2023-09-30T09:38:04.062201+0000","last_fullsized":"2023-09-30T09:38:04.062201+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:23:33.770354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":292,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062223+0000","last_change":"2023-09-30T09:34:37.839323+0000","last_active":"2023-09-30T09:38:04.062223+0000","last_peered":"2023-09-30T09:38:04.062223+0000","last_clean":"2023-09-30T09:38:04.062223+0000","last_became_active":"2023-09-30T09:34:37.838975+0000","last_became_peered":"2023-09-30T09:34:37.838975+0000","last_unstale":"2023-09-30T09:38:04.062223+0000","last_undegraded":"2023-09-30T09:38:04.062223+0000","last_fullsized":"2023-09-30T09:38:04.062223+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:32.630404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":48,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.063366+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:38:04.063366+0000","last_peered":"2023-09-30T09:38:04.063366+0000","last_clean":"2023-09-30T09:38:04.063366+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:38:04.063366+0000","last_undegraded":"2023-09-30T09:38:04.063366+0000","last_fullsized":"2023-09-30T09:38:04.063366+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"363'323","reported_seq":339,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:07.301906+0000","last_change":"2023-09-30T09:37:56.537364+0000","last_active":"2023-09-30T09:38:07.301906+0000","last_peered":"2023-09-30T09:38:07.301906+0000","last_clean":"2023-09-30T09:38:07.301906+0000","last_became_active":"2023-09-30T09:37:56.537053+0000","last_became_peered":"2023-09-30T09:37:56.537053+0000","last_unstale":"2023-09-30T09:38:07.301906+0000","last_undegraded":"2023-09-30T09:38:07.301906+0000","last_fullsized":"2023-09-30T09:38:07.301906+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:05:59.771104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21168128,"num_objects":323,"num_object_clones":0,"num_object_copies":969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":323,"num_write_kb":20672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":637,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.069728+0000","last_change":"2023-09-30T09:31:55.431747+0000","last_active":"2023-09-30T09:38:04.069728+0000","last_peered":"2023-09-30T09:38:04.069728+0000","last_clean":"2023-09-30T09:38:04.069728+0000","last_became_active":"2023-09-30T09:31:09.034731+0000","last_became_peered":"2023-09-30T09:31:09.034731+0000","last_unstale":"2023-09-30T09:38:04.069728+0000","last_undegraded":"2023-09-30T09:38:04.069728+0000","last_fullsized":"2023-09-30T09:38:04.069728+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:27.331851+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":854,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062548+0000","last_change":"2023-09-30T09:37:36.568108+0000","last_active":"2023-09-30T09:38:04.062548+0000","last_peered":"2023-09-30T09:38:04.062548+0000","last_clean":"2023-09-30T09:38:04.062548+0000","last_became_active":"2023-09-30T09:37:36.567318+0000","last_became_peered":"2023-09-30T09:37:36.567318+0000","last_unstale":"2023-09-30T09:38:04.062548+0000","last_undegraded":"2023-09-30T09:38:04.062548+0000","last_fullsized":"2023-09-30T09:38:04.062548+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:25:45.846255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":89,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068801+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:38:04.068801+0000","last_peered":"2023-09-30T09:38:04.068801+0000","last_clean":"2023-09-30T09:38:04.068801+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:38:04.068801+0000","last_undegraded":"2023-09-30T09:38:04.068801+0000","last_fullsized":"2023-09-30T09:38:04.068801+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":264,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062863+0000","last_change":"2023-09-30T09:34:07.492779+0000","last_active":"2023-09-30T09:38:04.062863+0000","last_peered":"2023-09-30T09:38:04.062863+0000","last_clean":"2023-09-30T09:38:04.062863+0000","last_became_active":"2023-09-30T09:34:07.492375+0000","last_became_peered":"2023-09-30T09:34:07.492375+0000","last_unstale":"2023-09-30T09:38:04.062863+0000","last_undegraded":"2023-09-30T09:38:04.062863+0000","last_fullsized":"2023-09-30T09:38:04.062863+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:09:56.187732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":16,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062472+0000","last_change":"2023-09-30T09:37:56.537712+0000","last_active":"2023-09-30T09:38:04.062472+0000","last_peered":"2023-09-30T09:38:04.062472+0000","last_clean":"2023-09-30T09:38:04.062472+0000","last_became_active":"2023-09-30T09:37:56.537406+0000","last_became_peered":"2023-09-30T09:37:56.537406+0000","last_unstale":"2023-09-30T09:38:04.062472+0000","last_undegraded":"2023-09-30T09:38:04.062472+0000","last_fullsized":"2023-09-30T09:38:04.062472+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:34:47.724492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":835,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065381+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:38:04.065381+0000","last_peered":"2023-09-30T09:38:04.065381+0000","last_clean":"2023-09-30T09:38:04.065381+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:38:04.065381+0000","last_undegraded":"2023-09-30T09:38:04.065381+0000","last_fullsized":"2023-09-30T09:38:04.065381+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":364,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065813+0000","last_change":"2023-09-30T09:34:07.503794+0000","last_active":"2023-09-30T09:38:04.065813+0000","last_peered":"2023-09-30T09:38:04.065813+0000","last_clean":"2023-09-30T09:38:04.065813+0000","last_became_active":"2023-09-30T09:34:07.488659+0000","last_became_peered":"2023-09-30T09:34:07.488659+0000","last_unstale":"2023-09-30T09:38:04.065813+0000","last_undegraded":"2023-09-30T09:38:04.065813+0000","last_fullsized":"2023-09-30T09:38:04.065813+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:33:24.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1129,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.069646+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:38:04.069646+0000","last_peered":"2023-09-30T09:38:04.069646+0000","last_clean":"2023-09-30T09:38:04.069646+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:38:04.069646+0000","last_undegraded":"2023-09-30T09:38:04.069646+0000","last_fullsized":"2023-09-30T09:38:04.069646+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":381,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064669+0000","last_change":"2023-09-30T09:35:10.651070+0000","last_active":"2023-09-30T09:38:04.064669+0000","last_peered":"2023-09-30T09:38:04.064669+0000","last_clean":"2023-09-30T09:38:04.064669+0000","last_became_active":"2023-09-30T09:35:10.649518+0000","last_became_peered":"2023-09-30T09:35:10.649518+0000","last_unstale":"2023-09-30T09:38:04.064669+0000","last_undegraded":"2023-09-30T09:38:04.064669+0000","last_fullsized":"2023-09-30T09:38:04.064669+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:57.021522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"363'701","reported_seq":714,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:09.309994+0000","last_change":"2023-09-30T09:37:56.540414+0000","last_active":"2023-09-30T09:38:09.309994+0000","last_peered":"2023-09-30T09:38:09.309994+0000","last_clean":"2023-09-30T09:38:09.309994+0000","last_became_active":"2023-09-30T09:37:56.540102+0000","last_became_peered":"2023-09-30T09:37:56.540102+0000","last_unstale":"2023-09-30T09:38:09.309994+0000","last_undegraded":"2023-09-30T09:38:09.309994+0000","last_fullsized":"2023-09-30T09:38:09.309994+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":701,"log_dups_size":0,"ondisk_log_size":701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:17.396707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45940736,"num_objects":701,"num_object_clones":0,"num_object_copies":2103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":701,"num_write_kb":44864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":829,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065507+0000","last_change":"2023-09-30T09:34:50.446718+0000","last_active":"2023-09-30T09:38:04.065507+0000","last_peered":"2023-09-30T09:38:04.065507+0000","last_clean":"2023-09-30T09:38:04.065507+0000","last_became_active":"2023-09-30T09:34:50.446313+0000","last_became_peered":"2023-09-30T09:34:50.446313+0000","last_unstale":"2023-09-30T09:38:04.065507+0000","last_undegraded":"2023-09-30T09:38:04.065507+0000","last_fullsized":"2023-09-30T09:38:04.065507+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:16:06.564958+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":688,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.069820+0000","last_change":"2023-09-30T09:34:47.649626+0000","last_active":"2023-09-30T09:38:04.069820+0000","last_peered":"2023-09-30T09:38:04.069820+0000","last_clean":"2023-09-30T09:38:04.069820+0000","last_became_active":"2023-09-30T09:34:07.485804+0000","last_became_peered":"2023-09-30T09:34:07.485804+0000","last_unstale":"2023-09-30T09:38:04.069820+0000","last_undegraded":"2023-09-30T09:38:04.069820+0000","last_fullsized":"2023-09-30T09:38:04.069820+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:43.314406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":51,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.063399+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:38:04.063399+0000","last_peered":"2023-09-30T09:38:04.063399+0000","last_clean":"2023-09-30T09:38:04.063399+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:38:04.063399+0000","last_undegraded":"2023-09-30T09:38:04.063399+0000","last_fullsized":"2023-09-30T09:38:04.063399+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":304,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064701+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:38:04.064701+0000","last_peered":"2023-09-30T09:38:04.064701+0000","last_clean":"2023-09-30T09:38:04.064701+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:38:04.064701+0000","last_undegraded":"2023-09-30T09:38:04.064701+0000","last_fullsized":"2023-09-30T09:38:04.064701+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":777,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064696+0000","last_change":"2023-09-30T09:35:13.701577+0000","last_active":"2023-09-30T09:38:04.064696+0000","last_peered":"2023-09-30T09:38:04.064696+0000","last_clean":"2023-09-30T09:38:04.064696+0000","last_became_active":"2023-09-30T09:35:13.699135+0000","last_became_peered":"2023-09-30T09:35:13.699135+0000","last_unstale":"2023-09-30T09:38:04.064696+0000","last_undegraded":"2023-09-30T09:38:04.064696+0000","last_fullsized":"2023-09-30T09:38:04.064696+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:21:08.085109+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":47,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062786+0000","last_change":"2023-09-30T09:35:10.646263+0000","last_active":"2023-09-30T09:38:04.062786+0000","last_peered":"2023-09-30T09:38:04.062786+0000","last_clean":"2023-09-30T09:38:04.062786+0000","last_became_active":"2023-09-30T09:35:10.645865+0000","last_became_peered":"2023-09-30T09:35:10.645865+0000","last_unstale":"2023-09-30T09:38:04.062786+0000","last_undegraded":"2023-09-30T09:38:04.062786+0000","last_fullsized":"2023-09-30T09:38:04.062786+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:00.685092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":41,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.068713+0000","last_change":"2023-09-30T09:35:23.854788+0000","last_active":"2023-09-30T09:38:04.068713+0000","last_peered":"2023-09-30T09:38:04.068713+0000","last_clean":"2023-09-30T09:38:04.068713+0000","last_became_active":"2023-09-30T09:35:23.854515+0000","last_became_peered":"2023-09-30T09:35:23.854515+0000","last_unstale":"2023-09-30T09:38:04.068713+0000","last_undegraded":"2023-09-30T09:38:04.068713+0000","last_fullsized":"2023-09-30T09:38:04.068713+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:00:01.243302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":549,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065208+0000","last_change":"2023-09-30T09:34:10.525750+0000","last_active":"2023-09-30T09:38:04.065208+0000","last_peered":"2023-09-30T09:38:04.065208+0000","last_clean":"2023-09-30T09:38:04.065208+0000","last_became_active":"2023-09-30T09:34:10.525400+0000","last_became_peered":"2023-09-30T09:34:10.525400+0000","last_unstale":"2023-09-30T09:38:04.065208+0000","last_undegraded":"2023-09-30T09:38:04.065208+0000","last_fullsized":"2023-09-30T09:38:04.065208+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:48:50.218215+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":341,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071049+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:38:04.071049+0000","last_peered":"2023-09-30T09:38:04.071049+0000","last_clean":"2023-09-30T09:38:04.071049+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:38:04.071049+0000","last_undegraded":"2023-09-30T09:38:04.071049+0000","last_fullsized":"2023-09-30T09:38:04.071049+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":57,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064895+0000","last_change":"2023-09-30T09:34:37.836267+0000","last_active":"2023-09-30T09:38:04.064895+0000","last_peered":"2023-09-30T09:38:04.064895+0000","last_clean":"2023-09-30T09:38:04.064895+0000","last_became_active":"2023-09-30T09:34:37.835944+0000","last_became_peered":"2023-09-30T09:34:37.835944+0000","last_unstale":"2023-09-30T09:38:04.064895+0000","last_undegraded":"2023-09-30T09:38:04.064895+0000","last_fullsized":"2023-09-30T09:38:04.064895+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:17.331511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":307,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.071198+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:38:04.071198+0000","last_peered":"2023-09-30T09:38:04.071198+0000","last_clean":"2023-09-30T09:38:04.071198+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:38:04.071198+0000","last_undegraded":"2023-09-30T09:38:04.071198+0000","last_fullsized":"2023-09-30T09:38:04.071198+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":57,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.064757+0000","last_change":"2023-09-30T09:34:37.838959+0000","last_active":"2023-09-30T09:38:04.064757+0000","last_peered":"2023-09-30T09:38:04.064757+0000","last_clean":"2023-09-30T09:38:04.064757+0000","last_became_active":"2023-09-30T09:34:37.838555+0000","last_became_peered":"2023-09-30T09:34:37.838555+0000","last_unstale":"2023-09-30T09:38:04.064757+0000","last_undegraded":"2023-09-30T09:38:04.064757+0000","last_fullsized":"2023-09-30T09:38:04.064757+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:59.369451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":501,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.067989+0000","last_change":"2023-09-30T09:37:54.087694+0000","last_active":"2023-09-30T09:38:04.067989+0000","last_peered":"2023-09-30T09:38:04.067989+0000","last_clean":"2023-09-30T09:38:04.067989+0000","last_became_active":"2023-09-30T09:34:10.497672+0000","last_became_peered":"2023-09-30T09:34:10.497672+0000","last_unstale":"2023-09-30T09:38:04.067989+0000","last_undegraded":"2023-09-30T09:38:04.067989+0000","last_fullsized":"2023-09-30T09:38:04.067989+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:54.087577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:54.087577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:07.339971+0000","scrub_duration":0.001456983,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":360,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065611+0000","last_change":"2023-09-30T09:33:10.362171+0000","last_active":"2023-09-30T09:38:04.065611+0000","last_peered":"2023-09-30T09:38:04.065611+0000","last_clean":"2023-09-30T09:38:04.065611+0000","last_became_active":"2023-09-30T09:33:10.361859+0000","last_became_peered":"2023-09-30T09:33:10.361859+0000","last_unstale":"2023-09-30T09:38:04.065611+0000","last_undegraded":"2023-09-30T09:38:04.065611+0000","last_fullsized":"2023-09-30T09:38:04.065611+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:39:31.555581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":384,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065270+0000","last_change":"2023-09-30T09:33:10.353925+0000","last_active":"2023-09-30T09:38:04.065270+0000","last_peered":"2023-09-30T09:38:04.065270+0000","last_clean":"2023-09-30T09:38:04.065270+0000","last_became_active":"2023-09-30T09:33:10.353591+0000","last_became_peered":"2023-09-30T09:33:10.353591+0000","last_unstale":"2023-09-30T09:38:04.065270+0000","last_undegraded":"2023-09-30T09:38:04.065270+0000","last_fullsized":"2023-09-30T09:38:04.065270+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:43.214669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":820,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.065090+0000","last_change":"2023-09-30T09:37:57.093368+0000","last_active":"2023-09-30T09:38:04.065090+0000","last_peered":"2023-09-30T09:38:04.065090+0000","last_clean":"2023-09-30T09:38:04.065090+0000","last_became_active":"2023-09-30T09:33:32.659395+0000","last_became_peered":"2023-09-30T09:33:32.659395+0000","last_unstale":"2023-09-30T09:38:04.065090+0000","last_undegraded":"2023-09-30T09:38:04.065090+0000","last_fullsized":"2023-09-30T09:38:04.065090+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:57.093254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:57.093254+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:34:37.285604+0000","scrub_duration":0.001722147,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":67,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-30T09:38:04.062846+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:38:04.062846+0000","last_peered":"2023-09-30T09:38:04.062846+0000","last_clean":"2023-09-30T09:38:04.062846+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:38:04.062846+0000","last_undegraded":"2023-09-30T09:38:04.062846+0000","last_fullsized":"2023-09-30T09:38:04.062846+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":348,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-30T09:38:05.412543+0000","last_change":"2023-09-30T09:38:05.412543+0000","last_active":"2023-09-30T09:38:05.412543+0000","last_peered":"2023-09-30T09:38:05.412543+0000","last_clean":"2023-09-30T09:38:05.412543+0000","last_became_active":"2023-09-30T09:38:05.412129+0000","last_became_peered":"2023-09-30T09:38:05.412129+0000","last_unstale":"2023-09-30T09:38:05.412543+0000","last_undegraded":"2023-09-30T09:38:05.412543+0000","last_fullsized":"2023-09-30T09:38:05.412543+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:49:02.515137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":6762,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":400162816,"num_objects":6106,"num_object_clones":0,"num_object_copies":18318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6106,"num_write_kb":390784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1104805888,"data_stored":1104805888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6106,"ondisk_log_size":6106,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630620,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":699802,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":162,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234318,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495955,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":655580,"kb_used_data":247380,"kb_used_omap":0,"kb_used_meta":408192,"kb_avail":93089572,"statfs":{"total":95995035648,"available":95323721728,"internally_reserved":0,"allocated":253317120,"data_stored":252091285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":417988608},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":2.4060000000000001,"15min":1.869},"min":{"1min":0.26700000000000002,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":4.6970000000000001,"5min":44.113,"15min":45.927999999999997},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.125,"5min":2.3140000000000001,"15min":2.0859999999999999},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.0460000000000003,"5min":44.164000000000001,"15min":45.316000000000003},"last":0.29799999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":2.5910000000000002,"15min":2.0649999999999999},"min":{"1min":0.316,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":52.893000000000001,"5min":52.893000000000001,"15min":52.893000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":4.673,"5min":2.8519999999999999,"15min":2.3260000000000001},"min":{"1min":0.47799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":52.905999999999999,"5min":52.905999999999999,"15min":52.905999999999999},"last":0.59599999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:37:31 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":2.2040000000000002,"15min":1.9910000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.1200000000000001,"5min":17.126000000000001,"15min":37.978000000000002},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.272,"5min":2.3399999999999999,"15min":1.8959999999999999},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2720000000000002,"5min":25.164999999999999,"15min":37.759999999999998},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":6.1580000000000004,"15min":3.605},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.27600000000000002},"max":{"1min":14.862,"5min":212.44800000000001,"15min":212.44800000000001},"last":0.746},{"interface":"front","average":{"1min":1.7090000000000001,"5min":6.7759999999999998,"15min":3.9950000000000001},"min":{"1min":0.39000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":5.2069999999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":0.371}]},{"osd":4,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.0129999999999999,"15min":1.6259999999999999},"min":{"1min":0.26900000000000002,"5min":0.26400000000000001,"15min":0.23400000000000001},"max":{"1min":4.0469999999999997,"5min":7.4359999999999999,"15min":73.656000000000006},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.91200000000000003,"15min":1.417},"min":{"1min":0.30599999999999999,"5min":0.29599999999999999,"15min":0.23000000000000001},"max":{"1min":1.5580000000000001,"5min":9.6590000000000007,"15min":73.882999999999996},"last":0.41599999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.2410000000000001,"15min":1.903},"min":{"1min":0.29899999999999999,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":11.428000000000001,"5min":11.428000000000001,"15min":76.613},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.25,"5min":1.143,"15min":1.8280000000000001},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7220000000000004,"5min":9.7750000000000004,"15min":76.329999999999998},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:37:31 2023","interfaces":[{"interface":"back","average":{"1min":8.1950000000000003,"5min":2.7679999999999998,"15min":2.6499999999999999},"min":{"1min":0.371,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":138.191,"5min":138.191,"15min":138.191},"last":0.64700000000000002},{"interface":"front","average":{"1min":8.0210000000000008,"5min":2.5619999999999998,"15min":2.5830000000000002},"min":{"1min":0.35099999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":140.33500000000001,"5min":140.33500000000001,"15min":140.33500000000001},"last":0.55300000000000005}]}]},{"osd":1,"up_from":301,"seq":1292785156152,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410852,"kb_used_data":87068,"kb_used_omap":9,"kb_used_meta":323766,"kb_avail":93334300,"statfs":{"total":95995035648,"available":95574323200,"internally_reserved":0,"allocated":89157632,"data_stored":87116204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":331536402},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":5.5,"5min":4.6900000000000004,"15min":3.258},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":59.115000000000002,"5min":138.43799999999999,"15min":138.43799999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":3.347,"5min":4.5780000000000003,"15min":3.5419999999999998},"min":{"1min":0.32600000000000001,"5min":0.25,"15min":0.25},"max":{"1min":12.332000000000001,"5min":138.482,"15min":138.482},"last":0.47199999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":2.895,"15min":2.8290000000000002},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":11.595000000000001,"5min":16.437000000000001,"15min":16.437000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":5.5129999999999999,"5min":3.6520000000000001,"15min":3.0209999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":59.078000000000003,"5min":59.078000000000003,"15min":59.078000000000003},"last":0.60299999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":14.836,"5min":9.0150000000000006,"15min":9.3030000000000008},"min":{"1min":0.32000000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":191.07300000000001,"5min":191.07300000000001,"15min":191.07300000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":13.679,"5min":8.9689999999999994,"15min":9.7739999999999991},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":191.11500000000001,"5min":191.11500000000001,"15min":191.11500000000001},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.0569999999999999,"15min":2.0099999999999998},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.1099999999999994,"5min":13.602,"15min":13.602},"last":0.69599999999999995},{"interface":"front","average":{"1min":4.8929999999999998,"5min":2.4319999999999999,"15min":1.6719999999999999},"min":{"1min":0.35099999999999998,"5min":0.311,"15min":0.311},"max":{"1min":58.939999999999998,"5min":58.939999999999998,"15min":58.939999999999998},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":3.0640000000000001,"5min":2.5419999999999998,"15min":2.2530000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":24.25,"5min":24.25,"15min":24.25},"last":0.75600000000000001},{"interface":"front","average":{"1min":6.4749999999999996,"5min":2.9630000000000001,"15min":2.0550000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":58.999000000000002,"5min":58.999000000000002,"15min":58.999000000000002},"last":0.64100000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":6.0739999999999998,"5min":9.3149999999999995,"15min":14.503},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":58.683999999999997,"5min":209.221,"15min":209.221},"last":0.66300000000000003},{"interface":"front","average":{"1min":6.4400000000000004,"5min":9.0920000000000005,"15min":14.374000000000001},"min":{"1min":0.40899999999999997,"5min":0.377,"15min":0.377},"max":{"1min":59.143000000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":0.73499999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:37:37 2023","interfaces":[{"interface":"back","average":{"1min":3.222,"5min":6.5190000000000001,"15min":10.468999999999999},"min":{"1min":0.495,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":11.73,"5min":165.59399999999999,"15min":165.59399999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.9969999999999999,"5min":6.6310000000000002,"15min":10.663},"min":{"1min":0.46000000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":11.692,"5min":165.833,"15min":165.833},"last":0.71899999999999997}]}]},{"osd":0,"up_from":94,"seq":403726926051,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":376400,"kb_used_data":9344,"kb_used_omap":5,"kb_used_meta":367034,"kb_avail":93368752,"statfs":{"total":95995035648,"available":95609602048,"internally_reserved":0,"allocated":9568256,"data_stored":7378154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":375843069},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":2.835,"15min":2.9430000000000001},"min":{"1min":0.20799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":7.4820000000000002,"5min":17.966000000000001,"15min":17.966000000000001},"last":0.311},{"interface":"front","average":{"1min":2.552,"5min":6.3799999999999999,"15min":8.4100000000000001},"min":{"1min":0.35199999999999998,"5min":0.22,"15min":0.22},"max":{"1min":7.9210000000000003,"5min":138.92400000000001,"15min":138.92400000000001},"last":0.48299999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:37:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9750000000000001,"5min":2.5209999999999999,"15min":1.9179999999999999},"min":{"1min":0.26600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":15.324,"5min":15.324,"15min":55.823999999999998},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.8799999999999999,"5min":2.976,"15min":2.0950000000000002},"min":{"1min":0.36199999999999999,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":14.848000000000001,"5min":40.128999999999998,"15min":55.790999999999997},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2480000000000002,"5min":8.9710000000000001,"15min":4.3929999999999998},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.221},"max":{"1min":9.4450000000000003,"5min":218.06999999999999,"15min":218.06999999999999},"last":0.372},{"interface":"front","average":{"1min":2.4449999999999998,"5min":6.5529999999999999,"15min":3.5499999999999998},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.245},"max":{"1min":9.4969999999999999,"5min":218.63200000000001,"15min":218.63200000000001},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:37:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.921,"15min":1.7709999999999999},"min":{"1min":0.41399999999999998,"5min":0.32900000000000001,"15min":0.29099999999999998},"max":{"1min":6.0179999999999998,"5min":10.614000000000001,"15min":121.486},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.4329999999999998,"5min":4.1600000000000001,"15min":2.7669999999999999},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.8819999999999997,"5min":136.262,"15min":136.262},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:37:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":4.4119999999999999,"15min":3.1829999999999998},"min":{"1min":0.33100000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.8439999999999994,"5min":136.34,"15min":136.34},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.4279999999999999,"5min":2.0009999999999999,"15min":2.3370000000000002},"min":{"1min":0.45500000000000002,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":4.5019999999999998,"5min":12.717000000000001,"15min":135.381},"last":0.60399999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:37:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":6.1539999999999999,"15min":3.302},"min":{"1min":0.47399999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":5.835,"5min":214.785,"15min":214.785},"last":0.501},{"interface":"front","average":{"1min":2.2120000000000002,"5min":6.3700000000000001,"15min":3.29},"min":{"1min":0.46500000000000002,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":7.8319999999999999,"5min":214.93899999999999,"15min":214.93899999999999},"last":0.624}]},{"osd":7,"last update":"Sat Sep 30 09:37:51 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":3.1480000000000001,"15min":3.2519999999999998},"min":{"1min":0.44800000000000001,"5min":0.38300000000000001,"15min":0.373},"max":{"1min":7.7889999999999997,"5min":97.382000000000005,"15min":150.96899999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.6269999999999998,"5min":3.2549999999999999,"15min":3.149},"min":{"1min":0.51100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.135999999999999,"5min":97.015000000000001,"15min":150.328},"last":0.57399999999999995}]}]},{"osd":2,"up_from":104,"seq":446676598992,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325100,"kb_used_data":4960,"kb_used_omap":5,"kb_used_meta":320122,"kb_avail":93420052,"statfs":{"total":95995035648,"available":95662133248,"internally_reserved":0,"allocated":5079040,"data_stored":3088104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":327805581},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.8420000000000001,"15min":2.242},"min":{"1min":0.20499999999999999,"5min":0.187,"15min":0.187},"max":{"1min":6.9480000000000004,"5min":77.379000000000005,"15min":77.379000000000005},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.8010000000000002,"15min":2.0750000000000002},"min":{"1min":0.29499999999999998,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":10.590999999999999,"5min":76.796999999999997,"15min":77.066999999999993},"last":0.26900000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":4.2720000000000002,"5min":4.9740000000000002,"15min":4.0389999999999997},"min":{"1min":0.35799999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":37.701999999999998,"5min":64.412999999999997,"15min":64.412999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.9129999999999998,"5min":4.2969999999999997,"15min":3.4700000000000002},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":37.728999999999999,"5min":63.247999999999998,"15min":63.247999999999998},"last":0.32900000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:37:54 2023","interfaces":[{"interface":"back","average":{"1min":4.9960000000000004,"5min":8.2409999999999997,"15min":3.7730000000000001},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":60.359999999999999,"5min":134.078,"15min":134.078},"last":0.37},{"interface":"front","average":{"1min":4.8029999999999999,"5min":7.8259999999999996,"15min":3.8780000000000001},"min":{"1min":0.35599999999999998,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":63.735999999999997,"5min":135.73400000000001,"15min":135.73400000000001},"last":0.55900000000000005}]},{"osd":4,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":2.1909999999999998,"15min":1.403},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":6.5979999999999999,"5min":35.972000000000001,"15min":35.972000000000001},"last":0.503},{"interface":"front","average":{"1min":1.099,"5min":1.917,"15min":1.6279999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":2.3759999999999999,"5min":36.518000000000001,"15min":36.518000000000001},"last":0.77400000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":2.2989999999999999,"15min":2.2200000000000002},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.6280000000000001,"5min":12.651999999999999,"15min":155.477},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.585,"5min":1.6850000000000001,"15min":2.0139999999999998},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":4.9450000000000003,"5min":8.0839999999999996,"15min":155.46000000000001},"last":0.57399999999999995}]},{"osd":6,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":5.5700000000000003,"15min":3.3690000000000002},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.8040000000000003,"5min":154.417,"15min":154.417},"last":0.69799999999999995},{"interface":"front","average":{"1min":2.3809999999999998,"5min":6.3570000000000002,"15min":3.4540000000000002},"min":{"1min":0.39000000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":7.6799999999999997,"5min":154.04400000000001,"15min":154.04400000000001},"last":0.71799999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":3.7839999999999998,"15min":3.1890000000000001},"min":{"1min":0.45000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0229999999999997,"5min":123.294,"15min":213.404},"last":0.73799999999999999},{"interface":"front","average":{"1min":2.2970000000000002,"5min":4.3479999999999999,"15min":3.2749999999999999},"min":{"1min":0.48099999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.71,"5min":123.16,"15min":213.27199999999999},"last":0.59899999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435465,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":802684,"kb_used_data":135732,"kb_used_omap":2,"kb_used_meta":666941,"kb_avail":92942468,"statfs":{"total":95995035648,"available":95173087232,"internally_reserved":0,"allocated":138989568,"data_stored":136864214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":682947943},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:03 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":3.9260000000000002,"15min":3.2480000000000002},"min":{"1min":0.24399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.539,"5min":60.726999999999997,"15min":130.946},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.907,"5min":3.7090000000000001,"15min":3.157},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.20899999999999999},"max":{"1min":9.0790000000000006,"5min":52.093000000000004,"15min":130.96899999999999},"last":0.33000000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":8.1340000000000003,"5min":11.768000000000001,"15min":18.100000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":98.826999999999998,"5min":273.99599999999998,"15min":273.99599999999998},"last":0.38900000000000001},{"interface":"front","average":{"1min":7.7400000000000002,"5min":11.098000000000001,"15min":16.379999999999999},"min":{"1min":0.28899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":98.918000000000006,"5min":274.21600000000001,"15min":274.21600000000001},"last":0.375}]},{"osd":2,"last update":"Sat Sep 30 09:38:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":4.8319999999999999,"15min":3.5},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25600000000000001},"max":{"1min":8.7880000000000003,"5min":76.974000000000004,"15min":145.75899999999999},"last":0.308},{"interface":"front","average":{"1min":2.6120000000000001,"5min":4.6749999999999998,"15min":3.3620000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":11.366,"5min":77.593999999999994,"15min":150.358},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:38:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":2.8330000000000002,"15min":1.764},"min":{"1min":0.36599999999999999,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":11.414999999999999,"5min":60.637,"15min":60.637},"last":0.63},{"interface":"front","average":{"1min":1.488,"5min":2.4289999999999998,"15min":1.6850000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.27400000000000002},"max":{"1min":9.2140000000000004,"5min":10.731,"15min":15.638},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:38:03 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":3.1899999999999999,"15min":2.0249999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.308},"max":{"1min":9.2469999999999999,"5min":37.485999999999997,"15min":37.485999999999997},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.891,"5min":3.431,"15min":2.1920000000000002},"min":{"1min":0.435,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":8.6999999999999993,"5min":37.448,"15min":37.448},"last":0.72799999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:38:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":3.0579999999999998,"15min":1.901},"min":{"1min":0.39500000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":11.452,"5min":60.698,"15min":60.698},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.9159999999999999,"5min":2.8900000000000001,"15min":1.982},"min":{"1min":0.435,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":9.2840000000000007,"5min":12.121,"15min":14.69},"last":0.68400000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:38:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":3.1400000000000001,"15min":2.8319999999999999},"min":{"1min":0.497,"5min":0.39100000000000001,"15min":0.32100000000000001},"max":{"1min":8.6530000000000005,"5min":24.687000000000001,"15min":226.03999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.881,"5min":3.0920000000000001,"15min":2.681},"min":{"1min":0.49399999999999999,"5min":0.38800000000000001,"15min":0.34799999999999998},"max":{"1min":11.504,"5min":60.670000000000002,"15min":226.06100000000001},"last":0.70399999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019228,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":452700,"kb_used_data":183124,"kb_used_omap":0,"kb_used_meta":269568,"kb_avail":93292452,"statfs":{"total":95995035648,"available":95531470848,"internally_reserved":0,"allocated":187518976,"data_stored":186293222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":276037632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:42 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":1.958,"15min":3.0649999999999999},"min":{"1min":0.314,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":26.309999999999999,"5min":26.309999999999999,"15min":128.79300000000001},"last":0.95799999999999996},{"interface":"front","average":{"1min":2.6230000000000002,"5min":2.2389999999999999,"15min":3.0129999999999999},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.248},"max":{"1min":26.433,"5min":26.433,"15min":128.38800000000001},"last":0.69799999999999995}]},{"osd":1,"last update":"Sat Sep 30 09:37:42 2023","interfaces":[{"interface":"back","average":{"1min":4.1150000000000002,"5min":5.7889999999999997,"15min":8.2650000000000006},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":25.663,"5min":160.69999999999999,"15min":160.69999999999999},"last":1.238},{"interface":"front","average":{"1min":2.8370000000000002,"5min":5.4500000000000002,"15min":8.1929999999999996},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":25.367999999999999,"5min":160.50700000000001,"15min":160.50700000000001},"last":0.91800000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":2.089,"15min":2.9980000000000002},"min":{"1min":0.309,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.984,"5min":19.391999999999999,"15min":140.601},"last":1.296},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.2080000000000002,"15min":3.0230000000000001},"min":{"1min":0.36899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":7.085,"5min":19.422999999999998,"15min":140.54900000000001},"last":0.88}]},{"osd":3,"last update":"Sat Sep 30 09:37:48 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":3.399,"15min":3.802},"min":{"1min":0.40500000000000003,"5min":0.315,"15min":0.26400000000000001},"max":{"1min":28.527000000000001,"5min":52.762,"15min":281.42200000000003},"last":1.028},{"interface":"front","average":{"1min":3.4390000000000001,"5min":3.4220000000000002,"15min":3.7599999999999998},"min":{"1min":0.39100000000000001,"5min":0.312,"15min":0.312},"max":{"1min":28.486000000000001,"5min":52.790999999999997,"15min":281.44999999999999},"last":0.432}]},{"osd":5,"last update":"Sat Sep 30 09:37:42 2023","interfaces":[{"interface":"back","average":{"1min":10.028,"5min":2.766,"15min":2.3500000000000001},"min":{"1min":0.30099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":172.97800000000001,"5min":172.97800000000001,"15min":172.97800000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":9.8870000000000005,"5min":2.8039999999999998,"15min":2.395},"min":{"1min":0.29199999999999998,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":172.92400000000001,"5min":172.92400000000001,"15min":172.92400000000001},"last":1.4410000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:38:05 2023","interfaces":[{"interface":"back","average":{"1min":10.061999999999999,"5min":4.8209999999999997,"15min":2.2250000000000001},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":182.10300000000001,"5min":202.72499999999999,"15min":202.72499999999999},"last":1.0660000000000001},{"interface":"front","average":{"1min":9.8759999999999994,"5min":4.6769999999999996,"15min":2.2130000000000001},"min":{"1min":0.34799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":181.76300000000001,"5min":202.42400000000001,"15min":202.42400000000001},"last":0.66800000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:37:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.0910000000000002,"15min":3.0619999999999998},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.30299999999999999},"max":{"1min":5.4059999999999997,"5min":85.114000000000004,"15min":230.90100000000001},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.22,"5min":1.772,"15min":2.5960000000000001},"min":{"1min":0.33200000000000002,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":7.1550000000000002,"5min":84.697999999999993,"15min":229.97},"last":0.84099999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757592,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":633880,"kb_used_data":338640,"kb_used_omap":0,"kb_used_meta":295232,"kb_avail":93111272,"statfs":{"total":95995035648,"available":95345942528,"internally_reserved":0,"allocated":346767360,"data_stored":345542228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":302317568},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":2.3260000000000001,"15min":2.508},"min":{"1min":0.249,"5min":0.223,"15min":0.22},"max":{"1min":9.1460000000000008,"5min":56.563000000000002,"15min":148.79900000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.204,"5min":2.3809999999999998,"15min":2.7949999999999999},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":8.4930000000000003,"5min":57.241999999999997,"15min":148.779},"last":0.23200000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:37:40 2023","interfaces":[{"interface":"back","average":{"1min":3.3079999999999998,"5min":2.956,"15min":2.21},"min":{"1min":0.27300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":13.363,"5min":60.799999999999997,"15min":60.799999999999997},"last":1.492},{"interface":"front","average":{"1min":2.5790000000000002,"5min":3.464,"15min":3.262},"min":{"1min":0.36199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":16.757000000000001,"5min":63.798999999999999,"15min":63.798999999999999},"last":0.78100000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:37:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9199999999999999,"15min":3.149},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.9180000000000001,"5min":30.613,"15min":165.768},"last":1.03},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.327,"15min":3.714},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.5439999999999996,"5min":36.488,"15min":166.24000000000001},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:38:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":3.7869999999999999,"15min":4.0780000000000003},"min":{"1min":0.38900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.351,"5min":49.875,"15min":226.34700000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.508,"5min":3.6619999999999999,"15min":3.887},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.313},"max":{"1min":7.3769999999999998,"5min":49.895000000000003,"15min":226.36000000000001},"last":1.522}]},{"osd":4,"last update":"Sat Sep 30 09:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.76900000000000002,"15min":0.85999999999999999},"min":{"1min":0.26400000000000001,"5min":0.25800000000000001,"15min":0.222},"max":{"1min":3.1360000000000001,"5min":5.7160000000000002,"15min":8.5359999999999996},"last":1.415},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.94799999999999995,"15min":0.95499999999999996},"min":{"1min":0.28499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.4790000000000001,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.28799999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":2.9980000000000002,"15min":1.7509999999999999},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":3.2069999999999999,"5min":155.602,"15min":155.602},"last":1.4470000000000001},{"interface":"front","average":{"1min":0.67400000000000004,"5min":3.012,"15min":1.867},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.9490000000000001,"5min":154.00800000000001,"15min":154.00800000000001},"last":0.35399999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":2.5590000000000002,"15min":1.659},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.883,"5min":133.40700000000001,"15min":133.40700000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":2.7229999999999999,"15min":1.5840000000000001},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":5.7880000000000003,"5min":132.797,"15min":132.797},"last":1.468}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":81526784,"data_stored":81526784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":136511488,"data_stored":136511488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":213975040,"data_stored":213975040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":410845184,"data_stored":410845184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":261947392,"data_stored":261947392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:38:14.197 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-30T09:38:14.197 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:38:14.197 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-30T09:38:14.197 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-30T09:38:14.234 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:14.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:14.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:14.340 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:14.380 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:14.462 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:14.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:14.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:14.509 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 9 16 20376 20360 141.344 105.062 0.00663651 0.00707017 2023-09-30T09:38:14.568 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:14.632 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:14.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:14 smithi167 ceph-mon[192880]: pgmap v941: 119 pgs: 119 active+clean; 809 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-30T09:38:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:14 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:14 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:14.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:14 smithi019 ceph-mon[216815]: pgmap v941: 119 pgs: 119 active+clean; 809 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-30T09:38:14.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:14 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:14.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:14 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:14.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:14 smithi019 ceph-mon[224043]: pgmap v941: 119 pgs: 119 active+clean; 809 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-30T09:38:14.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:14 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:14.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:14 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:15.513 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 10 16 21886 21870 136.578 94.375 0.00517132 0.00731211 2023-09-30T09:38:15.522 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:15.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:15.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:15.634 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:15.684 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:15.940 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:15.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:15.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:15.997 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:16.057 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:16.093 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:16.155 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:16.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:16.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:16.261 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:16.368 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:16.514 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 11 16 23413 23397 132.837 95.4375 0.00626566 0.00752131 2023-09-30T09:38:16.645 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:38:17.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:16 smithi167 ceph-mon[192880]: pgmap v942: 119 pgs: 119 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-30T09:38:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2244260239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:38:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:16 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:38:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:16 smithi019 ceph-mon[224043]: pgmap v942: 119 pgs: 119 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-30T09:38:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2244260239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:38:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:16 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:38:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:16 smithi019 ceph-mon[216815]: pgmap v942: 119 pgs: 119 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-30T09:38:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2244260239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:38:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:16 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:38:17.302 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:38:17.353 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:17.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:17.358 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:38:17.457 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:17.484 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:17.514 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 12 16 24643 24627 128.175 76.875 0.00297395 0.00769451 2023-09-30T09:38:17.529 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:17.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:17.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:17.596 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:38:17.635 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:17.644 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:38:17.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:17.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:38:17.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[216815]: failed to encode map e364 with expected crc 2023-09-30T09:38:17.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[216815]: osdmap e364: 8 total, 8 up, 7 in 2023-09-30T09:38:17.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[216815]: failed to encode map e364 with expected crc 2023-09-30T09:38:17.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[216815]: failed to encode map e364 with expected crc 2023-09-30T09:38:17.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/516564163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:17.674 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:17.717 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:17.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:17.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:17.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:17 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:38:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:17 smithi167 ceph-mon[192880]: failed to encode map e364 with expected crc 2023-09-30T09:38:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:17 smithi167 ceph-mon[192880]: osdmap e364: 8 total, 8 up, 7 in 2023-09-30T09:38:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:17 smithi167 ceph-mon[192880]: failed to encode map e364 with expected crc 2023-09-30T09:38:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:17 smithi167 ceph-mon[192880]: failed to encode map e364 with expected crc 2023-09-30T09:38:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/516564163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:17.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:38:17.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:38:17.818 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:38:17.824 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:17.889 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:18.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:38:18.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[224043]: failed to encode map e364 with expected crc 2023-09-30T09:38:18.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[224043]: osdmap e364: 8 total, 8 up, 7 in 2023-09-30T09:38:18.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[224043]: failed to encode map e364 with expected crc 2023-09-30T09:38:18.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[224043]: failed to encode map e364 with expected crc 2023-09-30T09:38:18.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/516564163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:18.309 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:18.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:18.414 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:18.492 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:18.514 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 13 16 26033 26017 124.998 86.875 0.00969983 0.00799434 2023-09-30T09:38:18.972 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:18 smithi167 ceph-mon[192880]: pgmap v944: 119 pgs: 119 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-30T09:38:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:19.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:18 smithi019 ceph-mon[216815]: pgmap v944: 119 pgs: 119 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-30T09:38:19.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:19.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:19.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:19.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:18 smithi019 ceph-mon[224043]: pgmap v944: 119 pgs: 119 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-30T09:38:19.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:19.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:19.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:19.318 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:19.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:19.427 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:19.463 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:19.516 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 14 16 27471 27455 122.49 89.875 0.00792776 0.00815815 2023-09-30T09:38:19.528 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:19.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:19.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:19.634 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:19.672 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:19.738 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:19.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:19.844 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:19.908 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:20.267 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:38:20.469 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:20.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:20.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:20.514 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 15 16 29370 29354 122.236 118.688 0.00811061 0.00817446 2023-09-30T09:38:20.576 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:20.683 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:20 smithi019 ceph-mon[216815]: pgmap v945: 119 pgs: 119 active+clean; 1.5 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-09-30T09:38:21.093 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:21.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:21.197 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:21.238 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:20 smithi167 ceph-mon[192880]: pgmap v945: 119 pgs: 119 active+clean; 1.5 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-09-30T09:38:21.315 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:21.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:21.334 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:38:21.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:20 smithi019 ceph-mon[224043]: pgmap v945: 119 pgs: 119 active+clean; 1.5 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-09-30T09:38:21.422 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:21.514 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:21.515 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 16 16 30755 30739 120.003 86.5625 0.0166353 0.00832592 2023-09-30T09:38:22.118 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:22.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:22.224 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:22.310 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:22.516 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 17 16 31791 31775 116.748 64.75 0.00405087 0.00855887 2023-09-30T09:38:22.653 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:22.760 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:22.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:22.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:22.872 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:22.902 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:22.957 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:22.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:22.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:22.985 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:38:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:38:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:38:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:38:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:145247 7.6 7:6ffe3934:::benchmark_data_smithi019_404601_object31969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-09-30T09:38:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.509374+0000", 2023-09-30T09:38:22.985 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 226.46662844100001, 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.269183332, 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145247 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.986 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509374+0000", 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509374+0000", 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509376+0000", 2023-09-30T09:38:22.987 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5999999999999999e-06 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509382+0000", 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5679999999999996e-06 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509399+0000", 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6305000000000001e-05 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.988 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.509411+0000", 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1966e-05 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.516749+0000", 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0073378269999999999 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:22.989 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.516898+0000", 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014945299999999999 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517008+0000", 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000110299 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524212+0000", 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0072039929999999997 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.990 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.529282+0000", 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050696489999999999 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.773766+0000", 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.24448402299999999 2023-09-30T09:38:22.991 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.773827+0000", 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1353999999999996e-05 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.778558+0000", 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0047302949999999998 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:22.992 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:145256 7.6 7:6d734290:::benchmark_data_smithi019_404601_object31978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.524177+0000", 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 226.45182531699999, 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25938216400000003, 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 145256 2023-09-30T09:38:22.993 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524177+0000", 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524177+0000", 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.994 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524179+0000", 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.268e-06 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524190+0000", 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0905e-05 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:22.995 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524203+0000", 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3406e-05 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524211+0000", 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2779999999999993e-06 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524257+0000", 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5769999999999997e-05 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.996 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524370+0000", 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000112953 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.524467+0000", 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6589999999999995e-05 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.529403+0000", 2023-09-30T09:38:22.997 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049365889999999999 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.529872+0000", 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00046835699999999997 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.778631+0000", 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.248759229 2023-09-30T09:38:22.998 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.778703+0000", 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2402000000000002e-05 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.783560+0000", 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004856417 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:22.999 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:181474 7.6 7:6809b434:::benchmark_data_smithi019_404601_object68196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:56.400375+0000", 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 146.57562754200001, 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.39013431399999998, 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 181474 2023-09-30T09:38:23.000 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.400375+0000", 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.400375+0000", 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.001 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.400376+0000", 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5200000000000002e-07 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.400378+0000", 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7049999999999999e-06 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.002 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.400384+0000", 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8000000000000004e-06 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.400389+0000", 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8670000000000002e-06 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.003 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.407127+0000", 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0067378960000000002 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.407299+0000", 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017238299999999999 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:38:23.004 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.407475+0000", 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017559300000000001 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.422395+0000", 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014919763000000001 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.425038+0000", 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002642883 2023-09-30T09:38:23.005 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.787936+0000", 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.362898094 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.787993+0000", 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7751000000000003e-05 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.006 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.790509+0000", 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025161269999999999 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:181485 7.6 7:67eeb4cd:::benchmark_data_smithi019_404601_object68207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:56.422854+0000", 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 146.55314887, 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.370218885, 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.007 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 181485 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.422854+0000", 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.008 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.422854+0000", 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.422855+0000", 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5260000000000001e-06 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.422861+0000", 2023-09-30T09:38:23.009 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9920000000000004e-06 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.422877+0000", 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5282e-05 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.422884+0000", 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3950000000000003e-06 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.010 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.425074+0000", 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021900399999999999 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.425265+0000", 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019049400000000001 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.425454+0000", 2023-09-30T09:38:23.011 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018952899999999999 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.430941+0000", 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0054865620000000004 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.430987+0000", 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5849000000000001e-05 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.012 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.790649+0000", 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.359662918 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.790705+0000", 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5995999999999997e-05 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.013 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:56.793073+0000", 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023673019999999999 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:182215 7.6 7:65ec635f:::benchmark_data_smithi019_404601_object68937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:58.375252+0000", 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 144.600750455, 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27563141800000002, 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.014 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 182215 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.375252+0000", 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.015 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.375252+0000", 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.375254+0000", 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5149999999999999e-06 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.016 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.375260+0000", 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0650000000000004e-06 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.375275+0000", 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5328e-05 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.375283+0000", 2023-09-30T09:38:23.017 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9160000000000001e-06 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.381547+0000", 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062641279999999999 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.381716+0000", 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016830099999999999 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.018 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.381872+0000", 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000156537 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.396482+0000", 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014609834 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.019 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.396560+0000", 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8195999999999995e-05 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.648497+0000", 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25193683700000002 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.648553+0000", 2023-09-30T09:38:23.020 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5930000000000002e-05 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:58.650884+0000", 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023308309999999998 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:197904 7.6 7:63fe7422:::benchmark_data_smithi019_404601_object84626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-30T09:38:23.021 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:35.301471+0000", 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.674531727, 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.310057843, 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 197904 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.022 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.301471+0000", 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.301471+0000", 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.301473+0000", 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8500000000000001e-06 2023-09-30T09:38:23.023 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.301480+0000", 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1949999999999997e-06 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.301494+0000", 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3747999999999999e-05 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.024 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.301504+0000", 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0434999999999999e-05 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.301863+0000", 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035891900000000002 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.025 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.302023+0000", 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000159469 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.302199+0000", 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017680899999999999 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305859+0000", 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003660085 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.026 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610340+0000", 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30448065499999999 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610392+0000", 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1867000000000003e-05 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.027 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610484+0000", 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1556000000000002e-05 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.611529+0000", 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0010452549999999999 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.028 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:197895 7.5 e339/327 7:a7dc683e:::benchmark_data_smithi019_404601_object84617:head v 339'12389, mlcod=292'5308)", 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:35.305700+0000", 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.67030294600001, 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30454469699999998, 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305700+0000", 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.029 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305700+0000", 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305700+0000", 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8999999999999996e-07 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.030 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305707+0000", 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3589999999999996e-06 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305717+0000", 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0042e-05 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305724+0000", 2023-09-30T09:38:23.031 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8140000000000003e-06 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305799+0000", 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5474000000000003e-05 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.305819+0000", 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0086e-05 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.310039+0000", 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0042198340000000004 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610244+0000", 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30020539800000001 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:197906 7.f e339/327 7:f82d676b:::benchmark_data_smithi019_404601_object84628:head v 339'12270, mlcod=339'12267)", 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:35.309862+0000", 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.66614080399999, 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31096722100000002, 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309862+0000", 2023-09-30T09:38:23.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309862+0000", 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309863+0000", 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.71e-07 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309870+0000", 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6790000000000005e-06 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309882+0000", 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1915e-05 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309891+0000", 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4800000000000001e-06 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.310039+0000", 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000147968 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610224+0000", 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30018524699999999 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.620781+0000", 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010556618 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.620829+0000", 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8442999999999997e-05 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop_reply(client.35424.0:197904 7.6 e339/320 ondisk, result = 0)", 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:35.309925+0000", 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.666078062, 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.300426688, 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309925+0000", 2023-09-30T09:38:23.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309925+0000", 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309925+0000", 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7599999999999998e-07 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309928+0000", 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9950000000000001e-06 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309934+0000", 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4699999999999999e-06 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309939+0000", 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8160000000000001e-06 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610234+0000", 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30029493000000002 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610331+0000", 2023-09-30T09:38:23.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6751999999999998e-05 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610351+0000", 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0449e-05 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:197912 7.f e339/327 7:fc5f7a4a:::benchmark_data_smithi019_404601_object84634:head v 339'12271, mlcod=339'12267)", 2023-09-30T09:38:23.043 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:35.309953+0000", 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.666049577, 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31091738099999999, 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309953+0000", 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309953+0000", 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309953+0000", 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9000000000000001e-07 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309957+0000", 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8419999999999998e-06 2023-09-30T09:38:23.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309963+0000", 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8100000000000003e-06 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.309967+0000", 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2749999999999997e-06 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.612538+0000", 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30257118599999999 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.612560+0000", 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1148000000000001e-05 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.620832+0000", 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0082724649999999997 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.620871+0000", 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8464999999999998e-05 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.048 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPushReply(7.6 339/320 [PushReplyOp(7:6648d98c:::benchmark_data_smithi019_404601_object103257:head)])", 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:35.311326+0000", 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.66467631499999, 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30133704700000002, 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.311326+0000", 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.049 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.311326+0000", 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.311327+0000", 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8599999999999997e-07 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.311335+0000", 2023-09-30T09:38:23.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3130000000000001e-06 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610204+0000", 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.29886948299999999 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610215+0000", 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0925e-05 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.611568+0000", 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0013530860000000001 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.612663+0000", 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001095354 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:197910 7.7 e339/279 7:eadd4151:::benchmark_data_smithi019_404601_object84632:head v 339'12483, mlcod=339'12481)", 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:35.311902+0000", 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.66410031700001, 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30900577200000001, 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.311902+0000", 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.311902+0000", 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.311903+0000", 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0800000000000004e-07 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.311909+0000", 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0299999999999999e-06 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610192+0000", 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.29828258800000002 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610203+0000", 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1177e-05 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.614355+0000", 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041517139999999999 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.614368+0000", 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3179999999999999e-05 2023-09-30T09:38:23.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.620873+0000", 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0065051370000000002 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.620908+0000", 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5237999999999997e-05 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.057 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:197914 7.f e339/327 7:f75f9790:::benchmark_data_smithi019_404601_object84636:head v 339'12272, mlcod=339'12267)", 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:35.314772+0000", 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.66123095499999, 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30624558800000001, 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.058 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.314772+0000", 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.314772+0000", 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.314773+0000", 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8200000000000008e-07 2023-09-30T09:38:23.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.314782+0000", 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7900000000000005e-06 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610176+0000", 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.29539464300000001 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.610196+0000", 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0296999999999999e-05 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.616469+0000", 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062721690000000002 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.616488+0000", 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9516000000000001e-05 2023-09-30T09:38:23.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.620985+0000", 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0044971289999999999 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:35.621017+0000", 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2061999999999999e-05 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.062 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:199698 7.6 7:6d837a4a:::benchmark_data_smithi019_404601_object86420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:39.646448+0000", 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 103.329554737, 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32026793399999998, 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:23.063 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 199698 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.646448+0000", 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.646448+0000", 2023-09-30T09:38:23.064 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.646449+0000", 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4729999999999999e-06 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.646456+0000", 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.426e-06 2023-09-30T09:38:23.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.646469+0000", 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2666e-05 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.646476+0000", 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3830000000000003e-06 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.066 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.647430+0000", 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000954214 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.647582+0000", 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015182999999999999 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.647720+0000", 2023-09-30T09:38:23.067 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013782099999999999 2023-09-30T09:38:23.069 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.069 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.658220+0000", 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010499730000000001 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.658332+0000", 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011209000000000001 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.070 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.965950+0000", 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30761807400000002 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.966011+0000", 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1803000000000004e-05 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:39.966716+0000", 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00070442400000000004 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.071 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:207875 7.6 7:616d2b76:::benchmark_data_smithi019_404601_object94597:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:58.472338+0000", 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 84.503664798000003, 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30838732600000002, 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.072 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 207875 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472338+0000", 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.073 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472338+0000", 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472339+0000", 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4810000000000001e-06 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472346+0000", 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4019999999999999e-06 2023-09-30T09:38:23.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472363+0000", 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6832000000000001e-05 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472371+0000", 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.799e-06 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.075 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472441+0000", 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9256000000000002e-05 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472621+0000", 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000180793 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.472802+0000", 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018102 2023-09-30T09:38:23.076 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.475909+0000", 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031067130000000001 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.480265+0000", 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043555089999999996 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.077 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.780387+0000", 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.300121945 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.780453+0000", 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6537000000000003e-05 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.780725+0000", 2023-09-30T09:38:23.078 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027203900000000002 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35424.0:207879 7.6 7:6491e80d:::benchmark_data_smithi019_404601_object94601:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:36:58.482765+0000", 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 84.493237789999995, 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30042815699999997, 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.079 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35424", 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 207879 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.482765+0000", 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.080 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.482765+0000", 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.482766+0000", 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0589999999999999e-06 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.482770+0000", 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3640000000000001e-06 2023-09-30T09:38:23.081 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.482783+0000", 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3030000000000001e-05 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.482789+0000", 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6670000000000001e-06 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.082 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.482836+0000", 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7169000000000002e-05 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.482953+0000", 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011663900000000001 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.483067+0000", 2023-09-30T09:38:23.083 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000114116 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.485681+0000", 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002613921 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.488296+0000", 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026154809999999998 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.084 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.782876+0000", 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.29457920999999998 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.782924+0000", 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8408000000000001e-05 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:36:58.783193+0000", 2023-09-30T09:38:23.085 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00026909299999999998 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:26047 8.e 8:72faff48:::benchmark_data_smithi019_478679_object26046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:18.525655+0000", 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 4.4503479800000001, 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.26303435800000002, 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.086 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 26047 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.525655+0000", 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.087 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.525655+0000", 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.525658+0000", 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9450000000000002e-06 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.525683+0000", 2023-09-30T09:38:23.088 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5496000000000001e-05 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.525701+0000", 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7580000000000001e-05 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.525708+0000", 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5689999999999999e-06 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.525756+0000", 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7919e-05 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.525882+0000", 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000125711 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:38:23.090 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.526010+0000", 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012782000000000001 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.532882+0000", 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0068725770000000004 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.532908+0000", 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6140999999999999e-05 2023-09-30T09:38:23.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.788588+0000", 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25567995500000001 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.788652+0000", 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3601000000000003e-05 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.788689+0000", 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7044e-05 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:26055 8.6 8:6cb4ddf5:::benchmark_data_smithi019_478679_object26054:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:18.527223+0000", 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 4.4487798700000001, 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.26234112100000001, 2023-09-30T09:38:23.093 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 26055 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.527223+0000", 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.094 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.527223+0000", 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.527224+0000", 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1200000000000001e-06 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.527247+0000", 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2697e-05 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.527257+0000", 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0376e-05 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.527264+0000", 2023-09-30T09:38:23.096 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4259999999999997e-06 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.529026+0000", 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0017614989999999999 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.529180+0000", 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015361800000000001 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.529328+0000", 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000148868 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.532917+0000", 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035886239999999999 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.098 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.535347+0000", 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024295760000000001 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.789480+0000", 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25413328400000001 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.789525+0000", 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5323e-05 2023-09-30T09:38:23.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.789564+0000", 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.871e-05 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:26068 8.1 8:85f4e42a:::benchmark_data_smithi019_478679_object26067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:18.533133+0000", 2023-09-30T09:38:23.100 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 4.4428694909999997, 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25573431800000002, 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 26068 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.533133+0000", 2023-09-30T09:38:23.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.533133+0000", 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.533135+0000", 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2920000000000001e-06 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.533178+0000", 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.371e-05 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.533216+0000", 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7945999999999999e-05 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.533225+0000", 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6230000000000003e-06 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.535035+0000", 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001810367 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.535145+0000", 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000109759 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.535262+0000", 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000116657 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.536856+0000", 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015939890000000001 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.537560+0000", 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00070414299999999996 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.788772+0000", 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25121204699999999 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.788831+0000", 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9101000000000002e-05 2023-09-30T09:38:23.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.788868+0000", 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6684000000000003e-05 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:26076 8.6 8:60ddb856:::benchmark_data_smithi019_478679_object26075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:18.538018+0000", 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 4.437984353, 2023-09-30T09:38:23.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.251690519, 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 26076 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538018+0000", 2023-09-30T09:38:23.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538018+0000", 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538020+0000", 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2839999999999999e-06 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.109 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538039+0000", 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9426999999999999e-05 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538055+0000", 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6277999999999999e-05 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538063+0000", 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0770000000000008e-06 2023-09-30T09:38:23.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538124+0000", 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0194e-05 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538277+0000", 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000153578 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.538437+0000", 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015938500000000001 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.539625+0000", 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011888140000000001 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.539952+0000", 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00032651799999999998 2023-09-30T09:38:23.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.789602+0000", 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.24965047500000001 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.789666+0000", 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3675000000000003e-05 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:38:23.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:38:23.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:18.789709+0000", 2023-09-30T09:38:23.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2814e-05 2023-09-30T09:38:23.114 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.114 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.114 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.114 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:38:23.114 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:38:23.114 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:38:23.114 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:23.115 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:22 smithi019 ceph-mon[216815]: pgmap v946: 119 pgs: 119 active+clean; 1.7 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:38:23.151 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:22 smithi167 ceph-mon[192880]: pgmap v946: 119 pgs: 119 active+clean; 1.7 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:38:23.151 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:23.213 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:23.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:23.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:23.323 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:22 smithi019 ceph-mon[224043]: pgmap v946: 119 pgs: 119 active+clean; 1.7 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:38:23.388 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:23.516 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 18 16 33297 33281 115.491 94.125 0.00389168 0.00865236 2023-09-30T09:38:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:38:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:38:23] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2023-09-30T09:38:24.516 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 19 16 34035 34019 111.842 46.125 0.00804499 0.00893639 2023-09-30T09:38:24.590 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:24.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:24.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:24.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:24.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:24 smithi019 ceph-mon[216815]: pgmap v947: 119 pgs: 119 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-30T09:38:24.784 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:24 smithi019 ceph-mon[224043]: pgmap v947: 119 pgs: 119 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-30T09:38:24.787 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:25.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:24 smithi167 ceph-mon[192880]: pgmap v947: 119 pgs: 119 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-30T09:38:25.449 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:38:25.519 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:38:25.519591+0000 min lat: 0.0012782 max lat: 0.313209 avg lat: 0.00905413 2023-09-30T09:38:25.519 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:38:25.519 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 20 16 35361 35345 110.376 82.875 0.00789575 0.00905413 2023-09-30T09:38:25.644 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:25.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:25.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:25.656 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:25.755 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:25.786 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:25.844 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:25.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:25.951 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:25.990 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:26.047 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:26.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:26.153 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:26.182 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:26.232 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:26.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:26.338 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:26.415 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:26.519 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 21 16 36536 36520 108.618 73.4375 0.00212538 0.00916725 2023-09-30T09:38:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:26 smithi019 conmon[216791]: 2023-09-30T09:38:26.309+0000 7faf52971700 -1 mon.a@0(leader).osd e364 definitely_dead 0 2023-09-30T09:38:26.659 INFO:teuthology.orchestra.run.smithi019.stderr:marked in osd.7. 2023-09-30T09:38:27.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:26 smithi167 ceph-mon[192880]: pgmap v948: 119 pgs: 119 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-30T09:38:27.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/194258891' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-30T09:38:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:26 smithi019 ceph-mon[216815]: pgmap v948: 119 pgs: 119 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-30T09:38:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/194258891' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-30T09:38:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:26 smithi019 ceph-mon[224043]: pgmap v948: 119 pgs: 119 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-30T09:38:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/194258891' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-30T09:38:27.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 22 16 36604 36588 103.876 4.25 0.00144376 0.00925321 2023-09-30T09:38:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/194258891' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-30T09:38:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[216815]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[216815]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[216815]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[216815]: osdmap e365: 8 total, 8 up, 8 in 2023-09-30T09:38:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[216815]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[216815]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[216815]: osdmap e366: 8 total, 8 up, 8 in 2023-09-30T09:38:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/194258891' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-30T09:38:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[224043]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[224043]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[224043]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[224043]: osdmap e365: 8 total, 8 up, 8 in 2023-09-30T09:38:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[224043]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[224043]: failed to encode map e365 with expected crc 2023-09-30T09:38:27.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:27 smithi019 ceph-mon[224043]: osdmap e366: 8 total, 8 up, 8 in 2023-09-30T09:38:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/194258891' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-30T09:38:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:27 smithi167 ceph-mon[192880]: failed to encode map e365 with expected crc 2023-09-30T09:38:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:27 smithi167 ceph-mon[192880]: failed to encode map e365 with expected crc 2023-09-30T09:38:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:27 smithi167 ceph-mon[192880]: failed to encode map e365 with expected crc 2023-09-30T09:38:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:27 smithi167 ceph-mon[192880]: osdmap e365: 8 total, 8 up, 8 in 2023-09-30T09:38:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:27 smithi167 ceph-mon[192880]: failed to encode map e365 with expected crc 2023-09-30T09:38:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:27 smithi167 ceph-mon[192880]: failed to encode map e365 with expected crc 2023-09-30T09:38:28.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:27 smithi167 ceph-mon[192880]: osdmap e366: 8 total, 8 up, 8 in 2023-09-30T09:38:28.182 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:28.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:28.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:28.286 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:28.314 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:28.358 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:28.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:28.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:28.464 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:28.494 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:28.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 23 16 36604 36588 99.3622 0 - 0.00925321 2023-09-30T09:38:28.656 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:38:28.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: pgmap v950: 119 pgs: 8 peering, 111 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-30T09:38:28.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.678 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.679 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.679 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.679 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.679 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.827 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: pgmap v950: 119 pgs: 8 peering, 111 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-30T09:38:28.827 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/311916434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:28.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:28.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:28.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:28.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[224043]: osdmap e367: 8 total, 8 up, 8 in 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: pgmap v950: 119 pgs: 8 peering, 111 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: failed to encode map e366 with expected crc 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/311916434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:28.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:28 smithi019 ceph-mon[216815]: osdmap e367: 8 total, 8 up, 8 in 2023-09-30T09:38:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: failed to encode map e366 with expected crc 2023-09-30T09:38:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/311916434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:28 smithi167 ceph-mon[192880]: osdmap e367: 8 total, 8 up, 8 in 2023-09-30T09:38:29.052 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:29.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:29.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:29.162 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:29.227 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:29.324 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:29.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:29.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:29.434 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:29.467 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:29.511 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:29.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:29.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 24 16 37421 37405 97.3506 25.5312 0.00342644 0.0102343 2023-09-30T09:38:29.616 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:29.672 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:29 smithi167 ceph-mon[192880]: failed to encode map e367 with expected crc 2023-09-30T09:38:29.774 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:29.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:29.879 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:29.942 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[216815]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:29 smithi019 ceph-mon[224043]: failed to encode map e367 with expected crc 2023-09-30T09:38:30.305 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-30T09:38:30.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 25 16 38574 38558 96.3393 72.0625 0.00459091 0.0103697 2023-09-30T09:38:30.742 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:30.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:30.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[224043]: pgmap v953: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 8 peering, 102 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 6999/109017 objects degraded (6.420%); 44 KiB/s, 1 keys/s, 4 objects/s recovering 2023-09-30T09:38:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 6999/109017 objects degraded (6.420%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[224043]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[224043]: osdmap e368: 8 total, 8 up, 8 in 2023-09-30T09:38:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[224043]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[224043]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[224043]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[224043]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[216815]: pgmap v953: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 8 peering, 102 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 6999/109017 objects degraded (6.420%); 44 KiB/s, 1 keys/s, 4 objects/s recovering 2023-09-30T09:38:30.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 6999/109017 objects degraded (6.420%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:30.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[216815]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[216815]: osdmap e368: 8 total, 8 up, 8 in 2023-09-30T09:38:30.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[216815]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[216815]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[216815]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:30 smithi019 ceph-mon[216815]: failed to encode map e368 with expected crc 2023-09-30T09:38:30.851 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:30.975 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:30 smithi167 ceph-mon[192880]: pgmap v953: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 8 peering, 102 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 6999/109017 objects degraded (6.420%); 44 KiB/s, 1 keys/s, 4 objects/s recovering 2023-09-30T09:38:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:30 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 6999/109017 objects degraded (6.420%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:30 smithi167 ceph-mon[192880]: failed to encode map e368 with expected crc 2023-09-30T09:38:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:30 smithi167 ceph-mon[192880]: osdmap e368: 8 total, 8 up, 8 in 2023-09-30T09:38:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:30 smithi167 ceph-mon[192880]: failed to encode map e368 with expected crc 2023-09-30T09:38:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:30 smithi167 ceph-mon[192880]: failed to encode map e368 with expected crc 2023-09-30T09:38:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:30 smithi167 ceph-mon[192880]: failed to encode map e368 with expected crc 2023-09-30T09:38:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:30 smithi167 ceph-mon[192880]: failed to encode map e368 with expected crc 2023-09-30T09:38:31.156 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:31.215 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:31.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:31.323 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:31.344 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:38:31.391 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:38:31.397 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:31.433 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:38:31.485 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:38:31.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 26 16 39664 39648 95.2544 68.125 0.0139564 0.0104881 2023-09-30T09:38:31.533 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:38:31.548 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:31.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:31.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:31.577 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:38:31.654 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:31.661 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:38:31.708 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:38:31.734 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:38:31.734 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:38:31.734 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:38:31.734 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:38:31.734 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:38:31.735 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:38:31.735 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:38:31.735 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:38:31.744 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:31.782 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.782 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.782 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: osdmap e369: 8 total, 8 up, 8 in 2023-09-30T09:38:31.782 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3401630600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:31.783 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: osdmap e369: 8 total, 8 up, 8 in 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: failed to encode map e369 with expected crc 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:31.784 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:31 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3401630600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: osdmap e369: 8 total, 8 up, 8 in 2023-09-30T09:38:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: failed to encode map e369 with expected crc 2023-09-30T09:38:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:32.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:31 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3401630600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:32.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 27 16 40780 40764 94.3093 69.75 0.00515351 0.0105982 2023-09-30T09:38:32.545 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:38:32.831 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:32.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:32.937 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:32.992 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:32 smithi167 ceph-mon[192880]: pgmap v956: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 peering, 98 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 9322/112089 objects degraded (8.317%); 2270/112089 objects misplaced (2.025%); 1.6 MiB/s, 2 keys/s, 33 objects/s recovering 2023-09-30T09:38:32.992 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:38:32.992 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:38:32.992 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:32.994 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:33.071 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:33.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:32 smithi019 ceph-mon[224043]: pgmap v956: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 peering, 98 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 9322/112089 objects degraded (8.317%); 2270/112089 objects misplaced (2.025%); 1.6 MiB/s, 2 keys/s, 33 objects/s recovering 2023-09-30T09:38:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:38:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:38:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:33.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:32 smithi019 ceph-mon[216815]: pgmap v956: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 peering, 98 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 9322/112089 objects degraded (8.317%); 2270/112089 objects misplaced (2.025%); 1.6 MiB/s, 2 keys/s, 33 objects/s recovering 2023-09-30T09:38:33.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:38:33.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:38:33.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:33.180 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:33.242 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:33.319 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:33.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:33.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:33.426 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:33.451 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:33.494 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:33.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:33.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:33.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 28 16 41889 41873 93.4167 69.3125 0.0151094 0.0106986 2023-09-30T09:38:33.600 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:33.652 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:33.722 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:33.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:33.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:33.833 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:33.890 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:33.961 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:33.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:33.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:33.979 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:38:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:38:33] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.43.0" 2023-09-30T09:38:34.065 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:34.114 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:34.198 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:34.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:34.311 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:34.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:34.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:34.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:34.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:34.370 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:34.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 29 16 43180 43164 92.9777 80.6875 0.00736546 0.0107487 2023-09-30T09:38:34.545 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:34.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:34.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:34.660 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:34.732 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:34.741 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:34.935 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:34.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:34.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:34.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:34 smithi019 ceph-mon[216815]: pgmap v957: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 1 peering, 106 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 9322/114993 objects degraded (8.107%); 2270/114993 objects misplaced (1.974%); 1.1 MiB/s, 1 keys/s, 22 objects/s recovering 2023-09-30T09:38:35.043 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:35.096 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:35.196 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:35.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:35.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:34 smithi167 ceph-mon[192880]: pgmap v957: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 1 peering, 106 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 9322/114993 objects degraded (8.107%); 2270/114993 objects misplaced (1.974%); 1.1 MiB/s, 1 keys/s, 22 objects/s recovering 2023-09-30T09:38:35.275 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:38:35.275 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:38:35.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:34 smithi019 ceph-mon[224043]: pgmap v957: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 1 peering, 106 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 9322/114993 objects degraded (8.107%); 2270/114993 objects misplaced (1.974%); 1.1 MiB/s, 1 keys/s, 22 objects/s recovering 2023-09-30T09:38:35.307 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:38:35.307 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:38:35.307 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [1, 3, 4, 5, 6, 0, 2, 7] 2023-09-30T09:38:35.307 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-30T09:38:35.309 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:35.354 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:35.438 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:35.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:35.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:35.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 30 16 44363 44347 92.3431 73.9375 0.0215845 0.0108222 2023-09-30T09:38:35.547 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:35.585 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:35.676 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:35.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:35.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:35.792 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:35.837 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:35.943 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:35.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:35.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:36.055 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:36.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:36 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9322/127971 objects degraded (7.284%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:36.114 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:36 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9322/127971 objects degraded (7.284%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:36.118 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:36.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:36 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9322/127971 objects degraded (7.284%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:36.520 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 31 16 45595 45579 91.8481 77 0.00822644 0.0108826 2023-09-30T09:38:36.549 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:38:37.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:37 smithi019 ceph-mon[216815]: pgmap v958: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.0 KiB/s rd, 70 MiB/s wr, 1.12k op/s; 9322/127971 objects degraded (7.284%); 1978/127971 objects misplaced (1.546%); 4.1 MiB/s, 1 keys/s, 63 objects/s recovering 2023-09-30T09:38:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:37 smithi019 ceph-mon[224043]: pgmap v958: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.0 KiB/s rd, 70 MiB/s wr, 1.12k op/s; 9322/127971 objects degraded (7.284%); 1978/127971 objects misplaced (1.546%); 4.1 MiB/s, 1 keys/s, 63 objects/s recovering 2023-09-30T09:38:37.425 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:37.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:37.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:37.522 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 32 16 46725 46709 91.1814 70.625 0.0166383 0.0109619 2023-09-30T09:38:37.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:37 smithi167 ceph-mon[192880]: pgmap v958: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.0 KiB/s rd, 70 MiB/s wr, 1.12k op/s; 9322/127971 objects degraded (7.284%); 1978/127971 objects misplaced (1.546%); 4.1 MiB/s, 1 keys/s, 63 objects/s recovering 2023-09-30T09:38:37.528 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:37.598 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:37.976 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:37.989 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:38:37.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:37.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:38.100 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:38.101 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:38.130 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:38.169 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:38.212 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:38.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:38.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:38.322 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:38.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:38 smithi019 ceph-mon[224043]: pgmap v959: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.7 KiB/s rd, 61 MiB/s wr, 982 op/s; 9322/132570 objects degraded (7.032%); 1978/132570 objects misplaced (1.492%); 3.6 MiB/s, 51 objects/s recovering 2023-09-30T09:38:38.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:38 smithi019 ceph-mon[216815]: pgmap v959: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.7 KiB/s rd, 61 MiB/s wr, 982 op/s; 9322/132570 objects degraded (7.032%); 1978/132570 objects misplaced (1.492%); 3.6 MiB/s, 51 objects/s recovering 2023-09-30T09:38:38.395 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:38.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:38 smithi167 ceph-mon[192880]: pgmap v959: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.7 KiB/s rd, 61 MiB/s wr, 982 op/s; 9322/132570 objects degraded (7.032%); 1978/132570 objects misplaced (1.492%); 3.6 MiB/s, 51 objects/s recovering 2023-09-30T09:38:38.525 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 33 16 47785 47769 90.426 66.25 0.0238899 0.0110516 2023-09-30T09:38:38.773 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:38.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:38.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:38.883 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:38.962 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:39.522 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 34 16 48920 48904 89.8528 70.9375 0.017076 0.0111236 2023-09-30T09:38:39.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:39 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:39.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:39 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:40.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:39 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:40.413 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:40.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:40.522 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 35 16 50069 50053 89.3374 71.8125 0.00275172 0.0111619 2023-09-30T09:38:40.524 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:40.569 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:40.585 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:38:40.692 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:40.703 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:40 smithi019 ceph-mon[216815]: pgmap v960: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 66 MiB/s wr, 1.05k op/s; 9322/140196 objects degraded (6.649%); 1978/140196 objects misplaced (1.411%); 2.9 MiB/s, 41 objects/s recovering 2023-09-30T09:38:40.703 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2236530495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:38:40.703 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:40 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:38:40.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:40.704 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:40 smithi019 ceph-mon[224043]: pgmap v960: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 66 MiB/s wr, 1.05k op/s; 9322/140196 objects degraded (6.649%); 1978/140196 objects misplaced (1.411%); 2.9 MiB/s, 41 objects/s recovering 2023-09-30T09:38:40.704 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2236530495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:38:40.704 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:40 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:38:40.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:40.836 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:41.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:40 smithi167 ceph-mon[192880]: pgmap v960: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 66 MiB/s wr, 1.05k op/s; 9322/140196 objects degraded (6.649%); 1978/140196 objects misplaced (1.411%); 2.9 MiB/s, 41 objects/s recovering 2023-09-30T09:38:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2236530495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:38:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:40 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:38:41.522 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 36 16 51088 51072 88.625 63.6875 0.00730233 0.0112752 2023-09-30T09:38:41.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:41.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:38:41.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[216815]: osdmap e370: 8 total, 8 up, 8 in 2023-09-30T09:38:41.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[216815]: failed to encode map e370 with expected crc 2023-09-30T09:38:41.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[216815]: failed to encode map e370 with expected crc 2023-09-30T09:38:41.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:41.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:38:41.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[224043]: osdmap e370: 8 total, 8 up, 8 in 2023-09-30T09:38:41.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[224043]: failed to encode map e370 with expected crc 2023-09-30T09:38:41.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:41 smithi019 ceph-mon[224043]: failed to encode map e370 with expected crc 2023-09-30T09:38:42.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:41 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:38:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:41 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:38:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:41 smithi167 ceph-mon[192880]: osdmap e370: 8 total, 8 up, 8 in 2023-09-30T09:38:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:41 smithi167 ceph-mon[192880]: failed to encode map e370 with expected crc 2023-09-30T09:38:42.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:41 smithi167 ceph-mon[192880]: failed to encode map e370 with expected crc 2023-09-30T09:38:42.444 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:42.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:42.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:42.522 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 37 16 51929 51913 87.6503 52.5625 0.0123269 0.0114035 2023-09-30T09:38:42.551 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:42.612 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:42.711 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:42.716 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:42.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:42.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:42.730 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:42 smithi019 ceph-mon[224043]: pgmap v962: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 70 MiB/s wr, 1.11k op/s; 9322/145539 objects degraded (6.405%); 1646/145539 objects misplaced (1.131%); 4.3 MiB/s, 63 objects/s recovering 2023-09-30T09:38:42.731 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:42 smithi019 ceph-mon[216815]: pgmap v962: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 70 MiB/s wr, 1.11k op/s; 9322/145539 objects degraded (6.405%); 1646/145539 objects misplaced (1.131%); 4.3 MiB/s, 63 objects/s recovering 2023-09-30T09:38:42.823 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:42.858 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:42 smithi167 ceph-mon[192880]: pgmap v962: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 70 MiB/s wr, 1.11k op/s; 9322/145539 objects degraded (6.405%); 1646/145539 objects misplaced (1.131%); 4.3 MiB/s, 63 objects/s recovering 2023-09-30T09:38:42.863 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:42.970 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:42.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:43.080 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:43.171 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:43.323 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:43.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:43.433 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:43.483 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:43.523 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:38:43.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 38 16 53000 52984 87.1052 66.9375 0.0260199 0.0114746 2023-09-30T09:38:43.559 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:38:43.565 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:43.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:43.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9322/145539 objects degraded (6.405%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:43.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1833670571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:43.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9322/145539 objects degraded (6.405%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:43.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1833670571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:43.619 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:38:43.663 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:38:43.670 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:43.712 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:38:43.715 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:43.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:38:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9322/145539 objects degraded (6.405%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1833670571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:43.792 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:43.796 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:38:43.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:43.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:43.835 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:38:43.838 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:38:43.838 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:38:43.838 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:38:43.838 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:38:43.838 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:38:43.838 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:38:43.838 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:38:43.839 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:38:43.869 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:38:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:38:43] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.43.0" 2023-09-30T09:38:43.900 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:43.955 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:44.025 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:44.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:44.131 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:44.197 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:44.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 39 16 54134 54118 86.689 70.875 0.00591005 0.01153 2023-09-30T09:38:44.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:44 smithi019 conmon[216791]: 2023-09-30T09:38:44.181+0000 7faf52971700 -1 mon.a@0(leader).osd e370 definitely_dead 0 2023-09-30T09:38:44.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:44 smithi019 ceph-mon[216815]: pgmap v963: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 72 MiB/s wr, 1.15k op/s; 9322/149439 objects degraded (6.238%); 1646/149439 objects misplaced (1.101%); 4.3 MiB/s, 63 objects/s recovering 2023-09-30T09:38:44.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3937173326' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-30T09:38:44.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:44 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-30T09:38:44.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:44 smithi019 ceph-mon[216815]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-30T09:38:44.607 INFO:teuthology.orchestra.run.smithi019.stderr:marked out osd.3. 2023-09-30T09:38:44.756 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:38:44.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:44 smithi019 ceph-mon[224043]: pgmap v963: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 72 MiB/s wr, 1.15k op/s; 9322/149439 objects degraded (6.238%); 1646/149439 objects misplaced (1.101%); 4.3 MiB/s, 63 objects/s recovering 2023-09-30T09:38:44.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3937173326' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-30T09:38:44.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:44 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-30T09:38:44.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:44 smithi019 ceph-mon[224043]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-30T09:38:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:44 smithi167 ceph-mon[192880]: pgmap v963: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 1.4 KiB/s rd, 72 MiB/s wr, 1.15k op/s; 9322/149439 objects degraded (6.238%); 1646/149439 objects misplaced (1.101%); 4.3 MiB/s, 63 objects/s recovering 2023-09-30T09:38:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3937173326' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-30T09:38:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:44 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-30T09:38:45.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:44 smithi167 ceph-mon[192880]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-30T09:38:45.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:38:45.523078+0000 min lat: 0.0012782 max lat: 2.31896 avg lat: 0.0115363 2023-09-30T09:38:45.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:38:45.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 40 16 54245 54229 84.6959 6.9375 0.00427843 0.0115363 2023-09-30T09:38:45.550 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:45.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:45.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:45.655 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:45.694 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:45.736 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:45.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:45.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:45.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-30T09:38:45.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[216815]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[216815]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[216815]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[216815]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[216815]: osdmap e371: 8 total, 8 up, 7 in 2023-09-30T09:38:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[216815]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[216815]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-30T09:38:45.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[224043]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[224043]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[224043]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[224043]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[224043]: osdmap e371: 8 total, 8 up, 7 in 2023-09-30T09:38:45.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[224043]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:45 smithi019 ceph-mon[224043]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.841 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:45.865 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:45 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-30T09:38:45.865 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:45 smithi167 ceph-mon[192880]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.865 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:45 smithi167 ceph-mon[192880]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.866 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:45 smithi167 ceph-mon[192880]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.866 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:45 smithi167 ceph-mon[192880]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.866 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:45 smithi167 ceph-mon[192880]: osdmap e371: 8 total, 8 up, 7 in 2023-09-30T09:38:45.866 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:45 smithi167 ceph-mon[192880]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.866 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:45 smithi167 ceph-mon[192880]: failed to encode map e371 with expected crc 2023-09-30T09:38:45.868 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:45.908 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:45.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:46.018 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:46.044 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:46.087 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:46.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:46.194 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:46.223 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:46.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 41 16 54245 54229 82.6309 0 - 0.0115363 2023-09-30T09:38:47.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: osdmap e372: 8 total, 8 up, 7 in 2023-09-30T09:38:47.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: pgmap v966: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 9322/160632 objects degraded (5.803%); 7.5 MiB/s, 119 objects/s recovering 2023-09-30T09:38:47.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:46 smithi167 ceph-mon[192880]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: osdmap e372: 8 total, 8 up, 7 in 2023-09-30T09:38:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: pgmap v966: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 9322/160632 objects degraded (5.803%); 7.5 MiB/s, 119 objects/s recovering 2023-09-30T09:38:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[224043]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: osdmap e372: 8 total, 8 up, 7 in 2023-09-30T09:38:47.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: pgmap v966: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 9322/160632 objects degraded (5.803%); 7.5 MiB/s, 119 objects/s recovering 2023-09-30T09:38:47.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:46 smithi019 ceph-mon[216815]: failed to encode map e372 with expected crc 2023-09-30T09:38:47.271 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:47.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:47.378 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:47.381 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:47.466 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:47.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 42 16 54906 54890 81.6473 20.6562 0.033292 0.0122421 2023-09-30T09:38:47.576 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:47.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:47.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:47.683 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:47.750 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 5 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:38:47.750 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.750 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.750 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.750 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.750 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.750 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.750 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.751 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.751 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.751 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: osdmap e373: 8 total, 8 up, 7 in 2023-09-30T09:38:47.751 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.751 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.751 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.751 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:47 smithi167 ceph-mon[192880]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.753 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 5 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:38:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: osdmap e373: 8 total, 8 up, 7 in 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[224043]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 5 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:38:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: osdmap e373: 8 total, 8 up, 7 in 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:47 smithi019 ceph-mon[216815]: failed to encode map e373 with expected crc 2023-09-30T09:38:47.870 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:47.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:47.990 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:48.090 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:38:48.090 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:38:48.091 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:38:48.091 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:38:48.091 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:38:48.096 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:48.477 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:48.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:48.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:48.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 43 16 55854 55838 81.1264 59.25 0.01412 0.0123215 2023-09-30T09:38:48.584 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:38:48.647 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:48.739 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:38:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:48.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:38:48.781 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:38:48.852 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:48.929 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: pgmap v968: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 9322/160632 objects degraded (5.803%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:38:48.929 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: failed to encode map e374 with expected crc 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: osdmap e374: 8 total, 8 up, 7 in 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: failed to encode map e374 with expected crc 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: failed to encode map e374 with expected crc 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: failed to encode map e374 with expected crc 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9322/160632 objects degraded (5.803%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: pgmap v968: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 9322/160632 objects degraded (5.803%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:38:48.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: failed to encode map e374 with expected crc 2023-09-30T09:38:48.931 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: osdmap e374: 8 total, 8 up, 7 in 2023-09-30T09:38:48.931 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: failed to encode map e374 with expected crc 2023-09-30T09:38:48.931 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: failed to encode map e374 with expected crc 2023-09-30T09:38:48.931 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: failed to encode map e374 with expected crc 2023-09-30T09:38:48.931 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:48.931 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:48.931 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9322/160632 objects degraded (5.803%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:48.947 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: pgmap v968: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 9322/160632 objects degraded (5.803%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:38:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: failed to encode map e374 with expected crc 2023-09-30T09:38:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: osdmap e374: 8 total, 8 up, 7 in 2023-09-30T09:38:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: failed to encode map e374 with expected crc 2023-09-30T09:38:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: failed to encode map e374 with expected crc 2023-09-30T09:38:49.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: failed to encode map e374 with expected crc 2023-09-30T09:38:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:38:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:38:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9322/160632 objects degraded (5.803%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:49.524 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 44 16 56695 56679 80.4774 52.5625 0.00472665 0.0124113 2023-09-30T09:38:49.676 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:49.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:49.782 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:49.827 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: osdmap e375: 8 total, 8 up, 7 in 2023-09-30T09:38:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:49 smithi167 ceph-mon[192880]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: osdmap e375: 8 total, 8 up, 7 in 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[216815]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: osdmap e375: 8 total, 8 up, 7 in 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:49 smithi019 ceph-mon[224043]: failed to encode map e375 with expected crc 2023-09-30T09:38:50.523 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 45 16 57393 57377 79.6586 43.625 0.00978682 0.0125477 2023-09-30T09:38:50.713 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:50.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:50.817 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:50.849 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:38:50.897 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:50 smithi019 ceph-mon[224043]: pgmap v971: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 8 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 32 MiB/s wr, 513 op/s; 20453/166809 objects degraded (12.261%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:38:50.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:50 smithi019 ceph-mon[216815]: pgmap v971: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 8 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 32 MiB/s wr, 513 op/s; 20453/166809 objects degraded (12.261%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:38:50.902 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:50 smithi167 ceph-mon[192880]: pgmap v971: 119 pgs: 1 remapped+peering, 3 unknown, 15 peering, 8 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 32 MiB/s wr, 513 op/s; 20453/166809 objects degraded (12.261%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:38:51.024 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:51.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:51.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:51.075 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:51.139 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:38:51.227 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:51.316 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:38:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:51.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:38:51.427 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:38:51.503 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:51.530 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 46 16 58311 58295 79.1646 57.375 0.0167861 0.0126252 2023-09-30T09:38:51.666 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:38:51.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:51.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:38:51.773 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:51.845 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:51.861 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:51 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 3 pgs peering) 2023-09-30T09:38:51.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:51 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 3 pgs peering) 2023-09-30T09:38:51.929 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:51.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:51.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:51 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 3 pgs peering) 2023-09-30T09:38:52.039 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:52.125 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:52.195 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 6, 0, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:38:52.195 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:38:52.195 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-09-30T09:38:52.196 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:38:52.206 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:52.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:52.312 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:52.382 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:52.487 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:52.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:52.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:52.529 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 47 16 59230 59214 78.7024 57.4375 0.00668425 0.0126996 2023-09-30T09:38:52.597 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:52.655 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:52.701 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:52.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:52.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:52.739 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:38:52.806 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:38:52.864 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:52.956 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:38:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:52.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:38:52.973 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:52 smithi167 ceph-mon[192880]: pgmap v972: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 23817/170313 objects degraded (13.984%); 1963/170313 objects misplaced (1.153%); 5.5 MiB/s, 89 objects/s recovering 2023-09-30T09:38:52.973 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3985544324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:53.074 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:52 smithi019 ceph-mon[216815]: pgmap v972: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 23817/170313 objects degraded (13.984%); 1963/170313 objects misplaced (1.153%); 5.5 MiB/s, 89 objects/s recovering 2023-09-30T09:38:53.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3985544324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:53.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:52 smithi019 ceph-mon[224043]: pgmap v972: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 23817/170313 objects degraded (13.984%); 1963/170313 objects misplaced (1.153%); 5.5 MiB/s, 89 objects/s recovering 2023-09-30T09:38:53.093 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3985544324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:38:53.132 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:53.529 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 48 16 60055 60039 78.1372 51.5625 0.00404088 0.0127828 2023-09-30T09:38:53.777 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:53.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:53.883 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:53.941 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:54.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:53 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:38:54.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3985544324' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:54.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:53 smithi167 ceph-mon[192880]: failed to encode map e376 with expected crc 2023-09-30T09:38:54.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:53 smithi167 ceph-mon[192880]: osdmap e376: 8 total, 8 up, 7 in 2023-09-30T09:38:54.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23817/170313 objects degraded (13.984%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:54.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:38:54.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3985544324' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:54.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[216815]: failed to encode map e376 with expected crc 2023-09-30T09:38:54.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[216815]: osdmap e376: 8 total, 8 up, 7 in 2023-09-30T09:38:54.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23817/170313 objects degraded (13.984%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:54.042 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:38:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:38:53] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2023-09-30T09:38:54.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:38:54.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3985544324' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:38:54.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[224043]: failed to encode map e376 with expected crc 2023-09-30T09:38:54.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[224043]: osdmap e376: 8 total, 8 up, 7 in 2023-09-30T09:38:54.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23817/170313 objects degraded (13.984%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:54.132 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:54.138 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:54.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:54.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:54.262 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:38:54.264 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:54.334 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:54.442 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:38:54.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:54.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:38:54.529 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 49 16 60963 60947 77.7008 56.75 0.0170885 0.0128644 2023-09-30T09:38:54.561 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:38:54.641 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:54.737 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:54 smithi167 ceph-mon[192880]: pgmap v974: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 23817/170313 objects degraded (13.984%); 1963/170313 objects misplaced (1.153%); 5.5 MiB/s, 89 objects/s recovering 2023-09-30T09:38:54.883 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:54 smithi019 ceph-mon[216815]: pgmap v974: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 23817/170313 objects degraded (13.984%); 1963/170313 objects misplaced (1.153%); 5.5 MiB/s, 89 objects/s recovering 2023-09-30T09:38:54.883 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:54 smithi019 ceph-mon[224043]: pgmap v974: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 23817/170313 objects degraded (13.984%); 1963/170313 objects misplaced (1.153%); 5.5 MiB/s, 89 objects/s recovering 2023-09-30T09:38:54.909 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:38:54.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:54.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:38:55.015 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:55.090 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:55.278 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:38:55.310 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:38:55.344 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:38:55.381 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:38:55.422 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:38:55.471 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:38:55.521 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:38:55.530 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 50 16 61896 61880 77.3131 58.3125 0.00662538 0.0129274 2023-09-30T09:38:55.554 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:38:55.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:38:55.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:38:55.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:38:55.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:38:55.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:38:55.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:38:55.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:38:55.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:38:55.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1580122966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:55.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1580122966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:56.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1580122966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:38:56.129 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:38:56.446 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:38:57.392 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:56 smithi167 ceph-mon[192880]: pgmap v975: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s; 23817/183351 objects degraded (12.990%); 1674/183351 objects misplaced (0.913%); 6.4 MiB/s, 103 objects/s recovering 2023-09-30T09:38:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:57.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:38:57.393 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 51 16 62614 62598 76.6772 44.875 0.0131091 0.013036 2023-09-30T09:38:57.394 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:56 smithi019 ceph-mon[216815]: pgmap v975: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s; 23817/183351 objects degraded (12.990%); 1674/183351 objects misplaced (0.913%); 6.4 MiB/s, 103 objects/s recovering 2023-09-30T09:38:57.394 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:56 smithi019 ceph-mon[224043]: pgmap v975: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s; 23817/183351 objects degraded (12.990%); 1674/183351 objects misplaced (0.913%); 6.4 MiB/s, 103 objects/s recovering 2023-09-30T09:38:57.493 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:57.529 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 52 16 63584 63568 76.3685 60.625 0.0243693 0.0130874 2023-09-30T09:38:57.578 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:58.530 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 53 16 64453 64437 75.9517 54.3125 0.0327094 0.0131605 2023-09-30T09:38:58.535 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:58.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:58.640 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:38:58.704 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:58.862 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:38:58.994 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:58 smithi019 ceph-mon[224043]: pgmap v976: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s; 23817/183351 objects degraded (12.990%); 1674/183351 objects misplaced (0.913%); 5.7 MiB/s, 92 objects/s recovering 2023-09-30T09:38:58.994 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:38:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23817/183351 objects degraded (12.990%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:58.994 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:58 smithi019 ceph-mon[216815]: pgmap v976: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s; 23817/183351 objects degraded (12.990%); 1674/183351 objects misplaced (0.913%); 5.7 MiB/s, 92 objects/s recovering 2023-09-30T09:38:58.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:38:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23817/183351 objects degraded (12.990%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:59.012 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:38:59.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:38:59.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:38:59.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:58 smithi167 ceph-mon[192880]: pgmap v976: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s; 23817/183351 objects degraded (12.990%); 1674/183351 objects misplaced (0.913%); 5.7 MiB/s, 92 objects/s recovering 2023-09-30T09:38:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:38:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23817/183351 objects degraded (12.990%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:38:59.059 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:38:59.059 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:38:59.120 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:38:59.211 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:38:59.531 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 54 16 65316 65300 75.5428 53.9375 0.0189936 0.0132315 2023-09-30T09:39:00.074 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:39:00.101 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:39:00.431 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:39:00.531 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 55 16 65939 65923 74.8774 38.9375 0.022174 0.013349 2023-09-30T09:39:00.760 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:00 smithi019 ceph-mon[216815]: pgmap v977: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 23817/191466 objects degraded (12.439%); 1674/191466 objects misplaced (0.874%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:39:00.760 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:00 smithi019 ceph-mon[216815]: from='client.35868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:39:00.761 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:00 smithi019 ceph-mon[224043]: pgmap v977: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 23817/191466 objects degraded (12.439%); 1674/191466 objects misplaced (0.874%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:39:00.761 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:00 smithi019 ceph-mon[224043]: from='client.35868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:39:00.817 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:00.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:00.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:00.924 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:00 smithi167 ceph-mon[192880]: pgmap v977: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 23817/191466 objects degraded (12.439%); 1674/191466 objects misplaced (0.874%); 4.0 MiB/s, 64 objects/s recovering 2023-09-30T09:39:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:00 smithi167 ceph-mon[192880]: from='client.35868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:39:01.056 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:01.151 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:01.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:01.255 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:01.284 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:01.531 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 56 16 66729 66713 74.4221 49.375 0.0103832 0.0134302 2023-09-30T09:39:01.617 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:39:02.132 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:02.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:02.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:02.236 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:02.320 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:02.396 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":977,"stamp":"2023-09-30T09:38:59.592988+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4185395181,"num_objects":63822,"num_object_clones":0,"num_object_copies":191466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23817,"num_objects_misplaced":1674,"num_objects_unfound":0,"num_objects_dirty":63822,"num_whiteouts":0,"num_read":5950,"num_read_kb":12160,"num_write":64706,"num_write_kb":4095642,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2212,"num_bytes_recovered":139330910,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69817,"ondisk_log_size":69817,"up":357,"acting":346,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":364,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13868452,"kb_used_data":11229160,"kb_used_omap":21,"kb_used_meta":2639210,"kb_avail":642347612,"statfs":{"total":671965249536,"available":657763954688,"internally_reserved":0,"allocated":11498659840,"data_stored":11487127669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21604,"internal_metadata":2702551964},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":538640384,"num_objects":8219,"num_object_clones":0,"num_object_copies":24657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3364,"num_objects_misplaced":1674,"num_objects_unfound":0,"num_objects_dirty":8219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8219,"num_write_kb":526016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":647,"num_bytes_recovered":41680896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006371"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":864,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.641118+0000","last_change":"2023-09-30T09:38:30.746866+0000","last_active":"2023-09-30T09:38:48.641118+0000","last_peered":"2023-09-30T09:38:48.641118+0000","last_clean":"2023-09-30T09:38:48.641118+0000","last_became_active":"2023-09-30T09:38:30.746361+0000","last_became_peered":"2023-09-30T09:38:30.746361+0000","last_unstale":"2023-09-30T09:38:48.641118+0000","last_undegraded":"2023-09-30T09:38:48.641118+0000","last_fullsized":"2023-09-30T09:38:48.641118+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:06:25.130460+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":394,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.652097+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:38:48.652097+0000","last_peered":"2023-09-30T09:38:48.652097+0000","last_clean":"2023-09-30T09:38:48.652097+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:38:48.652097+0000","last_undegraded":"2023-09-30T09:38:48.652097+0000","last_fullsized":"2023-09-30T09:38:48.652097+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":289,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.651060+0000","last_change":"2023-09-30T09:38:45.612048+0000","last_active":"2023-09-30T09:38:48.651060+0000","last_peered":"2023-09-30T09:38:48.651060+0000","last_clean":"2023-09-30T09:38:48.651060+0000","last_became_active":"2023-09-30T09:38:45.611678+0000","last_became_peered":"2023-09-30T09:38:45.611678+0000","last_unstale":"2023-09-30T09:38:48.651060+0000","last_undegraded":"2023-09-30T09:38:48.651060+0000","last_fullsized":"2023-09-30T09:38:48.651060+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:33.553781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":820,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.637625+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:38:48.637625+0000","last_peered":"2023-09-30T09:38:48.637625+0000","last_clean":"2023-09-30T09:38:48.637625+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:38:48.637625+0000","last_undegraded":"2023-09-30T09:38:48.637625+0000","last_fullsized":"2023-09-30T09:38:48.637625+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":196,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650783+0000","last_change":"2023-09-30T09:38:45.617492+0000","last_active":"2023-09-30T09:38:48.650783+0000","last_peered":"2023-09-30T09:38:48.650783+0000","last_clean":"2023-09-30T09:38:48.650783+0000","last_became_active":"2023-09-30T09:38:45.616672+0000","last_became_peered":"2023-09-30T09:38:45.616672+0000","last_unstale":"2023-09-30T09:38:48.650783+0000","last_undegraded":"2023-09-30T09:38:48.650783+0000","last_fullsized":"2023-09-30T09:38:48.650783+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:23:49.964602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":297,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.641435+0000","last_change":"2023-09-30T09:38:45.602891+0000","last_active":"2023-09-30T09:38:48.641435+0000","last_peered":"2023-09-30T09:38:48.641435+0000","last_clean":"2023-09-30T09:38:48.641435+0000","last_became_active":"2023-09-30T09:38:45.602518+0000","last_became_peered":"2023-09-30T09:38:45.602518+0000","last_unstale":"2023-09-30T09:38:48.641435+0000","last_undegraded":"2023-09-30T09:38:48.641435+0000","last_fullsized":"2023-09-30T09:38:48.641435+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:42.613824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":609,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.655026+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:38:48.655026+0000","last_peered":"2023-09-30T09:38:48.655026+0000","last_clean":"2023-09-30T09:38:48.655026+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:38:48.655026+0000","last_undegraded":"2023-09-30T09:38:48.655026+0000","last_fullsized":"2023-09-30T09:38:48.655026+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":381,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.631846+0000","last_change":"2023-09-30T09:38:45.593976+0000","last_active":"2023-09-30T09:38:48.631846+0000","last_peered":"2023-09-30T09:38:48.631846+0000","last_clean":"2023-09-30T09:38:48.631846+0000","last_became_active":"2023-09-30T09:38:45.593643+0000","last_became_peered":"2023-09-30T09:38:45.593643+0000","last_unstale":"2023-09-30T09:38:48.631846+0000","last_undegraded":"2023-09-30T09:38:48.631846+0000","last_fullsized":"2023-09-30T09:38:48.631846+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:27.551484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":352,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.653752+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:38:48.653752+0000","last_peered":"2023-09-30T09:38:48.653752+0000","last_clean":"2023-09-30T09:38:48.653752+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:38:48.653752+0000","last_undegraded":"2023-09-30T09:38:48.653752+0000","last_fullsized":"2023-09-30T09:38:48.653752+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":750,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.647775+0000","last_change":"2023-09-30T09:38:30.744203+0000","last_active":"2023-09-30T09:38:48.647775+0000","last_peered":"2023-09-30T09:38:48.647775+0000","last_clean":"2023-09-30T09:38:48.647775+0000","last_became_active":"2023-09-30T09:38:30.740607+0000","last_became_peered":"2023-09-30T09:38:30.740607+0000","last_unstale":"2023-09-30T09:38:48.647775+0000","last_undegraded":"2023-09-30T09:38:48.647775+0000","last_fullsized":"2023-09-30T09:38:48.647775+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.164029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":280,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632582+0000","last_change":"2023-09-30T09:38:45.617262+0000","last_active":"2023-09-30T09:38:48.632582+0000","last_peered":"2023-09-30T09:38:48.632582+0000","last_clean":"2023-09-30T09:38:48.632582+0000","last_became_active":"2023-09-30T09:38:45.616900+0000","last_became_peered":"2023-09-30T09:38:45.616900+0000","last_unstale":"2023-09-30T09:38:48.632582+0000","last_undegraded":"2023-09-30T09:38:48.632582+0000","last_fullsized":"2023-09-30T09:38:48.632582+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:17:19.381402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":361,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.636604+0000","last_change":"2023-09-30T09:38:45.608517+0000","last_active":"2023-09-30T09:38:48.636604+0000","last_peered":"2023-09-30T09:38:48.636604+0000","last_clean":"2023-09-30T09:38:48.636604+0000","last_became_active":"2023-09-30T09:38:45.608231+0000","last_became_peered":"2023-09-30T09:38:45.608231+0000","last_unstale":"2023-09-30T09:38:48.636604+0000","last_undegraded":"2023-09-30T09:38:48.636604+0000","last_fullsized":"2023-09-30T09:38:48.636604+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:08:14.623881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":562,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.631848+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:38:48.631848+0000","last_peered":"2023-09-30T09:38:48.631848+0000","last_clean":"2023-09-30T09:38:48.631848+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:38:48.631848+0000","last_undegraded":"2023-09-30T09:38:48.631848+0000","last_fullsized":"2023-09-30T09:38:48.631848+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":264,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.639683+0000","last_change":"2023-09-30T09:38:27.687229+0000","last_active":"2023-09-30T09:38:48.639683+0000","last_peered":"2023-09-30T09:38:48.639683+0000","last_clean":"2023-09-30T09:38:48.639683+0000","last_became_active":"2023-09-30T09:38:27.686912+0000","last_became_peered":"2023-09-30T09:38:27.686912+0000","last_unstale":"2023-09-30T09:38:48.639683+0000","last_undegraded":"2023-09-30T09:38:48.639683+0000","last_fullsized":"2023-09-30T09:38:48.639683+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:05:28.924932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":310,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.642309+0000","last_change":"2023-09-30T09:38:45.622002+0000","last_active":"2023-09-30T09:38:48.642309+0000","last_peered":"2023-09-30T09:38:48.642309+0000","last_clean":"2023-09-30T09:38:48.642309+0000","last_became_active":"2023-09-30T09:38:45.621679+0000","last_became_peered":"2023-09-30T09:38:45.621679+0000","last_unstale":"2023-09-30T09:38:48.642309+0000","last_undegraded":"2023-09-30T09:38:48.642309+0000","last_fullsized":"2023-09-30T09:38:48.642309+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:01:00.354953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":542,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.639987+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:38:48.639987+0000","last_peered":"2023-09-30T09:38:48.639987+0000","last_clean":"2023-09-30T09:38:48.639987+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:38:48.639987+0000","last_undegraded":"2023-09-30T09:38:48.639987+0000","last_fullsized":"2023-09-30T09:38:48.639987+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":42,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.647972+0000","last_change":"2023-09-30T09:38:45.666418+0000","last_active":"2023-09-30T09:38:48.647972+0000","last_peered":"2023-09-30T09:38:48.647972+0000","last_clean":"2023-09-30T09:38:48.647972+0000","last_became_active":"2023-09-30T09:38:45.612315+0000","last_became_peered":"2023-09-30T09:38:45.612315+0000","last_unstale":"2023-09-30T09:38:48.647972+0000","last_undegraded":"2023-09-30T09:38:48.647972+0000","last_fullsized":"2023-09-30T09:38:48.647972+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:16.379322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":289,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.640254+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:38:48.640254+0000","last_peered":"2023-09-30T09:38:48.640254+0000","last_clean":"2023-09-30T09:38:48.640254+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:38:48.640254+0000","last_undegraded":"2023-09-30T09:38:48.640254+0000","last_fullsized":"2023-09-30T09:38:48.640254+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":558,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.661938+0000","last_change":"2023-09-30T09:38:48.653405+0000","last_active":"2023-09-30T09:38:48.661938+0000","last_peered":"2023-09-30T09:38:48.661938+0000","last_clean":"2023-09-30T09:38:48.661938+0000","last_became_active":"2023-09-30T09:38:48.653094+0000","last_became_peered":"2023-09-30T09:38:48.653094+0000","last_unstale":"2023-09-30T09:38:48.661938+0000","last_undegraded":"2023-09-30T09:38:48.661938+0000","last_fullsized":"2023-09-30T09:38:48.661938+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:27:37.328591+0000","scrub_duration":0.0114254,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":380,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650636+0000","last_change":"2023-09-30T09:38:27.691178+0000","last_active":"2023-09-30T09:38:48.650636+0000","last_peered":"2023-09-30T09:38:48.650636+0000","last_clean":"2023-09-30T09:38:48.650636+0000","last_became_active":"2023-09-30T09:38:27.680565+0000","last_became_peered":"2023-09-30T09:38:27.680565+0000","last_unstale":"2023-09-30T09:38:48.650636+0000","last_undegraded":"2023-09-30T09:38:48.650636+0000","last_fullsized":"2023-09-30T09:38:48.650636+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:33:45.600275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9444,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":371,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.654549+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:38:48.654549+0000","last_peered":"2023-09-30T09:38:48.654549+0000","last_clean":"2023-09-30T09:38:48.654549+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:38:48.654549+0000","last_undegraded":"2023-09-30T09:38:48.654549+0000","last_fullsized":"2023-09-30T09:38:48.654549+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":676,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.640857+0000","last_change":"2023-09-30T09:38:48.640319+0000","last_active":"2023-09-30T09:38:48.640857+0000","last_peered":"2023-09-30T09:38:48.640857+0000","last_clean":"2023-09-30T09:38:48.640857+0000","last_became_active":"2023-09-30T09:38:48.639990+0000","last_became_peered":"2023-09-30T09:38:48.639990+0000","last_unstale":"2023-09-30T09:38:48.640857+0000","last_undegraded":"2023-09-30T09:38:48.640857+0000","last_fullsized":"2023-09-30T09:38:48.640857+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:56.889537+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":26,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.647345+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:38:48.647345+0000","last_peered":"2023-09-30T09:38:48.647345+0000","last_clean":"2023-09-30T09:38:48.647345+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:38:48.647345+0000","last_undegraded":"2023-09-30T09:38:48.647345+0000","last_fullsized":"2023-09-30T09:38:48.647345+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":401,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632673+0000","last_change":"2023-09-30T09:38:45.606327+0000","last_active":"2023-09-30T09:38:48.632673+0000","last_peered":"2023-09-30T09:38:48.632673+0000","last_clean":"2023-09-30T09:38:48.632673+0000","last_became_active":"2023-09-30T09:38:45.605910+0000","last_became_peered":"2023-09-30T09:38:45.605910+0000","last_unstale":"2023-09-30T09:38:48.632673+0000","last_undegraded":"2023-09-30T09:38:48.632673+0000","last_fullsized":"2023-09-30T09:38:48.632673+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:36:50.105906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":742,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.652526+0000","last_change":"2023-09-30T09:38:30.746770+0000","last_active":"2023-09-30T09:38:48.652526+0000","last_peered":"2023-09-30T09:38:48.652526+0000","last_clean":"2023-09-30T09:38:48.652526+0000","last_became_active":"2023-09-30T09:38:30.745926+0000","last_became_peered":"2023-09-30T09:38:30.745926+0000","last_unstale":"2023-09-30T09:38:48.652526+0000","last_undegraded":"2023-09-30T09:38:48.652526+0000","last_fullsized":"2023-09-30T09:38:48.652526+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:25:58.952467+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":296,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632409+0000","last_change":"2023-09-30T09:38:45.600099+0000","last_active":"2023-09-30T09:38:48.632409+0000","last_peered":"2023-09-30T09:38:48.632409+0000","last_clean":"2023-09-30T09:38:48.632409+0000","last_became_active":"2023-09-30T09:38:45.599769+0000","last_became_peered":"2023-09-30T09:38:45.599769+0000","last_unstale":"2023-09-30T09:38:48.632409+0000","last_undegraded":"2023-09-30T09:38:48.632409+0000","last_fullsized":"2023-09-30T09:38:48.632409+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:02:58.820333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":85,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.642670+0000","last_change":"2023-09-30T09:38:45.620080+0000","last_active":"2023-09-30T09:38:48.642670+0000","last_peered":"2023-09-30T09:38:48.642670+0000","last_clean":"2023-09-30T09:38:48.642670+0000","last_became_active":"2023-09-30T09:38:45.617392+0000","last_became_peered":"2023-09-30T09:38:45.617392+0000","last_unstale":"2023-09-30T09:38:48.642670+0000","last_undegraded":"2023-09-30T09:38:48.642670+0000","last_fullsized":"2023-09-30T09:38:48.642670+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:24.685400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":766,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.666215+0000","last_change":"2023-09-30T09:38:48.664123+0000","last_active":"2023-09-30T09:38:48.666215+0000","last_peered":"2023-09-30T09:38:48.666215+0000","last_clean":"2023-09-30T09:38:48.666215+0000","last_became_active":"2023-09-30T09:38:48.663799+0000","last_became_peered":"2023-09-30T09:38:48.663799+0000","last_unstale":"2023-09-30T09:38:48.666215+0000","last_undegraded":"2023-09-30T09:38:48.666215+0000","last_fullsized":"2023-09-30T09:38:48.666215+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.642458+0000","last_change":"2023-09-30T09:38:27.680269+0000","last_active":"2023-09-30T09:38:48.642458+0000","last_peered":"2023-09-30T09:38:48.642458+0000","last_clean":"2023-09-30T09:38:48.642458+0000","last_became_active":"2023-09-30T09:38:27.679931+0000","last_became_peered":"2023-09-30T09:38:27.679931+0000","last_unstale":"2023-09-30T09:38:48.642458+0000","last_undegraded":"2023-09-30T09:38:48.642458+0000","last_fullsized":"2023-09-30T09:38:48.642458+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:06.847392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":296,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.641521+0000","last_change":"2023-09-30T09:38:45.622309+0000","last_active":"2023-09-30T09:38:48.641521+0000","last_peered":"2023-09-30T09:38:48.641521+0000","last_clean":"2023-09-30T09:38:48.641521+0000","last_became_active":"2023-09-30T09:38:45.622020+0000","last_became_peered":"2023-09-30T09:38:45.622020+0000","last_unstale":"2023-09-30T09:38:48.641521+0000","last_undegraded":"2023-09-30T09:38:48.641521+0000","last_fullsized":"2023-09-30T09:38:48.641521+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:42:26.066681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":841,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.636958+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:38:48.636958+0000","last_peered":"2023-09-30T09:38:48.636958+0000","last_clean":"2023-09-30T09:38:48.636958+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:38:48.636958+0000","last_undegraded":"2023-09-30T09:38:48.636958+0000","last_fullsized":"2023-09-30T09:38:48.636958+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":535,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.644822+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:38:48.644822+0000","last_peered":"2023-09-30T09:38:48.644822+0000","last_clean":"2023-09-30T09:38:48.644822+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:38:48.644822+0000","last_undegraded":"2023-09-30T09:38:48.644822+0000","last_fullsized":"2023-09-30T09:38:48.644822+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":392,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.649121+0000","last_change":"2023-09-30T09:38:45.606456+0000","last_active":"2023-09-30T09:38:48.649121+0000","last_peered":"2023-09-30T09:38:48.649121+0000","last_clean":"2023-09-30T09:38:48.649121+0000","last_became_active":"2023-09-30T09:38:45.604792+0000","last_became_peered":"2023-09-30T09:38:45.604792+0000","last_unstale":"2023-09-30T09:38:48.649121+0000","last_undegraded":"2023-09-30T09:38:48.649121+0000","last_fullsized":"2023-09-30T09:38:48.649121+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:11.533941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"376'4068","reported_seq":4094,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:59.301330+0000","last_change":"2023-09-30T09:37:56.539717+0000","last_active":"2023-09-30T09:38:59.301330+0000","last_peered":"2023-09-30T09:38:59.301330+0000","last_clean":"2023-09-30T09:38:59.301330+0000","last_became_active":"2023-09-30T09:37:56.539331+0000","last_became_peered":"2023-09-30T09:37:56.539331+0000","last_unstale":"2023-09-30T09:38:59.301330+0000","last_undegraded":"2023-09-30T09:38:59.301330+0000","last_fullsized":"2023-09-30T09:38:59.301330+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4068,"log_dups_size":0,"ondisk_log_size":4068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:15:01.898944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266600448,"num_objects":4068,"num_object_clones":0,"num_object_copies":12204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4068,"num_write_kb":260352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":742,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.647828+0000","last_change":"2023-09-30T09:38:30.737676+0000","last_active":"2023-09-30T09:38:48.647828+0000","last_peered":"2023-09-30T09:38:48.647828+0000","last_clean":"2023-09-30T09:38:48.647828+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:38:48.647828+0000","last_undegraded":"2023-09-30T09:38:48.647828+0000","last_fullsized":"2023-09-30T09:38:48.647828+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.740851+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.740851+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":24,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.649358+0000","last_change":"2023-09-30T09:38:27.700442+0000","last_active":"2023-09-30T09:38:48.649358+0000","last_peered":"2023-09-30T09:38:48.649358+0000","last_clean":"2023-09-30T09:38:48.649358+0000","last_became_active":"2023-09-30T09:38:27.678601+0000","last_became_peered":"2023-09-30T09:38:27.678601+0000","last_unstale":"2023-09-30T09:38:48.649358+0000","last_undegraded":"2023-09-30T09:38:48.649358+0000","last_fullsized":"2023-09-30T09:38:48.649358+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:59:10.222887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":75,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.636751+0000","last_change":"2023-09-30T09:38:27.687124+0000","last_active":"2023-09-30T09:38:48.636751+0000","last_peered":"2023-09-30T09:38:48.636751+0000","last_clean":"2023-09-30T09:38:48.636751+0000","last_became_active":"2023-09-30T09:38:27.686794+0000","last_became_peered":"2023-09-30T09:38:27.686794+0000","last_unstale":"2023-09-30T09:38:48.636751+0000","last_undegraded":"2023-09-30T09:38:48.636751+0000","last_fullsized":"2023-09-30T09:38:48.636751+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:33.583208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"376'4057","reported_seq":4448,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:59.101538+0000","last_change":"2023-09-30T09:38:45.626369+0000","last_active":"2023-09-30T09:38:59.101538+0000","last_peered":"2023-09-30T09:38:59.101538+0000","last_clean":"2023-09-30T09:38:26.563035+0000","last_became_active":"2023-09-30T09:38:28.865949+0000","last_became_peered":"2023-09-30T09:38:28.865949+0000","last_unstale":"2023-09-30T09:38:59.101538+0000","last_undegraded":"2023-09-30T09:38:45.624667+0000","last_fullsized":"2023-09-30T09:38:28.655474+0000","mapping_epoch":366,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:57:56.059269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265879552,"num_objects":4057,"num_object_clones":0,"num_object_copies":12171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2094,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4057,"num_write_kb":259648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":176,"num_bytes_recovered":11534336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":2094},{"shards":"4,6","objects":1963}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":712,"reported_epoch":375,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:48.632810+0000","last_change":"2023-09-30T09:38:46.666950+0000","last_active":"2023-09-30T09:38:48.632810+0000","last_peered":"2023-09-30T09:38:48.632810+0000","last_clean":"2023-09-30T09:31:49.462297+0000","last_became_active":"2023-09-30T09:38:46.666594+0000","last_became_peered":"2023-09-30T09:38:46.666594+0000","last_unstale":"2023-09-30T09:38:48.632810+0000","last_undegraded":"2023-09-30T09:38:46.591792+0000","last_fullsized":"2023-09-30T09:38:46.591274+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":24,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.648482+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:38:48.648482+0000","last_peered":"2023-09-30T09:38:48.648482+0000","last_clean":"2023-09-30T09:38:48.648482+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:38:48.648482+0000","last_undegraded":"2023-09-30T09:38:48.648482+0000","last_fullsized":"2023-09-30T09:38:48.648482+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":388,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650866+0000","last_change":"2023-09-30T09:38:45.617752+0000","last_active":"2023-09-30T09:38:48.650866+0000","last_peered":"2023-09-30T09:38:48.650866+0000","last_clean":"2023-09-30T09:38:48.650866+0000","last_became_active":"2023-09-30T09:38:45.617146+0000","last_became_peered":"2023-09-30T09:38:45.617146+0000","last_unstale":"2023-09-30T09:38:48.650866+0000","last_undegraded":"2023-09-30T09:38:48.650866+0000","last_fullsized":"2023-09-30T09:38:48.650866+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:41:29.189921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"376'3896","reported_seq":5199,"reported_epoch":376,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:38:55.417663+0000","last_change":"2023-09-30T09:38:45.644472+0000","last_active":"2023-09-30T09:38:55.417663+0000","last_peered":"2023-09-30T09:38:55.417663+0000","last_clean":"2023-09-30T09:38:26.430308+0000","last_became_active":"2023-09-30T09:38:28.834509+0000","last_became_peered":"2023-09-30T09:38:28.834509+0000","last_unstale":"2023-09-30T09:38:55.417663+0000","last_undegraded":"2023-09-30T09:38:55.417663+0000","last_fullsized":"2023-09-30T09:38:28.651148+0000","mapping_epoch":366,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3896,"log_dups_size":0,"ondisk_log_size":3896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:48:25.057565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255328256,"num_objects":3896,"num_object_clones":0,"num_object_copies":11688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1674,"num_objects_unfound":0,"num_objects_dirty":3896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3896,"num_write_kb":249344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":635,"num_bytes_recovered":41615360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1674},{"shards":"0,6","objects":2222}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":659,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.653674+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:38:48.653674+0000","last_peered":"2023-09-30T09:38:48.653674+0000","last_clean":"2023-09-30T09:38:48.653674+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:38:48.653674+0000","last_undegraded":"2023-09-30T09:38:48.653674+0000","last_fullsized":"2023-09-30T09:38:48.653674+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":24,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.647632+0000","last_change":"2023-09-30T09:38:27.708392+0000","last_active":"2023-09-30T09:38:48.647632+0000","last_peered":"2023-09-30T09:38:48.647632+0000","last_clean":"2023-09-30T09:38:48.647632+0000","last_became_active":"2023-09-30T09:38:27.685766+0000","last_became_peered":"2023-09-30T09:38:27.685766+0000","last_unstale":"2023-09-30T09:38:48.647632+0000","last_undegraded":"2023-09-30T09:38:48.647632+0000","last_fullsized":"2023-09-30T09:38:48.647632+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:32.744901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":365,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.640141+0000","last_change":"2023-09-30T09:38:27.678428+0000","last_active":"2023-09-30T09:38:48.640141+0000","last_peered":"2023-09-30T09:38:48.640141+0000","last_clean":"2023-09-30T09:38:48.640141+0000","last_became_active":"2023-09-30T09:38:27.678086+0000","last_became_peered":"2023-09-30T09:38:27.678086+0000","last_unstale":"2023-09-30T09:38:48.640141+0000","last_undegraded":"2023-09-30T09:38:48.640141+0000","last_fullsized":"2023-09-30T09:38:48.640141+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:05.480763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"376'4113","reported_seq":4149,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:59.103495+0000","last_change":"2023-09-30T09:38:46.804449+0000","last_active":"2023-09-30T09:38:59.103495+0000","last_peered":"2023-09-30T09:38:59.103495+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:38:46.792139+0000","last_became_peered":"2023-09-30T09:38:46.792139+0000","last_unstale":"2023-09-30T09:38:59.103495+0000","last_undegraded":"2023-09-30T09:38:46.632535+0000","last_fullsized":"2023-09-30T09:38:46.612269+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4113,"log_dups_size":0,"ondisk_log_size":4113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:52:58.957782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269549568,"num_objects":4113,"num_object_clones":0,"num_object_copies":12339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3423,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4113,"num_write_kb":263232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":3423},{"shards":"1,6","objects":690}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":591,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.666310+0000","last_change":"2023-09-30T09:38:48.662225+0000","last_active":"2023-09-30T09:38:48.666310+0000","last_peered":"2023-09-30T09:38:48.666310+0000","last_clean":"2023-09-30T09:38:48.666310+0000","last_became_active":"2023-09-30T09:38:48.661883+0000","last_became_peered":"2023-09-30T09:38:48.661883+0000","last_unstale":"2023-09-30T09:38:48.666310+0000","last_undegraded":"2023-09-30T09:38:48.666310+0000","last_fullsized":"2023-09-30T09:38:48.666310+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:01.892446+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":345,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.639831+0000","last_change":"2023-09-30T09:38:45.614020+0000","last_active":"2023-09-30T09:38:48.639831+0000","last_peered":"2023-09-30T09:38:48.639831+0000","last_clean":"2023-09-30T09:38:48.639831+0000","last_became_active":"2023-09-30T09:38:45.613566+0000","last_became_peered":"2023-09-30T09:38:45.613566+0000","last_unstale":"2023-09-30T09:38:48.639831+0000","last_undegraded":"2023-09-30T09:38:48.639831+0000","last_fullsized":"2023-09-30T09:38:48.639831+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:56:47.788900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":283,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:52.731960+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:38:52.731960+0000","last_peered":"2023-09-30T09:38:52.731960+0000","last_clean":"2023-09-30T09:38:52.731960+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:38:52.731960+0000","last_undegraded":"2023-09-30T09:38:52.731960+0000","last_fullsized":"2023-09-30T09:38:52.731960+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"376'4189","reported_seq":4230,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:59.342422+0000","last_change":"2023-09-30T09:38:46.870625+0000","last_active":"2023-09-30T09:38:59.342422+0000","last_peered":"2023-09-30T09:38:59.342422+0000","last_clean":"2023-09-30T09:38:44.551281+0000","last_became_active":"2023-09-30T09:38:46.863561+0000","last_became_peered":"2023-09-30T09:38:46.863561+0000","last_unstale":"2023-09-30T09:38:59.342422+0000","last_undegraded":"2023-09-30T09:38:46.794041+0000","last_fullsized":"2023-09-30T09:38:46.698201+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4189,"log_dups_size":0,"ondisk_log_size":4189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:01:50.671330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274530304,"num_objects":4189,"num_object_clones":0,"num_object_copies":12567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3471,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4189,"num_write_kb":268096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":3471},{"shards":"1,5","objects":718}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":831,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.642388+0000","last_change":"2023-09-30T09:38:45.616475+0000","last_active":"2023-09-30T09:38:48.642388+0000","last_peered":"2023-09-30T09:38:48.642388+0000","last_clean":"2023-09-30T09:38:48.642388+0000","last_became_active":"2023-09-30T09:38:45.616163+0000","last_became_peered":"2023-09-30T09:38:45.616163+0000","last_unstale":"2023-09-30T09:38:48.642388+0000","last_undegraded":"2023-09-30T09:38:48.642388+0000","last_fullsized":"2023-09-30T09:38:48.642388+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:11:18.666981+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":362,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632199+0000","last_change":"2023-09-30T09:38:45.648614+0000","last_active":"2023-09-30T09:38:48.632199+0000","last_peered":"2023-09-30T09:38:48.632199+0000","last_clean":"2023-09-30T09:38:48.632199+0000","last_became_active":"2023-09-30T09:38:45.615581+0000","last_became_peered":"2023-09-30T09:38:45.615581+0000","last_unstale":"2023-09-30T09:38:48.632199+0000","last_undegraded":"2023-09-30T09:38:48.632199+0000","last_fullsized":"2023-09-30T09:38:48.632199+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:19:19.655673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":344,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650646+0000","last_change":"2023-09-30T09:38:45.612082+0000","last_active":"2023-09-30T09:38:48.650646+0000","last_peered":"2023-09-30T09:38:48.650646+0000","last_clean":"2023-09-30T09:38:48.650646+0000","last_became_active":"2023-09-30T09:38:45.611852+0000","last_became_peered":"2023-09-30T09:38:45.611852+0000","last_unstale":"2023-09-30T09:38:48.650646+0000","last_undegraded":"2023-09-30T09:38:48.650646+0000","last_fullsized":"2023-09-30T09:38:48.650646+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:49:35.044511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"376'3810","reported_seq":3839,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:55.384964+0000","last_change":"2023-09-30T09:37:56.535045+0000","last_active":"2023-09-30T09:38:55.384964+0000","last_peered":"2023-09-30T09:38:55.384964+0000","last_clean":"2023-09-30T09:38:55.384964+0000","last_became_active":"2023-09-30T09:37:56.534751+0000","last_became_peered":"2023-09-30T09:37:56.534751+0000","last_unstale":"2023-09-30T09:38:55.384964+0000","last_undegraded":"2023-09-30T09:38:55.384964+0000","last_fullsized":"2023-09-30T09:38:55.384964+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3810,"log_dups_size":0,"ondisk_log_size":3810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:27:36.529588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249692160,"num_objects":3810,"num_object_clones":0,"num_object_copies":11430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3810,"num_write_kb":243840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":812,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.666442+0000","last_change":"2023-09-30T09:38:48.666155+0000","last_active":"2023-09-30T09:38:48.666442+0000","last_peered":"2023-09-30T09:38:48.666442+0000","last_clean":"2023-09-30T09:38:48.666442+0000","last_became_active":"2023-09-30T09:38:48.665829+0000","last_became_peered":"2023-09-30T09:38:48.665829+0000","last_unstale":"2023-09-30T09:38:48.666442+0000","last_undegraded":"2023-09-30T09:38:48.666442+0000","last_fullsized":"2023-09-30T09:38:48.666442+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:09:04.988358+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":353,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.639991+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:38:48.639991+0000","last_peered":"2023-09-30T09:38:48.639991+0000","last_clean":"2023-09-30T09:38:48.639991+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:38:48.639991+0000","last_undegraded":"2023-09-30T09:38:48.639991+0000","last_fullsized":"2023-09-30T09:38:48.639991+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":70,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650473+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:38:48.650473+0000","last_peered":"2023-09-30T09:38:48.650473+0000","last_clean":"2023-09-30T09:38:48.650473+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:38:48.650473+0000","last_undegraded":"2023-09-30T09:38:48.650473+0000","last_fullsized":"2023-09-30T09:38:48.650473+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"376'3952","reported_seq":3978,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:59.134185+0000","last_change":"2023-09-30T09:37:56.538248+0000","last_active":"2023-09-30T09:38:59.134185+0000","last_peered":"2023-09-30T09:38:59.134185+0000","last_clean":"2023-09-30T09:38:59.134185+0000","last_became_active":"2023-09-30T09:37:56.537940+0000","last_became_peered":"2023-09-30T09:37:56.537940+0000","last_unstale":"2023-09-30T09:38:59.134185+0000","last_undegraded":"2023-09-30T09:38:59.134185+0000","last_fullsized":"2023-09-30T09:38:59.134185+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3952,"log_dups_size":0,"ondisk_log_size":3952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:56:16.621887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258998272,"num_objects":3952,"num_object_clones":0,"num_object_copies":11856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3952,"num_write_kb":252928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":820,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632289+0000","last_change":"2023-09-30T09:38:09.139181+0000","last_active":"2023-09-30T09:38:48.632289+0000","last_peered":"2023-09-30T09:38:48.632289+0000","last_clean":"2023-09-30T09:38:48.632289+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:38:48.632289+0000","last_undegraded":"2023-09-30T09:38:48.632289+0000","last_fullsized":"2023-09-30T09:38:48.632289+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:20:47.321880+0000","scrub_duration":0.0056612939999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":382,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650512+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:38:48.650512+0000","last_peered":"2023-09-30T09:38:48.650512+0000","last_clean":"2023-09-30T09:38:48.650512+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:38:48.650512+0000","last_undegraded":"2023-09-30T09:38:48.650512+0000","last_fullsized":"2023-09-30T09:38:48.650512+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":351,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.651369+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:38:48.651369+0000","last_peered":"2023-09-30T09:38:48.651369+0000","last_clean":"2023-09-30T09:38:48.651369+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:38:48.651369+0000","last_undegraded":"2023-09-30T09:38:48.651369+0000","last_fullsized":"2023-09-30T09:38:48.651369+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"376'4022","reported_seq":4048,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:59.116909+0000","last_change":"2023-09-30T09:37:56.538633+0000","last_active":"2023-09-30T09:38:59.116909+0000","last_peered":"2023-09-30T09:38:59.116909+0000","last_clean":"2023-09-30T09:38:59.116909+0000","last_became_active":"2023-09-30T09:37:56.538313+0000","last_became_peered":"2023-09-30T09:37:56.538313+0000","last_unstale":"2023-09-30T09:38:59.116909+0000","last_undegraded":"2023-09-30T09:38:59.116909+0000","last_fullsized":"2023-09-30T09:38:59.116909+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4022,"log_dups_size":0,"ondisk_log_size":4022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:12.090507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263585792,"num_objects":4022,"num_object_clones":0,"num_object_copies":12066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4022,"num_write_kb":257408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":534,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632435+0000","last_change":"2023-09-30T09:37:52.780622+0000","last_active":"2023-09-30T09:38:48.632435+0000","last_peered":"2023-09-30T09:38:48.632435+0000","last_clean":"2023-09-30T09:38:48.632435+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:38:48.632435+0000","last_undegraded":"2023-09-30T09:38:48.632435+0000","last_fullsized":"2023-09-30T09:38:48.632435+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:16:18.525289+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":373,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.643057+0000","last_change":"2023-09-30T09:38:45.620180+0000","last_active":"2023-09-30T09:38:48.643057+0000","last_peered":"2023-09-30T09:38:48.643057+0000","last_clean":"2023-09-30T09:38:48.643057+0000","last_became_active":"2023-09-30T09:38:45.619776+0000","last_became_peered":"2023-09-30T09:38:45.619776+0000","last_unstale":"2023-09-30T09:38:48.643057+0000","last_undegraded":"2023-09-30T09:38:48.643057+0000","last_fullsized":"2023-09-30T09:38:48.643057+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:25:08.739481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":76,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.651696+0000","last_change":"2023-09-30T09:38:45.610897+0000","last_active":"2023-09-30T09:38:48.651696+0000","last_peered":"2023-09-30T09:38:48.651696+0000","last_clean":"2023-09-30T09:38:48.651696+0000","last_became_active":"2023-09-30T09:38:45.610628+0000","last_became_peered":"2023-09-30T09:38:45.610628+0000","last_unstale":"2023-09-30T09:38:48.651696+0000","last_undegraded":"2023-09-30T09:38:48.651696+0000","last_fullsized":"2023-09-30T09:38:48.651696+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:04.928128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.e","version":"376'3792","reported_seq":3821,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:55.392272+0000","last_change":"2023-09-30T09:37:56.536019+0000","last_active":"2023-09-30T09:38:55.392272+0000","last_peered":"2023-09-30T09:38:55.392272+0000","last_clean":"2023-09-30T09:38:55.392272+0000","last_became_active":"2023-09-30T09:37:56.535667+0000","last_became_peered":"2023-09-30T09:37:56.535667+0000","last_unstale":"2023-09-30T09:38:55.392272+0000","last_undegraded":"2023-09-30T09:38:55.392272+0000","last_fullsized":"2023-09-30T09:38:55.392272+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3792,"log_dups_size":0,"ondisk_log_size":3792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:35:07.105737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248512512,"num_objects":3792,"num_object_clones":0,"num_object_copies":11376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3792,"num_write_kb":242688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":700,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.653832+0000","last_change":"2023-09-30T09:38:45.624734+0000","last_active":"2023-09-30T09:38:48.653832+0000","last_peered":"2023-09-30T09:38:48.653832+0000","last_clean":"2023-09-30T09:38:48.653832+0000","last_became_active":"2023-09-30T09:38:45.624345+0000","last_became_peered":"2023-09-30T09:38:45.624345+0000","last_unstale":"2023-09-30T09:38:48.653832+0000","last_undegraded":"2023-09-30T09:38:48.653832+0000","last_fullsized":"2023-09-30T09:38:48.653832+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:40:11.986970+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":598,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:52.735384+0000","last_change":"2023-09-30T09:38:27.714032+0000","last_active":"2023-09-30T09:38:52.735384+0000","last_peered":"2023-09-30T09:38:52.735384+0000","last_clean":"2023-09-30T09:38:52.735384+0000","last_became_active":"2023-09-30T09:38:27.684654+0000","last_became_peered":"2023-09-30T09:38:27.684654+0000","last_unstale":"2023-09-30T09:38:52.735384+0000","last_undegraded":"2023-09-30T09:38:52.735384+0000","last_fullsized":"2023-09-30T09:38:52.735384+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:09:43.539211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":382,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.642862+0000","last_change":"2023-09-30T09:38:45.608476+0000","last_active":"2023-09-30T09:38:48.642862+0000","last_peered":"2023-09-30T09:38:48.642862+0000","last_clean":"2023-09-30T09:38:48.642862+0000","last_became_active":"2023-09-30T09:38:45.608152+0000","last_became_peered":"2023-09-30T09:38:45.608152+0000","last_unstale":"2023-09-30T09:38:48.642862+0000","last_undegraded":"2023-09-30T09:38:48.642862+0000","last_fullsized":"2023-09-30T09:38:48.642862+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:40:39.508922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.636674+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:38:48.636674+0000","last_peered":"2023-09-30T09:38:48.636674+0000","last_clean":"2023-09-30T09:38:48.636674+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:38:48.636674+0000","last_undegraded":"2023-09-30T09:38:48.636674+0000","last_fullsized":"2023-09-30T09:38:48.636674+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"376'4133","reported_seq":4178,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:59.328745+0000","last_change":"2023-09-30T09:38:46.863036+0000","last_active":"2023-09-30T09:38:59.328745+0000","last_peered":"2023-09-30T09:38:59.328745+0000","last_clean":"2023-09-30T09:38:26.554137+0000","last_became_active":"2023-09-30T09:38:46.841212+0000","last_became_peered":"2023-09-30T09:38:46.841212+0000","last_unstale":"2023-09-30T09:38:59.328745+0000","last_undegraded":"2023-09-30T09:38:46.696157+0000","last_fullsized":"2023-09-30T09:38:46.602700+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4133,"log_dups_size":0,"ondisk_log_size":4133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:05:30.574870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270860288,"num_objects":4133,"num_object_clones":0,"num_object_copies":12399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3448,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4133,"num_write_kb":264512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"0,3,5","objects":2348},{"shards":"3,5,7","objects":1100},{"shards":"5,7","objects":685}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":838,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.636582+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:38:48.636582+0000","last_peered":"2023-09-30T09:38:48.636582+0000","last_clean":"2023-09-30T09:38:48.636582+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:38:48.636582+0000","last_undegraded":"2023-09-30T09:38:48.636582+0000","last_fullsized":"2023-09-30T09:38:48.636582+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":367,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.640068+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:38:48.640068+0000","last_peered":"2023-09-30T09:38:48.640068+0000","last_clean":"2023-09-30T09:38:48.640068+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:38:48.640068+0000","last_undegraded":"2023-09-30T09:38:48.640068+0000","last_fullsized":"2023-09-30T09:38:48.640068+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":355,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.654593+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:38:48.654593+0000","last_peered":"2023-09-30T09:38:48.654593+0000","last_clean":"2023-09-30T09:38:48.654593+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:38:48.654593+0000","last_undegraded":"2023-09-30T09:38:48.654593+0000","last_fullsized":"2023-09-30T09:38:48.654593+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":102,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.631933+0000","last_change":"2023-09-30T09:38:45.615879+0000","last_active":"2023-09-30T09:38:48.631933+0000","last_peered":"2023-09-30T09:38:48.631933+0000","last_clean":"2023-09-30T09:38:48.631933+0000","last_became_active":"2023-09-30T09:38:45.615545+0000","last_became_peered":"2023-09-30T09:38:45.615545+0000","last_unstale":"2023-09-30T09:38:48.631933+0000","last_undegraded":"2023-09-30T09:38:48.631933+0000","last_fullsized":"2023-09-30T09:38:48.631933+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:39:23.834391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"376'3991","reported_seq":4029,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:59.118278+0000","last_change":"2023-09-30T09:38:46.747921+0000","last_active":"2023-09-30T09:38:59.118278+0000","last_peered":"2023-09-30T09:38:59.118278+0000","last_clean":"2023-09-30T09:38:44.556600+0000","last_became_active":"2023-09-30T09:38:46.733175+0000","last_became_peered":"2023-09-30T09:38:46.733175+0000","last_unstale":"2023-09-30T09:38:59.118278+0000","last_undegraded":"2023-09-30T09:38:46.628065+0000","last_fullsized":"2023-09-30T09:38:46.604027+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3991,"log_dups_size":0,"ondisk_log_size":3991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:41.383870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261554176,"num_objects":3991,"num_object_clones":0,"num_object_copies":11973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3313,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3991,"num_write_kb":255424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":3313},{"shards":"1,4","objects":678}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":755,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.642081+0000","last_change":"2023-09-30T09:38:30.737659+0000","last_active":"2023-09-30T09:38:48.642081+0000","last_peered":"2023-09-30T09:38:48.642081+0000","last_clean":"2023-09-30T09:38:48.642081+0000","last_became_active":"2023-09-30T09:38:30.737311+0000","last_became_peered":"2023-09-30T09:38:30.737311+0000","last_unstale":"2023-09-30T09:38:48.642081+0000","last_undegraded":"2023-09-30T09:38:48.642081+0000","last_fullsized":"2023-09-30T09:38:48.642081+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:08:13.108463+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":591,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.643152+0000","last_change":"2023-09-30T09:38:48.641903+0000","last_active":"2023-09-30T09:38:48.643152+0000","last_peered":"2023-09-30T09:38:48.643152+0000","last_clean":"2023-09-30T09:38:48.643152+0000","last_became_active":"2023-09-30T09:38:48.641372+0000","last_became_peered":"2023-09-30T09:38:48.641372+0000","last_unstale":"2023-09-30T09:38:48.643152+0000","last_undegraded":"2023-09-30T09:38:48.643152+0000","last_fullsized":"2023-09-30T09:38:48.643152+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:49:32.411360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.630470+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:38:48.630470+0000","last_peered":"2023-09-30T09:38:48.630470+0000","last_clean":"2023-09-30T09:38:48.630470+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:38:48.630470+0000","last_undegraded":"2023-09-30T09:38:48.630470+0000","last_fullsized":"2023-09-30T09:38:48.630470+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":295,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.640277+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:38:48.640277+0000","last_peered":"2023-09-30T09:38:48.640277+0000","last_clean":"2023-09-30T09:38:48.640277+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:38:48.640277+0000","last_undegraded":"2023-09-30T09:38:48.640277+0000","last_fullsized":"2023-09-30T09:38:48.640277+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"376'4033","reported_seq":4058,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:59.116629+0000","last_change":"2023-09-30T09:37:56.537696+0000","last_active":"2023-09-30T09:38:59.116629+0000","last_peered":"2023-09-30T09:38:59.116629+0000","last_clean":"2023-09-30T09:38:59.116629+0000","last_became_active":"2023-09-30T09:37:56.537311+0000","last_became_peered":"2023-09-30T09:37:56.537311+0000","last_unstale":"2023-09-30T09:38:59.116629+0000","last_undegraded":"2023-09-30T09:38:59.116629+0000","last_fullsized":"2023-09-30T09:38:59.116629+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4033,"log_dups_size":0,"ondisk_log_size":4033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:04:15.717625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264306688,"num_objects":4033,"num_object_clones":0,"num_object_copies":12099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4033,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1002,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.637713+0000","last_change":"2023-09-30T09:38:30.748839+0000","last_active":"2023-09-30T09:38:48.637713+0000","last_peered":"2023-09-30T09:38:48.637713+0000","last_clean":"2023-09-30T09:38:48.637713+0000","last_became_active":"2023-09-30T09:38:30.748346+0000","last_became_peered":"2023-09-30T09:38:30.748346+0000","last_unstale":"2023-09-30T09:38:48.637713+0000","last_undegraded":"2023-09-30T09:38:48.637713+0000","last_fullsized":"2023-09-30T09:38:48.637713+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:46:58.191800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":659,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.643259+0000","last_change":"2023-09-30T09:38:30.743421+0000","last_active":"2023-09-30T09:38:48.643259+0000","last_peered":"2023-09-30T09:38:48.643259+0000","last_clean":"2023-09-30T09:38:48.643259+0000","last_became_active":"2023-09-30T09:38:30.743110+0000","last_became_peered":"2023-09-30T09:38:30.743110+0000","last_unstale":"2023-09-30T09:38:48.643259+0000","last_undegraded":"2023-09-30T09:38:48.643259+0000","last_fullsized":"2023-09-30T09:38:48.643259+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:13.366523+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":30,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.637070+0000","last_change":"2023-09-30T09:38:45.608979+0000","last_active":"2023-09-30T09:38:48.637070+0000","last_peered":"2023-09-30T09:38:48.637070+0000","last_clean":"2023-09-30T09:38:48.637070+0000","last_became_active":"2023-09-30T09:38:45.608817+0000","last_became_peered":"2023-09-30T09:38:45.608817+0000","last_unstale":"2023-09-30T09:38:48.637070+0000","last_undegraded":"2023-09-30T09:38:48.637070+0000","last_fullsized":"2023-09-30T09:38:48.637070+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:03.733113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":338,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.645391+0000","last_change":"2023-09-30T09:38:27.682465+0000","last_active":"2023-09-30T09:38:48.645391+0000","last_peered":"2023-09-30T09:38:48.645391+0000","last_clean":"2023-09-30T09:38:48.645391+0000","last_became_active":"2023-09-30T09:38:27.682121+0000","last_became_peered":"2023-09-30T09:38:27.682121+0000","last_unstale":"2023-09-30T09:38:48.645391+0000","last_undegraded":"2023-09-30T09:38:48.645391+0000","last_fullsized":"2023-09-30T09:38:48.645391+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:56:49.578285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"376'3820","reported_seq":3849,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-30T09:38:55.396665+0000","last_change":"2023-09-30T09:37:56.539888+0000","last_active":"2023-09-30T09:38:55.396665+0000","last_peered":"2023-09-30T09:38:55.396665+0000","last_clean":"2023-09-30T09:38:55.396665+0000","last_became_active":"2023-09-30T09:37:56.539552+0000","last_became_peered":"2023-09-30T09:37:56.539552+0000","last_unstale":"2023-09-30T09:38:55.396665+0000","last_undegraded":"2023-09-30T09:38:55.396665+0000","last_fullsized":"2023-09-30T09:38:55.396665+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3820,"log_dups_size":0,"ondisk_log_size":3820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:00:59.888085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250347520,"num_objects":3820,"num_object_clones":0,"num_object_copies":11460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3820,"num_write_kb":244480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":531,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.640834+0000","last_change":"2023-09-30T09:37:53.041461+0000","last_active":"2023-09-30T09:38:48.640834+0000","last_peered":"2023-09-30T09:38:48.640834+0000","last_clean":"2023-09-30T09:38:48.640834+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:38:48.640834+0000","last_undegraded":"2023-09-30T09:38:48.640834+0000","last_fullsized":"2023-09-30T09:38:48.640834+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:03:14.451639+0000","scrub_duration":0.0016789540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":763,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.643190+0000","last_change":"2023-09-30T09:38:30.736806+0000","last_active":"2023-09-30T09:38:48.643190+0000","last_peered":"2023-09-30T09:38:48.643190+0000","last_clean":"2023-09-30T09:38:48.643190+0000","last_became_active":"2023-09-30T09:38:30.736355+0000","last_became_peered":"2023-09-30T09:38:30.736355+0000","last_unstale":"2023-09-30T09:38:48.643190+0000","last_undegraded":"2023-09-30T09:38:48.643190+0000","last_fullsized":"2023-09-30T09:38:48.643190+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:46.413975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":313,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.634241+0000","last_change":"2023-09-30T09:38:45.620156+0000","last_active":"2023-09-30T09:38:48.634241+0000","last_peered":"2023-09-30T09:38:48.634241+0000","last_clean":"2023-09-30T09:38:48.634241+0000","last_became_active":"2023-09-30T09:38:45.619887+0000","last_became_peered":"2023-09-30T09:38:45.619887+0000","last_unstale":"2023-09-30T09:38:48.634241+0000","last_undegraded":"2023-09-30T09:38:48.634241+0000","last_fullsized":"2023-09-30T09:38:48.634241+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:37.503622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":61,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.641234+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:38:48.641234+0000","last_peered":"2023-09-30T09:38:48.641234+0000","last_clean":"2023-09-30T09:38:48.641234+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:38:48.641234+0000","last_undegraded":"2023-09-30T09:38:48.641234+0000","last_fullsized":"2023-09-30T09:38:48.641234+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"376'3767","reported_seq":3803,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:55.397383+0000","last_change":"2023-09-30T09:38:46.795169+0000","last_active":"2023-09-30T09:38:55.397383+0000","last_peered":"2023-09-30T09:38:55.397383+0000","last_clean":"2023-09-30T09:38:44.567377+0000","last_became_active":"2023-09-30T09:38:46.789998+0000","last_became_peered":"2023-09-30T09:38:46.789998+0000","last_unstale":"2023-09-30T09:38:55.397383+0000","last_undegraded":"2023-09-30T09:38:46.692264+0000","last_fullsized":"2023-09-30T09:38:46.597825+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3767,"log_dups_size":0,"ondisk_log_size":3767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:36.136530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246874112,"num_objects":3767,"num_object_clones":0,"num_object_copies":11301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3767,"num_write_kb":241088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":479},{"shards":"1,2,3","objects":3288}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":670,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.677851+0000","last_change":"2023-09-30T09:38:48.664915+0000","last_active":"2023-09-30T09:38:48.677851+0000","last_peered":"2023-09-30T09:38:48.677851+0000","last_clean":"2023-09-30T09:38:48.677851+0000","last_became_active":"2023-09-30T09:38:48.664604+0000","last_became_peered":"2023-09-30T09:38:48.664604+0000","last_unstale":"2023-09-30T09:38:48.677851+0000","last_undegraded":"2023-09-30T09:38:48.677851+0000","last_fullsized":"2023-09-30T09:38:48.677851+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:13:22.591021+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":872,"reported_epoch":375,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:48.632087+0000","last_change":"2023-09-30T09:38:28.863157+0000","last_active":"2023-09-30T09:38:48.632087+0000","last_peered":"2023-09-30T09:38:48.632087+0000","last_clean":"2023-09-30T09:38:16.639813+0000","last_became_active":"2023-09-30T09:38:28.862687+0000","last_became_peered":"2023-09-30T09:38:28.862687+0000","last_unstale":"2023-09-30T09:38:48.632087+0000","last_undegraded":"2023-09-30T09:38:28.721828+0000","last_fullsized":"2023-09-30T09:38:28.650935+0000","mapping_epoch":366,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:00:11.723777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.631740+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:38:48.631740+0000","last_peered":"2023-09-30T09:38:48.631740+0000","last_clean":"2023-09-30T09:38:48.631740+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:38:48.631740+0000","last_undegraded":"2023-09-30T09:38:48.631740+0000","last_fullsized":"2023-09-30T09:38:48.631740+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":285,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632667+0000","last_change":"2023-09-30T09:38:27.672393+0000","last_active":"2023-09-30T09:38:48.632667+0000","last_peered":"2023-09-30T09:38:48.632667+0000","last_clean":"2023-09-30T09:38:48.632667+0000","last_became_active":"2023-09-30T09:38:27.672044+0000","last_became_peered":"2023-09-30T09:38:27.672044+0000","last_unstale":"2023-09-30T09:38:48.632667+0000","last_undegraded":"2023-09-30T09:38:48.632667+0000","last_fullsized":"2023-09-30T09:38:48.632667+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:14:24.444433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"376'3975","reported_seq":6234,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:55.421318+0000","last_change":"2023-09-30T09:38:46.679360+0000","last_active":"2023-09-30T09:38:55.421318+0000","last_peered":"2023-09-30T09:38:55.421318+0000","last_clean":"2023-09-30T09:38:26.563448+0000","last_became_active":"2023-09-30T09:38:46.670968+0000","last_became_peered":"2023-09-30T09:38:46.670968+0000","last_unstale":"2023-09-30T09:38:55.421318+0000","last_undegraded":"2023-09-30T09:38:46.664891+0000","last_fullsized":"2023-09-30T09:38:46.591242+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3975,"log_dups_size":0,"ondisk_log_size":3975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:42:55.605938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260505600,"num_objects":3975,"num_object_clones":0,"num_object_copies":11925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2384,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3975,"num_write_kb":254400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1108,"num_bytes_recovered":72613888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,5],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3,5","objects":1259},{"shards":"0,3,7","objects":1125},{"shards":"0,5","objects":1591}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":848,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650637+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:38:48.650637+0000","last_peered":"2023-09-30T09:38:48.650637+0000","last_clean":"2023-09-30T09:38:48.650637+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:38:48.650637+0000","last_undegraded":"2023-09-30T09:38:48.650637+0000","last_fullsized":"2023-09-30T09:38:48.650637+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":387,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.649214+0000","last_change":"2023-09-30T09:38:45.668143+0000","last_active":"2023-09-30T09:38:48.649214+0000","last_peered":"2023-09-30T09:38:48.649214+0000","last_clean":"2023-09-30T09:38:48.649214+0000","last_became_active":"2023-09-30T09:38:45.624556+0000","last_became_peered":"2023-09-30T09:38:45.624556+0000","last_unstale":"2023-09-30T09:38:48.649214+0000","last_undegraded":"2023-09-30T09:38:48.649214+0000","last_fullsized":"2023-09-30T09:38:48.649214+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:39.105840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1142,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.645581+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:38:48.645581+0000","last_peered":"2023-09-30T09:38:48.645581+0000","last_clean":"2023-09-30T09:38:48.645581+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:38:48.645581+0000","last_undegraded":"2023-09-30T09:38:48.645581+0000","last_fullsized":"2023-09-30T09:38:48.645581+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":400,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.643123+0000","last_change":"2023-09-30T09:38:45.620031+0000","last_active":"2023-09-30T09:38:48.643123+0000","last_peered":"2023-09-30T09:38:48.643123+0000","last_clean":"2023-09-30T09:38:48.643123+0000","last_became_active":"2023-09-30T09:38:45.619374+0000","last_became_peered":"2023-09-30T09:38:45.619374+0000","last_unstale":"2023-09-30T09:38:48.643123+0000","last_undegraded":"2023-09-30T09:38:48.643123+0000","last_fullsized":"2023-09-30T09:38:48.643123+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:16.452264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"376'4177","reported_seq":4210,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:38:59.326905+0000","last_change":"2023-09-30T09:38:28.879709+0000","last_active":"2023-09-30T09:38:59.326905+0000","last_peered":"2023-09-30T09:38:59.326905+0000","last_clean":"2023-09-30T09:38:26.438920+0000","last_became_active":"2023-09-30T09:38:28.865945+0000","last_became_peered":"2023-09-30T09:38:28.865945+0000","last_unstale":"2023-09-30T09:38:59.326905+0000","last_undegraded":"2023-09-30T09:38:28.675959+0000","last_fullsized":"2023-09-30T09:38:28.657614+0000","mapping_epoch":366,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4177,"log_dups_size":0,"ondisk_log_size":4177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:22:08.971450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273743872,"num_objects":4177,"num_object_clones":0,"num_object_copies":12531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2381,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4177,"num_write_kb":267328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":1796},{"shards":"2,5,6","objects":2381}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":789,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.639537+0000","last_change":"2023-09-30T09:38:45.613829+0000","last_active":"2023-09-30T09:38:48.639537+0000","last_peered":"2023-09-30T09:38:48.639537+0000","last_clean":"2023-09-30T09:38:48.639537+0000","last_became_active":"2023-09-30T09:38:45.613119+0000","last_became_peered":"2023-09-30T09:38:45.613119+0000","last_unstale":"2023-09-30T09:38:48.639537+0000","last_undegraded":"2023-09-30T09:38:48.639537+0000","last_fullsized":"2023-09-30T09:38:48.639537+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:02.320545+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":728,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.653971+0000","last_change":"2023-09-30T09:38:27.708928+0000","last_active":"2023-09-30T09:38:48.653971+0000","last_peered":"2023-09-30T09:38:48.653971+0000","last_clean":"2023-09-30T09:38:48.653971+0000","last_became_active":"2023-09-30T09:38:27.682036+0000","last_became_peered":"2023-09-30T09:38:27.682036+0000","last_unstale":"2023-09-30T09:38:48.653971+0000","last_undegraded":"2023-09-30T09:38:48.653971+0000","last_fullsized":"2023-09-30T09:38:48.653971+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:26:59.892254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":64,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.641350+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:38:48.641350+0000","last_peered":"2023-09-30T09:38:48.641350+0000","last_clean":"2023-09-30T09:38:48.641350+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:38:48.641350+0000","last_undegraded":"2023-09-30T09:38:48.641350+0000","last_fullsized":"2023-09-30T09:38:48.641350+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":317,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650942+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:38:48.650942+0000","last_peered":"2023-09-30T09:38:48.650942+0000","last_clean":"2023-09-30T09:38:48.650942+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:38:48.650942+0000","last_undegraded":"2023-09-30T09:38:48.650942+0000","last_fullsized":"2023-09-30T09:38:48.650942+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":749,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.649521+0000","last_change":"2023-09-30T09:38:48.648959+0000","last_active":"2023-09-30T09:38:48.649521+0000","last_peered":"2023-09-30T09:38:48.649521+0000","last_clean":"2023-09-30T09:38:48.649521+0000","last_became_active":"2023-09-30T09:38:48.648731+0000","last_became_peered":"2023-09-30T09:38:48.648731+0000","last_unstale":"2023-09-30T09:38:48.649521+0000","last_undegraded":"2023-09-30T09:38:48.649521+0000","last_fullsized":"2023-09-30T09:38:48.649521+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:20:36.548251+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":68,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632582+0000","last_change":"2023-09-30T09:38:27.682550+0000","last_active":"2023-09-30T09:38:48.632582+0000","last_peered":"2023-09-30T09:38:48.632582+0000","last_clean":"2023-09-30T09:38:48.632582+0000","last_became_active":"2023-09-30T09:38:27.682171+0000","last_became_peered":"2023-09-30T09:38:27.682171+0000","last_unstale":"2023-09-30T09:38:48.632582+0000","last_undegraded":"2023-09-30T09:38:48.632582+0000","last_fullsized":"2023-09-30T09:38:48.632582+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:47:58.913725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":62,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.631650+0000","last_change":"2023-09-30T09:38:45.603597+0000","last_active":"2023-09-30T09:38:48.631650+0000","last_peered":"2023-09-30T09:38:48.631650+0000","last_clean":"2023-09-30T09:38:48.631650+0000","last_became_active":"2023-09-30T09:38:45.603355+0000","last_became_peered":"2023-09-30T09:38:45.603355+0000","last_unstale":"2023-09-30T09:38:48.631650+0000","last_undegraded":"2023-09-30T09:38:48.631650+0000","last_fullsized":"2023-09-30T09:38:48.631650+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:51.273460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":563,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.661277+0000","last_change":"2023-09-30T09:38:48.656922+0000","last_active":"2023-09-30T09:38:48.661277+0000","last_peered":"2023-09-30T09:38:48.661277+0000","last_clean":"2023-09-30T09:38:48.661277+0000","last_became_active":"2023-09-30T09:38:48.656616+0000","last_became_peered":"2023-09-30T09:38:48.656616+0000","last_unstale":"2023-09-30T09:38:48.661277+0000","last_undegraded":"2023-09-30T09:38:48.661277+0000","last_fullsized":"2023-09-30T09:38:48.661277+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":354,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.639759+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:38:48.639759+0000","last_peered":"2023-09-30T09:38:48.639759+0000","last_clean":"2023-09-30T09:38:48.639759+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:38:48.639759+0000","last_undegraded":"2023-09-30T09:38:48.639759+0000","last_fullsized":"2023-09-30T09:38:48.639759+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":80,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.654326+0000","last_change":"2023-09-30T09:38:45.595684+0000","last_active":"2023-09-30T09:38:48.654326+0000","last_peered":"2023-09-30T09:38:48.654326+0000","last_clean":"2023-09-30T09:38:48.654326+0000","last_became_active":"2023-09-30T09:38:45.595231+0000","last_became_peered":"2023-09-30T09:38:45.595231+0000","last_unstale":"2023-09-30T09:38:48.654326+0000","last_undegraded":"2023-09-30T09:38:48.654326+0000","last_fullsized":"2023-09-30T09:38:48.654326+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:43:29.805772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":320,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.642795+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:38:48.642795+0000","last_peered":"2023-09-30T09:38:48.642795+0000","last_clean":"2023-09-30T09:38:48.642795+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:38:48.642795+0000","last_undegraded":"2023-09-30T09:38:48.642795+0000","last_fullsized":"2023-09-30T09:38:48.642795+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":80,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.652023+0000","last_change":"2023-09-30T09:38:45.604276+0000","last_active":"2023-09-30T09:38:48.652023+0000","last_peered":"2023-09-30T09:38:48.652023+0000","last_clean":"2023-09-30T09:38:48.652023+0000","last_became_active":"2023-09-30T09:38:45.603925+0000","last_became_peered":"2023-09-30T09:38:45.603925+0000","last_unstale":"2023-09-30T09:38:48.652023+0000","last_undegraded":"2023-09-30T09:38:48.652023+0000","last_fullsized":"2023-09-30T09:38:48.652023+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:33:04.729338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":658,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.656084+0000","last_change":"2023-09-30T09:38:30.745007+0000","last_active":"2023-09-30T09:38:48.656084+0000","last_peered":"2023-09-30T09:38:48.656084+0000","last_clean":"2023-09-30T09:38:48.656084+0000","last_became_active":"2023-09-30T09:38:30.744685+0000","last_became_peered":"2023-09-30T09:38:30.744685+0000","last_unstale":"2023-09-30T09:38:48.656084+0000","last_undegraded":"2023-09-30T09:38:48.656084+0000","last_fullsized":"2023-09-30T09:38:48.656084+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:54.087577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:54.087577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":27,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.645657+0000","last_change":"2023-09-30T09:38:45.614444+0000","last_active":"2023-09-30T09:38:48.645657+0000","last_peered":"2023-09-30T09:38:48.645657+0000","last_clean":"2023-09-30T09:38:48.645657+0000","last_became_active":"2023-09-30T09:38:45.614114+0000","last_became_peered":"2023-09-30T09:38:45.614114+0000","last_unstale":"2023-09-30T09:38:48.645657+0000","last_undegraded":"2023-09-30T09:38:48.645657+0000","last_fullsized":"2023-09-30T09:38:48.645657+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:14.962666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":321,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.631844+0000","last_change":"2023-09-30T09:38:45.600268+0000","last_active":"2023-09-30T09:38:48.631844+0000","last_peered":"2023-09-30T09:38:48.631844+0000","last_clean":"2023-09-30T09:38:48.631844+0000","last_became_active":"2023-09-30T09:38:45.599829+0000","last_became_peered":"2023-09-30T09:38:45.599829+0000","last_unstale":"2023-09-30T09:38:48.631844+0000","last_undegraded":"2023-09-30T09:38:48.631844+0000","last_fullsized":"2023-09-30T09:38:48.631844+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:39:20.321856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":751,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.665450+0000","last_change":"2023-09-30T09:38:48.664091+0000","last_active":"2023-09-30T09:38:48.665450+0000","last_peered":"2023-09-30T09:38:48.665450+0000","last_clean":"2023-09-30T09:38:48.665450+0000","last_became_active":"2023-09-30T09:38:48.663723+0000","last_became_peered":"2023-09-30T09:38:48.663723+0000","last_unstale":"2023-09-30T09:38:48.665450+0000","last_undegraded":"2023-09-30T09:38:48.665450+0000","last_fullsized":"2023-09-30T09:38:48.665450+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:57.093254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:57.093254+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":80,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.632744+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:38:48.632744+0000","last_peered":"2023-09-30T09:38:48.632744+0000","last_clean":"2023-09-30T09:38:48.632744+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:38:48.632744+0000","last_undegraded":"2023-09-30T09:38:48.632744+0000","last_fullsized":"2023-09-30T09:38:48.632744+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":361,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-30T09:38:48.650240+0000","last_change":"2023-09-30T09:38:05.412543+0000","last_active":"2023-09-30T09:38:48.650240+0000","last_peered":"2023-09-30T09:38:48.650240+0000","last_clean":"2023-09-30T09:38:48.650240+0000","last_became_active":"2023-09-30T09:38:05.412129+0000","last_became_peered":"2023-09-30T09:38:05.412129+0000","last_unstale":"2023-09-30T09:38:48.650240+0000","last_undegraded":"2023-09-30T09:38:48.650240+0000","last_fullsized":"2023-09-30T09:38:48.650240+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:49:02.515137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":11608,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4180869120,"num_objects":63795,"num_object_clones":0,"num_object_copies":191385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23802,"num_objects_misplaced":1674,"num_objects_unfound":0,"num_objects_dirty":63795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63795,"num_write_kb":4082880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":125894656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12666142720,"data_stored":12666142720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63795,"ondisk_log_size":63795,"up":48,"acting":39,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":326246,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":841734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":211,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234328,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1323136,"kb_used_data":932088,"kb_used_omap":0,"kb_used_meta":391040,"kb_avail":92422016,"statfs":{"total":95995035648,"available":94640144384,"internally_reserved":0,"allocated":954458112,"data_stored":953176170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":400424960},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:46 2023","interfaces":[{"interface":"back","average":{"1min":5.5099999999999998,"5min":2.7730000000000001,"15min":1.899},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.224},"max":{"1min":95.418999999999997,"5min":95.418999999999997,"15min":95.418999999999997},"last":3.1909999999999998},{"interface":"front","average":{"1min":5.3399999999999999,"5min":2.6230000000000002,"15min":1.7190000000000001},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.23899999999999999},"max":{"1min":95.230999999999995,"5min":95.230999999999995,"15min":95.230999999999995},"last":0.35599999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:38:46 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":3.0070000000000001,"15min":3.0350000000000001},"min":{"1min":0.27000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":89.308000000000007,"5min":89.308000000000007,"15min":89.308000000000007},"last":1.5409999999999999},{"interface":"front","average":{"1min":5.2510000000000003,"5min":3.343,"15min":3.5870000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":89.325999999999993,"5min":89.325999999999993,"15min":89.325999999999993},"last":0.38}]},{"osd":2,"last update":"Sat Sep 30 09:38:27 2023","interfaces":[{"interface":"back","average":{"1min":6.1109999999999998,"5min":2.9350000000000001,"15min":2.1579999999999999},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.24299999999999999},"max":{"1min":95.602999999999994,"5min":95.602999999999994,"15min":95.602999999999994},"last":0.86499999999999999},{"interface":"front","average":{"1min":5.5670000000000002,"5min":2.8610000000000002,"15min":2.161},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.28299999999999997},"max":{"1min":95.682000000000002,"5min":95.682000000000002,"15min":95.682000000000002},"last":2.98}]},{"osd":3,"last update":"Sat Sep 30 09:38:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":11.641,"15min":6.5940000000000003},"min":{"1min":0.40999999999999998,"5min":0.31,"15min":0.28899999999999998},"max":{"1min":6.516,"5min":277.57799999999997,"15min":277.57799999999997},"last":1.5960000000000001},{"interface":"front","average":{"1min":1.452,"5min":12.132999999999999,"15min":5.9699999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.9220000000000006,"5min":278.69299999999998,"15min":278.69299999999998},"last":0.65700000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:38:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.75600000000000001,"15min":0.95099999999999996},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23000000000000001},"max":{"1min":2.9260000000000002,"5min":6.9029999999999996,"15min":8.9329999999999998},"last":4.3550000000000004},{"interface":"front","average":{"1min":0.437,"5min":0.66400000000000003,"15min":0.876},"min":{"1min":0.252,"5min":0.252,"15min":0.251},"max":{"1min":0.65100000000000002,"5min":5.6319999999999997,"15min":7.8890000000000002},"last":0.63100000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:38:25 2023","interfaces":[{"interface":"back","average":{"1min":10.090999999999999,"5min":2.9319999999999999,"15min":1.752},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":188.41,"5min":188.41,"15min":188.41},"last":0.57899999999999996},{"interface":"front","average":{"1min":10.218,"5min":2.8410000000000002,"15min":1.7829999999999999},"min":{"1min":0.40100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":188.43000000000001,"5min":188.43000000000001,"15min":188.43000000000001},"last":1.5069999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:38:25 2023","interfaces":[{"interface":"back","average":{"1min":11.065,"5min":3.2490000000000001,"15min":1.8460000000000001},"min":{"1min":0.40799999999999997,"5min":0.33200000000000002,"15min":0.32400000000000001},"max":{"1min":206.244,"5min":206.244,"15min":206.244},"last":1.5700000000000001},{"interface":"front","average":{"1min":10.643000000000001,"5min":3.2799999999999998,"15min":1.8069999999999999},"min":{"1min":0.38,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":202.78800000000001,"5min":202.78800000000001,"15min":202.78800000000001},"last":0.60299999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495965,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1883756,"kb_used_data":1454244,"kb_used_omap":0,"kb_used_meta":429504,"kb_avail":91861396,"statfs":{"total":95995035648,"available":94066069504,"internally_reserved":0,"allocated":1489145856,"data_stored":1487883135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":439812096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":2.4060000000000001,"15min":1.869},"min":{"1min":0.26700000000000002,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":4.6970000000000001,"5min":44.113,"15min":45.927999999999997},"last":5.157},{"interface":"front","average":{"1min":1.125,"5min":2.3140000000000001,"15min":2.0859999999999999},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.0460000000000003,"5min":44.164000000000001,"15min":45.316000000000003},"last":2.9079999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:38:39 2023","interfaces":[{"interface":"back","average":{"1min":4.8029999999999999,"5min":3.1909999999999998,"15min":2.2650000000000001},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":69.569999999999993,"5min":69.569999999999993,"15min":69.569999999999993},"last":2.169},{"interface":"front","average":{"1min":4.7640000000000002,"5min":3.3919999999999999,"15min":2.5059999999999998},"min":{"1min":0.33600000000000002,"5min":0.313,"15min":0.313},"max":{"1min":70.471999999999994,"5min":70.471999999999994,"15min":70.471999999999994},"last":2.153}]},{"osd":2,"last update":"Sat Sep 30 09:38:32 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":2.0369999999999999,"15min":2.0129999999999999},"min":{"1min":0.33800000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.6200000000000001,"5min":17.126000000000001,"15min":37.978000000000002},"last":3.9359999999999999},{"interface":"front","average":{"1min":1.4099999999999999,"5min":2.367,"15min":1.9510000000000001},"min":{"1min":0.36899999999999999,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":5.6619999999999999,"5min":25.164999999999999,"15min":37.759999999999998},"last":2.1339999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":6.1580000000000004,"15min":3.605},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.27600000000000002},"max":{"1min":14.862,"5min":212.44800000000001,"15min":212.44800000000001},"last":4.5750000000000002},{"interface":"front","average":{"1min":1.7090000000000001,"5min":6.7759999999999998,"15min":3.9950000000000001},"min":{"1min":0.39000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":5.2069999999999999,"5min":210.84299999999999,"15min":210.84299999999999},"last":2.9359999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.0129999999999999,"15min":1.6259999999999999},"min":{"1min":0.26900000000000002,"5min":0.26400000000000001,"15min":0.23400000000000001},"max":{"1min":4.0469999999999997,"5min":7.4359999999999999,"15min":73.656000000000006},"last":2.1150000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.91200000000000003,"15min":1.417},"min":{"1min":0.30599999999999999,"5min":0.29599999999999999,"15min":0.23000000000000001},"max":{"1min":1.5580000000000001,"5min":9.6590000000000007,"15min":73.882999999999996},"last":4.6859999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.2410000000000001,"15min":1.903},"min":{"1min":0.29899999999999999,"5min":0.28699999999999998,"15min":0.28499999999999998},"max":{"1min":11.428000000000001,"5min":11.428000000000001,"15min":76.613},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.25,"5min":1.143,"15min":1.8280000000000001},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7220000000000004,"5min":9.7750000000000004,"15min":76.329999999999998},"last":4.6210000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:38:32 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":2.4689999999999999,"15min":2.6589999999999998},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":2.649,"5min":138.191,"15min":138.191},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.751,"5min":2.3610000000000002,"15min":2.5910000000000002},"min":{"1min":0.38400000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.1219999999999999,"5min":140.33500000000001,"15min":140.33500000000001},"last":4.6550000000000002}]}]},{"osd":1,"up_from":301,"seq":1292785156162,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2096544,"kb_used_data":1736664,"kb_used_omap":9,"kb_used_meta":359862,"kb_avail":91648608,"statfs":{"total":95995035648,"available":93848174592,"internally_reserved":0,"allocated":1778343936,"data_stored":1776229227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":368498706},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":4.4829999999999997,"15min":3.1890000000000001},"min":{"1min":0.20899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":14.992000000000001,"5min":138.43799999999999,"15min":138.43799999999999},"last":3.2519999999999998},{"interface":"front","average":{"1min":1.462,"5min":4.266,"15min":3.4380000000000002},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":15.029,"5min":138.482,"15min":138.482},"last":3.23}]},{"osd":2,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":2.6869999999999998,"15min":2.7589999999999999},"min":{"1min":0.25700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.597,"5min":16.437000000000001,"15min":16.437000000000001},"last":2.6560000000000001},{"interface":"front","average":{"1min":1.603,"5min":3.431,"15min":2.9470000000000001},"min":{"1min":0.27800000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":15.063000000000001,"5min":59.078000000000003,"15min":59.078000000000003},"last":3.177}]},{"osd":3,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":7.6390000000000002,"15min":8.8439999999999994},"min":{"1min":0.32400000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":19.733000000000001,"5min":191.07300000000001,"15min":191.07300000000001},"last":3.0760000000000001},{"interface":"front","average":{"1min":1.837,"5min":7.3010000000000002,"15min":9.218},"min":{"1min":0.32000000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":15.192,"5min":191.11500000000001,"15min":191.11500000000001},"last":1.577}]},{"osd":4,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.9419999999999999,"15min":1.972},"min":{"1min":0.41699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":15.138,"5min":15.138,"15min":15.138},"last":3.206},{"interface":"front","average":{"1min":2.2759999999999998,"5min":2.629,"15min":1.738},"min":{"1min":0.312,"5min":0.311,"15min":0.311},"max":{"1min":20.966000000000001,"5min":58.939999999999998,"15min":58.939999999999998},"last":1.8220000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":2.04,"5min":2.528,"15min":2.2480000000000002},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":23.257999999999999,"5min":24.25,"15min":24.25},"last":6.758},{"interface":"front","average":{"1min":1.9610000000000001,"5min":3.0350000000000001,"15min":2.0790000000000002},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.385999999999999,"5min":58.999000000000002,"15min":58.999000000000002},"last":1.6910000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":6.218,"15min":13.471},"min":{"1min":0.46500000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":15.101000000000001,"5min":209.221,"15min":209.221},"last":3.2730000000000001},{"interface":"front","average":{"1min":1.9199999999999999,"5min":6.0730000000000004,"15min":13.368},"min":{"1min":0.54200000000000004,"5min":0.377,"15min":0.377},"max":{"1min":23.204000000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":3.1480000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0150000000000001,"5min":4.4329999999999998,"15min":9.7739999999999991},"min":{"1min":0.498,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":15.587,"5min":165.59399999999999,"15min":165.59399999999999},"last":1.611},{"interface":"front","average":{"1min":1.9450000000000001,"5min":4.484,"15min":9.9469999999999992},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":15.226000000000001,"5min":165.833,"15min":165.833},"last":1.7190000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926061,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2226132,"kb_used_data":1786888,"kb_used_omap":5,"kb_used_meta":439226,"kb_avail":91519020,"statfs":{"total":95995035648,"available":93715476480,"internally_reserved":0,"allocated":1829773312,"data_stored":1827498625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":449767677},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:38:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":2.3660000000000001,"15min":2.7869999999999999},"min":{"1min":0.251,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.6000000000000001,"5min":17.966000000000001,"15min":17.966000000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.389,"5min":4.7729999999999997,"15min":7.8739999999999997},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":6.1219999999999999,"5min":138.92400000000001,"15min":138.92400000000001},"last":0.312}]},{"osd":2,"last update":"Sat Sep 30 09:38:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":2.6920000000000002,"15min":2.0190000000000001},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":9.5269999999999992,"5min":15.324,"15min":55.823999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.679,"5min":3.0979999999999999,"15min":2.177},"min":{"1min":0.26800000000000002,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":6.2549999999999999,"5min":40.128999999999998,"15min":55.790999999999997},"last":0.42699999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:37:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2480000000000002,"5min":8.9710000000000001,"15min":4.3929999999999998},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.221},"max":{"1min":9.4450000000000003,"5min":218.06999999999999,"15min":218.06999999999999},"last":1.839},{"interface":"front","average":{"1min":2.4449999999999998,"5min":6.5529999999999999,"15min":3.5499999999999998},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.245},"max":{"1min":9.4969999999999999,"5min":218.63200000000001,"15min":218.63200000000001},"last":1.8169999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.883,"15min":1.806},"min":{"1min":0.35299999999999998,"5min":0.33200000000000002,"15min":0.29099999999999998},"max":{"1min":6.2210000000000001,"5min":10.614000000000001,"15min":121.486},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.1970000000000001,"5min":4.0599999999999996,"15min":2.7930000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":4.7599999999999998,"5min":136.262,"15min":136.262},"last":1.3819999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":4.1920000000000002,"15min":3.21},"min":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.6100000000000003,"5min":136.34,"15min":136.34},"last":1.333},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.778,"15min":2.3319999999999999},"min":{"1min":0.41999999999999998,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":3.875,"5min":12.717000000000001,"15min":135.381},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":6.0410000000000004,"15min":3.2949999999999999},"min":{"1min":0.42899999999999999,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":3.5419999999999998,"5min":214.785,"15min":214.785},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.4490000000000001,"5min":6.3360000000000003,"15min":3.335},"min":{"1min":0.44500000000000001,"5min":0.40600000000000003,"15min":0.34200000000000003},"max":{"1min":6.1719999999999997,"5min":214.93899999999999,"15min":214.93899999999999},"last":2.141}]},{"osd":7,"last update":"Sat Sep 30 09:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":2.9889999999999999,"15min":3.2530000000000001},"min":{"1min":0.48999999999999999,"5min":0.42999999999999999,"15min":0.373},"max":{"1min":4.5469999999999997,"5min":97.382000000000005,"15min":150.96899999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.395,"5min":3.1890000000000001,"15min":3.1949999999999998},"min":{"1min":0.47799999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.2789999999999999,"5min":97.015000000000001,"15min":150.328},"last":1.7949999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599002,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2079912,"kb_used_data":1686560,"kb_used_omap":5,"kb_used_meta":393338,"kb_avail":91665240,"statfs":{"total":95995035648,"available":93865205760,"internally_reserved":0,"allocated":1727037440,"data_stored":1724972110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":402778765},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":2.7890000000000001,"15min":2.2450000000000001},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.187},"max":{"1min":1.728,"5min":77.379000000000005,"15min":77.379000000000005},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.3160000000000001,"5min":2.8599999999999999,"15min":2.1379999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19700000000000001},"max":{"1min":15.548999999999999,"5min":76.796999999999997,"15min":77.066999999999993},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":4.4530000000000003,"15min":3.8650000000000002},"min":{"1min":0.27600000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.5350000000000001,"5min":64.412999999999997,"15min":64.412999999999997},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.0089999999999999,"5min":3.8879999999999999,"15min":3.3340000000000001},"min":{"1min":0.29999999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.2679999999999998,"5min":63.247999999999998,"15min":63.247999999999998},"last":0.32800000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:37:54 2023","interfaces":[{"interface":"back","average":{"1min":4.9960000000000004,"5min":8.2409999999999997,"15min":3.7730000000000001},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":60.359999999999999,"5min":134.078,"15min":134.078},"last":1.333},{"interface":"front","average":{"1min":4.8029999999999999,"5min":7.8259999999999996,"15min":3.8780000000000001},"min":{"1min":0.35599999999999998,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":63.735999999999997,"5min":135.73400000000001,"15min":135.73400000000001},"last":1.3700000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":2.0880000000000001,"15min":1.415},"min":{"1min":0.37,"5min":0.34499999999999997,"15min":0.28999999999999998},"max":{"1min":4.4630000000000001,"5min":35.972000000000001,"15min":35.972000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.8009999999999999,"15min":1.6200000000000001},"min":{"1min":0.378,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":0.90000000000000002,"5min":36.518000000000001,"15min":36.518000000000001},"last":0.98799999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":2.254,"15min":2.2440000000000002},"min":{"1min":0.41999999999999998,"5min":0.377,"15min":0.34999999999999998},"max":{"1min":3.3679999999999999,"5min":12.651999999999999,"15min":155.477},"last":2.2469999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.655,"15min":2.0270000000000001},"min":{"1min":0.40500000000000003,"5min":0.35899999999999999,"15min":0.33100000000000002},"max":{"1min":4.4080000000000004,"5min":8.0839999999999996,"15min":155.46000000000001},"last":1.5580000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":3.661,"15min":3.3620000000000001},"min":{"1min":0.374,"5min":0.35499999999999998,"15min":0.33100000000000002},"max":{"1min":0.89200000000000002,"5min":98.519000000000005,"15min":154.417},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":4.5350000000000001,"15min":3.4769999999999999},"min":{"1min":0.41099999999999998,"5min":0.39000000000000001,"15min":0.36899999999999999},"max":{"1min":3.411,"5min":98.293000000000006,"15min":154.04400000000001},"last":4.2430000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":2.0859999999999999,"15min":3.1779999999999999},"min":{"1min":0.39400000000000002,"5min":0.39000000000000001,"15min":0.35899999999999999},"max":{"1min":0.93200000000000005,"5min":47.677999999999997,"15min":213.404},"last":6.2889999999999997},{"interface":"front","average":{"1min":0.95799999999999996,"5min":2.754,"15min":3.2970000000000002},"min":{"1min":0.45800000000000002,"5min":0.438,"15min":0.40000000000000002},"max":{"1min":3.1699999999999999,"5min":50.725999999999999,"15min":213.27199999999999},"last":2.5390000000000001}]}]},{"osd":3,"up_from":0,"seq":468151435475,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019238,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1663664,"kb_used_data":1365928,"kb_used_omap":0,"kb_used_meta":297728,"kb_avail":92081488,"statfs":{"total":95995035648,"available":94291443712,"internally_reserved":0,"allocated":1398710272,"data_stored":1397443570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304873472},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:44 2023","interfaces":[{"interface":"back","average":{"1min":6.6360000000000001,"5min":3.081,"15min":3.4700000000000002},"min":{"1min":0.315,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":105.179,"5min":105.179,"15min":128.79300000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":6.3959999999999999,"5min":3.2050000000000001,"15min":3.3980000000000001},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.248},"max":{"1min":104.70999999999999,"5min":104.70999999999999,"15min":128.38800000000001},"last":1.5720000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:38:44 2023","interfaces":[{"interface":"back","average":{"1min":4.7320000000000002,"5min":4.8339999999999996,"15min":7.9470000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":84.105999999999995,"5min":160.69999999999999,"15min":160.69999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":4.7220000000000004,"5min":4.4820000000000002,"15min":7.8700000000000001},"min":{"1min":0.35099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":83.941000000000003,"5min":160.50700000000001,"15min":160.50700000000001},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.7849999999999999,"15min":3.0259999999999998},"min":{"1min":0.41199999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.1520000000000001,"5min":11.096,"15min":140.601},"last":3.5329999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.9370000000000001,"15min":3.0510000000000002},"min":{"1min":0.434,"5min":0.34200000000000003,"15min":0.309},"max":{"1min":4.1390000000000002,"5min":11.057,"15min":140.54900000000001},"last":1.794}]},{"osd":3,"last update":"Sat Sep 30 09:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":3.2250000000000001,"15min":3.8330000000000002},"min":{"1min":0.377,"5min":0.315,"15min":0.26400000000000001},"max":{"1min":13.84,"5min":52.762,"15min":281.42200000000003},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.3029999999999999,"5min":3.2069999999999999,"15min":3.7799999999999998},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.312},"max":{"1min":13.776999999999999,"5min":52.790999999999997,"15min":281.44999999999999},"last":1.871}]},{"osd":5,"last update":"Sat Sep 30 09:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":2.6179999999999999,"15min":2.3490000000000002},"min":{"1min":0.29299999999999998,"5min":0.219,"15min":0.219},"max":{"1min":1.4410000000000001,"5min":172.97800000000001,"15min":172.97800000000001},"last":2.3250000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.5870000000000002,"15min":2.395},"min":{"1min":0.36399999999999999,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":1.4410000000000001,"5min":172.92400000000001,"15min":172.92400000000001},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:38:05 2023","interfaces":[{"interface":"back","average":{"1min":10.061999999999999,"5min":4.8209999999999997,"15min":2.2250000000000001},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":182.10300000000001,"5min":202.72499999999999,"15min":202.72499999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":9.8759999999999994,"5min":4.6769999999999996,"15min":2.2130000000000001},"min":{"1min":0.34799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":181.76300000000001,"5min":202.42400000000001,"15min":202.42400000000001},"last":2.8239999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.837,"15min":3.0630000000000002},"min":{"1min":0.35499999999999998,"5min":0.314,"15min":0.30299999999999999},"max":{"1min":2.948,"5min":85.114000000000004,"15min":230.90100000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.6599999999999999,"15min":2.5960000000000001},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":0.84499999999999997,"5min":84.697999999999993,"15min":229.97},"last":0.58399999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757602,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2595308,"kb_used_data":2266788,"kb_used_omap":0,"kb_used_meta":328512,"kb_avail":91149844,"statfs":{"total":95995035648,"available":93337440256,"internally_reserved":0,"allocated":2321190912,"data_stored":2319924832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":336396288},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:54 2023","interfaces":[{"interface":"back","average":{"1min":13.266999999999999,"5min":4.4210000000000003,"15min":3.3610000000000002},"min":{"1min":0.26200000000000001,"5min":0.223,"15min":0.22},"max":{"1min":98.548000000000002,"5min":98.548000000000002,"15min":148.79900000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":13.592000000000001,"5min":4.6609999999999996,"15min":3.6619999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":98.388000000000005,"5min":98.388000000000005,"15min":148.779},"last":0.29899999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:38:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.7930000000000001,"15min":2.1560000000000001},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.0640000000000001,"5min":60.799999999999997,"15min":60.799999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3,"15min":3.1080000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7120000000000002,"5min":63.798999999999999,"15min":63.798999999999999},"last":1.179}]},{"osd":2,"last update":"Sat Sep 30 09:38:27 2023","interfaces":[{"interface":"back","average":{"1min":15.347,"5min":4.8319999999999999,"15min":4.1379999999999999},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.27700000000000002},"max":{"1min":227.18000000000001,"5min":227.18000000000001,"15min":227.18000000000001},"last":2.2959999999999998},{"interface":"front","average":{"1min":15.637,"5min":5.0730000000000004,"15min":4.7199999999999998},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":227.46600000000001,"5min":227.46600000000001,"15min":227.46600000000001},"last":10.813000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:38:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":3.7869999999999999,"15min":4.0780000000000003},"min":{"1min":0.38900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.351,"5min":49.875,"15min":226.34700000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.508,"5min":3.6619999999999999,"15min":3.887},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.313},"max":{"1min":7.3769999999999998,"5min":49.895000000000003,"15min":226.36000000000001},"last":0.443}]},{"osd":4,"last update":"Sat Sep 30 09:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.67700000000000005,"15min":0.85899999999999999},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.222},"max":{"1min":1.415,"5min":5.7160000000000002,"15min":8.5359999999999996},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.77500000000000002,"15min":0.96399999999999997},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.6299999999999999,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":0.64200000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:38:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.96599999999999997,"15min":1.76},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":2.4390000000000001,"5min":6.4459999999999997,"15min":155.602},"last":3.7080000000000002},{"interface":"front","average":{"1min":0.752,"5min":1.0029999999999999,"15min":1.873},"min":{"1min":0.35399999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.5760000000000001,"5min":6.4930000000000003,"15min":154.00800000000001},"last":1.329}]},{"osd":7,"last update":"Sat Sep 30 09:38:09 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":2.5619999999999998,"15min":1.675},"min":{"1min":0.34599999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.0710000000000002,"5min":133.40700000000001,"15min":133.40700000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.85199999999999998,"5min":2.573,"15min":1.599},"min":{"1min":0.378,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":3.895,"5min":132.797,"15min":132.797},"last":0.38}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1846214656,"data_stored":1846214656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1787035648,"data_stored":1787035648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1724317696,"data_stored":1724317696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1339424768,"data_stored":1339424768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1405353984,"data_stored":1405353984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2340093952,"data_stored":2340093952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1488584704,"data_stored":1488584704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":735117312,"data_stored":735117312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:39:02.404 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:39:02.404 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-09-30T09:39:02.460 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:02.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:02.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:02.531 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 57 16 67559 67543 74.0266 51.875 0.0279941 0.0135022 2023-09-30T09:39:02.569 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:02.610 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:02.696 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:02.812 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:02.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:02.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:02.830 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:02.916 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:02.985 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:02 smithi167 ceph-mon[192880]: pgmap v978: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 23817/196200 objects degraded (12.139%); 1397/196200 objects misplaced (0.712%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:39:02.991 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:03.015 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:02 smithi019 ceph-mon[216815]: pgmap v978: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 23817/196200 objects degraded (12.139%); 1397/196200 objects misplaced (0.712%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:39:03.017 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:02 smithi019 ceph-mon[224043]: pgmap v978: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 23817/196200 objects degraded (12.139%); 1397/196200 objects misplaced (0.712%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:39:03.098 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:03.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:03.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:03.276 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:03.532 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 58 16 68413 68397 73.6706 53.375 0.0130463 0.0135692 2023-09-30T09:39:03.754 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:03.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:03.871 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:03.924 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:03.992 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:39:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:39:03] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-09-30T09:39:04.106 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:04.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:04.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:04.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:04.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23817/196200 objects degraded (12.139%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:04.277 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:04.277 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:04.277 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23817/196200 objects degraded (12.139%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:04.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:04.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:04.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23817/196200 objects degraded (12.139%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:04.295 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:04.534 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 59 16 69044 69028 73.0882 39.4375 0.00416324 0.0136576 2023-09-30T09:39:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:04 smithi167 ceph-mon[192880]: pgmap v979: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s; 23817/196200 objects degraded (12.139%); 1397/196200 objects misplaced (0.712%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:39:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3098830772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:04 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:04 smithi019 ceph-mon[224043]: pgmap v979: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s; 23817/196200 objects degraded (12.139%); 1397/196200 objects misplaced (0.712%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:39:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3098830772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:04 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:04 smithi019 ceph-mon[216815]: pgmap v979: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s; 23817/196200 objects degraded (12.139%); 1397/196200 objects misplaced (0.712%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:39:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3098830772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:04 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:05.430 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:05.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:05.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:05.446 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:05.533 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:39:05.533806+0000 min lat: 0.0012782 max lat: 2.31896 avg lat: 0.0137572 2023-09-30T09:39:05.534 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:39:05.534 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 60 16 69761 69745 72.617 44.8125 0.00719862 0.0137572 2023-09-30T09:39:05.535 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:05.566 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:05.596 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:39:06.248 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:06.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:05 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:06.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:05 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:39:06.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:05 smithi019 ceph-mon[216815]: osdmap e377: 8 total, 8 up, 7 in 2023-09-30T09:39:06.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:05 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:06.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:05 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:39:06.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:05 smithi019 ceph-mon[224043]: osdmap e377: 8 total, 8 up, 7 in 2023-09-30T09:39:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:06.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:05 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:05 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:39:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:05 smithi167 ceph-mon[192880]: osdmap e377: 8 total, 8 up, 7 in 2023-09-30T09:39:06.353 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:06.386 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:06.534 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 61 16 70451 70435 72.1333 43.125 0.195527 0.013822 2023-09-30T09:39:06.828 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:06.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:06.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:06.933 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:07.017 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:06 smithi019 ceph-mon[216815]: failed to encode map e377 with expected crc 2023-09-30T09:39:07.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:06 smithi019 ceph-mon[216815]: pgmap v981: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 23817/204696 objects degraded (11.635%); 1397/204696 objects misplaced (0.682%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:39:07.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:06 smithi019 ceph-mon[224043]: failed to encode map e377 with expected crc 2023-09-30T09:39:07.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:06 smithi019 ceph-mon[224043]: pgmap v981: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 23817/204696 objects degraded (11.635%); 1397/204696 objects misplaced (0.682%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:39:07.133 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:07.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:07.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:07.253 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:06 smithi167 ceph-mon[192880]: failed to encode map e377 with expected crc 2023-09-30T09:39:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:06 smithi167 ceph-mon[192880]: pgmap v981: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 23817/204696 objects degraded (11.635%); 1397/204696 objects misplaced (0.682%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:39:07.346 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:07.447 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:07.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:07.534 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 62 16 71330 71314 71.8559 54.9375 0.0276112 0.013911 2023-09-30T09:39:07.554 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:07.632 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:07.753 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:07.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:07.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:07.843 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:39:07.867 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:07.898 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:39:07.913 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:07.929 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:39:07.965 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:39:08.009 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:39:08.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2567210652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:08.022 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:08.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:08.132 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:08.156 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2567210652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:08.158 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:08.163 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:39:08.163 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:39:08.163 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 0 2023-09-30T09:39:08.163 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:39:08.186 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:39:08.187 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:39:08.277 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:08.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:08.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2567210652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:08.388 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:08.480 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:08.534 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 63 16 72121 72105 71.5001 49.4375 0.0276228 0.0139812 2023-09-30T09:39:08.575 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:08.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:08.691 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:08.749 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:09.013 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:09 smithi019 ceph-mon[216815]: pgmap v982: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 806 op/s; 23817/207567 objects degraded (11.474%); 1061/207567 objects misplaced (0.511%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:39:09.013 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23817/207567 objects degraded (11.474%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:09.105 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:39:09.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:09 smithi167 ceph-mon[192880]: pgmap v982: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 806 op/s; 23817/207567 objects degraded (11.474%); 1061/207567 objects misplaced (0.511%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:39:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23817/207567 objects degraded (11.474%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:09 smithi019 ceph-mon[224043]: pgmap v982: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 806 op/s; 23817/207567 objects degraded (11.474%); 1061/207567 objects misplaced (0.511%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:39:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23817/207567 objects degraded (11.474%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:09.534 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 64 16 73009 72993 71.2501 55.5 0.01655 0.01403 2023-09-30T09:39:10.241 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:10.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:10.351 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:10.427 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:10.534 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:10.535 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 65 16 73734 73718 70.8502 45.3125 0.0222409 0.0141085 2023-09-30T09:39:10.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:10.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:10.592 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:39:10.644 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:10.740 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:11.042 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:11.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:11.079 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:11 smithi019 ceph-mon[216815]: pgmap v983: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s; 23817/214839 objects degraded (11.086%); 1061/214839 objects misplaced (0.494%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:39:11.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:11 smithi019 ceph-mon[224043]: pgmap v983: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s; 23817/214839 objects degraded (11.086%); 1061/214839 objects misplaced (0.494%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:39:11.151 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:11.260 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:11.375 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:11.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:11.482 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:11 smithi167 ceph-mon[192880]: pgmap v983: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s; 23817/214839 objects degraded (11.086%); 1061/214839 objects misplaced (0.494%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:39:11.535 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 66 16 74541 74525 70.541 50.4375 0.0112063 0.0141706 2023-09-30T09:39:11.574 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:11.690 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:11.692 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:11.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:11.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:11.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:11.877 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:12.064 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 8 pg_num to 26 2023-09-30T09:39:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1866816602' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:39:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:12 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:39:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1866816602' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:39:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:12 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:39:12.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1866816602' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:39:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:12 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-30T09:39:12.537 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 67 16 75276 75260 70.1718 45.9375 0.210672 0.0142432 2023-09-30T09:39:12.710 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:39:12.884 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:12.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:12.991 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:13.059 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:13.377 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:13.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:13.377 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[224043]: pgmap v984: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 23817/219402 objects degraded (10.855%); 744/219402 objects misplaced (0.339%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:39:13.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-30T09:39:13.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[224043]: osdmap e378: 8 total, 8 up, 7 in 2023-09-30T09:39:13.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[224043]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[224043]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[224043]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.379 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[224043]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.379 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[216815]: pgmap v984: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 23817/219402 objects degraded (10.855%); 744/219402 objects misplaced (0.339%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:39:13.379 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-30T09:39:13.379 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[216815]: osdmap e378: 8 total, 8 up, 7 in 2023-09-30T09:39:13.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[216815]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[216815]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[216815]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:13 smithi019 ceph-mon[216815]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.478 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:13 smithi167 ceph-mon[192880]: pgmap v984: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 23817/219402 objects degraded (10.855%); 744/219402 objects misplaced (0.339%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:39:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:13 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-30T09:39:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:13 smithi167 ceph-mon[192880]: osdmap e378: 8 total, 8 up, 7 in 2023-09-30T09:39:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:13 smithi167 ceph-mon[192880]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:13 smithi167 ceph-mon[192880]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:13 smithi167 ceph-mon[192880]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:13 smithi167 ceph-mon[192880]: failed to encode map e378 with expected crc 2023-09-30T09:39:13.524 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:13.537 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 68 16 76133 76117 69.9276 53.5625 0.0181952 0.0142944 2023-09-30T09:39:13.806 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:39:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:39:13] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-09-30T09:39:13.981 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:14.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:14.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23817/219402 objects degraded (10.855%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:14 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-30T09:39:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23817/219402 objects degraded (10.855%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:14 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-30T09:39:14.104 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:14.175 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:14.302 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:14.319 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23817/219402 objects degraded (10.855%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:14.319 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:14 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-30T09:39:14.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:14.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:14.422 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:14.458 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:14.537 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 69 16 76890 76874 69.5999 47.3125 0.00622369 0.0143432 2023-09-30T09:39:14.946 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:14.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:14.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:15.052 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:15.081 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:15.202 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:15 smithi019 ceph-mon[216815]: pgmap v986: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 609 op/s; 23817/219402 objects degraded (10.855%); 744/219402 objects misplaced (0.339%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:39:15.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:15 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-30T09:39:15.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:15 smithi019 ceph-mon[216815]: osdmap e379: 8 total, 8 up, 7 in 2023-09-30T09:39:15.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:15 smithi019 ceph-mon[216815]: failed to encode map e379 with expected crc 2023-09-30T09:39:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:15 smithi019 ceph-mon[224043]: pgmap v986: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 609 op/s; 23817/219402 objects degraded (10.855%); 744/219402 objects misplaced (0.339%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:39:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:15 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-30T09:39:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:15 smithi019 ceph-mon[224043]: osdmap e379: 8 total, 8 up, 7 in 2023-09-30T09:39:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:15 smithi019 ceph-mon[224043]: failed to encode map e379 with expected crc 2023-09-30T09:39:15.238 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:15.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:15.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:15.340 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:15.342 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:15.423 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:15 smithi167 ceph-mon[192880]: pgmap v986: 119 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 609 op/s; 23817/219402 objects degraded (10.855%); 744/219402 objects misplaced (0.339%); 5.1 MiB/s, 81 objects/s recovering 2023-09-30T09:39:15.423 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:15 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-30T09:39:15.423 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:15 smithi167 ceph-mon[192880]: osdmap e379: 8 total, 8 up, 7 in 2023-09-30T09:39:15.423 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:15 smithi167 ceph-mon[192880]: failed to encode map e379 with expected crc 2023-09-30T09:39:15.425 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:15.470 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:15.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:15.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:15.538 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 70 16 76902 76886 68.6167 0.75 0.0389595 0.0143667 2023-09-30T09:39:15.575 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:15.626 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:15.893 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:15.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:15.997 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:16.019 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:39:16.019 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:39:16.019 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:39:16.019 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:39:16.019 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:39:16.033 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:16.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[224043]: failed to encode map e379 with expected crc 2023-09-30T09:39:16.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[224043]: failed to encode map e379 with expected crc 2023-09-30T09:39:16.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[224043]: osdmap e380: 8 total, 8 up, 7 in 2023-09-30T09:39:16.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[224043]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[224043]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.297 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[224043]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[216815]: failed to encode map e379 with expected crc 2023-09-30T09:39:16.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[216815]: failed to encode map e379 with expected crc 2023-09-30T09:39:16.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[216815]: osdmap e380: 8 total, 8 up, 7 in 2023-09-30T09:39:16.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[216815]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[216815]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:16 smithi019 ceph-mon[216815]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:16 smithi167 ceph-mon[192880]: failed to encode map e379 with expected crc 2023-09-30T09:39:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:16 smithi167 ceph-mon[192880]: failed to encode map e379 with expected crc 2023-09-30T09:39:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:16 smithi167 ceph-mon[192880]: osdmap e380: 8 total, 8 up, 7 in 2023-09-30T09:39:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:16 smithi167 ceph-mon[192880]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:16 smithi167 ceph-mon[192880]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:16 smithi167 ceph-mon[192880]: failed to encode map e380 with expected crc 2023-09-30T09:39:16.538 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 71 16 76993 76977 67.7307 5.6875 0.00309216 0.0145325 2023-09-30T09:39:16.562 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:39:16.701 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:16.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:16.809 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:16.856 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:17.171 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:17.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: pgmap v989: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 23817/228159 objects degraded (10.439%); 744/228159 objects misplaced (0.326%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:39:17.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/344567949' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:17.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:17.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:17.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:17.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: failed to encode map e381 with expected crc 2023-09-30T09:39:17.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[216815]: osdmap e381: 8 total, 8 up, 7 in 2023-09-30T09:39:17.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: pgmap v989: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 23817/228159 objects degraded (10.439%); 744/228159 objects misplaced (0.326%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/344567949' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:17.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: failed to encode map e381 with expected crc 2023-09-30T09:39:17.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:17 smithi019 ceph-mon[224043]: osdmap e381: 8 total, 8 up, 7 in 2023-09-30T09:39:17.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:17.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:17.281 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:17.311 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:17.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: failed to encode map e380 with expected crc 2023-09-30T09:39:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: pgmap v989: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 23817/228159 objects degraded (10.439%); 744/228159 objects misplaced (0.326%); 3.3 MiB/s, 52 objects/s recovering 2023-09-30T09:39:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/344567949' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: failed to encode map e381 with expected crc 2023-09-30T09:39:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:17 smithi167 ceph-mon[192880]: osdmap e381: 8 total, 8 up, 7 in 2023-09-30T09:39:17.537 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 72 16 77372 77356 67.1192 23.6875 0.00252297 0.0146837 2023-09-30T09:39:18.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[216815]: failed to encode map e381 with expected crc 2023-09-30T09:39:18.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[216815]: failed to encode map e381 with expected crc 2023-09-30T09:39:18.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[216815]: osdmap e382: 8 total, 8 up, 7 in 2023-09-30T09:39:18.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[216815]: failed to encode map e382 with expected crc 2023-09-30T09:39:18.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:18.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:18.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[224043]: failed to encode map e381 with expected crc 2023-09-30T09:39:18.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[224043]: failed to encode map e381 with expected crc 2023-09-30T09:39:18.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[224043]: osdmap e382: 8 total, 8 up, 7 in 2023-09-30T09:39:18.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[224043]: failed to encode map e382 with expected crc 2023-09-30T09:39:18.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:18.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:18.317 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:39:18.323 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:18.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:18.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:18 smithi167 ceph-mon[192880]: failed to encode map e381 with expected crc 2023-09-30T09:39:18.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:18 smithi167 ceph-mon[192880]: failed to encode map e381 with expected crc 2023-09-30T09:39:18.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:18 smithi167 ceph-mon[192880]: osdmap e382: 8 total, 8 up, 7 in 2023-09-30T09:39:18.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:18 smithi167 ceph-mon[192880]: failed to encode map e382 with expected crc 2023-09-30T09:39:18.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:18.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:18.538 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 73 16 78255 78239 66.9557 55.1875 0.0192876 0.0149304 2023-09-30T09:39:18.586 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:18.659 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:18.754 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:18.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:18.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:18.866 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:18.868 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:39:18.906 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:39:18.931 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:18.943 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:39:18.974 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:39:18.994 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:19.014 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:39:19.018 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 6, 0, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:39:19.018 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:39:19.018 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-09-30T09:39:19.019 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:39:19.055 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:39:19.064 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:19.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:19.105 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:39:19.140 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:39:19.442 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:19.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:19 smithi167 ceph-mon[192880]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:19 smithi167 ceph-mon[192880]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:19 smithi167 ceph-mon[192880]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:19 smithi167 ceph-mon[192880]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:19 smithi167 ceph-mon[192880]: pgmap v992: 129 pgs: 9 unknown, 11 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 914 op/s; 24044/230385 objects degraded (10.436%); 4.4 MiB/s, 70 objects/s recovering 2023-09-30T09:39:19.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 23817/228159 objects degraded (10.439%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:19.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3018434936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:19.444 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:39:19.444 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:39:19.444 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:39:19.444 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:39:19.444 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:39:19.444 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:39:19.445 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:39:19.445 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:39:19.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[216815]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[216815]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[216815]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[216815]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[216815]: pgmap v992: 129 pgs: 9 unknown, 11 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 914 op/s; 24044/230385 objects degraded (10.436%); 4.4 MiB/s, 70 objects/s recovering 2023-09-30T09:39:19.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 23817/228159 objects degraded (10.439%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:19.446 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3018434936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[224043]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[224043]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[224043]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[224043]: failed to encode map e382 with expected crc 2023-09-30T09:39:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[224043]: pgmap v992: 129 pgs: 9 unknown, 11 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 914 op/s; 24044/230385 objects degraded (10.436%); 4.4 MiB/s, 70 objects/s recovering 2023-09-30T09:39:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 23817/228159 objects degraded (10.439%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3018434936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:19.513 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:19.538 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 74 16 79249 79233 66.8904 62.125 0.0124482 0.0149438 2023-09-30T09:39:19.605 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:19.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:19.711 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:19.785 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:19.883 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:19.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:19.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:19.991 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:20.065 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:20.126 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:20.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:20.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:20.231 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:20.264 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:20.487 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:39:20.541 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 75 16 80115 80099 66.7171 54.125 0.0224343 0.0149831 2023-09-30T09:39:21.290 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:21.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:21.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:21 smithi019 ceph-mon[216815]: pgmap v993: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 unknown, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 18882/234633 objects degraded (8.047%); 4414/234633 objects misplaced (1.881%); 13 MiB/s, 203 objects/s recovering 2023-09-30T09:39:21.300 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:21 smithi019 ceph-mon[224043]: pgmap v993: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 unknown, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 18882/234633 objects degraded (8.047%); 4414/234633 objects misplaced (1.881%); 13 MiB/s, 203 objects/s recovering 2023-09-30T09:39:21.398 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:21.475 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:21 smithi167 ceph-mon[192880]: pgmap v993: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 unknown, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 18882/234633 objects degraded (8.047%); 4414/234633 objects misplaced (1.881%); 13 MiB/s, 203 objects/s recovering 2023-09-30T09:39:21.542 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 76 16 81071 81055 66.6254 59.75 0.019752 0.0150027 2023-09-30T09:39:21.552 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:21.639 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:21.645 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:21.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:21.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:21.750 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:21.805 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:21.944 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:21.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:21.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:22.060 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:39:22.060 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:39:22.063 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:22.140 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:22.257 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:22.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:22.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:22.375 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:22.443 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:22.542 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 77 16 82072 82056 66.5724 62.5625 0.0138466 0.0150163 2023-09-30T09:39:22.566 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:22.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:22.681 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:22.712 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:23.082 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:23 smithi019 ceph-mon[216815]: pgmap v994: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 19115/238350 objects degraded (8.020%); 4414/238350 objects misplaced (1.852%); 10 MiB/s, 170 objects/s recovering 2023-09-30T09:39:23.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:23 smithi019 ceph-mon[224043]: pgmap v994: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 19115/238350 objects degraded (8.020%); 4414/238350 objects misplaced (1.852%); 10 MiB/s, 170 objects/s recovering 2023-09-30T09:39:23.151 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:39:23.312 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:23.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:23.375 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:39:23.418 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:23.421 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:23 smithi167 ceph-mon[192880]: pgmap v994: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 19115/238350 objects degraded (8.020%); 4414/238350 objects misplaced (1.852%); 10 MiB/s, 170 objects/s recovering 2023-09-30T09:39:23.496 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:23.542 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 78 16 82860 82844 66.3504 49.25 0.0225817 0.0150653 2023-09-30T09:39:23.659 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:23.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:23.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:23.770 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:23.799 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:23.848 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:23.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:23.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:23.915 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:39:23.953 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:23.983 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:39:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:39:23] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.43.0" 2023-09-30T09:39:23.987 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:24.083 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 19115/238350 objects degraded (8.020%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:24.083 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:24 smithi167 ceph-mon[192880]: pgmap v995: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 27 MiB/s wr, 424 op/s; 19115/238350 objects degraded (8.020%); 4414/238350 objects misplaced (1.852%); 7.9 MiB/s, 128 objects/s recovering 2023-09-30T09:39:24.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 19115/238350 objects degraded (8.020%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:24.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:24 smithi019 ceph-mon[216815]: pgmap v995: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 27 MiB/s wr, 424 op/s; 19115/238350 objects degraded (8.020%); 4414/238350 objects misplaced (1.852%); 7.9 MiB/s, 128 objects/s recovering 2023-09-30T09:39:24.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 19115/238350 objects degraded (8.020%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:24.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:24 smithi019 ceph-mon[224043]: pgmap v995: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 27 MiB/s wr, 424 op/s; 19115/238350 objects degraded (8.020%); 4414/238350 objects misplaced (1.852%); 7.9 MiB/s, 128 objects/s recovering 2023-09-30T09:39:24.542 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 79 16 83575 83559 66.0762 44.6875 0.0103445 0.01512 2023-09-30T09:39:24.784 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:24.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:24.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:24.890 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:25.002 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:25.113 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:25.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:25.221 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:25.285 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:25.358 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:25.383 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:25.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:25.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:25.503 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:25.545 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:39:25.545208+0000 min lat: 0.0012782 max lat: 3.13734 avg lat: 0.0151701 2023-09-30T09:39:25.545 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:39:25.545 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 80 16 84398 84382 65.8909 51.4375 0.0133071 0.0151701 2023-09-30T09:39:25.563 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:25.711 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:25.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:25.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:25.821 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:25.864 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:25.994 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:25.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:26.100 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:26.208 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:26.225 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:39:26.241 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:39:26.355 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:26.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:26.473 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:26.542 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:26.550 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 81 16 85339 85323 65.799 58.8125 0.0269205 0.015193 2023-09-30T09:39:26.686 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:39:27.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:26 smithi167 ceph-mon[192880]: pgmap v996: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 19115/248046 objects degraded (7.706%); 3761/248046 objects misplaced (1.516%); 9.6 MiB/s, 154 objects/s recovering 2023-09-30T09:39:27.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3275099531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:39:27.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:26 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:39:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:26 smithi019 ceph-mon[224043]: pgmap v996: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 19115/248046 objects degraded (7.706%); 3761/248046 objects misplaced (1.516%); 9.6 MiB/s, 154 objects/s recovering 2023-09-30T09:39:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3275099531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:39:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:26 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:39:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:26 smithi019 ceph-mon[216815]: pgmap v996: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 19115/248046 objects degraded (7.706%); 3761/248046 objects misplaced (1.516%); 9.6 MiB/s, 154 objects/s recovering 2023-09-30T09:39:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3275099531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:39:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:26 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:39:27.550 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 82 16 86197 86181 65.6506 53.625 0.00752375 0.0152245 2023-09-30T09:39:27.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[216815]: from='client.35889 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:39:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:39:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[216815]: osdmap e383: 8 total, 8 up, 7 in 2023-09-30T09:39:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[216815]: failed to encode map e383 with expected crc 2023-09-30T09:39:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[216815]: failed to encode map e383 with expected crc 2023-09-30T09:39:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[216815]: failed to encode map e383 with expected crc 2023-09-30T09:39:27.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[216815]: failed to encode map e383 with expected crc 2023-09-30T09:39:27.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[224043]: from='client.35889 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:39:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:39:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[224043]: osdmap e383: 8 total, 8 up, 7 in 2023-09-30T09:39:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[224043]: failed to encode map e383 with expected crc 2023-09-30T09:39:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[224043]: failed to encode map e383 with expected crc 2023-09-30T09:39:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[224043]: failed to encode map e383 with expected crc 2023-09-30T09:39:27.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:27 smithi019 ceph-mon[224043]: failed to encode map e383 with expected crc 2023-09-30T09:39:28.015 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":996,"stamp":"2023-09-30T09:39:25.607144+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5421404141,"num_objects":82682,"num_object_clones":0,"num_object_copies":248046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19115,"num_objects_misplaced":3761,"num_objects_unfound":0,"num_objects_dirty":82682,"num_whiteouts":0,"num_read":5950,"num_read_kb":12160,"num_write":83566,"num_write_kb":5302682,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4826,"num_bytes_recovered":309724510,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136404,"ondisk_log_size":136404,"up":387,"acting":380,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":396,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17810944,"kb_used_data":14992136,"kb_used_omap":21,"kb_used_meta":2818666,"kb_avail":638405120,"statfs":{"total":671965249536,"available":653726842880,"internally_reserved":0,"allocated":15351947264,"data_stored":15340218929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21604,"internal_metadata":2886314908},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":385810432,"num_objects":5887,"num_object_clones":0,"num_object_copies":17661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-4929,"num_objects_misplaced":3761,"num_objects_unfound":0,"num_objects_dirty":5887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5887,"num_write_kb":376768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1401,"num_bytes_recovered":90898432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.051516"},"pg_stats":[{"pgid":"8.11","version":"382'4850","reported_seq":4895,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:20.561300+0000","last_change":"2023-09-30T09:39:15.602701+0000","last_active":"2023-09-30T09:39:20.561300+0000","last_peered":"2023-09-30T09:39:20.561300+0000","last_clean":"2023-09-30T09:39:20.561300+0000","last_became_active":"2023-09-30T09:39:15.594688+0000","last_became_peered":"2023-09-30T09:39:15.594688+0000","last_unstale":"2023-09-30T09:39:20.561300+0000","last_undegraded":"2023-09-30T09:39:20.561300+0000","last_fullsized":"2023-09-30T09:39:20.561300+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162627584,"num_objects":2482,"num_object_clones":0,"num_object_copies":7446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2482,"num_write_kb":158816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":870,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.567592+0000","last_change":"2023-09-30T09:38:30.746866+0000","last_active":"2023-09-30T09:39:16.567592+0000","last_peered":"2023-09-30T09:39:16.567592+0000","last_clean":"2023-09-30T09:39:16.567592+0000","last_became_active":"2023-09-30T09:38:30.746361+0000","last_became_peered":"2023-09-30T09:38:30.746361+0000","last_unstale":"2023-09-30T09:39:16.567592+0000","last_undegraded":"2023-09-30T09:39:16.567592+0000","last_fullsized":"2023-09-30T09:39:16.567592+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:06:25.130460+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":400,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.566127+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:39:16.566127+0000","last_peered":"2023-09-30T09:39:16.566127+0000","last_clean":"2023-09-30T09:39:16.566127+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:39:16.566127+0000","last_undegraded":"2023-09-30T09:39:16.566127+0000","last_fullsized":"2023-09-30T09:39:16.566127+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":295,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.568446+0000","last_change":"2023-09-30T09:38:45.612048+0000","last_active":"2023-09-30T09:39:16.568446+0000","last_peered":"2023-09-30T09:39:16.568446+0000","last_clean":"2023-09-30T09:39:16.568446+0000","last_became_active":"2023-09-30T09:38:45.611678+0000","last_became_peered":"2023-09-30T09:38:45.611678+0000","last_unstale":"2023-09-30T09:39:16.568446+0000","last_undegraded":"2023-09-30T09:39:16.568446+0000","last_fullsized":"2023-09-30T09:39:16.568446+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:33.553781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.12","version":"382'4846","reported_seq":4886,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:24.105909+0000","last_change":"2023-09-30T09:39:15.631668+0000","last_active":"2023-09-30T09:39:24.105909+0000","last_peered":"2023-09-30T09:39:24.105909+0000","last_clean":"2023-09-30T09:39:24.105909+0000","last_became_active":"2023-09-30T09:39:15.629300+0000","last_became_peered":"2023-09-30T09:39:15.629300+0000","last_unstale":"2023-09-30T09:39:24.105909+0000","last_undegraded":"2023-09-30T09:39:24.105909+0000","last_fullsized":"2023-09-30T09:39:24.105909+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4846,"log_dups_size":0,"ondisk_log_size":4846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164593664,"num_objects":2512,"num_object_clones":0,"num_object_copies":7536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2512,"num_write_kb":160736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":826,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559653+0000","last_change":"2023-09-30T09:32:01.514155+0000","last_active":"2023-09-30T09:39:16.559653+0000","last_peered":"2023-09-30T09:39:16.559653+0000","last_clean":"2023-09-30T09:39:16.559653+0000","last_became_active":"2023-09-30T09:31:55.918101+0000","last_became_peered":"2023-09-30T09:31:55.918101+0000","last_unstale":"2023-09-30T09:39:16.559653+0000","last_undegraded":"2023-09-30T09:39:16.559653+0000","last_fullsized":"2023-09-30T09:39:16.559653+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:26.060404+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":202,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565532+0000","last_change":"2023-09-30T09:38:45.617492+0000","last_active":"2023-09-30T09:39:16.565532+0000","last_peered":"2023-09-30T09:39:16.565532+0000","last_clean":"2023-09-30T09:39:16.565532+0000","last_became_active":"2023-09-30T09:38:45.616672+0000","last_became_peered":"2023-09-30T09:38:45.616672+0000","last_unstale":"2023-09-30T09:39:16.565532+0000","last_undegraded":"2023-09-30T09:39:16.565532+0000","last_fullsized":"2023-09-30T09:39:16.565532+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:23:49.964602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":304,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.626412+0000","last_change":"2023-09-30T09:38:45.602891+0000","last_active":"2023-09-30T09:39:16.626412+0000","last_peered":"2023-09-30T09:39:16.626412+0000","last_clean":"2023-09-30T09:39:16.626412+0000","last_became_active":"2023-09-30T09:38:45.602518+0000","last_became_peered":"2023-09-30T09:38:45.602518+0000","last_unstale":"2023-09-30T09:39:16.626412+0000","last_undegraded":"2023-09-30T09:39:16.626412+0000","last_fullsized":"2023-09-30T09:39:16.626412+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:42.613824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"382'4944","reported_seq":4986,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:24.072290+0000","last_change":"2023-09-30T09:39:15.633126+0000","last_active":"2023-09-30T09:39:24.072290+0000","last_peered":"2023-09-30T09:39:24.072290+0000","last_clean":"2023-09-30T09:39:24.072290+0000","last_became_active":"2023-09-30T09:39:15.628418+0000","last_became_peered":"2023-09-30T09:39:15.628418+0000","last_unstale":"2023-09-30T09:39:24.072290+0000","last_undegraded":"2023-09-30T09:39:24.072290+0000","last_fullsized":"2023-09-30T09:39:24.072290+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4944,"log_dups_size":0,"ondisk_log_size":4944,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167870464,"num_objects":2562,"num_object_clones":0,"num_object_copies":7686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2562,"num_write_kb":163936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":615,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.566293+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:39:16.566293+0000","last_peered":"2023-09-30T09:39:16.566293+0000","last_clean":"2023-09-30T09:39:16.566293+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:39:16.566293+0000","last_undegraded":"2023-09-30T09:39:16.566293+0000","last_fullsized":"2023-09-30T09:39:16.566293+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":387,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.568238+0000","last_change":"2023-09-30T09:38:45.593976+0000","last_active":"2023-09-30T09:39:16.568238+0000","last_peered":"2023-09-30T09:39:16.568238+0000","last_clean":"2023-09-30T09:39:16.568238+0000","last_became_active":"2023-09-30T09:38:45.593643+0000","last_became_peered":"2023-09-30T09:38:45.593643+0000","last_unstale":"2023-09-30T09:39:16.568238+0000","last_undegraded":"2023-09-30T09:39:16.568238+0000","last_fullsized":"2023-09-30T09:39:16.568238+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:27.551484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":358,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.575633+0000","last_change":"2023-09-30T09:33:32.632150+0000","last_active":"2023-09-30T09:39:16.575633+0000","last_peered":"2023-09-30T09:39:16.575633+0000","last_clean":"2023-09-30T09:39:16.575633+0000","last_became_active":"2023-09-30T09:33:32.631853+0000","last_became_peered":"2023-09-30T09:33:32.631853+0000","last_unstale":"2023-09-30T09:39:16.575633+0000","last_undegraded":"2023-09-30T09:39:16.575633+0000","last_fullsized":"2023-09-30T09:39:16.575633+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:54:30.446910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":756,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565528+0000","last_change":"2023-09-30T09:38:30.744203+0000","last_active":"2023-09-30T09:39:16.565528+0000","last_peered":"2023-09-30T09:39:16.565528+0000","last_clean":"2023-09-30T09:39:16.565528+0000","last_became_active":"2023-09-30T09:38:30.740607+0000","last_became_peered":"2023-09-30T09:38:30.740607+0000","last_unstale":"2023-09-30T09:39:16.565528+0000","last_undegraded":"2023-09-30T09:39:16.565528+0000","last_fullsized":"2023-09-30T09:39:16.565528+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.164029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":286,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.626080+0000","last_change":"2023-09-30T09:38:45.617262+0000","last_active":"2023-09-30T09:39:16.626080+0000","last_peered":"2023-09-30T09:39:16.626080+0000","last_clean":"2023-09-30T09:39:16.626080+0000","last_became_active":"2023-09-30T09:38:45.616900+0000","last_became_peered":"2023-09-30T09:38:45.616900+0000","last_unstale":"2023-09-30T09:39:16.626080+0000","last_undegraded":"2023-09-30T09:39:16.626080+0000","last_fullsized":"2023-09-30T09:39:16.626080+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:17:19.381402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":367,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.567339+0000","last_change":"2023-09-30T09:38:45.608517+0000","last_active":"2023-09-30T09:39:16.567339+0000","last_peered":"2023-09-30T09:39:16.567339+0000","last_clean":"2023-09-30T09:39:16.567339+0000","last_became_active":"2023-09-30T09:38:45.608231+0000","last_became_peered":"2023-09-30T09:38:45.608231+0000","last_unstale":"2023-09-30T09:39:16.567339+0000","last_undegraded":"2023-09-30T09:39:16.567339+0000","last_fullsized":"2023-09-30T09:39:16.567339+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:08:14.623881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":568,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.621116+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:39:16.621116+0000","last_peered":"2023-09-30T09:39:16.621116+0000","last_clean":"2023-09-30T09:39:16.621116+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:39:16.621116+0000","last_undegraded":"2023-09-30T09:39:16.621116+0000","last_fullsized":"2023-09-30T09:39:16.621116+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":270,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559853+0000","last_change":"2023-09-30T09:38:27.687229+0000","last_active":"2023-09-30T09:39:16.559853+0000","last_peered":"2023-09-30T09:39:16.559853+0000","last_clean":"2023-09-30T09:39:16.559853+0000","last_became_active":"2023-09-30T09:38:27.686912+0000","last_became_peered":"2023-09-30T09:38:27.686912+0000","last_unstale":"2023-09-30T09:39:16.559853+0000","last_undegraded":"2023-09-30T09:39:16.559853+0000","last_fullsized":"2023-09-30T09:39:16.559853+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:05:28.924932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":316,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564529+0000","last_change":"2023-09-30T09:38:45.622002+0000","last_active":"2023-09-30T09:39:16.564529+0000","last_peered":"2023-09-30T09:39:16.564529+0000","last_clean":"2023-09-30T09:39:16.564529+0000","last_became_active":"2023-09-30T09:38:45.621679+0000","last_became_peered":"2023-09-30T09:38:45.621679+0000","last_unstale":"2023-09-30T09:39:16.564529+0000","last_undegraded":"2023-09-30T09:39:16.564529+0000","last_fullsized":"2023-09-30T09:39:16.564529+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:01:00.354953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":548,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.560133+0000","last_change":"2023-09-30T09:31:59.523565+0000","last_active":"2023-09-30T09:39:16.560133+0000","last_peered":"2023-09-30T09:39:16.560133+0000","last_clean":"2023-09-30T09:39:16.560133+0000","last_became_active":"2023-09-30T09:31:55.923681+0000","last_became_peered":"2023-09-30T09:31:55.923681+0000","last_unstale":"2023-09-30T09:39:16.560133+0000","last_undegraded":"2023-09-30T09:39:16.560133+0000","last_fullsized":"2023-09-30T09:39:16.560133+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:38:26.434723+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":48,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565821+0000","last_change":"2023-09-30T09:38:45.666418+0000","last_active":"2023-09-30T09:39:16.565821+0000","last_peered":"2023-09-30T09:39:16.565821+0000","last_clean":"2023-09-30T09:39:16.565821+0000","last_became_active":"2023-09-30T09:38:45.612315+0000","last_became_peered":"2023-09-30T09:38:45.612315+0000","last_unstale":"2023-09-30T09:39:16.565821+0000","last_undegraded":"2023-09-30T09:39:16.565821+0000","last_fullsized":"2023-09-30T09:39:16.565821+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:21:16.379322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":295,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569235+0000","last_change":"2023-09-30T09:33:33.606515+0000","last_active":"2023-09-30T09:39:16.569235+0000","last_peered":"2023-09-30T09:39:16.569235+0000","last_clean":"2023-09-30T09:39:16.569235+0000","last_became_active":"2023-09-30T09:33:33.604490+0000","last_became_peered":"2023-09-30T09:33:33.604490+0000","last_unstale":"2023-09-30T09:39:16.569235+0000","last_undegraded":"2023-09-30T09:39:16.569235+0000","last_fullsized":"2023-09-30T09:39:16.569235+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:28:23.167304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":565,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.626672+0000","last_change":"2023-09-30T09:38:48.653405+0000","last_active":"2023-09-30T09:39:16.626672+0000","last_peered":"2023-09-30T09:39:16.626672+0000","last_clean":"2023-09-30T09:39:16.626672+0000","last_became_active":"2023-09-30T09:38:48.653094+0000","last_became_peered":"2023-09-30T09:38:48.653094+0000","last_unstale":"2023-09-30T09:39:16.626672+0000","last_undegraded":"2023-09-30T09:39:16.626672+0000","last_fullsized":"2023-09-30T09:39:16.626672+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:27:37.328591+0000","scrub_duration":0.0114254,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":386,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565885+0000","last_change":"2023-09-30T09:38:27.691178+0000","last_active":"2023-09-30T09:39:16.565885+0000","last_peered":"2023-09-30T09:39:16.565885+0000","last_clean":"2023-09-30T09:39:16.565885+0000","last_became_active":"2023-09-30T09:38:27.680565+0000","last_became_peered":"2023-09-30T09:38:27.680565+0000","last_unstale":"2023-09-30T09:39:16.565885+0000","last_undegraded":"2023-09-30T09:39:16.565885+0000","last_fullsized":"2023-09-30T09:39:16.565885+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:33:45.600275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9444,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":377,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.572794+0000","last_change":"2023-09-30T09:33:32.749181+0000","last_active":"2023-09-30T09:39:16.572794+0000","last_peered":"2023-09-30T09:39:16.572794+0000","last_clean":"2023-09-30T09:39:16.572794+0000","last_became_active":"2023-09-30T09:33:32.748835+0000","last_became_peered":"2023-09-30T09:33:32.748835+0000","last_unstale":"2023-09-30T09:39:16.572794+0000","last_undegraded":"2023-09-30T09:39:16.572794+0000","last_fullsized":"2023-09-30T09:39:16.572794+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:54.055507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"382'5068","reported_seq":6884,"reported_epoch":382,"state":"active+recovering","last_fresh":"2023-09-30T09:39:24.364072+0000","last_change":"2023-09-30T09:39:15.742608+0000","last_active":"2023-09-30T09:39:24.364072+0000","last_peered":"2023-09-30T09:39:24.364072+0000","last_clean":"2023-09-30T09:38:26.554137+0000","last_became_active":"2023-09-30T09:39:15.682660+0000","last_became_peered":"2023-09-30T09:39:15.682660+0000","last_unstale":"2023-09-30T09:39:24.364072+0000","last_undegraded":"2023-09-30T09:39:24.364072+0000","last_fullsized":"2023-09-30T09:39:24.364072+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5068,"log_dups_size":0,"ondisk_log_size":5068,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172916736,"num_objects":2639,"num_object_clones":0,"num_object_copies":7917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":854,"num_objects_unfound":0,"num_objects_dirty":2639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2639,"num_write_kb":168864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":57671680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":["5","0","3"],"object_location_counts":[{"shards":"0,3,5","objects":854},{"shards":"0,5,7","objects":1785}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":682,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.622569+0000","last_change":"2023-09-30T09:38:48.640319+0000","last_active":"2023-09-30T09:39:16.622569+0000","last_peered":"2023-09-30T09:39:16.622569+0000","last_clean":"2023-09-30T09:39:16.622569+0000","last_became_active":"2023-09-30T09:38:48.639990+0000","last_became_peered":"2023-09-30T09:38:48.639990+0000","last_unstale":"2023-09-30T09:39:16.622569+0000","last_undegraded":"2023-09-30T09:39:16.622569+0000","last_fullsized":"2023-09-30T09:39:16.622569+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:56.889537+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":32,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564505+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:39:16.564505+0000","last_peered":"2023-09-30T09:39:16.564505+0000","last_clean":"2023-09-30T09:39:16.564505+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:39:16.564505+0000","last_undegraded":"2023-09-30T09:39:16.564505+0000","last_fullsized":"2023-09-30T09:39:16.564505+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":407,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.625636+0000","last_change":"2023-09-30T09:38:45.606327+0000","last_active":"2023-09-30T09:39:16.625636+0000","last_peered":"2023-09-30T09:39:16.625636+0000","last_clean":"2023-09-30T09:39:16.625636+0000","last_became_active":"2023-09-30T09:38:45.605910+0000","last_became_peered":"2023-09-30T09:38:45.605910+0000","last_unstale":"2023-09-30T09:39:16.625636+0000","last_undegraded":"2023-09-30T09:39:16.625636+0000","last_fullsized":"2023-09-30T09:39:16.625636+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:36:50.105906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.19","version":"382'4989","reported_seq":5029,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:24.030000+0000","last_change":"2023-09-30T09:39:15.631047+0000","last_active":"2023-09-30T09:39:24.030000+0000","last_peered":"2023-09-30T09:39:24.030000+0000","last_clean":"2023-09-30T09:39:24.030000+0000","last_became_active":"2023-09-30T09:39:15.628359+0000","last_became_peered":"2023-09-30T09:39:15.628359+0000","last_unstale":"2023-09-30T09:39:24.030000+0000","last_undegraded":"2023-09-30T09:39:24.030000+0000","last_fullsized":"2023-09-30T09:39:24.030000+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4989,"log_dups_size":0,"ondisk_log_size":4989,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170426368,"num_objects":2601,"num_object_clones":0,"num_object_copies":7803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2601,"num_write_kb":166432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":748,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.590581+0000","last_change":"2023-09-30T09:38:30.746770+0000","last_active":"2023-09-30T09:39:16.590581+0000","last_peered":"2023-09-30T09:39:16.590581+0000","last_clean":"2023-09-30T09:39:16.590581+0000","last_became_active":"2023-09-30T09:38:30.745926+0000","last_became_peered":"2023-09-30T09:38:30.745926+0000","last_unstale":"2023-09-30T09:39:16.590581+0000","last_undegraded":"2023-09-30T09:39:16.590581+0000","last_fullsized":"2023-09-30T09:39:16.590581+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:25:58.952467+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":302,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.622769+0000","last_change":"2023-09-30T09:38:45.600099+0000","last_active":"2023-09-30T09:39:16.622769+0000","last_peered":"2023-09-30T09:39:16.622769+0000","last_clean":"2023-09-30T09:39:16.622769+0000","last_became_active":"2023-09-30T09:38:45.599769+0000","last_became_peered":"2023-09-30T09:38:45.599769+0000","last_unstale":"2023-09-30T09:39:16.622769+0000","last_undegraded":"2023-09-30T09:39:16.622769+0000","last_fullsized":"2023-09-30T09:39:16.622769+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:02:58.820333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.561540+0000","last_change":"2023-09-30T09:38:45.620080+0000","last_active":"2023-09-30T09:39:16.561540+0000","last_peered":"2023-09-30T09:39:16.561540+0000","last_clean":"2023-09-30T09:39:16.561540+0000","last_became_active":"2023-09-30T09:38:45.617392+0000","last_became_peered":"2023-09-30T09:38:45.617392+0000","last_unstale":"2023-09-30T09:39:16.561540+0000","last_undegraded":"2023-09-30T09:39:16.561540+0000","last_fullsized":"2023-09-30T09:39:16.561540+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:24.685400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":772,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.572316+0000","last_change":"2023-09-30T09:38:48.664123+0000","last_active":"2023-09-30T09:39:16.572316+0000","last_peered":"2023-09-30T09:39:16.572316+0000","last_clean":"2023-09-30T09:39:16.572316+0000","last_became_active":"2023-09-30T09:38:48.663799+0000","last_became_peered":"2023-09-30T09:38:48.663799+0000","last_unstale":"2023-09-30T09:39:16.572316+0000","last_undegraded":"2023-09-30T09:39:16.572316+0000","last_fullsized":"2023-09-30T09:39:16.572316+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":26,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564667+0000","last_change":"2023-09-30T09:38:27.680269+0000","last_active":"2023-09-30T09:39:16.564667+0000","last_peered":"2023-09-30T09:39:16.564667+0000","last_clean":"2023-09-30T09:39:16.564667+0000","last_became_active":"2023-09-30T09:38:27.679931+0000","last_became_peered":"2023-09-30T09:38:27.679931+0000","last_unstale":"2023-09-30T09:39:16.564667+0000","last_undegraded":"2023-09-30T09:39:16.564667+0000","last_fullsized":"2023-09-30T09:39:16.564667+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:06.847392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":302,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.626470+0000","last_change":"2023-09-30T09:38:45.622309+0000","last_active":"2023-09-30T09:39:16.626470+0000","last_peered":"2023-09-30T09:39:16.626470+0000","last_clean":"2023-09-30T09:39:16.626470+0000","last_became_active":"2023-09-30T09:38:45.622020+0000","last_became_peered":"2023-09-30T09:38:45.622020+0000","last_unstale":"2023-09-30T09:39:16.626470+0000","last_undegraded":"2023-09-30T09:39:16.626470+0000","last_fullsized":"2023-09-30T09:39:16.626470+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:42:26.066681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":847,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559606+0000","last_change":"2023-09-30T09:33:32.658135+0000","last_active":"2023-09-30T09:39:16.559606+0000","last_peered":"2023-09-30T09:39:16.559606+0000","last_clean":"2023-09-30T09:39:16.559606+0000","last_became_active":"2023-09-30T09:33:32.657918+0000","last_became_peered":"2023-09-30T09:33:32.657918+0000","last_unstale":"2023-09-30T09:39:16.559606+0000","last_undegraded":"2023-09-30T09:39:16.559606+0000","last_fullsized":"2023-09-30T09:39:16.559606+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:29:48.120096+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":541,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.568513+0000","last_change":"2023-09-30T09:35:10.771912+0000","last_active":"2023-09-30T09:39:16.568513+0000","last_peered":"2023-09-30T09:39:16.568513+0000","last_clean":"2023-09-30T09:39:16.568513+0000","last_became_active":"2023-09-30T09:35:10.651432+0000","last_became_peered":"2023-09-30T09:35:10.651432+0000","last_unstale":"2023-09-30T09:39:16.568513+0000","last_undegraded":"2023-09-30T09:39:16.568513+0000","last_fullsized":"2023-09-30T09:39:16.568513+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:49.111689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":398,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565299+0000","last_change":"2023-09-30T09:38:45.606456+0000","last_active":"2023-09-30T09:39:16.565299+0000","last_peered":"2023-09-30T09:39:16.565299+0000","last_clean":"2023-09-30T09:39:16.565299+0000","last_became_active":"2023-09-30T09:38:45.604792+0000","last_became_peered":"2023-09-30T09:38:45.604792+0000","last_unstale":"2023-09-30T09:39:16.565299+0000","last_undegraded":"2023-09-30T09:39:16.565299+0000","last_fullsized":"2023-09-30T09:39:16.565299+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:11.533941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"382'5032","reported_seq":5075,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:24.336878+0000","last_change":"2023-09-30T09:39:15.603264+0000","last_active":"2023-09-30T09:39:24.336878+0000","last_peered":"2023-09-30T09:39:24.336878+0000","last_clean":"2023-09-30T09:39:24.336878+0000","last_became_active":"2023-09-30T09:39:15.600826+0000","last_became_peered":"2023-09-30T09:39:15.600826+0000","last_unstale":"2023-09-30T09:39:24.336878+0000","last_undegraded":"2023-09-30T09:39:24.336878+0000","last_fullsized":"2023-09-30T09:39:24.336878+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172097536,"num_objects":2626,"num_object_clones":0,"num_object_copies":7878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2626,"num_write_kb":168064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":748,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565671+0000","last_change":"2023-09-30T09:38:30.737676+0000","last_active":"2023-09-30T09:39:16.565671+0000","last_peered":"2023-09-30T09:39:16.565671+0000","last_clean":"2023-09-30T09:39:16.565671+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:39:16.565671+0000","last_undegraded":"2023-09-30T09:39:16.565671+0000","last_fullsized":"2023-09-30T09:39:16.565671+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.740851+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.740851+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":30,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565487+0000","last_change":"2023-09-30T09:38:27.700442+0000","last_active":"2023-09-30T09:39:16.565487+0000","last_peered":"2023-09-30T09:39:16.565487+0000","last_clean":"2023-09-30T09:39:16.565487+0000","last_became_active":"2023-09-30T09:38:27.678601+0000","last_became_peered":"2023-09-30T09:38:27.678601+0000","last_unstale":"2023-09-30T09:39:16.565487+0000","last_undegraded":"2023-09-30T09:39:16.565487+0000","last_fullsized":"2023-09-30T09:39:16.565487+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:59:10.222887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.567495+0000","last_change":"2023-09-30T09:38:27.687124+0000","last_active":"2023-09-30T09:39:16.567495+0000","last_peered":"2023-09-30T09:39:16.567495+0000","last_clean":"2023-09-30T09:39:16.567495+0000","last_became_active":"2023-09-30T09:38:27.686794+0000","last_became_peered":"2023-09-30T09:38:27.686794+0000","last_unstale":"2023-09-30T09:39:16.567495+0000","last_undegraded":"2023-09-30T09:39:16.567495+0000","last_fullsized":"2023-09-30T09:39:16.567495+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:33.583208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"382'4989","reported_seq":5397,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:39:24.133068+0000","last_change":"2023-09-30T09:39:16.580810+0000","last_active":"2023-09-30T09:39:24.133068+0000","last_peered":"2023-09-30T09:39:24.133068+0000","last_clean":"2023-09-30T09:38:26.563035+0000","last_became_active":"2023-09-30T09:39:16.577321+0000","last_became_peered":"2023-09-30T09:39:16.577321+0000","last_unstale":"2023-09-30T09:39:24.133068+0000","last_undegraded":"2023-09-30T09:39:16.572808+0000","last_fullsized":"2023-09-30T09:39:24.133068+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4989,"log_dups_size":0,"ondisk_log_size":4989,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169541632,"num_objects":2587,"num_object_clones":0,"num_object_copies":7761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1063,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2587,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":5767168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":1063},{"shards":"4,6,7","objects":1524}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":718,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:39:16.626279+0000","last_change":"2023-09-30T09:38:46.666950+0000","last_active":"2023-09-30T09:39:16.626279+0000","last_peered":"2023-09-30T09:39:16.626279+0000","last_clean":"2023-09-30T09:31:49.462297+0000","last_became_active":"2023-09-30T09:38:46.666594+0000","last_became_peered":"2023-09-30T09:38:46.666594+0000","last_unstale":"2023-09-30T09:39:16.626279+0000","last_undegraded":"2023-09-30T09:38:46.591792+0000","last_fullsized":"2023-09-30T09:38:46.591274+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":30,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.566712+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:39:16.566712+0000","last_peered":"2023-09-30T09:39:16.566712+0000","last_clean":"2023-09-30T09:39:16.566712+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:39:16.566712+0000","last_undegraded":"2023-09-30T09:39:16.566712+0000","last_fullsized":"2023-09-30T09:39:16.566712+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":394,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565772+0000","last_change":"2023-09-30T09:38:45.617752+0000","last_active":"2023-09-30T09:39:16.565772+0000","last_peered":"2023-09-30T09:39:16.565772+0000","last_clean":"2023-09-30T09:39:16.565772+0000","last_became_active":"2023-09-30T09:38:45.617146+0000","last_became_peered":"2023-09-30T09:38:45.617146+0000","last_unstale":"2023-09-30T09:39:16.565772+0000","last_undegraded":"2023-09-30T09:39:16.565772+0000","last_fullsized":"2023-09-30T09:39:16.565772+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:41:29.189921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"382'4977","reported_seq":8722,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:39:20.473896+0000","last_change":"2023-09-30T09:39:15.569999+0000","last_active":"2023-09-30T09:39:20.473896+0000","last_peered":"2023-09-30T09:39:20.473896+0000","last_clean":"2023-09-30T09:38:26.430308+0000","last_became_active":"2023-09-30T09:39:15.567420+0000","last_became_peered":"2023-09-30T09:39:15.567420+0000","last_unstale":"2023-09-30T09:39:20.473896+0000","last_undegraded":"2023-09-30T09:39:15.554887+0000","last_fullsized":"2023-09-30T09:39:15.554454+0000","mapping_epoch":366,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4977,"log_dups_size":0,"ondisk_log_size":4977,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166199296,"num_objects":2536,"num_object_clones":0,"num_object_copies":7608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":227,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2536,"num_write_kb":162304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":924,"num_bytes_recovered":60555264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":227},{"shards":"0,6","objects":2309}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":665,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.575553+0000","last_change":"2023-09-30T09:32:05.450100+0000","last_active":"2023-09-30T09:39:16.575553+0000","last_peered":"2023-09-30T09:39:16.575553+0000","last_clean":"2023-09-30T09:39:16.575553+0000","last_became_active":"2023-09-30T09:31:55.929831+0000","last_became_peered":"2023-09-30T09:31:55.929831+0000","last_unstale":"2023-09-30T09:39:16.575553+0000","last_undegraded":"2023-09-30T09:39:16.575553+0000","last_fullsized":"2023-09-30T09:39:16.575553+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:18.345710+0000","scrub_duration":0.003470454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":30,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565201+0000","last_change":"2023-09-30T09:38:27.708392+0000","last_active":"2023-09-30T09:39:16.565201+0000","last_peered":"2023-09-30T09:39:16.565201+0000","last_clean":"2023-09-30T09:39:16.565201+0000","last_became_active":"2023-09-30T09:38:27.685766+0000","last_became_peered":"2023-09-30T09:38:27.685766+0000","last_unstale":"2023-09-30T09:39:16.565201+0000","last_undegraded":"2023-09-30T09:39:16.565201+0000","last_fullsized":"2023-09-30T09:39:16.565201+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:32.744901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":371,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569156+0000","last_change":"2023-09-30T09:38:27.678428+0000","last_active":"2023-09-30T09:39:16.569156+0000","last_peered":"2023-09-30T09:39:16.569156+0000","last_clean":"2023-09-30T09:39:16.569156+0000","last_became_active":"2023-09-30T09:38:27.678086+0000","last_became_peered":"2023-09-30T09:38:27.678086+0000","last_unstale":"2023-09-30T09:39:16.569156+0000","last_undegraded":"2023-09-30T09:39:16.569156+0000","last_fullsized":"2023-09-30T09:39:16.569156+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:05.480763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"382'5064","reported_seq":5118,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:39:24.083581+0000","last_change":"2023-09-30T09:39:16.630854+0000","last_active":"2023-09-30T09:39:24.083581+0000","last_peered":"2023-09-30T09:39:24.083581+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:39:16.624151+0000","last_became_peered":"2023-09-30T09:39:16.624151+0000","last_unstale":"2023-09-30T09:39:24.083581+0000","last_undegraded":"2023-09-30T09:39:16.574604+0000","last_fullsized":"2023-09-30T09:39:24.083581+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172261376,"num_objects":2628,"num_object_clones":0,"num_object_copies":7884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1728,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2628,"num_write_kb":168224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,2,6","objects":900},{"shards":"1,3,6","objects":1728}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":597,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.567827+0000","last_change":"2023-09-30T09:38:48.662225+0000","last_active":"2023-09-30T09:39:16.567827+0000","last_peered":"2023-09-30T09:39:16.567827+0000","last_clean":"2023-09-30T09:39:16.567827+0000","last_became_active":"2023-09-30T09:38:48.661883+0000","last_became_peered":"2023-09-30T09:38:48.661883+0000","last_unstale":"2023-09-30T09:39:16.567827+0000","last_undegraded":"2023-09-30T09:39:16.567827+0000","last_fullsized":"2023-09-30T09:39:16.567827+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:01.892446+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":351,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.560019+0000","last_change":"2023-09-30T09:38:45.614020+0000","last_active":"2023-09-30T09:39:16.560019+0000","last_peered":"2023-09-30T09:39:16.560019+0000","last_clean":"2023-09-30T09:39:16.560019+0000","last_became_active":"2023-09-30T09:38:45.613566+0000","last_became_peered":"2023-09-30T09:38:45.613566+0000","last_unstale":"2023-09-30T09:39:16.560019+0000","last_undegraded":"2023-09-30T09:39:16.560019+0000","last_fullsized":"2023-09-30T09:39:16.560019+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:56:47.788900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":289,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.625596+0000","last_change":"2023-09-30T09:33:32.609497+0000","last_active":"2023-09-30T09:39:16.625596+0000","last_peered":"2023-09-30T09:39:16.625596+0000","last_clean":"2023-09-30T09:39:16.625596+0000","last_became_active":"2023-09-30T09:33:32.609128+0000","last_became_peered":"2023-09-30T09:33:32.609128+0000","last_unstale":"2023-09-30T09:39:16.625596+0000","last_undegraded":"2023-09-30T09:39:16.625596+0000","last_fullsized":"2023-09-30T09:39:16.625596+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:03.804343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"382'5167","reported_seq":5225,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:39:24.335373+0000","last_change":"2023-09-30T09:39:16.613964+0000","last_active":"2023-09-30T09:39:24.335373+0000","last_peered":"2023-09-30T09:39:24.335373+0000","last_clean":"2023-09-30T09:38:44.551281+0000","last_became_active":"2023-09-30T09:39:16.603809+0000","last_became_peered":"2023-09-30T09:39:16.603809+0000","last_unstale":"2023-09-30T09:39:24.335373+0000","last_undegraded":"2023-09-30T09:39:16.562159+0000","last_fullsized":"2023-09-30T09:39:24.335373+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5167,"log_dups_size":0,"ondisk_log_size":5167,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176390144,"num_objects":2691,"num_object_clones":0,"num_object_copies":8073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2691,"num_write_kb":172256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":1729},{"shards":"1,4,5","objects":962}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":837,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564598+0000","last_change":"2023-09-30T09:38:45.616475+0000","last_active":"2023-09-30T09:39:16.564598+0000","last_peered":"2023-09-30T09:39:16.564598+0000","last_clean":"2023-09-30T09:39:16.564598+0000","last_became_active":"2023-09-30T09:38:45.616163+0000","last_became_peered":"2023-09-30T09:38:45.616163+0000","last_unstale":"2023-09-30T09:39:16.564598+0000","last_undegraded":"2023-09-30T09:39:16.564598+0000","last_fullsized":"2023-09-30T09:39:16.564598+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:11:18.666981+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":368,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.625373+0000","last_change":"2023-09-30T09:38:45.648614+0000","last_active":"2023-09-30T09:39:16.625373+0000","last_peered":"2023-09-30T09:39:16.625373+0000","last_clean":"2023-09-30T09:39:16.625373+0000","last_became_active":"2023-09-30T09:38:45.615581+0000","last_became_peered":"2023-09-30T09:38:45.615581+0000","last_unstale":"2023-09-30T09:39:16.625373+0000","last_undegraded":"2023-09-30T09:39:16.625373+0000","last_fullsized":"2023-09-30T09:39:16.625373+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:19:19.655673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":350,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565970+0000","last_change":"2023-09-30T09:38:45.612082+0000","last_active":"2023-09-30T09:39:16.565970+0000","last_peered":"2023-09-30T09:39:16.565970+0000","last_clean":"2023-09-30T09:39:16.565970+0000","last_became_active":"2023-09-30T09:38:45.611852+0000","last_became_peered":"2023-09-30T09:38:45.611852+0000","last_unstale":"2023-09-30T09:39:16.565970+0000","last_undegraded":"2023-09-30T09:39:16.565970+0000","last_fullsized":"2023-09-30T09:39:16.565970+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:49:35.044511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"382'4839","reported_seq":4884,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:20.494048+0000","last_change":"2023-09-30T09:39:15.563035+0000","last_active":"2023-09-30T09:39:20.494048+0000","last_peered":"2023-09-30T09:39:20.494048+0000","last_clean":"2023-09-30T09:39:20.494048+0000","last_became_active":"2023-09-30T09:39:15.560626+0000","last_became_peered":"2023-09-30T09:39:15.560626+0000","last_unstale":"2023-09-30T09:39:20.494048+0000","last_undegraded":"2023-09-30T09:39:20.494048+0000","last_fullsized":"2023-09-30T09:39:20.494048+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4839,"log_dups_size":0,"ondisk_log_size":4839,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161906688,"num_objects":2470,"num_object_clones":0,"num_object_copies":7410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2470,"num_write_kb":158112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":818,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.620392+0000","last_change":"2023-09-30T09:38:48.666155+0000","last_active":"2023-09-30T09:39:16.620392+0000","last_peered":"2023-09-30T09:39:16.620392+0000","last_clean":"2023-09-30T09:39:16.620392+0000","last_became_active":"2023-09-30T09:38:48.665829+0000","last_became_peered":"2023-09-30T09:38:48.665829+0000","last_unstale":"2023-09-30T09:39:16.620392+0000","last_undegraded":"2023-09-30T09:39:16.620392+0000","last_fullsized":"2023-09-30T09:39:16.620392+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:09:04.988358+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":359,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569051+0000","last_change":"2023-09-30T09:33:32.608779+0000","last_active":"2023-09-30T09:39:16.569051+0000","last_peered":"2023-09-30T09:39:16.569051+0000","last_clean":"2023-09-30T09:39:16.569051+0000","last_became_active":"2023-09-30T09:33:32.608335+0000","last_became_peered":"2023-09-30T09:33:32.608335+0000","last_unstale":"2023-09-30T09:39:16.569051+0000","last_undegraded":"2023-09-30T09:39:16.569051+0000","last_fullsized":"2023-09-30T09:39:16.569051+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:17.640936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565530+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:39:16.565530+0000","last_peered":"2023-09-30T09:39:16.565530+0000","last_clean":"2023-09-30T09:39:16.565530+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:39:16.565530+0000","last_undegraded":"2023-09-30T09:39:16.565530+0000","last_fullsized":"2023-09-30T09:39:16.565530+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"382'4874","reported_seq":4915,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:24.093576+0000","last_change":"2023-09-30T09:39:15.580661+0000","last_active":"2023-09-30T09:39:24.093576+0000","last_peered":"2023-09-30T09:39:24.093576+0000","last_clean":"2023-09-30T09:39:24.093576+0000","last_became_active":"2023-09-30T09:39:15.577294+0000","last_became_peered":"2023-09-30T09:39:15.577294+0000","last_unstale":"2023-09-30T09:39:24.093576+0000","last_undegraded":"2023-09-30T09:39:24.093576+0000","last_fullsized":"2023-09-30T09:39:24.093576+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4874,"log_dups_size":0,"ondisk_log_size":4874,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166428672,"num_objects":2539,"num_object_clones":0,"num_object_copies":7617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2539,"num_write_kb":162528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":826,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.625470+0000","last_change":"2023-09-30T09:38:09.139181+0000","last_active":"2023-09-30T09:39:16.625470+0000","last_peered":"2023-09-30T09:39:16.625470+0000","last_clean":"2023-09-30T09:39:16.625470+0000","last_became_active":"2023-09-30T09:33:32.622822+0000","last_became_peered":"2023-09-30T09:33:32.622822+0000","last_unstale":"2023-09-30T09:39:16.625470+0000","last_undegraded":"2023-09-30T09:39:16.625470+0000","last_fullsized":"2023-09-30T09:39:16.625470+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:20:47.321880+0000","scrub_duration":0.0056612939999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":388,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565663+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:39:16.565663+0000","last_peered":"2023-09-30T09:39:16.565663+0000","last_clean":"2023-09-30T09:39:16.565663+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:39:16.565663+0000","last_undegraded":"2023-09-30T09:39:16.565663+0000","last_fullsized":"2023-09-30T09:39:16.565663+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":357,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.567960+0000","last_change":"2023-09-30T09:33:32.613016+0000","last_active":"2023-09-30T09:39:16.567960+0000","last_peered":"2023-09-30T09:39:16.567960+0000","last_clean":"2023-09-30T09:39:16.567960+0000","last_became_active":"2023-09-30T09:33:32.612710+0000","last_became_peered":"2023-09-30T09:33:32.612710+0000","last_unstale":"2023-09-30T09:39:16.567960+0000","last_undegraded":"2023-09-30T09:39:16.567960+0000","last_fullsized":"2023-09-30T09:39:16.567960+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:57.504850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"382'4968","reported_seq":5011,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:24.058933+0000","last_change":"2023-09-30T09:39:15.577825+0000","last_active":"2023-09-30T09:39:24.058933+0000","last_peered":"2023-09-30T09:39:24.058933+0000","last_clean":"2023-09-30T09:39:24.058933+0000","last_became_active":"2023-09-30T09:39:15.575559+0000","last_became_peered":"2023-09-30T09:39:15.575559+0000","last_unstale":"2023-09-30T09:39:24.058933+0000","last_undegraded":"2023-09-30T09:39:24.058933+0000","last_fullsized":"2023-09-30T09:39:24.058933+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4968,"log_dups_size":0,"ondisk_log_size":4968,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169443328,"num_objects":2585,"num_object_clones":0,"num_object_copies":7755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2585,"num_write_kb":165472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":540,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.620860+0000","last_change":"2023-09-30T09:37:52.780622+0000","last_active":"2023-09-30T09:39:16.620860+0000","last_peered":"2023-09-30T09:39:16.620860+0000","last_clean":"2023-09-30T09:39:16.620860+0000","last_became_active":"2023-09-30T09:33:13.388503+0000","last_became_peered":"2023-09-30T09:33:13.388503+0000","last_unstale":"2023-09-30T09:39:16.620860+0000","last_undegraded":"2023-09-30T09:39:16.620860+0000","last_fullsized":"2023-09-30T09:39:16.620860+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:16:18.525289+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":379,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564179+0000","last_change":"2023-09-30T09:38:45.620180+0000","last_active":"2023-09-30T09:39:16.564179+0000","last_peered":"2023-09-30T09:39:16.564179+0000","last_clean":"2023-09-30T09:39:16.564179+0000","last_became_active":"2023-09-30T09:38:45.619776+0000","last_became_peered":"2023-09-30T09:38:45.619776+0000","last_unstale":"2023-09-30T09:39:16.564179+0000","last_undegraded":"2023-09-30T09:39:16.564179+0000","last_fullsized":"2023-09-30T09:39:16.564179+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:25:08.739481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":82,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.568243+0000","last_change":"2023-09-30T09:38:45.610897+0000","last_active":"2023-09-30T09:39:16.568243+0000","last_peered":"2023-09-30T09:39:16.568243+0000","last_clean":"2023-09-30T09:39:16.568243+0000","last_became_active":"2023-09-30T09:38:45.610628+0000","last_became_peered":"2023-09-30T09:38:45.610628+0000","last_unstale":"2023-09-30T09:39:16.568243+0000","last_undegraded":"2023-09-30T09:39:16.568243+0000","last_fullsized":"2023-09-30T09:39:16.568243+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:04.928128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.e","version":"382'4943","reported_seq":4978,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:20.602865+0000","last_change":"2023-09-30T09:37:56.536019+0000","last_active":"2023-09-30T09:39:20.602865+0000","last_peered":"2023-09-30T09:39:20.602865+0000","last_clean":"2023-09-30T09:39:20.602865+0000","last_became_active":"2023-09-30T09:37:56.535667+0000","last_became_peered":"2023-09-30T09:37:56.535667+0000","last_unstale":"2023-09-30T09:39:20.602865+0000","last_undegraded":"2023-09-30T09:39:20.602865+0000","last_fullsized":"2023-09-30T09:39:20.602865+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:02:24.984163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323944448,"num_objects":4943,"num_object_clones":0,"num_object_copies":14829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4943,"num_write_kb":316352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":706,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.575718+0000","last_change":"2023-09-30T09:38:45.624734+0000","last_active":"2023-09-30T09:39:16.575718+0000","last_peered":"2023-09-30T09:39:16.575718+0000","last_clean":"2023-09-30T09:39:16.575718+0000","last_became_active":"2023-09-30T09:38:45.624345+0000","last_became_peered":"2023-09-30T09:38:45.624345+0000","last_unstale":"2023-09-30T09:39:16.575718+0000","last_undegraded":"2023-09-30T09:39:16.575718+0000","last_fullsized":"2023-09-30T09:39:16.575718+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:40:11.986970+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":604,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.626396+0000","last_change":"2023-09-30T09:38:27.714032+0000","last_active":"2023-09-30T09:39:16.626396+0000","last_peered":"2023-09-30T09:39:16.626396+0000","last_clean":"2023-09-30T09:39:16.626396+0000","last_became_active":"2023-09-30T09:38:27.684654+0000","last_became_peered":"2023-09-30T09:38:27.684654+0000","last_unstale":"2023-09-30T09:39:16.626396+0000","last_undegraded":"2023-09-30T09:39:16.626396+0000","last_fullsized":"2023-09-30T09:39:16.626396+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:09:43.539211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":388,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.561761+0000","last_change":"2023-09-30T09:38:45.608476+0000","last_active":"2023-09-30T09:39:16.561761+0000","last_peered":"2023-09-30T09:39:16.561761+0000","last_clean":"2023-09-30T09:39:16.561761+0000","last_became_active":"2023-09-30T09:38:45.608152+0000","last_became_peered":"2023-09-30T09:38:45.608152+0000","last_unstale":"2023-09-30T09:39:16.561761+0000","last_undegraded":"2023-09-30T09:39:16.561761+0000","last_fullsized":"2023-09-30T09:39:16.561761+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:40:39.508922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559403+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:39:16.559403+0000","last_peered":"2023-09-30T09:39:16.559403+0000","last_clean":"2023-09-30T09:39:16.559403+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:39:16.559403+0000","last_undegraded":"2023-09-30T09:39:16.559403+0000","last_fullsized":"2023-09-30T09:39:16.559403+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"382'5066","reported_seq":5124,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:39:24.328032+0000","last_change":"2023-09-30T09:39:16.635223+0000","last_active":"2023-09-30T09:39:24.328032+0000","last_peered":"2023-09-30T09:39:24.328032+0000","last_clean":"2023-09-30T09:38:26.554137+0000","last_became_active":"2023-09-30T09:39:16.631064+0000","last_became_peered":"2023-09-30T09:39:16.631064+0000","last_unstale":"2023-09-30T09:39:24.328032+0000","last_undegraded":"2023-09-30T09:39:16.589691+0000","last_fullsized":"2023-09-30T09:39:24.328032+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172785664,"num_objects":2636,"num_object_clones":0,"num_object_copies":7908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1714,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2636,"num_write_kb":168736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"0,3,5","objects":1142},{"shards":"0,5,7","objects":922},{"shards":"3,5,7","objects":572}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":844,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559706+0000","last_change":"2023-09-30T09:33:13.387054+0000","last_active":"2023-09-30T09:39:16.559706+0000","last_peered":"2023-09-30T09:39:16.559706+0000","last_clean":"2023-09-30T09:39:16.559706+0000","last_became_active":"2023-09-30T09:33:13.386698+0000","last_became_peered":"2023-09-30T09:33:13.386698+0000","last_unstale":"2023-09-30T09:39:16.559706+0000","last_undegraded":"2023-09-30T09:39:16.559706+0000","last_fullsized":"2023-09-30T09:39:16.559706+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:02.431648+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":373,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569112+0000","last_change":"2023-09-30T09:33:32.622160+0000","last_active":"2023-09-30T09:39:16.569112+0000","last_peered":"2023-09-30T09:39:16.569112+0000","last_clean":"2023-09-30T09:39:16.569112+0000","last_became_active":"2023-09-30T09:33:32.621793+0000","last_became_peered":"2023-09-30T09:33:32.621793+0000","last_unstale":"2023-09-30T09:39:16.569112+0000","last_undegraded":"2023-09-30T09:39:16.569112+0000","last_fullsized":"2023-09-30T09:39:16.569112+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:49.549740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":361,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.572876+0000","last_change":"2023-09-30T09:23:08.017773+0000","last_active":"2023-09-30T09:39:16.572876+0000","last_peered":"2023-09-30T09:39:16.572876+0000","last_clean":"2023-09-30T09:39:16.572876+0000","last_became_active":"2023-09-30T09:23:08.017422+0000","last_became_peered":"2023-09-30T09:23:08.017422+0000","last_unstale":"2023-09-30T09:39:16.572876+0000","last_undegraded":"2023-09-30T09:39:16.572876+0000","last_fullsized":"2023-09-30T09:39:16.572876+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:14:21.885567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:17.563463+0000","last_change":"2023-09-30T09:38:45.615879+0000","last_active":"2023-09-30T09:39:17.563463+0000","last_peered":"2023-09-30T09:39:17.563463+0000","last_clean":"2023-09-30T09:39:17.563463+0000","last_became_active":"2023-09-30T09:38:45.615545+0000","last_became_peered":"2023-09-30T09:38:45.615545+0000","last_unstale":"2023-09-30T09:39:17.563463+0000","last_undegraded":"2023-09-30T09:39:17.563463+0000","last_fullsized":"2023-09-30T09:39:17.563463+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:39:23.834391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"382'5134","reported_seq":5991,"reported_epoch":382,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:39:24.174347+0000","last_change":"2023-09-30T09:39:15.720508+0000","last_active":"2023-09-30T09:39:24.174347+0000","last_peered":"2023-09-30T09:39:24.174347+0000","last_clean":"2023-09-30T09:38:44.556600+0000","last_became_active":"2023-09-30T09:38:46.733175+0000","last_became_peered":"2023-09-30T09:38:46.733175+0000","last_unstale":"2023-09-30T09:39:24.174347+0000","last_undegraded":"2023-09-30T09:39:24.174347+0000","last_fullsized":"2023-09-30T09:38:46.604027+0000","mapping_epoch":372,"log_start":"363'200","ondisk_log_start":"363'200","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:41.383870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336461824,"num_objects":5134,"num_object_clones":0,"num_object_copies":15402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2907,"num_objects_unfound":0,"num_objects_dirty":5134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5134,"num_write_kb":328576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":406,"num_bytes_recovered":26607616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":2907},{"shards":"1,4","objects":2227}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":761,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564842+0000","last_change":"2023-09-30T09:38:30.737659+0000","last_active":"2023-09-30T09:39:16.564842+0000","last_peered":"2023-09-30T09:39:16.564842+0000","last_clean":"2023-09-30T09:39:16.564842+0000","last_became_active":"2023-09-30T09:38:30.737311+0000","last_became_peered":"2023-09-30T09:38:30.737311+0000","last_unstale":"2023-09-30T09:39:16.564842+0000","last_undegraded":"2023-09-30T09:39:16.564842+0000","last_fullsized":"2023-09-30T09:39:16.564842+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:08:13.108463+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":598,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:17.563031+0000","last_change":"2023-09-30T09:38:48.641903+0000","last_active":"2023-09-30T09:39:17.563031+0000","last_peered":"2023-09-30T09:39:17.563031+0000","last_clean":"2023-09-30T09:39:17.563031+0000","last_became_active":"2023-09-30T09:38:48.641372+0000","last_became_peered":"2023-09-30T09:38:48.641372+0000","last_unstale":"2023-09-30T09:39:17.563031+0000","last_undegraded":"2023-09-30T09:39:17.563031+0000","last_fullsized":"2023-09-30T09:39:17.563031+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:49:32.411360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":109,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.620424+0000","last_change":"2023-09-30T09:33:33.595241+0000","last_active":"2023-09-30T09:39:16.620424+0000","last_peered":"2023-09-30T09:39:16.620424+0000","last_clean":"2023-09-30T09:39:16.620424+0000","last_became_active":"2023-09-30T09:33:33.594945+0000","last_became_peered":"2023-09-30T09:33:33.594945+0000","last_unstale":"2023-09-30T09:39:16.620424+0000","last_undegraded":"2023-09-30T09:39:16.620424+0000","last_fullsized":"2023-09-30T09:39:16.620424+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:10:21.541406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":301,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.560359+0000","last_change":"2023-09-30T09:35:10.646622+0000","last_active":"2023-09-30T09:39:16.560359+0000","last_peered":"2023-09-30T09:39:16.560359+0000","last_clean":"2023-09-30T09:39:16.560359+0000","last_became_active":"2023-09-30T09:35:10.646338+0000","last_became_peered":"2023-09-30T09:35:10.646338+0000","last_unstale":"2023-09-30T09:39:16.560359+0000","last_undegraded":"2023-09-30T09:39:16.560359+0000","last_fullsized":"2023-09-30T09:39:16.560359+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:57.476585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"382'4964","reported_seq":5005,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:24.116869+0000","last_change":"2023-09-30T09:39:15.576294+0000","last_active":"2023-09-30T09:39:24.116869+0000","last_peered":"2023-09-30T09:39:24.116869+0000","last_clean":"2023-09-30T09:39:24.116869+0000","last_became_active":"2023-09-30T09:39:15.573564+0000","last_became_peered":"2023-09-30T09:39:15.573564+0000","last_unstale":"2023-09-30T09:39:24.116869+0000","last_undegraded":"2023-09-30T09:39:24.116869+0000","last_fullsized":"2023-09-30T09:39:24.116869+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4964,"log_dups_size":0,"ondisk_log_size":4964,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168787968,"num_objects":2575,"num_object_clones":0,"num_object_copies":7725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2575,"num_write_kb":164832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1008,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559736+0000","last_change":"2023-09-30T09:38:30.748839+0000","last_active":"2023-09-30T09:39:16.559736+0000","last_peered":"2023-09-30T09:39:16.559736+0000","last_clean":"2023-09-30T09:39:16.559736+0000","last_became_active":"2023-09-30T09:38:30.748346+0000","last_became_peered":"2023-09-30T09:38:30.748346+0000","last_unstale":"2023-09-30T09:39:16.559736+0000","last_undegraded":"2023-09-30T09:39:16.559736+0000","last_fullsized":"2023-09-30T09:39:16.559736+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:46:58.191800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":665,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564464+0000","last_change":"2023-09-30T09:38:30.743421+0000","last_active":"2023-09-30T09:39:16.564464+0000","last_peered":"2023-09-30T09:39:16.564464+0000","last_clean":"2023-09-30T09:39:16.564464+0000","last_became_active":"2023-09-30T09:38:30.743110+0000","last_became_peered":"2023-09-30T09:38:30.743110+0000","last_unstale":"2023-09-30T09:39:16.564464+0000","last_undegraded":"2023-09-30T09:39:16.564464+0000","last_fullsized":"2023-09-30T09:39:16.564464+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:13.366523+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":36,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559881+0000","last_change":"2023-09-30T09:38:45.608979+0000","last_active":"2023-09-30T09:39:16.559881+0000","last_peered":"2023-09-30T09:39:16.559881+0000","last_clean":"2023-09-30T09:39:16.559881+0000","last_became_active":"2023-09-30T09:38:45.608817+0000","last_became_peered":"2023-09-30T09:38:45.608817+0000","last_unstale":"2023-09-30T09:39:16.559881+0000","last_undegraded":"2023-09-30T09:39:16.559881+0000","last_fullsized":"2023-09-30T09:39:16.559881+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:03.733113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":344,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.570693+0000","last_change":"2023-09-30T09:38:27.682465+0000","last_active":"2023-09-30T09:39:16.570693+0000","last_peered":"2023-09-30T09:39:16.570693+0000","last_clean":"2023-09-30T09:39:16.570693+0000","last_became_active":"2023-09-30T09:38:27.682121+0000","last_became_peered":"2023-09-30T09:38:27.682121+0000","last_unstale":"2023-09-30T09:39:16.570693+0000","last_undegraded":"2023-09-30T09:39:16.570693+0000","last_fullsized":"2023-09-30T09:39:16.570693+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:56:49.578285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"382'5203","reported_seq":5240,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:25.372655+0000","last_change":"2023-09-30T09:37:56.539888+0000","last_active":"2023-09-30T09:39:25.372655+0000","last_peered":"2023-09-30T09:39:25.372655+0000","last_clean":"2023-09-30T09:39:25.372655+0000","last_became_active":"2023-09-30T09:37:56.539552+0000","last_became_peered":"2023-09-30T09:37:56.539552+0000","last_unstale":"2023-09-30T09:39:25.372655+0000","last_undegraded":"2023-09-30T09:39:25.372655+0000","last_fullsized":"2023-09-30T09:39:25.372655+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5203,"log_dups_size":0,"ondisk_log_size":5203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:37.074768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340983808,"num_objects":5203,"num_object_clones":0,"num_object_copies":15609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5203,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":537,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569698+0000","last_change":"2023-09-30T09:37:53.041461+0000","last_active":"2023-09-30T09:39:16.569698+0000","last_peered":"2023-09-30T09:39:16.569698+0000","last_clean":"2023-09-30T09:39:16.569698+0000","last_became_active":"2023-09-30T09:33:32.748520+0000","last_became_peered":"2023-09-30T09:33:32.748520+0000","last_unstale":"2023-09-30T09:39:16.569698+0000","last_undegraded":"2023-09-30T09:39:16.569698+0000","last_fullsized":"2023-09-30T09:39:16.569698+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:03:14.451639+0000","scrub_duration":0.0016789540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":769,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564369+0000","last_change":"2023-09-30T09:38:30.736806+0000","last_active":"2023-09-30T09:39:16.564369+0000","last_peered":"2023-09-30T09:39:16.564369+0000","last_clean":"2023-09-30T09:39:16.564369+0000","last_became_active":"2023-09-30T09:38:30.736355+0000","last_became_peered":"2023-09-30T09:38:30.736355+0000","last_unstale":"2023-09-30T09:39:16.564369+0000","last_undegraded":"2023-09-30T09:39:16.564369+0000","last_fullsized":"2023-09-30T09:39:16.564369+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:46.413975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":319,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.622281+0000","last_change":"2023-09-30T09:38:45.620156+0000","last_active":"2023-09-30T09:39:16.622281+0000","last_peered":"2023-09-30T09:39:16.622281+0000","last_clean":"2023-09-30T09:39:16.622281+0000","last_became_active":"2023-09-30T09:38:45.619887+0000","last_became_peered":"2023-09-30T09:38:45.619887+0000","last_unstale":"2023-09-30T09:39:16.622281+0000","last_undegraded":"2023-09-30T09:39:16.622281+0000","last_fullsized":"2023-09-30T09:39:16.622281+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:37.503622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.626303+0000","last_change":"2023-09-30T09:35:10.655859+0000","last_active":"2023-09-30T09:39:16.626303+0000","last_peered":"2023-09-30T09:39:16.626303+0000","last_clean":"2023-09-30T09:39:16.626303+0000","last_became_active":"2023-09-30T09:35:10.655369+0000","last_became_peered":"2023-09-30T09:35:10.655369+0000","last_unstale":"2023-09-30T09:39:16.626303+0000","last_undegraded":"2023-09-30T09:39:16.626303+0000","last_fullsized":"2023-09-30T09:39:16.626303+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:09.917229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"382'5216","reported_seq":5261,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:39:25.343848+0000","last_change":"2023-09-30T09:38:46.795169+0000","last_active":"2023-09-30T09:39:25.343848+0000","last_peered":"2023-09-30T09:39:25.343848+0000","last_clean":"2023-09-30T09:38:44.567377+0000","last_became_active":"2023-09-30T09:38:46.789998+0000","last_became_peered":"2023-09-30T09:38:46.789998+0000","last_unstale":"2023-09-30T09:39:25.343848+0000","last_undegraded":"2023-09-30T09:38:46.692264+0000","last_fullsized":"2023-09-30T09:38:46.597825+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5216,"log_dups_size":0,"ondisk_log_size":5216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:12.989149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341835776,"num_objects":5216,"num_object_clones":0,"num_object_copies":15648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5216,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":1928},{"shards":"1,2,3","objects":3288}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":676,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569517+0000","last_change":"2023-09-30T09:38:48.664915+0000","last_active":"2023-09-30T09:39:16.569517+0000","last_peered":"2023-09-30T09:39:16.569517+0000","last_clean":"2023-09-30T09:39:16.569517+0000","last_became_active":"2023-09-30T09:38:48.664604+0000","last_became_peered":"2023-09-30T09:38:48.664604+0000","last_unstale":"2023-09-30T09:39:16.569517+0000","last_undegraded":"2023-09-30T09:39:16.569517+0000","last_fullsized":"2023-09-30T09:39:16.569517+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:13:22.591021+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":870,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:17.578611+0000","last_change":"2023-09-30T09:39:17.578611+0000","last_active":"2023-09-30T09:39:17.578611+0000","last_peered":"2023-09-30T09:39:17.578611+0000","last_clean":"2023-09-30T09:39:17.578611+0000","last_became_active":"2023-09-30T09:39:17.578096+0000","last_became_peered":"2023-09-30T09:39:17.578096+0000","last_unstale":"2023-09-30T09:39:17.578611+0000","last_undegraded":"2023-09-30T09:39:17.578611+0000","last_fullsized":"2023-09-30T09:39:17.578611+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:38.025468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":109,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:17.563288+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:39:17.563288+0000","last_peered":"2023-09-30T09:39:17.563288+0000","last_clean":"2023-09-30T09:39:17.563288+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:39:17.563288+0000","last_undegraded":"2023-09-30T09:39:17.563288+0000","last_fullsized":"2023-09-30T09:39:17.563288+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":291,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.625681+0000","last_change":"2023-09-30T09:38:27.672393+0000","last_active":"2023-09-30T09:39:16.625681+0000","last_peered":"2023-09-30T09:39:16.625681+0000","last_clean":"2023-09-30T09:39:16.625681+0000","last_became_active":"2023-09-30T09:38:27.672044+0000","last_became_peered":"2023-09-30T09:38:27.672044+0000","last_unstale":"2023-09-30T09:39:16.625681+0000","last_undegraded":"2023-09-30T09:39:16.625681+0000","last_fullsized":"2023-09-30T09:39:16.625681+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:14:24.444433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"382'5119","reported_seq":7386,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:39:20.578830+0000","last_change":"2023-09-30T09:38:46.679360+0000","last_active":"2023-09-30T09:39:20.578830+0000","last_peered":"2023-09-30T09:39:20.578830+0000","last_clean":"2023-09-30T09:38:26.563448+0000","last_became_active":"2023-09-30T09:38:46.670968+0000","last_became_peered":"2023-09-30T09:38:46.670968+0000","last_unstale":"2023-09-30T09:39:20.578830+0000","last_undegraded":"2023-09-30T09:38:46.664891+0000","last_fullsized":"2023-09-30T09:38:46.591242+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5119,"log_dups_size":0,"ondisk_log_size":5119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:35:29.816698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335478784,"num_objects":5119,"num_object_clones":0,"num_object_copies":15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2384,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5119,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1108,"num_bytes_recovered":72613888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,5],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3,5","objects":1259},{"shards":"0,3,7","objects":1125},{"shards":"0,5","objects":2735}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":854,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.567917+0000","last_change":"2023-09-30T09:33:13.389491+0000","last_active":"2023-09-30T09:39:16.567917+0000","last_peered":"2023-09-30T09:39:16.567917+0000","last_clean":"2023-09-30T09:39:16.567917+0000","last_became_active":"2023-09-30T09:33:13.389096+0000","last_became_peered":"2023-09-30T09:33:13.389096+0000","last_unstale":"2023-09-30T09:39:16.567917+0000","last_undegraded":"2023-09-30T09:39:16.567917+0000","last_fullsized":"2023-09-30T09:39:16.567917+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:01:34.365990+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":393,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565392+0000","last_change":"2023-09-30T09:38:45.668143+0000","last_active":"2023-09-30T09:39:16.565392+0000","last_peered":"2023-09-30T09:39:16.565392+0000","last_clean":"2023-09-30T09:39:16.565392+0000","last_became_active":"2023-09-30T09:38:45.624556+0000","last_became_peered":"2023-09-30T09:38:45.624556+0000","last_unstale":"2023-09-30T09:39:16.565392+0000","last_undegraded":"2023-09-30T09:39:16.565392+0000","last_fullsized":"2023-09-30T09:39:16.565392+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:47:39.105840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1148,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569251+0000","last_change":"2023-09-30T09:33:32.614518+0000","last_active":"2023-09-30T09:39:16.569251+0000","last_peered":"2023-09-30T09:39:16.569251+0000","last_clean":"2023-09-30T09:39:16.569251+0000","last_became_active":"2023-09-30T09:33:32.613803+0000","last_became_peered":"2023-09-30T09:33:32.613803+0000","last_unstale":"2023-09-30T09:39:16.569251+0000","last_undegraded":"2023-09-30T09:39:16.569251+0000","last_fullsized":"2023-09-30T09:39:16.569251+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:33:53.781634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":406,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.564254+0000","last_change":"2023-09-30T09:38:45.620031+0000","last_active":"2023-09-30T09:39:16.564254+0000","last_peered":"2023-09-30T09:39:16.564254+0000","last_clean":"2023-09-30T09:39:16.564254+0000","last_became_active":"2023-09-30T09:38:45.619374+0000","last_became_peered":"2023-09-30T09:38:45.619374+0000","last_unstale":"2023-09-30T09:39:16.564254+0000","last_undegraded":"2023-09-30T09:39:16.564254+0000","last_fullsized":"2023-09-30T09:39:16.564254+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:16.452264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"382'5316","reported_seq":5560,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:39:24.332931+0000","last_change":"2023-09-30T09:39:15.708467+0000","last_active":"2023-09-30T09:39:24.332931+0000","last_peered":"2023-09-30T09:39:24.332931+0000","last_clean":"2023-09-30T09:38:26.438920+0000","last_became_active":"2023-09-30T09:38:28.865945+0000","last_became_peered":"2023-09-30T09:38:28.865945+0000","last_unstale":"2023-09-30T09:39:24.332931+0000","last_undegraded":"2023-09-30T09:39:15.705293+0000","last_fullsized":"2023-09-30T09:38:28.657614+0000","mapping_epoch":366,"log_start":"363'200","ondisk_log_start":"363'200","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5116,"log_dups_size":0,"ondisk_log_size":5116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:22:08.971450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348389376,"num_objects":5316,"num_object_clones":0,"num_object_copies":15948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2280,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5316,"num_write_kb":340224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":6619136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":3036},{"shards":"2,5,6","objects":2280}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":795,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559718+0000","last_change":"2023-09-30T09:38:45.613829+0000","last_active":"2023-09-30T09:39:16.559718+0000","last_peered":"2023-09-30T09:39:16.559718+0000","last_clean":"2023-09-30T09:39:16.559718+0000","last_became_active":"2023-09-30T09:38:45.613119+0000","last_became_peered":"2023-09-30T09:38:45.613119+0000","last_unstale":"2023-09-30T09:39:16.559718+0000","last_undegraded":"2023-09-30T09:39:16.559718+0000","last_fullsized":"2023-09-30T09:39:16.559718+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:02.320545+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":734,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.575890+0000","last_change":"2023-09-30T09:38:27.708928+0000","last_active":"2023-09-30T09:39:16.575890+0000","last_peered":"2023-09-30T09:39:16.575890+0000","last_clean":"2023-09-30T09:39:16.575890+0000","last_became_active":"2023-09-30T09:38:27.682036+0000","last_became_peered":"2023-09-30T09:38:27.682036+0000","last_unstale":"2023-09-30T09:39:16.575890+0000","last_undegraded":"2023-09-30T09:39:16.575890+0000","last_fullsized":"2023-09-30T09:39:16.575890+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:26:59.892254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":70,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.626362+0000","last_change":"2023-09-30T09:35:10.761121+0000","last_active":"2023-09-30T09:39:16.626362+0000","last_peered":"2023-09-30T09:39:16.626362+0000","last_clean":"2023-09-30T09:39:16.626362+0000","last_became_active":"2023-09-30T09:35:10.649915+0000","last_became_peered":"2023-09-30T09:35:10.649915+0000","last_unstale":"2023-09-30T09:39:16.626362+0000","last_undegraded":"2023-09-30T09:39:16.626362+0000","last_fullsized":"2023-09-30T09:39:16.626362+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:40.921924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":323,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.565728+0000","last_change":"2023-09-30T09:12:09.017023+0000","last_active":"2023-09-30T09:39:16.565728+0000","last_peered":"2023-09-30T09:39:16.565728+0000","last_clean":"2023-09-30T09:39:16.565728+0000","last_became_active":"2023-09-30T09:12:09.016505+0000","last_became_peered":"2023-09-30T09:12:09.016505+0000","last_unstale":"2023-09-30T09:39:16.565728+0000","last_undegraded":"2023-09-30T09:39:16.565728+0000","last_fullsized":"2023-09-30T09:39:16.565728+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:27.872412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"382'5129","reported_seq":5184,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:39:24.303468+0000","last_change":"2023-09-30T09:39:15.664356+0000","last_active":"2023-09-30T09:39:24.303468+0000","last_peered":"2023-09-30T09:39:24.303468+0000","last_clean":"2023-09-30T09:38:44.551281+0000","last_became_active":"2023-09-30T09:39:15.641278+0000","last_became_peered":"2023-09-30T09:39:15.641278+0000","last_unstale":"2023-09-30T09:39:24.303468+0000","last_undegraded":"2023-09-30T09:39:15.626302+0000","last_fullsized":"2023-09-30T09:39:24.303468+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5129,"log_dups_size":0,"ondisk_log_size":5129,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173899776,"num_objects":2654,"num_object_clones":0,"num_object_copies":7962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1742,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2654,"num_write_kb":169824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":1742},{"shards":"1,4,5","objects":912}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":755,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.622158+0000","last_change":"2023-09-30T09:38:48.648959+0000","last_active":"2023-09-30T09:39:16.622158+0000","last_peered":"2023-09-30T09:39:16.622158+0000","last_clean":"2023-09-30T09:39:16.622158+0000","last_became_active":"2023-09-30T09:38:48.648731+0000","last_became_peered":"2023-09-30T09:38:48.648731+0000","last_unstale":"2023-09-30T09:39:16.622158+0000","last_undegraded":"2023-09-30T09:39:16.622158+0000","last_fullsized":"2023-09-30T09:39:16.622158+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:20:36.548251+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":74,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.625618+0000","last_change":"2023-09-30T09:38:27.682550+0000","last_active":"2023-09-30T09:39:16.625618+0000","last_peered":"2023-09-30T09:39:16.625618+0000","last_clean":"2023-09-30T09:39:16.625618+0000","last_became_active":"2023-09-30T09:38:27.682171+0000","last_became_peered":"2023-09-30T09:38:27.682171+0000","last_unstale":"2023-09-30T09:39:16.625618+0000","last_undegraded":"2023-09-30T09:39:16.625618+0000","last_fullsized":"2023-09-30T09:39:16.625618+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:47:58.913725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":68,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.620926+0000","last_change":"2023-09-30T09:38:45.603597+0000","last_active":"2023-09-30T09:39:16.620926+0000","last_peered":"2023-09-30T09:39:16.620926+0000","last_clean":"2023-09-30T09:39:16.620926+0000","last_became_active":"2023-09-30T09:38:45.603355+0000","last_became_peered":"2023-09-30T09:38:45.603355+0000","last_unstale":"2023-09-30T09:39:16.620926+0000","last_undegraded":"2023-09-30T09:39:16.620926+0000","last_fullsized":"2023-09-30T09:39:16.620926+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:51.273460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"382'5071","reported_seq":5122,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:39:24.127263+0000","last_change":"2023-09-30T09:39:15.652833+0000","last_active":"2023-09-30T09:39:24.127263+0000","last_peered":"2023-09-30T09:39:24.127263+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:39:15.628186+0000","last_became_peered":"2023-09-30T09:39:15.628186+0000","last_unstale":"2023-09-30T09:39:24.127263+0000","last_undegraded":"2023-09-30T09:39:15.623368+0000","last_fullsized":"2023-09-30T09:39:24.127263+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5071,"log_dups_size":0,"ondisk_log_size":5071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172720128,"num_objects":2636,"num_object_clones":0,"num_object_copies":7908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2636,"num_write_kb":168672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,2,6","objects":941},{"shards":"1,3,6","objects":1695}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":569,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569550+0000","last_change":"2023-09-30T09:38:48.656922+0000","last_active":"2023-09-30T09:39:16.569550+0000","last_peered":"2023-09-30T09:39:16.569550+0000","last_clean":"2023-09-30T09:39:16.569550+0000","last_became_active":"2023-09-30T09:38:48.656616+0000","last_became_peered":"2023-09-30T09:38:48.656616+0000","last_unstale":"2023-09-30T09:39:16.569550+0000","last_undegraded":"2023-09-30T09:39:16.569550+0000","last_fullsized":"2023-09-30T09:39:16.569550+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":360,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.559949+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:39:16.559949+0000","last_peered":"2023-09-30T09:39:16.559949+0000","last_clean":"2023-09-30T09:39:16.559949+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:39:16.559949+0000","last_undegraded":"2023-09-30T09:39:16.559949+0000","last_fullsized":"2023-09-30T09:39:16.559949+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.572426+0000","last_change":"2023-09-30T09:38:45.595684+0000","last_active":"2023-09-30T09:39:16.572426+0000","last_peered":"2023-09-30T09:39:16.572426+0000","last_clean":"2023-09-30T09:39:16.572426+0000","last_became_active":"2023-09-30T09:38:45.595231+0000","last_became_peered":"2023-09-30T09:38:45.595231+0000","last_unstale":"2023-09-30T09:39:16.572426+0000","last_undegraded":"2023-09-30T09:39:16.572426+0000","last_fullsized":"2023-09-30T09:39:16.572426+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:43:29.805772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"382'5000","reported_seq":8745,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:39:20.579499+0000","last_change":"2023-09-30T09:39:17.571978+0000","last_active":"2023-09-30T09:39:20.579499+0000","last_peered":"2023-09-30T09:39:20.579499+0000","last_clean":"2023-09-30T09:38:26.430308+0000","last_became_active":"2023-09-30T09:39:17.569050+0000","last_became_peered":"2023-09-30T09:39:17.569050+0000","last_unstale":"2023-09-30T09:39:20.579499+0000","last_undegraded":"2023-09-30T09:39:17.561437+0000","last_fullsized":"2023-09-30T09:39:17.560906+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5000,"log_dups_size":0,"ondisk_log_size":5000,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167706624,"num_objects":2559,"num_object_clones":0,"num_object_copies":7677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2559,"num_write_kb":163776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":924,"num_bytes_recovered":60555264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":233},{"shards":"0,6","objects":2326}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":326,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.561684+0000","last_change":"2023-09-30T09:12:07.940608+0000","last_active":"2023-09-30T09:39:16.561684+0000","last_peered":"2023-09-30T09:39:16.561684+0000","last_clean":"2023-09-30T09:39:16.561684+0000","last_became_active":"2023-09-30T09:12:07.940301+0000","last_became_peered":"2023-09-30T09:12:07.940301+0000","last_unstale":"2023-09-30T09:39:16.561684+0000","last_undegraded":"2023-09-30T09:39:16.561684+0000","last_fullsized":"2023-09-30T09:39:16.561684+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:08.950935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.566040+0000","last_change":"2023-09-30T09:38:45.604276+0000","last_active":"2023-09-30T09:39:16.566040+0000","last_peered":"2023-09-30T09:39:16.566040+0000","last_clean":"2023-09-30T09:39:16.566040+0000","last_became_active":"2023-09-30T09:38:45.603925+0000","last_became_peered":"2023-09-30T09:38:45.603925+0000","last_unstale":"2023-09-30T09:39:16.566040+0000","last_undegraded":"2023-09-30T09:39:16.566040+0000","last_fullsized":"2023-09-30T09:39:16.566040+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:33:04.729338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"382'5003","reported_seq":5408,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:39:24.116567+0000","last_change":"2023-09-30T09:39:15.613281+0000","last_active":"2023-09-30T09:39:24.116567+0000","last_peered":"2023-09-30T09:39:24.116567+0000","last_clean":"2023-09-30T09:38:26.563035+0000","last_became_active":"2023-09-30T09:39:15.610064+0000","last_became_peered":"2023-09-30T09:39:15.610064+0000","last_unstale":"2023-09-30T09:39:24.116567+0000","last_undegraded":"2023-09-30T09:39:15.594785+0000","last_fullsized":"2023-09-30T09:39:24.116567+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170459136,"num_objects":2601,"num_object_clones":0,"num_object_copies":7803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1031,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2601,"num_write_kb":166464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":5767168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":1031},{"shards":"4,6,7","objects":1570}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":664,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.566126+0000","last_change":"2023-09-30T09:38:30.745007+0000","last_active":"2023-09-30T09:39:16.566126+0000","last_peered":"2023-09-30T09:39:16.566126+0000","last_clean":"2023-09-30T09:39:16.566126+0000","last_became_active":"2023-09-30T09:38:30.744685+0000","last_became_peered":"2023-09-30T09:38:30.744685+0000","last_unstale":"2023-09-30T09:39:16.566126+0000","last_undegraded":"2023-09-30T09:39:16.566126+0000","last_fullsized":"2023-09-30T09:39:16.566126+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:54.087577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:54.087577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":33,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.569345+0000","last_change":"2023-09-30T09:38:45.614444+0000","last_active":"2023-09-30T09:39:16.569345+0000","last_peered":"2023-09-30T09:39:16.569345+0000","last_clean":"2023-09-30T09:39:16.569345+0000","last_became_active":"2023-09-30T09:38:45.614114+0000","last_became_peered":"2023-09-30T09:38:45.614114+0000","last_unstale":"2023-09-30T09:39:16.569345+0000","last_undegraded":"2023-09-30T09:39:16.569345+0000","last_fullsized":"2023-09-30T09:39:16.569345+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:14.962666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":327,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.620670+0000","last_change":"2023-09-30T09:38:45.600268+0000","last_active":"2023-09-30T09:39:16.620670+0000","last_peered":"2023-09-30T09:39:16.620670+0000","last_clean":"2023-09-30T09:39:16.620670+0000","last_became_active":"2023-09-30T09:38:45.599829+0000","last_became_peered":"2023-09-30T09:38:45.599829+0000","last_unstale":"2023-09-30T09:39:16.620670+0000","last_undegraded":"2023-09-30T09:39:16.620670+0000","last_fullsized":"2023-09-30T09:39:16.620670+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:39:20.321856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"382'5011","reported_seq":5053,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:24.360592+0000","last_change":"2023-09-30T09:39:15.638244+0000","last_active":"2023-09-30T09:39:24.360592+0000","last_peered":"2023-09-30T09:39:24.360592+0000","last_clean":"2023-09-30T09:39:24.360592+0000","last_became_active":"2023-09-30T09:39:15.634600+0000","last_became_peered":"2023-09-30T09:39:15.634600+0000","last_unstale":"2023-09-30T09:39:24.360592+0000","last_undegraded":"2023-09-30T09:39:24.360592+0000","last_fullsized":"2023-09-30T09:39:24.360592+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5011,"log_dups_size":0,"ondisk_log_size":5011,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170721280,"num_objects":2605,"num_object_clones":0,"num_object_copies":7815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2605,"num_write_kb":166720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":757,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.567205+0000","last_change":"2023-09-30T09:38:48.664091+0000","last_active":"2023-09-30T09:39:16.567205+0000","last_peered":"2023-09-30T09:39:16.567205+0000","last_clean":"2023-09-30T09:39:16.567205+0000","last_became_active":"2023-09-30T09:38:48.663723+0000","last_became_peered":"2023-09-30T09:38:48.663723+0000","last_unstale":"2023-09-30T09:39:16.567205+0000","last_undegraded":"2023-09-30T09:39:16.567205+0000","last_fullsized":"2023-09-30T09:39:16.567205+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:57.093254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:57.093254+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":87,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-30T09:39:17.561222+0000","last_change":"2023-09-30T09:35:23.871379+0000","last_active":"2023-09-30T09:39:17.561222+0000","last_peered":"2023-09-30T09:39:17.561222+0000","last_clean":"2023-09-30T09:39:17.561222+0000","last_became_active":"2023-09-30T09:35:23.842277+0000","last_became_peered":"2023-09-30T09:35:23.842277+0000","last_unstale":"2023-09-30T09:39:17.561222+0000","last_undegraded":"2023-09-30T09:39:17.561222+0000","last_fullsized":"2023-09-30T09:39:17.561222+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:46.244170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":367,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-30T09:39:16.566991+0000","last_change":"2023-09-30T09:38:05.412543+0000","last_active":"2023-09-30T09:39:16.566991+0000","last_peered":"2023-09-30T09:39:16.566991+0000","last_clean":"2023-09-30T09:39:16.566991+0000","last_became_active":"2023-09-30T09:38:05.412129+0000","last_became_peered":"2023-09-30T09:38:05.412129+0000","last_unstale":"2023-09-30T09:39:16.566991+0000","last_undegraded":"2023-09-30T09:39:16.566991+0000","last_fullsized":"2023-09-30T09:39:16.566991+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:49:02.515137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":11608,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":5416878080,"num_objects":82655,"num_object_clones":0,"num_object_copies":247965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19114,"num_objects_misplaced":3761,"num_objects_unfound":0,"num_objects_dirty":82655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82655,"num_write_kb":5289920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4521,"num_bytes_recovered":296288256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16564027392,"data_stored":16564027392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130382,"ondisk_log_size":130382,"up":78,"acting":72,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":326246,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":841734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":225,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234333,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1747848,"kb_used_data":1347712,"kb_used_omap":0,"kb_used_meta":400064,"kb_avail":91997304,"statfs":{"total":95995035648,"available":94205239296,"internally_reserved":0,"allocated":1380057088,"data_stored":1378752126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":409665536},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:46 2023","interfaces":[{"interface":"back","average":{"1min":5.5099999999999998,"5min":2.7730000000000001,"15min":1.899},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.224},"max":{"1min":95.418999999999997,"5min":95.418999999999997,"15min":95.418999999999997},"last":0.29099999999999998},{"interface":"front","average":{"1min":5.3399999999999999,"5min":2.6230000000000002,"15min":1.7190000000000001},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.23899999999999999},"max":{"1min":95.230999999999995,"5min":95.230999999999995,"15min":95.230999999999995},"last":0.68999999999999995}]},{"osd":1,"last update":"Sat Sep 30 09:38:46 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":3.0070000000000001,"15min":3.0350000000000001},"min":{"1min":0.27000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":89.308000000000007,"5min":89.308000000000007,"15min":89.308000000000007},"last":0.32800000000000001},{"interface":"front","average":{"1min":5.2510000000000003,"5min":3.343,"15min":3.5870000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":89.325999999999993,"5min":89.325999999999993,"15min":89.325999999999993},"last":0.34499999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:38:27 2023","interfaces":[{"interface":"back","average":{"1min":6.1109999999999998,"5min":2.9350000000000001,"15min":2.1579999999999999},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.24299999999999999},"max":{"1min":95.602999999999994,"5min":95.602999999999994,"15min":95.602999999999994},"last":0.379},{"interface":"front","average":{"1min":5.5670000000000002,"5min":2.8610000000000002,"15min":2.161},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.28299999999999997},"max":{"1min":95.682000000000002,"5min":95.682000000000002,"15min":95.682000000000002},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:39:05 2023","interfaces":[{"interface":"back","average":{"1min":19.452000000000002,"5min":13.77,"15min":7.8070000000000004},"min":{"1min":0.34399999999999997,"5min":0.34200000000000003,"15min":0.28899999999999998},"max":{"1min":248.691,"5min":277.57799999999997,"15min":277.57799999999997},"last":0.53600000000000003},{"interface":"front","average":{"1min":19.899999999999999,"5min":14.314,"15min":7.2300000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":249.36500000000001,"5min":278.69299999999998,"15min":278.69299999999998},"last":1.1830000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:38:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.75600000000000001,"15min":0.95099999999999996},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23000000000000001},"max":{"1min":2.9260000000000002,"5min":6.9029999999999996,"15min":8.9329999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.66400000000000003,"15min":0.876},"min":{"1min":0.252,"5min":0.252,"15min":0.251},"max":{"1min":0.65100000000000002,"5min":5.6319999999999997,"15min":7.8890000000000002},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:38:25 2023","interfaces":[{"interface":"back","average":{"1min":10.090999999999999,"5min":2.9319999999999999,"15min":1.752},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":188.41,"5min":188.41,"15min":188.41},"last":0.56100000000000005},{"interface":"front","average":{"1min":10.218,"5min":2.8410000000000002,"15min":1.7829999999999999},"min":{"1min":0.40100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":188.43000000000001,"5min":188.43000000000001,"15min":188.43000000000001},"last":0.63200000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:38:25 2023","interfaces":[{"interface":"back","average":{"1min":11.065,"5min":3.2490000000000001,"15min":1.8460000000000001},"min":{"1min":0.40799999999999997,"5min":0.33200000000000002,"15min":0.32400000000000001},"max":{"1min":206.244,"5min":206.244,"15min":206.244},"last":0.39200000000000002},{"interface":"front","average":{"1min":10.643000000000001,"5min":3.2799999999999998,"15min":1.8069999999999999},"min":{"1min":0.38,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":202.78800000000001,"5min":202.78800000000001,"15min":202.78800000000001},"last":0.65000000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495970,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2253940,"kb_used_data":1809516,"kb_used_omap":0,"kb_used_meta":444416,"kb_avail":91491212,"statfs":{"total":95995035648,"available":93687001088,"internally_reserved":0,"allocated":1852944384,"data_stored":1851658643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":455081984},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:39:01 2023","interfaces":[{"interface":"back","average":{"1min":14.414999999999999,"5min":4.8410000000000002,"15min":2.778},"min":{"1min":0.27000000000000002,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":138.95699999999999,"5min":138.95699999999999,"15min":138.95699999999999},"last":187.19800000000001},{"interface":"front","average":{"1min":13.981999999999999,"5min":4.5359999999999996,"15min":2.9750000000000001},"min":{"1min":0.29799999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":138.94399999999999,"5min":138.94399999999999,"15min":138.94399999999999},"last":187.46700000000001}]},{"osd":1,"last update":"Sat Sep 30 09:38:39 2023","interfaces":[{"interface":"back","average":{"1min":4.8029999999999999,"5min":3.1909999999999998,"15min":2.2650000000000001},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":69.569999999999993,"5min":69.569999999999993,"15min":69.569999999999993},"last":0.27000000000000002},{"interface":"front","average":{"1min":4.7640000000000002,"5min":3.3919999999999999,"15min":2.5059999999999998},"min":{"1min":0.33600000000000002,"5min":0.313,"15min":0.313},"max":{"1min":70.471999999999994,"5min":70.471999999999994,"15min":70.471999999999994},"last":3.2890000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:38:32 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":2.0369999999999999,"15min":2.0129999999999999},"min":{"1min":0.33800000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.6200000000000001,"5min":17.126000000000001,"15min":37.978000000000002},"last":1.3600000000000001},{"interface":"front","average":{"1min":1.4099999999999999,"5min":2.367,"15min":1.9510000000000001},"min":{"1min":0.36899999999999999,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":5.6619999999999999,"5min":25.164999999999999,"15min":37.759999999999998},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:39:01 2023","interfaces":[{"interface":"back","average":{"1min":5.7480000000000002,"5min":6.8769999999999998,"15min":3.9340000000000002},"min":{"1min":0.42299999999999999,"5min":0.34699999999999998,"15min":0.27600000000000002},"max":{"1min":43.984999999999999,"5min":212.44800000000001,"15min":212.44800000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":5.4720000000000004,"5min":7.407,"15min":4.2409999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.313},"max":{"1min":42.899000000000001,"5min":210.84299999999999,"15min":210.84299999999999},"last":3.367}]},{"osd":4,"last update":"Sat Sep 30 09:39:01 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.78700000000000003,"15min":1.6439999999999999},"min":{"1min":0.29799999999999999,"5min":0.26400000000000001,"15min":0.23400000000000001},"max":{"1min":3.1120000000000001,"5min":7.234,"15min":73.656000000000006},"last":8.4779999999999998},{"interface":"front","average":{"1min":1.2769999999999999,"5min":0.89600000000000002,"15min":1.4630000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23000000000000001},"max":{"1min":4.6859999999999999,"5min":9.6590000000000007,"15min":73.882999999999996},"last":0.376}]},{"osd":5,"last update":"Sat Sep 30 09:39:01 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.056,"15min":1.9199999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.2149999999999999,"5min":11.428000000000001,"15min":76.613},"last":1.397},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.0129999999999999,"15min":1.8600000000000001},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.6210000000000004,"5min":9.7750000000000004,"15min":76.329999999999998},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:38:32 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":2.4689999999999999,"15min":2.6589999999999998},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":2.649,"5min":138.191,"15min":138.191},"last":3.3460000000000001},{"interface":"front","average":{"1min":0.751,"5min":2.3610000000000002,"15min":2.5910000000000002},"min":{"1min":0.38400000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.1219999999999999,"5min":140.33500000000001,"15min":140.33500000000001},"last":0.41399999999999998}]}]},{"osd":1,"up_from":301,"seq":1292785156167,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2648052,"kb_used_data":2255916,"kb_used_omap":9,"kb_used_meta":392118,"kb_avail":91097100,"statfs":{"total":95995035648,"available":93283430400,"internally_reserved":0,"allocated":2310057984,"data_stored":2307909693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":401528850},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":4.4829999999999997,"15min":3.1890000000000001},"min":{"1min":0.20899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":14.992000000000001,"5min":138.43799999999999,"15min":138.43799999999999},"last":51.490000000000002},{"interface":"front","average":{"1min":1.462,"5min":4.266,"15min":3.4380000000000002},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":15.029,"5min":138.482,"15min":138.482},"last":51.432000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":2.6869999999999998,"15min":2.7589999999999999},"min":{"1min":0.25700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.597,"5min":16.437000000000001,"15min":16.437000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.603,"5min":3.431,"15min":2.9470000000000001},"min":{"1min":0.27800000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":15.063000000000001,"5min":59.078000000000003,"15min":59.078000000000003},"last":0.314}]},{"osd":3,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":7.6390000000000002,"15min":8.8439999999999994},"min":{"1min":0.32400000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":19.733000000000001,"5min":191.07300000000001,"15min":191.07300000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.837,"5min":7.3010000000000002,"15min":9.218},"min":{"1min":0.32000000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":15.192,"5min":191.11500000000001,"15min":191.11500000000001},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.9419999999999999,"15min":1.972},"min":{"1min":0.41699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":15.138,"5min":15.138,"15min":15.138},"last":2.3809999999999998},{"interface":"front","average":{"1min":2.2759999999999998,"5min":2.629,"15min":1.738},"min":{"1min":0.312,"5min":0.311,"15min":0.311},"max":{"1min":20.966000000000001,"5min":58.939999999999998,"15min":58.939999999999998},"last":2.3340000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":2.04,"5min":2.528,"15min":2.2480000000000002},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":23.257999999999999,"5min":24.25,"15min":24.25},"last":3.9329999999999998},{"interface":"front","average":{"1min":1.9610000000000001,"5min":3.0350000000000001,"15min":2.0790000000000002},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.385999999999999,"5min":58.999000000000002,"15min":58.999000000000002},"last":0.872}]},{"osd":6,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":6.218,"15min":13.471},"min":{"1min":0.46500000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":15.101000000000001,"5min":209.221,"15min":209.221},"last":3.6200000000000001},{"interface":"front","average":{"1min":1.9199999999999999,"5min":6.0730000000000004,"15min":13.368},"min":{"1min":0.54200000000000004,"5min":0.377,"15min":0.377},"max":{"1min":23.204000000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":3.3530000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0150000000000001,"5min":4.4329999999999998,"15min":9.7739999999999991},"min":{"1min":0.498,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":15.587,"5min":165.59399999999999,"15min":165.59399999999999},"last":2.4630000000000001},{"interface":"front","average":{"1min":1.9450000000000001,"5min":4.484,"15min":9.9469999999999992},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":15.226000000000001,"5min":165.833,"15min":165.833},"last":1.0780000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926066,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2833832,"kb_used_data":2358496,"kb_used_omap":5,"kb_used_meta":475322,"kb_avail":90911320,"statfs":{"total":95995035648,"available":93093191680,"internally_reserved":0,"allocated":2415099904,"data_stored":2412788297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":486729981},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:38:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":2.3660000000000001,"15min":2.7869999999999999},"min":{"1min":0.251,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.6000000000000001,"5min":17.966000000000001,"15min":17.966000000000001},"last":2.8159999999999998},{"interface":"front","average":{"1min":1.389,"5min":4.7729999999999997,"15min":7.8739999999999997},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":6.1219999999999999,"5min":138.92400000000001,"15min":138.92400000000001},"last":2.0419999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:38:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":2.6920000000000002,"15min":2.0190000000000001},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":9.5269999999999992,"5min":15.324,"15min":55.823999999999998},"last":2.6099999999999999},{"interface":"front","average":{"1min":1.679,"5min":3.0979999999999999,"15min":2.177},"min":{"1min":0.26800000000000002,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":6.2549999999999999,"5min":40.128999999999998,"15min":55.790999999999997},"last":4.7080000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:38:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":8.7300000000000004,"15min":4.4169999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.221},"max":{"1min":4.5620000000000003,"5min":218.06999999999999,"15min":218.06999999999999},"last":2.1429999999999998},{"interface":"front","average":{"1min":1.2869999999999999,"5min":6.3029999999999999,"15min":3.552},"min":{"1min":0.31900000000000001,"5min":0.27800000000000002,"15min":0.245},"max":{"1min":4.5350000000000001,"5min":218.63200000000001,"15min":218.63200000000001},"last":2.7280000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.883,"15min":1.806},"min":{"1min":0.35299999999999998,"5min":0.33200000000000002,"15min":0.29099999999999998},"max":{"1min":6.2210000000000001,"5min":10.614000000000001,"15min":121.486},"last":2.1150000000000002},{"interface":"front","average":{"1min":1.1970000000000001,"5min":4.0599999999999996,"15min":2.7930000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":4.7599999999999998,"5min":136.262,"15min":136.262},"last":2.093}]},{"osd":5,"last update":"Sat Sep 30 09:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":4.1920000000000002,"15min":3.21},"min":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.6100000000000003,"5min":136.34,"15min":136.34},"last":4.3179999999999996},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.778,"15min":2.3319999999999999},"min":{"1min":0.41999999999999998,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":3.875,"5min":12.717000000000001,"15min":135.381},"last":3.968}]},{"osd":6,"last update":"Sat Sep 30 09:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":6.0410000000000004,"15min":3.2949999999999999},"min":{"1min":0.42899999999999999,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":3.5419999999999998,"5min":214.785,"15min":214.785},"last":8.0329999999999995},{"interface":"front","average":{"1min":1.4490000000000001,"5min":6.3360000000000003,"15min":3.335},"min":{"1min":0.44500000000000001,"5min":0.40600000000000003,"15min":0.34200000000000003},"max":{"1min":6.1719999999999997,"5min":214.93899999999999,"15min":214.93899999999999},"last":7.9779999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":2.9889999999999999,"15min":3.2530000000000001},"min":{"1min":0.48999999999999999,"5min":0.42999999999999999,"15min":0.373},"max":{"1min":4.5469999999999997,"5min":97.382000000000005,"15min":150.96899999999999},"last":2.7799999999999998},{"interface":"front","average":{"1min":1.395,"5min":3.1890000000000001,"15min":3.1949999999999998},"min":{"1min":0.47799999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.2789999999999999,"5min":97.015000000000001,"15min":150.328},"last":2.181}]}]},{"osd":2,"up_from":104,"seq":446676599008,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2844284,"kb_used_data":2410676,"kb_used_omap":5,"kb_used_meta":433594,"kb_avail":90900868,"statfs":{"total":95995035648,"available":93082488832,"internally_reserved":0,"allocated":2468532224,"data_stored":2466433312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":444000909},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":2.7890000000000001,"15min":2.2450000000000001},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.187},"max":{"1min":1.728,"5min":77.379000000000005,"15min":77.379000000000005},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.3160000000000001,"5min":2.8599999999999999,"15min":2.1379999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19700000000000001},"max":{"1min":15.548999999999999,"5min":76.796999999999997,"15min":77.066999999999993},"last":3.645}]},{"osd":1,"last update":"Sat Sep 30 09:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":4.4530000000000003,"15min":3.8650000000000002},"min":{"1min":0.27600000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.5350000000000001,"5min":64.412999999999997,"15min":64.412999999999997},"last":3.5609999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":3.8879999999999999,"15min":3.3340000000000001},"min":{"1min":0.29999999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.2679999999999998,"5min":63.247999999999998,"15min":63.247999999999998},"last":3.7970000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:38:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":7.2519999999999998,"15min":3.762},"min":{"1min":0.313,"5min":0.30599999999999999,"15min":0.28000000000000003},"max":{"1min":5.0289999999999999,"5min":134.078,"15min":134.078},"last":3.899},{"interface":"front","average":{"1min":1.0920000000000001,"5min":6.9560000000000004,"15min":3.8690000000000002},"min":{"1min":0.375,"5min":0.28899999999999998,"15min":0.22500000000000001},"max":{"1min":4.6740000000000004,"5min":135.73400000000001,"15min":135.73400000000001},"last":3.4249999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":2.0880000000000001,"15min":1.415},"min":{"1min":0.37,"5min":0.34499999999999997,"15min":0.28999999999999998},"max":{"1min":4.4630000000000001,"5min":35.972000000000001,"15min":35.972000000000001},"last":3.9889999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.8009999999999999,"15min":1.6200000000000001},"min":{"1min":0.378,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":0.90000000000000002,"5min":36.518000000000001,"15min":36.518000000000001},"last":3.4860000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":2.254,"15min":2.2440000000000002},"min":{"1min":0.41999999999999998,"5min":0.377,"15min":0.34999999999999998},"max":{"1min":3.3679999999999999,"5min":12.651999999999999,"15min":155.477},"last":5.1059999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.655,"15min":2.0270000000000001},"min":{"1min":0.40500000000000003,"5min":0.35899999999999999,"15min":0.33100000000000002},"max":{"1min":4.4080000000000004,"5min":8.0839999999999996,"15min":155.46000000000001},"last":3.9609999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":3.661,"15min":3.3620000000000001},"min":{"1min":0.374,"5min":0.35499999999999998,"15min":0.33100000000000002},"max":{"1min":0.89200000000000002,"5min":98.519000000000005,"15min":154.417},"last":3.5419999999999998},{"interface":"front","average":{"1min":0.96599999999999997,"5min":4.5350000000000001,"15min":3.4769999999999999},"min":{"1min":0.41099999999999998,"5min":0.39000000000000001,"15min":0.36899999999999999},"max":{"1min":3.411,"5min":98.293000000000006,"15min":154.04400000000001},"last":3.6150000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":2.0859999999999999,"15min":3.1779999999999999},"min":{"1min":0.39400000000000002,"5min":0.39000000000000001,"15min":0.35899999999999999},"max":{"1min":0.93200000000000005,"5min":47.677999999999997,"15min":213.404},"last":3.589},{"interface":"front","average":{"1min":0.95799999999999996,"5min":2.754,"15min":3.2970000000000002},"min":{"1min":0.45800000000000002,"5min":0.438,"15min":0.40000000000000002},"max":{"1min":3.1699999999999999,"5min":50.725999999999999,"15min":213.27199999999999},"last":5.0800000000000001}]}]},{"osd":3,"up_from":0,"seq":468151435480,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019243,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2188344,"kb_used_data":1881648,"kb_used_omap":0,"kb_used_meta":306688,"kb_avail":91556808,"statfs":{"total":95995035648,"available":93754171392,"internally_reserved":0,"allocated":1926807552,"data_stored":1925517830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":314048512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:44 2023","interfaces":[{"interface":"back","average":{"1min":6.6360000000000001,"5min":3.081,"15min":3.4700000000000002},"min":{"1min":0.315,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":105.179,"5min":105.179,"15min":128.79300000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":6.3959999999999999,"5min":3.2050000000000001,"15min":3.3980000000000001},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.248},"max":{"1min":104.70999999999999,"5min":104.70999999999999,"15min":128.38800000000001},"last":1.3580000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:38:44 2023","interfaces":[{"interface":"back","average":{"1min":4.7320000000000002,"5min":4.8339999999999996,"15min":7.9470000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":84.105999999999995,"5min":160.69999999999999,"15min":160.69999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":4.7220000000000004,"5min":4.4820000000000002,"15min":7.8700000000000001},"min":{"1min":0.35099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":83.941000000000003,"5min":160.50700000000001,"15min":160.50700000000001},"last":2.419}]},{"osd":2,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.7849999999999999,"15min":3.0259999999999998},"min":{"1min":0.41199999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.1520000000000001,"5min":11.096,"15min":140.601},"last":1.931},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.9370000000000001,"15min":3.0510000000000002},"min":{"1min":0.434,"5min":0.34200000000000003,"15min":0.309},"max":{"1min":4.1390000000000002,"5min":11.057,"15min":140.54900000000001},"last":2.335}]},{"osd":3,"last update":"Sat Sep 30 09:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":3.2250000000000001,"15min":3.8330000000000002},"min":{"1min":0.377,"5min":0.315,"15min":0.26400000000000001},"max":{"1min":13.84,"5min":52.762,"15min":281.42200000000003},"last":2.3700000000000001},{"interface":"front","average":{"1min":1.3029999999999999,"5min":3.2069999999999999,"15min":3.7799999999999998},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.312},"max":{"1min":13.776999999999999,"5min":52.790999999999997,"15min":281.44999999999999},"last":1.3}]},{"osd":5,"last update":"Sat Sep 30 09:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":2.6179999999999999,"15min":2.3490000000000002},"min":{"1min":0.29299999999999998,"5min":0.219,"15min":0.219},"max":{"1min":1.4410000000000001,"5min":172.97800000000001,"15min":172.97800000000001},"last":1.5509999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.5870000000000002,"15min":2.395},"min":{"1min":0.36399999999999999,"5min":0.26200000000000001,"15min":0.26100000000000001},"max":{"1min":1.4410000000000001,"5min":172.92400000000001,"15min":172.92400000000001},"last":2.444}]},{"osd":6,"last update":"Sat Sep 30 09:39:06 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":2.8479999999999999,"15min":2.2530000000000001},"min":{"1min":0.34100000000000003,"5min":0.30299999999999999,"15min":0.23799999999999999},"max":{"1min":3.8050000000000002,"5min":182.10300000000001,"15min":202.72499999999999},"last":2.395},{"interface":"front","average":{"1min":0.75900000000000001,"5min":2.7040000000000002,"15min":2.2240000000000002},"min":{"1min":0.377,"5min":0.30199999999999999,"15min":0.249},"max":{"1min":2.8239999999999998,"5min":181.76300000000001,"15min":202.42400000000001},"last":1.333}]},{"osd":7,"last update":"Sat Sep 30 09:38:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.837,"15min":3.0630000000000002},"min":{"1min":0.35499999999999998,"5min":0.314,"15min":0.30299999999999999},"max":{"1min":2.948,"5min":85.114000000000004,"15min":230.90100000000001},"last":2.1640000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.6599999999999999,"15min":2.5960000000000001},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":0.84499999999999997,"5min":84.697999999999993,"15min":229.97},"last":2.528}]}]},{"osd":5,"up_from":38,"seq":163208757607,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3294644,"kb_used_data":2928172,"kb_used_omap":0,"kb_used_meta":366464,"kb_avail":90450508,"statfs":{"total":95995035648,"available":92621320192,"internally_reserved":0,"allocated":2998448128,"data_stored":2997159028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":375259136},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:38:54 2023","interfaces":[{"interface":"back","average":{"1min":13.266999999999999,"5min":4.4210000000000003,"15min":3.3610000000000002},"min":{"1min":0.26200000000000001,"5min":0.223,"15min":0.22},"max":{"1min":98.548000000000002,"5min":98.548000000000002,"15min":148.79900000000001},"last":4.1130000000000004},{"interface":"front","average":{"1min":13.592000000000001,"5min":4.6609999999999996,"15min":3.6619999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":98.388000000000005,"5min":98.388000000000005,"15min":148.779},"last":4.1879999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:38:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.7930000000000001,"15min":2.1560000000000001},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.0640000000000001,"5min":60.799999999999997,"15min":60.799999999999997},"last":3.4910000000000001},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3,"15min":3.1080000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7120000000000002,"5min":63.798999999999999,"15min":63.798999999999999},"last":9.5860000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:38:27 2023","interfaces":[{"interface":"back","average":{"1min":15.347,"5min":4.8319999999999999,"15min":4.1379999999999999},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.27700000000000002},"max":{"1min":227.18000000000001,"5min":227.18000000000001,"15min":227.18000000000001},"last":4.2460000000000004},{"interface":"front","average":{"1min":15.637,"5min":5.0730000000000004,"15min":4.7199999999999998},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":227.46600000000001,"5min":227.46600000000001,"15min":227.46600000000001},"last":0.27400000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:39:02 2023","interfaces":[{"interface":"back","average":{"1min":14.996,"5min":5.6980000000000004,"15min":4.9379999999999997},"min":{"1min":0.32800000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":154.56700000000001,"5min":154.56700000000001,"15min":226.34700000000001},"last":9.5579999999999998},{"interface":"front","average":{"1min":14.015000000000001,"5min":5.452,"15min":4.7199999999999998},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.313},"max":{"1min":154.58000000000001,"5min":154.58000000000001,"15min":226.36000000000001},"last":8.9320000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.67700000000000005,"15min":0.85899999999999999},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.222},"max":{"1min":1.415,"5min":5.7160000000000002,"15min":8.5359999999999996},"last":3.399},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.77500000000000002,"15min":0.96399999999999997},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.6299999999999999,"5min":5.1580000000000004,"15min":9.9640000000000004},"last":3.907}]},{"osd":6,"last update":"Sat Sep 30 09:39:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.91100000000000003,"15min":1.774},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.25900000000000001},"max":{"1min":4.0410000000000004,"5min":5.7539999999999996,"15min":155.602},"last":4.0309999999999997},{"interface":"front","average":{"1min":2.0550000000000002,"5min":1.0840000000000001,"15min":1.927},"min":{"1min":0.38,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":10.880000000000001,"5min":10.880000000000001,"15min":154.00800000000001},"last":5.2549999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:39:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1509999999999998,"5min":1.0760000000000001,"15min":1.734},"min":{"1min":0.34499999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.606,"5min":11.606,"15min":133.40700000000001},"last":4.2160000000000002},{"interface":"front","average":{"1min":0.75,"5min":0.90800000000000003,"15min":1.5700000000000001},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":5.7350000000000003,"5min":5.7880000000000003,"15min":132.797},"last":3.4420000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2405761024,"data_stored":2405761024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2317877248,"data_stored":2317877248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2490892288,"data_stored":2490892288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1339424768,"data_stored":1339424768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1929904128,"data_stored":1929904128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3010985984,"data_stored":3010985984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1864433664,"data_stored":1864433664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1204748288,"data_stored":1204748288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:39:28.024 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 26 2023-09-30T09:39:28.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:27 smithi167 ceph-mon[192880]: from='client.35889 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:39:28.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:27 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:28.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:27 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:39:28.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:27 smithi167 ceph-mon[192880]: osdmap e383: 8 total, 8 up, 7 in 2023-09-30T09:39:28.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:27 smithi167 ceph-mon[192880]: failed to encode map e383 with expected crc 2023-09-30T09:39:28.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:27 smithi167 ceph-mon[192880]: failed to encode map e383 with expected crc 2023-09-30T09:39:28.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:27 smithi167 ceph-mon[192880]: failed to encode map e383 with expected crc 2023-09-30T09:39:28.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:27 smithi167 ceph-mon[192880]: failed to encode map e383 with expected crc 2023-09-30T09:39:28.096 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:28.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:28.205 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:28.269 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:28.383 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:28.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:28.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:28.499 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:28.550 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:28.551 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 83 16 87141 87125 65.5704 59 0.00790512 0.0152459 2023-09-30T09:39:28.784 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:28.787 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:28 smithi019 ceph-mon[216815]: pgmap v998: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 696 op/s; 19115/251292 objects degraded (7.607%); 3761/251292 objects misplaced (1.497%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:39:28.787 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 19115/251292 objects degraded (7.607%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:28.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:28 smithi019 ceph-mon[224043]: pgmap v998: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 696 op/s; 19115/251292 objects degraded (7.607%); 3761/251292 objects misplaced (1.497%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:39:28.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 19115/251292 objects degraded (7.607%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:28.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:28.894 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:28 smithi167 ceph-mon[192880]: pgmap v998: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 696 op/s; 19115/251292 objects degraded (7.607%); 3761/251292 objects misplaced (1.497%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:39:29.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 19115/251292 objects degraded (7.607%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:39:29.171 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:29.231 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:29.269 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:29.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:29.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:29.344 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:39:29.377 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:29.378 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:39:29.410 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:39:29.442 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:29.443 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:39:29.473 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:39:29.502 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:39:29.544 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:39:29.550 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 84 16 88132 88116 65.5271 61.9375 0.00992915 0.015256 2023-09-30T09:39:29.557 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:29.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:29.590 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:39:29.590 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:39:29.590 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:39:29.591 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:39:29.591 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:39:29.591 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:39:29.591 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:39:29.591 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:39:29.591 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:39:29.666 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:29.741 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:29.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:29 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/24241912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:29.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/24241912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:30.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:29 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/24241912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:30.499 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:39:30.499 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:39:30.499 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:39:30.499 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:39:30.551 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 85 16 88896 88880 65.318 47.75 0.0162832 0.0152804 2023-09-30T09:39:30.807 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:39:30.897 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:30.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:30.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:30.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:30 smithi019 ceph-mon[224043]: pgmap v999: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 19115/259779 objects degraded (7.358%); 2969/259779 objects misplaced (1.143%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:30.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:30 smithi019 ceph-mon[216815]: pgmap v999: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 19115/259779 objects degraded (7.358%); 2969/259779 objects misplaced (1.143%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:31.003 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:31.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:30 smithi167 ceph-mon[192880]: pgmap v999: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 19115/259779 objects degraded (7.358%); 2969/259779 objects misplaced (1.143%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:31.086 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:31.143 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:31.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:31.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:31.247 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:31.274 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:31.551 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 86 16 89675 89659 65.1246 48.6875 0.0331891 0.0153492 2023-09-30T09:39:31.675 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:31.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:31.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:31.780 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:31.856 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:31.861 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:31.930 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:31.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:31.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:32.034 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:32.110 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:32.311 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:32.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:32.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:32.421 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:32.524 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:39:32.531 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:32.551 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 87 16 90391 90375 64.8905 44.75 0.0282701 0.0154036 2023-09-30T09:39:32.741 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:32.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:32.763 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:32 smithi167 ceph-mon[192880]: pgmap v1000: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 19115/264771 objects degraded (7.219%); 2969/264771 objects misplaced (1.121%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:32.763 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:39:32.864 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:32.908 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:32.969 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:32.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:32.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:33.076 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:33.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:32 smithi019 ceph-mon[216815]: pgmap v1000: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 19115/264771 objects degraded (7.219%); 2969/264771 objects misplaced (1.121%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:33.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:39:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:32 smithi019 ceph-mon[224043]: pgmap v1000: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 19115/264771 objects degraded (7.219%); 2969/264771 objects misplaced (1.121%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:39:33.144 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:33.175 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:39:33.302 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:33.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:33.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:33.410 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:33.413 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:39:33.484 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:33.551 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 88 16 91053 91037 64.6234 41.375 0.0261179 0.0154683 2023-09-30T09:39:33.717 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:33.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:33.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:33.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:33 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1953557069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-30T09:39:33.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:33 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-30T09:39:33.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:33.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 19115/264771 objects degraded (7.219%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:33.746 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 8 pgp_num to 26 2023-09-30T09:39:33.822 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:33.901 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:33 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1953557069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-30T09:39:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:33 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-30T09:39:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 19115/264771 objects degraded (7.219%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:34.025 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:39:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:39:33] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-09-30T09:39:34.025 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:33 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1953557069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-30T09:39:34.026 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:33 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-30T09:39:34.026 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:34.026 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 19115/264771 objects degraded (7.219%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:34.352 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:34.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:34.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:34.459 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:34.529 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:34.551 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 89 16 91864 91848 64.4668 50.6875 0.00999749 0.0155066 2023-09-30T09:39:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:34 smithi167 ceph-mon[192880]: pgmap v1001: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 19115/264771 objects degraded (7.219%); 2969/264771 objects misplaced (1.121%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:34 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2023-09-30T09:39:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:34 smithi167 ceph-mon[192880]: osdmap e384: 8 total, 8 up, 7 in 2023-09-30T09:39:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:34 smithi167 ceph-mon[192880]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:34 smithi167 ceph-mon[192880]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:34 smithi167 ceph-mon[192880]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:34 smithi167 ceph-mon[192880]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[224043]: pgmap v1001: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 19115/264771 objects degraded (7.219%); 2969/264771 objects misplaced (1.121%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2023-09-30T09:39:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[224043]: osdmap e384: 8 total, 8 up, 7 in 2023-09-30T09:39:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[224043]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[224043]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[224043]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[224043]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[216815]: pgmap v1001: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 19115/264771 objects degraded (7.219%); 2969/264771 objects misplaced (1.121%); 9.0 MiB/s, 144 objects/s recovering 2023-09-30T09:39:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2023-09-30T09:39:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[216815]: osdmap e384: 8 total, 8 up, 7 in 2023-09-30T09:39:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[216815]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[216815]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[216815]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:34 smithi019 ceph-mon[216815]: failed to encode map e384 with expected crc 2023-09-30T09:39:35.179 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:35.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:35.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:35.285 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:35.341 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:35.528 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:35.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:35.551 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total time run: 90.0166 2023-09-30T09:39:35.551 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total writes made: 92445 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Write size: 65536 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object size: 65536 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Bandwidth (MB/sec): 64.1861 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Bandwidth: 35.5615 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max bandwidth (MB/sec): 195.5 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min bandwidth (MB/sec): 0 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average IOPS: 1026 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev IOPS: 568.997 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max IOPS: 3128 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min IOPS: 0 2023-09-30T09:39:35.552 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average Latency(s): 0.0155752 2023-09-30T09:39:35.553 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Latency(s): 0.0511874 2023-09-30T09:39:35.553 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max latency(s): 3.13734 2023-09-30T09:39:35.553 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min latency(s): 0.0012782 2023-09-30T09:39:35.559 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Cleaning up (deleting benchmark objects) 2023-09-30T09:39:35.638 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:35.717 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:35.727 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:35.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2613742558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:35.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2613742558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:35.755 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:39:35.870 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:35.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:35.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:35.987 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:36.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2613742558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:36.034 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:36.061 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:39:36.061 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:39:36.061 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:39:36.062 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:39:36.062 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:39:36.564 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:36.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:36.671 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:36.762 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:36.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: pgmap v1003: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 18084/273219 objects degraded (6.619%); 3249/273219 objects misplaced (1.189%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:39:36.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:39:36.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2613742558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:36.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: osdmap e385: 8 total, 8 up, 7 in 2023-09-30T09:39:36.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[224043]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: pgmap v1003: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 18084/273219 objects degraded (6.619%); 3249/273219 objects misplaced (1.189%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:39:36.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:39:36.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2613742558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:36.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: osdmap e385: 8 total, 8 up, 7 in 2023-09-30T09:39:36.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: failed to encode map e385 with expected crc 2023-09-30T09:39:36.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:36 smithi019 ceph-mon[216815]: failed to encode map e385 with expected crc 2023-09-30T09:39:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: pgmap v1003: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 18084/273219 objects degraded (6.619%); 3249/273219 objects misplaced (1.189%); 11 MiB/s, 172 objects/s recovering 2023-09-30T09:39:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:39:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2613742558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: osdmap e385: 8 total, 8 up, 7 in 2023-09-30T09:39:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: failed to encode map e385 with expected crc 2023-09-30T09:39:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: failed to encode map e385 with expected crc 2023-09-30T09:39:37.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: failed to encode map e385 with expected crc 2023-09-30T09:39:37.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: failed to encode map e385 with expected crc 2023-09-30T09:39:37.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:36 smithi167 ceph-mon[192880]: failed to encode map e385 with expected crc 2023-09-30T09:39:37.565 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:37.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:37.672 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:37.694 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:39:37.753 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:37.964 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:37.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:38.079 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:38.148 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:38.273 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:38.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:38.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:38.378 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:38.429 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:38.514 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:38.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:38.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:38.598 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:39:38.623 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:38.628 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:39:38.654 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:39:38.660 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:39:38.665 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:38.696 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:39:38.732 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:39:38.766 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:39:38.772 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:38.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:38.810 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:39:38.842 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:39:38.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:39:38.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:39:38.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:39:38.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:39:38.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:39:38.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:39:38.846 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:39:38.846 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:39:38.877 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:38.921 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:38 smithi019 ceph-mon[224043]: pgmap v1005: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 669 op/s; 18084/275787 objects degraded (6.557%); 3249/275787 objects misplaced (1.178%); 5.9 MiB/s, 93 objects/s recovering 2023-09-30T09:39:38.921 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18084/275787 objects degraded (6.557%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:38.921 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/792329175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:38.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:38 smithi019 ceph-mon[216815]: pgmap v1005: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 669 op/s; 18084/275787 objects degraded (6.557%); 3249/275787 objects misplaced (1.178%); 5.9 MiB/s, 93 objects/s recovering 2023-09-30T09:39:38.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18084/275787 objects degraded (6.557%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:38.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/792329175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:38.925 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:38 smithi167 ceph-mon[192880]: pgmap v1005: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 669 op/s; 18084/275787 objects degraded (6.557%); 3249/275787 objects misplaced (1.178%); 5.9 MiB/s, 93 objects/s recovering 2023-09-30T09:39:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18084/275787 objects degraded (6.557%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:39.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/792329175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:39.037 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:39:39.794 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:39.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:39.900 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:39.963 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:40.072 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:40.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:40.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:40.183 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:40.223 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:40.234 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:40.445 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:40.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:40.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:40.548 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 6, 0, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:39:40.548 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:39:40.548 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [1, 4, 5, 6, 0, 2, 7] 2023-09-30T09:39:40.548 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-30T09:39:40.554 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:40.630 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:41.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:40 smithi167 ceph-mon[192880]: pgmap v1006: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 33 MiB/s wr, 741 op/s; 17676/272256 objects degraded (6.492%); 2451/272256 objects misplaced (0.900%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:39:41.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:40 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:41.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:40 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:40 smithi019 ceph-mon[216815]: pgmap v1006: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 33 MiB/s wr, 741 op/s; 17676/272256 objects degraded (6.492%); 2451/272256 objects misplaced (0.900%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:39:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:40 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:40 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:40 smithi019 ceph-mon[224043]: pgmap v1006: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 33 MiB/s wr, 741 op/s; 17676/272256 objects degraded (6.492%); 2451/272256 objects misplaced (0.900%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:39:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:40 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:40 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:41.977 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:39:41.986 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:41.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:42.093 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:42.114 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:42.171 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:42.302 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:42.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:42.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:42.409 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:42.470 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:42.663 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:42.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:42.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:42.770 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:42.823 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:39:42.824 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:39:42.891 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:43.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[224043]: pgmap v1007: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 33 MiB/s wr, 910 op/s; 17676/268218 objects degraded (6.590%); 2451/268218 objects misplaced (0.914%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:39:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[224043]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[224043]: osdmap e386: 8 total, 8 up, 7 in 2023-09-30T09:39:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[224043]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[224043]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[216815]: pgmap v1007: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 33 MiB/s wr, 910 op/s; 17676/268218 objects degraded (6.590%); 2451/268218 objects misplaced (0.914%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:39:43.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[216815]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[216815]: osdmap e386: 8 total, 8 up, 7 in 2023-09-30T09:39:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[216815]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:42 smithi019 ceph-mon[216815]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.121 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:43.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:43.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:43.225 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:43.267 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:43.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:42 smithi167 ceph-mon[192880]: pgmap v1007: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 33 MiB/s wr, 910 op/s; 17676/268218 objects degraded (6.590%); 2451/268218 objects misplaced (0.914%); 15 MiB/s, 244 objects/s recovering 2023-09-30T09:39:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:42 smithi167 ceph-mon[192880]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:42 smithi167 ceph-mon[192880]: osdmap e386: 8 total, 8 up, 7 in 2023-09-30T09:39:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:42 smithi167 ceph-mon[192880]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:42 smithi167 ceph-mon[192880]: failed to encode map e386 with expected crc 2023-09-30T09:39:43.357 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:43.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:43.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:43.470 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:43.524 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:43.797 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:39:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[216815]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[216815]: osdmap e387: 8 total, 8 up, 7 in 2023-09-30T09:39:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[216815]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[216815]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[216815]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[216815]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3237360023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 17676/268218 objects degraded (6.590%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[224043]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[224043]: osdmap e387: 8 total, 8 up, 7 in 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[224043]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[224043]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[224043]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[224043]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3237360023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 17676/268218 objects degraded (6.590%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:44.087 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:39:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:39:43] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-09-30T09:39:44.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:43 smithi167 ceph-mon[192880]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:43 smithi167 ceph-mon[192880]: osdmap e387: 8 total, 8 up, 7 in 2023-09-30T09:39:44.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:43 smithi167 ceph-mon[192880]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:43 smithi167 ceph-mon[192880]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:43 smithi167 ceph-mon[192880]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:43 smithi167 ceph-mon[192880]: failed to encode map e387 with expected crc 2023-09-30T09:39:44.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3237360023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:39:44.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 17676/268218 objects degraded (6.590%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:39:44.412 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:44.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:44.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:44.523 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:44.551 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:39:44.559 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:44.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[216815]: pgmap v1010: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 130 B/s rd, 4.2 MiB/s wr, 455 op/s; 17676/268218 objects degraded (6.590%); 2451/268218 objects misplaced (0.914%); 9.6 MiB/s, 153 objects/s recovering 2023-09-30T09:39:44.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:44.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3237360023' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:39:44.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[216815]: failed to encode map e388 with expected crc 2023-09-30T09:39:44.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[216815]: osdmap e388: 8 total, 8 up, 7 in 2023-09-30T09:39:44.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:44.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:44.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[224043]: pgmap v1010: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 130 B/s rd, 4.2 MiB/s wr, 455 op/s; 17676/268218 objects degraded (6.590%); 2451/268218 objects misplaced (0.914%); 9.6 MiB/s, 153 objects/s recovering 2023-09-30T09:39:44.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:44.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3237360023' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:39:44.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[224043]: failed to encode map e388 with expected crc 2023-09-30T09:39:44.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[224043]: osdmap e388: 8 total, 8 up, 7 in 2023-09-30T09:39:44.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:44.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:44 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:44.988 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:44.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:44.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:45.092 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:45.176 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:44 smithi167 ceph-mon[192880]: pgmap v1010: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 130 B/s rd, 4.2 MiB/s wr, 455 op/s; 17676/268218 objects degraded (6.590%); 2451/268218 objects misplaced (0.914%); 9.6 MiB/s, 153 objects/s recovering 2023-09-30T09:39:45.176 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:44 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:45.176 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3237360023' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:39:45.177 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:44 smithi167 ceph-mon[192880]: failed to encode map e388 with expected crc 2023-09-30T09:39:45.177 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:44 smithi167 ceph-mon[192880]: osdmap e388: 8 total, 8 up, 7 in 2023-09-30T09:39:45.177 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:44 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:45.177 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:44 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:45.181 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:45.265 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:39:45.266 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:45.286 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:45.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:45.396 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:45.488 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:45.599 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:45.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:45.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:45.708 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:45.773 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:45.914 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:45 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:39:45.914 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:45 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:39:45.914 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:45 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:45.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:45 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:39:45.915 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:45 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:39:45.915 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:45 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:46.012 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:46.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:45 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:39:46.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:45 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:39:46.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:45 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:39:46.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:46.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:46.125 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:46.178 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:45 smithi019 conmon[216791]: 2023-09-30T09:39:45.909+0000 7faf52971700 -1 mon.a@0(leader).osd e388 definitely_dead 0 2023-09-30T09:39:46.181 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:46.262 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:46.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:46.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:46.365 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:46.393 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:46.439 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:46.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:46.543 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:46.600 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:46.645 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:46.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:46.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:46.748 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:46.800 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:46.834 INFO:teuthology.orchestra.run.smithi019.stderr:marked out osd.6. 2023-09-30T09:39:46.882 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:46.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:46.988 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:47.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:46 smithi167 ceph-mon[192880]: pgmap v1012: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 16944/259926 objects degraded (6.519%); 1975/259926 objects misplaced (0.760%); 13 MiB/s, 201 objects/s recovering 2023-09-30T09:39:47.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2959861654' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-30T09:39:47.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:46 smithi167 ceph-mon[192880]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-30T09:39:47.034 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:47.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:46 smithi019 ceph-mon[216815]: pgmap v1012: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 16944/259926 objects degraded (6.519%); 1975/259926 objects misplaced (0.760%); 13 MiB/s, 201 objects/s recovering 2023-09-30T09:39:47.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2959861654' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-30T09:39:47.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:46 smithi019 ceph-mon[216815]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-30T09:39:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:46 smithi019 ceph-mon[224043]: pgmap v1012: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 16944/259926 objects degraded (6.519%); 1975/259926 objects misplaced (0.760%); 13 MiB/s, 201 objects/s recovering 2023-09-30T09:39:47.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2959861654' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-30T09:39:47.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:46 smithi019 ceph-mon[224043]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-30T09:39:47.139 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:47.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:47.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:47.152 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:39:47.247 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:47.275 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:47.328 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:47.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:47.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:47.436 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:47.467 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:47.741 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:39:47.741 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:39:47.764 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:47.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:47.828 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2959861654' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-30T09:39:47.829 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: osdmap e389: 8 total, 8 up, 6 in 2023-09-30T09:39:47.829 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:47.867 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:47.898 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:47.997 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:48.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:48.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2959861654' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: osdmap e389: 8 total, 8 up, 6 in 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[216815]: osdmap e390: 8 total, 8 up, 6 in 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2959861654' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: osdmap e389: 8 total, 8 up, 6 in 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.033 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.033 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.033 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.033 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:47 smithi019 ceph-mon[224043]: osdmap e390: 8 total, 8 up, 6 in 2023-09-30T09:39:48.105 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:48.141 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:48.194 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:48.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:48.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:48.204 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:39:48.224 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:39:48.245 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:39:48.256 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:48.267 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:39:48.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: failed to encode map e389 with expected crc 2023-09-30T09:39:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:47 smithi167 ceph-mon[192880]: osdmap e390: 8 total, 8 up, 6 in 2023-09-30T09:39:48.290 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:39:48.297 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:48.312 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:39:48.333 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:39:48.336 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:48.352 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:39:48.354 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:39:48.354 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:39:48.354 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:39:48.354 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:39:48.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:39:48.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:39:48.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:39:48.355 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:39:48.386 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:48.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:48.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:48.492 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:48.522 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:48.571 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:48.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:48.676 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:48.706 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:48.756 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:48.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:48.861 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:48.932 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: pgmap v1014: 129 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 14559/257205 objects degraded (5.660%); 4158/257205 objects misplaced (1.617%); 15 MiB/s, 241 objects/s recovering 2023-09-30T09:39:49.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1465068268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 14559/257205 objects degraded (5.660%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:49.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:48 smithi167 ceph-mon[192880]: osdmap e391: 8 total, 8 up, 6 in 2023-09-30T09:39:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: pgmap v1014: 129 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 14559/257205 objects degraded (5.660%); 4158/257205 objects misplaced (1.617%); 15 MiB/s, 241 objects/s recovering 2023-09-30T09:39:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:49.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1465068268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:49.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 14559/257205 objects degraded (5.660%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:49.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[224043]: osdmap e391: 8 total, 8 up, 6 in 2023-09-30T09:39:49.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: pgmap v1014: 129 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 14559/257205 objects degraded (5.660%); 4158/257205 objects misplaced (1.617%); 15 MiB/s, 241 objects/s recovering 2023-09-30T09:39:49.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: failed to encode map e390 with expected crc 2023-09-30T09:39:49.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:39:49.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1465068268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:49.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 14559/257205 objects degraded (5.660%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:49.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:48 smithi019 ceph-mon[216815]: osdmap e391: 8 total, 8 up, 6 in 2023-09-30T09:39:49.274 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:49.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:49.380 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:49.387 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:39:49.412 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:49.465 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:49.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:49.570 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:49.602 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:49.664 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:49.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:49.769 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:49.802 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:49.866 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:49 smithi167 ceph-mon[192880]: failed to encode map e391 with expected crc 2023-09-30T09:39:49.867 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:49 smithi167 ceph-mon[192880]: failed to encode map e391 with expected crc 2023-09-30T09:39:49.867 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:49 smithi167 ceph-mon[192880]: failed to encode map e391 with expected crc 2023-09-30T09:39:49.867 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:49 smithi167 ceph-mon[192880]: failed to encode map e391 with expected crc 2023-09-30T09:39:49.867 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:49 smithi167 ceph-mon[192880]: failed to encode map e391 with expected crc 2023-09-30T09:39:49.867 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:49 smithi167 ceph-mon[192880]: failed to encode map e391 with expected crc 2023-09-30T09:39:49.867 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:49 smithi167 ceph-mon[192880]: osdmap e392: 8 total, 8 up, 6 in 2023-09-30T09:39:49.883 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:39:49.928 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:49.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:49.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:50.036 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:50.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[224043]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[224043]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[224043]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[224043]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[224043]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[224043]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[224043]: osdmap e392: 8 total, 8 up, 6 in 2023-09-30T09:39:50.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[216815]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.062 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[216815]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.062 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[216815]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.062 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[216815]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.062 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[216815]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.062 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[216815]: failed to encode map e391 with expected crc 2023-09-30T09:39:50.062 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:49 smithi019 ceph-mon[216815]: osdmap e392: 8 total, 8 up, 6 in 2023-09-30T09:39:50.063 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:50.369 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:50.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:50.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:50.474 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:50.507 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:50.555 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:50.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:50.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:50.658 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:50.687 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:50.736 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:50.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:50.840 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:50.877 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:50.930 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:50.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:50.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:50 smithi167 ceph-mon[192880]: pgmap v1017: 129 pgs: 2 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 4.8 KiB/s rd, 13 KiB/s wr, 784 op/s; 22412/254562 objects degraded (8.804%); 5172/254562 objects misplaced (2.032%); 25 MiB/s, 397 objects/s recovering 2023-09-30T09:39:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:50 smithi167 ceph-mon[192880]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:50 smithi167 ceph-mon[192880]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:50 smithi167 ceph-mon[192880]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:50 smithi167 ceph-mon[192880]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:50 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-30T09:39:51.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:50 smithi167 ceph-mon[192880]: osdmap e393: 8 total, 8 up, 6 in 2023-09-30T09:39:51.035 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:51.071 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:51.130 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:51.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[224043]: pgmap v1017: 129 pgs: 2 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 4.8 KiB/s rd, 13 KiB/s wr, 784 op/s; 22412/254562 objects degraded (8.804%); 5172/254562 objects misplaced (2.032%); 25 MiB/s, 397 objects/s recovering 2023-09-30T09:39:51.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[224043]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[224043]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[224043]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[224043]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[224043]: osdmap e393: 8 total, 8 up, 6 in 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[216815]: pgmap v1017: 129 pgs: 2 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 4.8 KiB/s rd, 13 KiB/s wr, 784 op/s; 22412/254562 objects degraded (8.804%); 5172/254562 objects misplaced (2.032%); 25 MiB/s, 397 objects/s recovering 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[216815]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[216815]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[216815]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[216815]: failed to encode map e392 with expected crc 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-30T09:39:51.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:50 smithi019 ceph-mon[216815]: osdmap e393: 8 total, 8 up, 6 in 2023-09-30T09:39:51.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:51.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:51.236 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:51.267 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:51.322 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:51.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:51.427 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:51.457 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:51.502 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:51.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:51.606 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:51.637 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:51.688 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:51.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:51.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:51.794 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:51.824 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:51.877 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:51.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:51.889 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:39:51.981 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:52.011 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3688390728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:52.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:51 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:52.108 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:52.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:52.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:52.211 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:52.241 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:52.291 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:52.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:52.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3688390728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:52.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:52.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: failed to encode map e393 with expected crc 2023-09-30T09:39:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3688390728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:51 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:39:52.347 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:39:52.395 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:52.422 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:52.695 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:52.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:52.800 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:52.822 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:52.832 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:52.923 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:52.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:52.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:52 smithi167 ceph-mon[192880]: pgmap v1020: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 375 op/s; 39962/252807 objects degraded (15.807%); 2989/252807 objects misplaced (1.182%); 18 MiB/s, 295 objects/s recovering 2023-09-30T09:39:52.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:52 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:52.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:52.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:52 smithi167 ceph-mon[192880]: failed to encode map e394 with expected crc 2023-09-30T09:39:52.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:52 smithi167 ceph-mon[192880]: osdmap e394: 8 total, 8 up, 6 in 2023-09-30T09:39:53.028 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:53.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[224043]: pgmap v1020: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 375 op/s; 39962/252807 objects degraded (15.807%); 2989/252807 objects misplaced (1.182%); 18 MiB/s, 295 objects/s recovering 2023-09-30T09:39:53.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:53.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:53.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[224043]: failed to encode map e394 with expected crc 2023-09-30T09:39:53.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[224043]: osdmap e394: 8 total, 8 up, 6 in 2023-09-30T09:39:53.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[216815]: pgmap v1020: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 375 op/s; 39962/252807 objects degraded (15.807%); 2989/252807 objects misplaced (1.182%); 18 MiB/s, 295 objects/s recovering 2023-09-30T09:39:53.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:53.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:39:53.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[216815]: failed to encode map e394 with expected crc 2023-09-30T09:39:53.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:52 smithi019 ceph-mon[216815]: osdmap e394: 8 total, 8 up, 6 in 2023-09-30T09:39:53.057 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 0, 2, 7] out_osds: [3, 6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:39:53.057 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:39:53.057 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.5747900230718067 2023-09-30T09:39:53.057 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 0.5747900230718067 2023-09-30T09:39:53.060 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:53.161 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:53.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:53.243 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:39:53.243 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:39:53.243 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:39:53.243 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:39:53.244 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:39:53.265 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:53.297 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:53.352 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:53.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:53.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:53.487 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:53.587 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:53.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:53.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:53.690 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:53.718 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:53.877 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:39:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:39:53] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-09-30T09:39:54.058 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:54.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:54.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:54.162 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:54.171 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:54.188 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 39962/252807 objects degraded (15.807%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:54.191 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:54.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 39962/252807 objects degraded (15.807%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 39962/252807 objects degraded (15.807%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:54.357 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:54.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:54.462 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:54.489 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:54.533 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:54.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:54.637 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:54.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:39:54.660 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:39:54.665 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:54.680 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:39:54.718 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:39:54.753 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:39:54.768 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:39:54.773 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:39:54.803 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:39:54.822 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:39:54.824 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:39:54.824 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:39:54.825 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:39:54.825 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:39:54.825 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:39:54.825 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:39:54.825 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:39:54.825 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:39:54.912 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:54.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:54.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:54.958 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:39:55.016 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:55.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:54 smithi019 ceph-mon[224043]: pgmap v1022: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 312 op/s; 39962/252807 objects degraded (15.807%); 2989/252807 objects misplaced (1.182%); 15 MiB/s, 246 objects/s recovering 2023-09-30T09:39:55.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:54 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/549129597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:55.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:54 smithi019 ceph-mon[216815]: pgmap v1022: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 312 op/s; 39962/252807 objects degraded (15.807%); 2989/252807 objects misplaced (1.182%); 15 MiB/s, 246 objects/s recovering 2023-09-30T09:39:55.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:54 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/549129597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:55.049 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:55.105 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:55.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:55.211 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:55.243 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:55.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:54 smithi167 ceph-mon[192880]: pgmap v1022: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 312 op/s; 39962/252807 objects degraded (15.807%); 2989/252807 objects misplaced (1.182%); 15 MiB/s, 246 objects/s recovering 2023-09-30T09:39:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:54 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/549129597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:39:55.353 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:39:55.497 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:55.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:55.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:55.601 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:39:55.636 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:56.336 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:39:56.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:56.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:39:56.440 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:56.476 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:56.476 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:56.545 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:56.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:56.650 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:56.685 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:56.945 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:56.948 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:56 smithi167 ceph-mon[192880]: pgmap v1023: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 3.3 KiB/s rd, 9.2 KiB/s wr, 100 op/s; 39962/252807 objects degraded (15.807%); 1013/252807 objects misplaced (0.401%); 23 MiB/s, 375 objects/s recovering 2023-09-30T09:39:56.948 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:56 smithi167 ceph-mon[192880]: osdmap e395: 8 total, 8 up, 6 in 2023-09-30T09:39:56.948 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:56 smithi167 ceph-mon[192880]: failed to encode map e395 with expected crc 2023-09-30T09:39:56.948 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:56 smithi167 ceph-mon[192880]: failed to encode map e395 with expected crc 2023-09-30T09:39:56.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:56.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:56.982 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:56 smithi019 ceph-mon[224043]: pgmap v1023: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 3.3 KiB/s rd, 9.2 KiB/s wr, 100 op/s; 39962/252807 objects degraded (15.807%); 1013/252807 objects misplaced (0.401%); 23 MiB/s, 375 objects/s recovering 2023-09-30T09:39:56.982 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:56 smithi019 ceph-mon[224043]: osdmap e395: 8 total, 8 up, 6 in 2023-09-30T09:39:56.982 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:56 smithi019 ceph-mon[224043]: failed to encode map e395 with expected crc 2023-09-30T09:39:56.982 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:56 smithi019 ceph-mon[224043]: failed to encode map e395 with expected crc 2023-09-30T09:39:56.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:56 smithi019 ceph-mon[216815]: pgmap v1023: 129 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 3.3 KiB/s rd, 9.2 KiB/s wr, 100 op/s; 39962/252807 objects degraded (15.807%); 1013/252807 objects misplaced (0.401%); 23 MiB/s, 375 objects/s recovering 2023-09-30T09:39:56.984 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:56 smithi019 ceph-mon[216815]: osdmap e395: 8 total, 8 up, 6 in 2023-09-30T09:39:56.984 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:56 smithi019 ceph-mon[216815]: failed to encode map e395 with expected crc 2023-09-30T09:39:56.984 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:56 smithi019 ceph-mon[216815]: failed to encode map e395 with expected crc 2023-09-30T09:39:57.050 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:57.089 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:57.165 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:39:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:57.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:39:57.268 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:39:57.298 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:57.352 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:39:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:57.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:39:57.458 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:57.491 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:57.542 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:57.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:57.578 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:39:57.579 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:39:57.647 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:39:57.698 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:57.768 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:39:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:57.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:39:57.875 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:39:57.905 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:57 smithi167 ceph-mon[192880]: failed to encode map e396 with expected crc 2023-09-30T09:39:57.905 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:57 smithi167 ceph-mon[192880]: osdmap e396: 8 total, 8 up, 6 in 2023-09-30T09:39:57.905 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:57 smithi167 ceph-mon[192880]: failed to encode map e396 with expected crc 2023-09-30T09:39:57.905 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:57 smithi167 ceph-mon[192880]: failed to encode map e396 with expected crc 2023-09-30T09:39:57.905 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:57 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/529539551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]: dispatch 2023-09-30T09:39:57.905 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:57 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]: dispatch 2023-09-30T09:39:57.924 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:57.927 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.0 to 0.57479 (9325) 2023-09-30T09:39:57.969 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:39:57.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:57.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:39:58.074 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:58.099 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:39:58.107 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:58.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[216815]: failed to encode map e396 with expected crc 2023-09-30T09:39:58.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[216815]: osdmap e396: 8 total, 8 up, 6 in 2023-09-30T09:39:58.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[216815]: failed to encode map e396 with expected crc 2023-09-30T09:39:58.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[216815]: failed to encode map e396 with expected crc 2023-09-30T09:39:58.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/529539551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]: dispatch 2023-09-30T09:39:58.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]: dispatch 2023-09-30T09:39:58.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[224043]: failed to encode map e396 with expected crc 2023-09-30T09:39:58.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[224043]: osdmap e396: 8 total, 8 up, 6 in 2023-09-30T09:39:58.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[224043]: failed to encode map e396 with expected crc 2023-09-30T09:39:58.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[224043]: failed to encode map e396 with expected crc 2023-09-30T09:39:58.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/529539551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]: dispatch 2023-09-30T09:39:58.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:57 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]: dispatch 2023-09-30T09:39:58.627 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:58.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:58.726 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:39:58.732 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:39:58.780 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:58.869 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:39:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:58.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:39:58.977 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:39:59.011 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:39:59.018 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: pgmap v1026: 129 pgs: 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 20 op/s; 39916/252435 objects degraded (15.812%); 1013/252435 objects misplaced (0.401%); 21 MiB/s, 332 objects/s recovering 2023-09-30T09:39:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]': finished 2023-09-30T09:39:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: osdmap e397: 8 total, 8 up, 6 in 2023-09-30T09:39:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:59.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:58 smithi167 ceph-mon[192880]: osdmap e398: 8 total, 8 up, 6 in 2023-09-30T09:39:59.087 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:39:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:59.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:39:59.123 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:39:59.193 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:59.223 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:59.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: pgmap v1026: 129 pgs: 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 20 op/s; 39916/252435 objects degraded (15.812%); 1013/252435 objects misplaced (0.401%); 21 MiB/s, 332 objects/s recovering 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]': finished 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: osdmap e397: 8 total, 8 up, 6 in 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[216815]: osdmap e398: 8 total, 8 up, 6 in 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: pgmap v1026: 129 pgs: 2 activating+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 20 op/s; 39916/252435 objects degraded (15.812%); 1013/252435 objects misplaced (0.401%); 21 MiB/s, 332 objects/s recovering 2023-09-30T09:39:59.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5747900230718067}]': finished 2023-09-30T09:39:59.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: osdmap e397: 8 total, 8 up, 6 in 2023-09-30T09:39:59.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: failed to encode map e397 with expected crc 2023-09-30T09:39:59.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:39:59.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:58 smithi019 ceph-mon[224043]: osdmap e398: 8 total, 8 up, 6 in 2023-09-30T09:39:59.320 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:59.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:59.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:59.424 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:39:59.453 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:59.578 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:39:59.682 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:39:59.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:39:59.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:39:59.788 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:39:59.817 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:39:59.904 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: failed to encode map e398 with expected crc 2023-09-30T09:39:59.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: failed to encode map e398 with expected crc 2023-09-30T09:39:59.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: failed to encode map e398 with expected crc 2023-09-30T09:39:59.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: failed to encode map e398 with expected crc 2023-09-30T09:39:59.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: failed to encode map e398 with expected crc 2023-09-30T09:39:59.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1160338798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:39:59.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:39:59.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1160338798' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:39:59.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[216815]: osdmap e399: 8 total, 8 up, 6 in 2023-09-30T09:40:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1160338798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1160338798' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:39:59 smithi167 ceph-mon[192880]: osdmap e399: 8 total, 8 up, 6 in 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: failed to encode map e398 with expected crc 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1160338798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1160338798' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:40:00.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:39:59 smithi019 ceph-mon[224043]: osdmap e399: 8 total, 8 up, 6 in 2023-09-30T09:40:00.526 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:00.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:00.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:00.635 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:00.669 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:00.726 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:00.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:00.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:00.830 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:00.861 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:00.913 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:00.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:00.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:01.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:01.048 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 396'6463 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pgmap v1030: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.9 KiB/s rd, 15 KiB/s wr, 164 op/s; 28041/250911 objects degraded (11.176%); 2583/250911 objects misplaced (1.029%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 2 pgs inactive; Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 1.7 is stuck inactive for 90s, current state activating+undersized+remapped, last acting [7,5] 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 1.f is stuck inactive for 8m, current state activating+undersized+remapped, last acting [5,4] 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: [WRN] PG_DEGRADED: Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized 2023-09-30T09:40:01.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 1.c is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 2.2 is active+recovery_wait+degraded, acting [0,5,1] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 2.6 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 2.7 is active+recovery_wait+degraded, acting [7,2,1] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.0 is active+recovery_wait+degraded, acting [5,1,4] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.3 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.5 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.8 is active+recovery_wait+degraded, acting [5,7,0] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.9 is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.b is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,1] 2023-09-30T09:40:01.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.c is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.d is stuck undersized for 88s, current state active+recovering+undersized+degraded+remapped, last acting [5,2] 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 396'6463 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pgmap v1030: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.9 KiB/s rd, 15 KiB/s wr, 164 op/s; 28041/250911 objects degraded (11.176%); 2583/250911 objects misplaced (1.029%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 2 pgs inactive; Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 1.7 is stuck inactive for 90s, current state activating+undersized+remapped, last acting [7,5] 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 1.f is stuck inactive for 8m, current state activating+undersized+remapped, last acting [5,4] 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: [WRN] PG_DEGRADED: Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 1.c is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 2.2 is active+recovery_wait+degraded, acting [0,5,1] 2023-09-30T09:40:01.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 2.6 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 2.7 is active+recovery_wait+degraded, acting [7,2,1] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.0 is active+recovery_wait+degraded, acting [5,1,4] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.3 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.5 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.8 is active+recovery_wait+degraded, acting [5,7,0] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.9 is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.b is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,1] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.c is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-30T09:40:01.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.d is stuck undersized for 88s, current state active+recovering+undersized+degraded+remapped, last acting [5,2] 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.f is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.13 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.15 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.16 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.17 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: pg 8.19 is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[224043]: osdmap e400: 8 total, 8 up, 6 in 2023-09-30T09:40:01.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.f is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-09-30T09:40:01.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.13 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2023-09-30T09:40:01.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.15 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-30T09:40:01.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.16 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2023-09-30T09:40:01.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.17 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-09-30T09:40:01.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: pg 8.19 is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-30T09:40:01.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:40:01.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:00 smithi019 ceph-mon[216815]: osdmap e400: 8 total, 8 up, 6 in 2023-09-30T09:40:01.171 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: failed to encode map e399 with expected crc 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 396'6463 2023-09-30T09:40:01.172 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pgmap v1030: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.9 KiB/s rd, 15 KiB/s wr, 164 op/s; 28041/250911 objects degraded (11.176%); 2583/250911 objects misplaced (1.029%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 2 pgs inactive; Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 1.7 is stuck inactive for 90s, current state activating+undersized+remapped, last acting [7,5] 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 1.f is stuck inactive for 8m, current state activating+undersized+remapped, last acting [5,4] 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: [WRN] PG_DEGRADED: Degraded data redundancy: 39916/252435 objects degraded (15.812%), 20 pgs degraded, 4 pgs undersized 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 1.c is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 2.2 is active+recovery_wait+degraded, acting [0,5,1] 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 2.6 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 2.7 is active+recovery_wait+degraded, acting [7,2,1] 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.0 is active+recovery_wait+degraded, acting [5,1,4] 2023-09-30T09:40:01.173 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.3 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.5 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.8 is active+recovery_wait+degraded, acting [5,7,0] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.9 is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.b is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,1] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.c is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.d is stuck undersized for 88s, current state active+recovering+undersized+degraded+remapped, last acting [5,2] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.f is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.13 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.15 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-30T09:40:01.174 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.16 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2023-09-30T09:40:01.175 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.17 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-09-30T09:40:01.175 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: pg 8.19 is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-30T09:40:01.175 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:40:01.175 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:00 smithi167 ceph-mon[192880]: osdmap e400: 8 total, 8 up, 6 in 2023-09-30T09:40:01.318 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:01.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:01.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:01.423 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:01.454 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:01.821 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:40:01.821 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:40:02.138 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:02.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:02.198 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.198 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[224043]: osdmap e401: 8 total, 8 up, 6 in 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.199 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:01 smithi019 ceph-mon[216815]: osdmap e401: 8 total, 8 up, 6 in 2023-09-30T09:40:02.242 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: failed to encode map e400 with expected crc 2023-09-30T09:40:02.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:01 smithi167 ceph-mon[192880]: osdmap e401: 8 total, 8 up, 6 in 2023-09-30T09:40:02.274 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:02.329 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:02.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:02.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:02.432 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:02.464 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:02.511 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:02.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:02.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:02.615 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:02.641 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:02.682 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:02.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:02.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:02.735 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:40:02.756 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:40:02.777 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:40:02.785 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:02.799 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:40:02.818 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:02.822 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:40:02.850 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:40:02.872 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:40:02.872 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:02.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:02.891 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:40:02.892 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:40:02.893 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:40:02.893 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:40:02.893 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:40:02.893 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:40:02.893 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:40:02.893 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:40:02.893 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:40:02.977 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:03.005 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: pgmap v1033: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.9 KiB/s rd, 15 KiB/s wr, 229 op/s; 25065/249921 objects degraded (10.029%); 4854/249921 objects misplaced (1.942%); 21 MiB/s, 339 objects/s recovering 2023-09-30T09:40:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2441314976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: pgmap v1033: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.9 KiB/s rd, 15 KiB/s wr, 229 op/s; 25065/249921 objects degraded (10.029%); 4854/249921 objects misplaced (1.942%); 21 MiB/s, 339 objects/s recovering 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:02 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2441314976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:03.120 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:03.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:03.224 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:03.255 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:03.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: pgmap v1033: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.9 KiB/s rd, 15 KiB/s wr, 229 op/s; 25065/249921 objects degraded (10.029%); 4854/249921 objects misplaced (1.942%); 21 MiB/s, 339 objects/s recovering 2023-09-30T09:40:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: failed to encode map e401 with expected crc 2023-09-30T09:40:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:02 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2441314976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:03.303 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:03.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:03.407 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:03.437 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:03.451 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:40:03.497 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:03.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:03.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:03.603 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:03.633 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:03.693 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:03.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:03.768 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 0, 2, 7] out_osds: [3, 6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:40:03.768 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:40:03.769 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-30T09:40:03.769 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:40:03.769 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:40:03.769 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-30T09:40:03.800 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:03.867 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:03.979 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:03.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:03.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:04.005 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:40:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 25065/249921 objects degraded (10.029%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:40:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:40:03] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-09-30T09:40:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 25065/249921 objects degraded (10.029%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:04.086 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:04.120 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:04.221 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:40:04.242 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:04.242 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:04.242 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 25065/249921 objects degraded (10.029%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:04.387 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:04.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:04.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:04.493 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:04.540 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:04.676 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:04.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:04.783 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:04.788 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:04.834 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:04.889 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:04.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:04.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:04.996 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:05.040 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:05.112 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:05.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:05.221 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:05.251 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:04 smithi167 ceph-mon[192880]: pgmap v1034: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 169 op/s; 25065/249921 objects degraded (10.029%); 4854/249921 objects misplaced (1.942%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:40:05.253 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:05.315 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:05.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:05.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:04 smithi019 ceph-mon[216815]: pgmap v1034: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 169 op/s; 25065/249921 objects degraded (10.029%); 4854/249921 objects misplaced (1.942%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:40:05.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:04 smithi019 ceph-mon[224043]: pgmap v1034: 129 pgs: 1 active+recovering+undersized+remapped, 12 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 169 op/s; 25065/249921 objects degraded (10.029%); 4854/249921 objects misplaced (1.942%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:40:05.418 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:05.488 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:05.619 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:05.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:05.724 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:05.760 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:05.987 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:05.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:05.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:06.033 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:06 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-30T09:40:06.081 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:40:06.092 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:06.149 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:06.245 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:06.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:06.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:06.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:06 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-30T09:40:06.349 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:06.387 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:06.415 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:06 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-30T09:40:06.443 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:06.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:06.484 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:40:06.548 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:06.598 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:06.839 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:40:06.839 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:40:06.839 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:40:06.839 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:40:06.839 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:40:06.939 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:06.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:06.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:07.043 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:07.071 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:07 smithi167 ceph-mon[192880]: pgmap v1035: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 45245/249270 objects degraded (18.151%); 9549/249270 objects misplaced (3.831%); 11 MiB/s, 174 objects/s recovering 2023-09-30T09:40:07.074 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:07.132 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:07.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:07.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:07.163 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:07.236 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:07.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:07 smithi019 ceph-mon[216815]: pgmap v1035: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 45245/249270 objects degraded (18.151%); 9549/249270 objects misplaced (3.831%); 11 MiB/s, 174 objects/s recovering 2023-09-30T09:40:07.254 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:07 smithi019 ceph-mon[224043]: pgmap v1035: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 45245/249270 objects degraded (18.151%); 9549/249270 objects misplaced (3.831%); 11 MiB/s, 174 objects/s recovering 2023-09-30T09:40:07.265 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:08.121 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:40:08.137 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:08.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:08.245 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:08.271 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:08 smithi167 ceph-mon[192880]: pgmap v1036: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 45142/248100 objects degraded (18.195%); 8966/248100 objects misplaced (3.614%); 13 MiB/s, 210 objects/s recovering 2023-09-30T09:40:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4121372081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:08 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:08.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:08 smithi019 ceph-mon[216815]: pgmap v1036: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 45142/248100 objects degraded (18.195%); 8966/248100 objects misplaced (3.614%); 13 MiB/s, 210 objects/s recovering 2023-09-30T09:40:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4121372081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:08.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:08 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:08 smithi019 ceph-mon[224043]: pgmap v1036: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 45142/248100 objects degraded (18.195%); 8966/248100 objects misplaced (3.614%); 13 MiB/s, 210 objects/s recovering 2023-09-30T09:40:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4121372081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:08.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:08 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:08.721 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:08.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:08.804 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:40:08.825 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:08.868 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:09.294 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:40:09.330 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:40:09.331 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:40:09.331 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[224043]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[224043]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[224043]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[224043]: osdmap e402: 8 total, 8 up, 6 in 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[224043]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 45142/248100 objects degraded (18.195%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:40:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[216815]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[216815]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[216815]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[216815]: osdmap e402: 8 total, 8 up, 6 in 2023-09-30T09:40:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[216815]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 45142/248100 objects degraded (18.195%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:09.361 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:09 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:40:09.361 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:09 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:09.361 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:09 smithi167 ceph-mon[192880]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.362 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:09 smithi167 ceph-mon[192880]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.362 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:09 smithi167 ceph-mon[192880]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.362 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:09 smithi167 ceph-mon[192880]: osdmap e402: 8 total, 8 up, 6 in 2023-09-30T09:40:09.362 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:09 smithi167 ceph-mon[192880]: failed to encode map e402 with expected crc 2023-09-30T09:40:09.362 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 45142/248100 objects degraded (18.195%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:09.701 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:09.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:09.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:09.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:09.851 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:10.377 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:10.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:10.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:10.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:10 smithi019 ceph-mon[224043]: failed to encode map e402 with expected crc 2023-09-30T09:40:10.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:10 smithi019 ceph-mon[224043]: pgmap v1038: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 44852/245739 objects degraded (18.252%); 8262/245739 objects misplaced (3.362%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:40:10.383 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:10 smithi019 ceph-mon[216815]: failed to encode map e402 with expected crc 2023-09-30T09:40:10.383 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:10 smithi019 ceph-mon[216815]: pgmap v1038: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 44852/245739 objects degraded (18.252%); 8262/245739 objects misplaced (3.362%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:40:10.482 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:10.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:10 smithi167 ceph-mon[192880]: failed to encode map e402 with expected crc 2023-09-30T09:40:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:10 smithi167 ceph-mon[192880]: pgmap v1038: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 44852/245739 objects degraded (18.252%); 8262/245739 objects misplaced (3.362%); 16 MiB/s, 255 objects/s recovering 2023-09-30T09:40:10.542 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:10.681 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:10.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:10.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:10.787 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:10.829 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:10.934 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:10.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:11.040 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:11.085 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:11.162 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:40:11.167 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:11.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:11.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:11.191 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:40:11.216 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:40:11.243 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:40:11.274 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:11.301 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:11.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2598384159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2598384159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:11.343 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:11.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:11.391 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:40:11.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:40:11.445 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:40:11.447 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:11.470 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:40:11.474 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:40:11.474 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:40:11.474 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:40:11.474 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:40:11.475 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:40:11.475 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:40:11.475 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:40:11.475 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:40:11.483 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2598384159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:11.697 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:11.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:11.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:11.803 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:11.845 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:11.965 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:11.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:11.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:12.072 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:12.113 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:12.170 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:12.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:12.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:12.268 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:40:12.268 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:40:12.274 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:12.294 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:12 smithi167 ceph-mon[192880]: pgmap v1039: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 44684/244302 objects degraded (18.290%); 7633/244302 objects misplaced (3.124%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:40:12.307 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:12.466 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:12.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:12.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:12.500 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:40:12.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:12 smithi019 ceph-mon[224043]: pgmap v1039: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 44684/244302 objects degraded (18.290%); 7633/244302 objects misplaced (3.124%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:40:12.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:12 smithi019 ceph-mon[216815]: pgmap v1039: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 44684/244302 objects degraded (18.290%); 7633/244302 objects misplaced (3.124%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:40:12.570 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:12.619 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:12.948 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:12.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:13.065 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:13.092 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:13.100 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:13.385 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:13.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:13.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:13.448 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:40:13.491 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:13.535 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:13.588 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:13.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:13.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 44684/244302 objects degraded (18.290%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:13.695 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:13.742 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:13.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 44684/244302 objects degraded (18.290%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:13.856 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:13.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:13.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:13.870 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:40:13.960 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:13.987 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:40:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:40:13] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-09-30T09:40:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 44684/244302 objects degraded (18.290%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:14.096 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:14.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:14.205 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:14.252 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:14.304 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:14.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:14.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:14.408 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:14.455 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:14.503 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:14.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:14.607 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:14.661 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:14.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:14 smithi167 ceph-mon[192880]: pgmap v1040: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 44684/244302 objects degraded (18.290%); 7633/244302 objects misplaced (3.124%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:40:14.930 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:14 smithi019 ceph-mon[216815]: pgmap v1040: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 44684/244302 objects degraded (18.290%); 7633/244302 objects misplaced (3.124%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:40:14.931 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:14 smithi019 ceph-mon[224043]: pgmap v1040: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 44684/244302 objects degraded (18.290%); 7633/244302 objects misplaced (3.124%); 17 MiB/s, 277 objects/s recovering 2023-09-30T09:40:14.957 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:14.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:14.969 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:40:15.065 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:15.096 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:15.338 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:15.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:15.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:15.442 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:15.485 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:15.608 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:15.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:15.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:15.641 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 0, 2, 7] out_osds: [3, 6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:40:15.641 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:40:15.641 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [1, 4, 5, 0, 2, 7] 2023-09-30T09:40:15.641 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd out 0 2023-09-30T09:40:15.660 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:15.715 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:15.793 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:15.862 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:40:15.862 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:40:16.017 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:16.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:16.125 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:16.161 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:16.215 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:16.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:16.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:16.320 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:16.359 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:16.426 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:16.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:16.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:16.530 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:16.580 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:16.649 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:16.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:16.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:16.717 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:40:16.756 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:16.792 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:16.845 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:16.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:16.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:16.904 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:40:16.949 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:16.974 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:16 smithi019 ceph-mon[216815]: pgmap v1041: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 44432/242376 objects degraded (18.332%); 6746/242376 objects misplaced (2.783%); 19 MiB/s, 308 objects/s recovering 2023-09-30T09:40:16.974 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1240155136' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:16.975 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:16 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:16.975 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:16 smithi019 ceph-mon[224043]: pgmap v1041: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 44432/242376 objects degraded (18.332%); 6746/242376 objects misplaced (2.783%); 19 MiB/s, 308 objects/s recovering 2023-09-30T09:40:16.975 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1240155136' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:16.975 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:16 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:16.977 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:17.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:16 smithi167 ceph-mon[192880]: pgmap v1041: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 44432/242376 objects degraded (18.332%); 6746/242376 objects misplaced (2.783%); 19 MiB/s, 308 objects/s recovering 2023-09-30T09:40:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1240155136' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:16 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:17.332 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:40:17.586 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:17.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:17.694 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:17.723 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:17.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:17.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:40:17.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[224043]: osdmap e403: 8 total, 8 up, 6 in 2023-09-30T09:40:17.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[224043]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[224043]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[224043]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[224043]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[224043]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:40:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[216815]: osdmap e403: 8 total, 8 up, 6 in 2023-09-30T09:40:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[216815]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[216815]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[216815]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[216815]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:17 smithi019 ceph-mon[216815]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:17 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:17.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:40:17.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:17 smithi167 ceph-mon[192880]: osdmap e403: 8 total, 8 up, 6 in 2023-09-30T09:40:17.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:17 smithi167 ceph-mon[192880]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.838 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:17 smithi167 ceph-mon[192880]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.839 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:17 smithi167 ceph-mon[192880]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.839 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:17 smithi167 ceph-mon[192880]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.839 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:17 smithi167 ceph-mon[192880]: failed to encode map e403 with expected crc 2023-09-30T09:40:17.881 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:17.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:17.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:17.986 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:18.049 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:18.335 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:18.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:18.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:18.443 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:18.444 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:18.489 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:18.662 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:18.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:18.737 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: pgmap v1043: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 44267/240795 objects degraded (18.384%); 5970/240795 objects misplaced (2.479%); 22 MiB/s, 347 objects/s recovering 2023-09-30T09:40:18.738 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: failed to encode map e404 with expected crc 2023-09-30T09:40:18.768 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:18.817 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:18.931 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:18.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:18.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:19.009 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: pgmap v1043: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 44267/240795 objects degraded (18.384%); 5970/240795 objects misplaced (2.479%); 22 MiB/s, 347 objects/s recovering 2023-09-30T09:40:19.009 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.009 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: osdmap e404: 8 total, 8 up, 6 in 2023-09-30T09:40:19.009 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 44267/240795 objects degraded (18.384%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: pgmap v1043: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 44267/240795 objects degraded (18.384%); 5970/240795 objects misplaced (2.479%); 22 MiB/s, 347 objects/s recovering 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: osdmap e404: 8 total, 8 up, 6 in 2023-09-30T09:40:19.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:19.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:19.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:18 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 44267/240795 objects degraded (18.384%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:19.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: osdmap e404: 8 total, 8 up, 6 in 2023-09-30T09:40:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: failed to encode map e404 with expected crc 2023-09-30T09:40:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:18 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 44267/240795 objects degraded (18.384%), 20 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:19.045 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:19.052 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:40:19.077 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:40:19.086 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:19.106 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:40:19.143 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:40:19.168 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:40:19.195 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:40:19.219 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:40:19.244 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:40:19.247 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:40:19.247 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:40:19.247 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:40:19.247 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:40:19.247 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:40:19.247 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:40:19.247 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:40:19.247 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:40:19.280 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:40:19.280 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:40:19.280 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:40:19.280 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:40:19.280 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:40:19.364 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:19.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:19.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:19.471 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:19.513 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:19.581 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:19.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:19.687 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:19.729 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:19 smithi167 ceph-mon[192880]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:19 smithi167 ceph-mon[192880]: osdmap e405: 8 total, 8 up, 6 in 2023-09-30T09:40:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:19 smithi167 ceph-mon[192880]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:19 smithi167 ceph-mon[192880]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:19 smithi167 ceph-mon[192880]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:19 smithi167 ceph-mon[192880]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4208173624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[216815]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[216815]: osdmap e405: 8 total, 8 up, 6 in 2023-09-30T09:40:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[216815]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[216815]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[216815]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[216815]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4208173624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[224043]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[224043]: osdmap e405: 8 total, 8 up, 6 in 2023-09-30T09:40:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[224043]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[224043]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[224043]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[224043]: failed to encode map e405 with expected crc 2023-09-30T09:40:20.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4208173624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:20.342 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:40:20.435 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:40:20.461 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:20.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:20.567 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:20.594 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:20.656 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:20.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:20.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:20.764 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:20.835 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:20.894 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:40:20.979 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[216815]: pgmap v1046: 129 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 317 op/s; 43674/238590 objects degraded (18.305%); 5325/238590 objects misplaced (2.232%); 30 MiB/s, 513 objects/s recovering 2023-09-30T09:40:20.979 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[216815]: failed to encode map e406 with expected crc 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[216815]: failed to encode map e406 with expected crc 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[216815]: osdmap e406: 8 total, 8 up, 6 in 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[216815]: failed to encode map e406 with expected crc 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[216815]: failed to encode map e406 with expected crc 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[224043]: pgmap v1046: 129 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 317 op/s; 43674/238590 objects degraded (18.305%); 5325/238590 objects misplaced (2.232%); 30 MiB/s, 513 objects/s recovering 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[224043]: failed to encode map e406 with expected crc 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[224043]: failed to encode map e406 with expected crc 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[224043]: osdmap e406: 8 total, 8 up, 6 in 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[224043]: failed to encode map e406 with expected crc 2023-09-30T09:40:20.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:20 smithi019 ceph-mon[224043]: failed to encode map e406 with expected crc 2023-09-30T09:40:20.994 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:21.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:21.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:21.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:20 smithi167 ceph-mon[192880]: pgmap v1046: 129 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 317 op/s; 43674/238590 objects degraded (18.305%); 5325/238590 objects misplaced (2.232%); 30 MiB/s, 513 objects/s recovering 2023-09-30T09:40:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:20 smithi167 ceph-mon[192880]: failed to encode map e406 with expected crc 2023-09-30T09:40:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:20 smithi167 ceph-mon[192880]: failed to encode map e406 with expected crc 2023-09-30T09:40:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:20 smithi167 ceph-mon[192880]: osdmap e406: 8 total, 8 up, 6 in 2023-09-30T09:40:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:20 smithi167 ceph-mon[192880]: failed to encode map e406 with expected crc 2023-09-30T09:40:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:20 smithi167 ceph-mon[192880]: failed to encode map e406 with expected crc 2023-09-30T09:40:21.104 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:21.146 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:21.200 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:21.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:21.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:21.304 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:21.337 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:21.342 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:21.408 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:21.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:21.514 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:21.543 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:21.592 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:21.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:21.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:21.695 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:21.723 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:21.770 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:21.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:21.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:21.791 INFO:teuthology.orchestra.run.smithi019.stderr:marked out osd.0. 2023-09-30T09:40:21.875 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:21.908 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:21.954 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:21.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:21.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:22.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:21 smithi167 ceph-mon[192880]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:21 smithi167 ceph-mon[192880]: osdmap e407: 8 total, 8 up, 6 in 2023-09-30T09:40:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:21 smithi167 ceph-mon[192880]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:21 smithi167 ceph-mon[192880]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:21 smithi167 ceph-mon[192880]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/942938366' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:40:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:21 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:40:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:21 smithi167 ceph-mon[192880]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-30T09:40:22.058 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[224043]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[224043]: osdmap e407: 8 total, 8 up, 6 in 2023-09-30T09:40:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[224043]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[224043]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[224043]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/942938366' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:40:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:40:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[224043]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 conmon[216791]: 2023-09-30T09:40:21.651+0000 7faf52971700 -1 mon.a@0(leader).osd e407 definitely_dead 0 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[216815]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[216815]: osdmap e407: 8 total, 8 up, 6 in 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[216815]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[216815]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[216815]: failed to encode map e407 with expected crc 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/942938366' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-30T09:40:22.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:21 smithi019 ceph-mon[216815]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-30T09:40:22.087 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:22.132 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:22.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:22.237 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:22.268 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:22.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: pgmap v1049: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 43479/236919 objects degraded (18.352%); 5042/236919 objects misplaced (2.128%); 22 MiB/s, 481 objects/s recovering 2023-09-30T09:40:22.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-30T09:40:22.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.797 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[216815]: osdmap e408: 8 total, 8 up, 5 in 2023-09-30T09:40:22.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: pgmap v1049: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 43479/236919 objects degraded (18.352%); 5042/236919 objects misplaced (2.128%); 22 MiB/s, 481 objects/s recovering 2023-09-30T09:40:22.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-30T09:40:22.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: failed to encode map e408 with expected crc 2023-09-30T09:40:22.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:22 smithi019 ceph-mon[224043]: osdmap e408: 8 total, 8 up, 5 in 2023-09-30T09:40:22.978 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:40:22.978 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:40:22.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:40:22.978 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:40:22.978 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55766 7.5 7:aa9fdb5d:::benchmark_data_smithi019_404601_object55765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 467.49004268599998, 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0102513550000001, 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55766 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.979 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483748+0000", 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483746+0000", 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483780+0000", 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4174999999999997e-05 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483797+0000", 2023-09-30T09:40:22.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6626999999999999e-05 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483805+0000", 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3229999999999998e-06 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:22.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.483862+0000", 2023-09-30T09:40:22.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6236000000000002e-05 2023-09-30T09:40:22.983 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.983 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:22.983 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.484005+0000", 2023-09-30T09:40:22.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014357899999999999 2023-09-30T09:40:22.983 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.984 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:40:22.984 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.484145+0000", 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013967499999999999 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491724+0000", 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075794119999999998 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.509743+0000", 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.018018790999999999 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.509753+0000", 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7790000000000007e-06 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:22.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493928+0000", 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98417540599999997 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493999+0000", 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0858999999999994e-05 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.987 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55755 7.9 7:9e422c00:::benchmark_data_smithi019_404601_object55754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:40:22.988 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:40:22.990 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 467.48729758500002, 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0082494440000001, 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55755 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:22.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486493+0000", 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486491+0000", 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:22.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486532+0000", 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0222000000000002e-05 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486545+0000", 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3288999999999999e-05 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486552+0000", 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6000000000000001e-06 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486609+0000", 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6268999999999999e-05 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:40:22.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.486616+0000", 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0260000000000002e-06 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.495776+0000", 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091603880000000002 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.495784+0000", 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3149999999999992e-06 2023-09-30T09:40:22.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.493615+0000", 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99783076999999998 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.494742+0000", 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001126993 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:22.996 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35424.0:55773 7.9 7:9be0e4ac:::benchmark_data_smithi019_404601_object55772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 467.48654382299998, 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0225480730000001, 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35424", 2023-09-30T09:40:22.997 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4222949558", 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55773 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:22.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487247+0000", 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487246+0000", 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487313+0000", 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7360999999999994e-05 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:22.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487328+0000", 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5197999999999999e-05 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.487335+0000", 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.596e-06 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491649+0000", 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043143920000000002 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.491656+0000", 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.092e-06 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.496453+0000", 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047967569999999996 2023-09-30T09:40:23.001 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:35.496469+0000", 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5926e-05 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.494766+0000", 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99829705800000001 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.495026+0000", 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00026024099999999998 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 6", 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.495185+0000", 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015893299999999999 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.500813+0000", 2023-09-30T09:40:23.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0056275379999999996 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509659+0000", 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088464240000000003 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509708+0000", 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8619000000000003e-05 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:32:36.509795+0000", 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6776999999999999e-05 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101707 8.c 8:32eb269e:::benchmark_data_smithi019_478679_object9259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.658900352, 2023-09-30T09:40:23.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.50943429299999998, 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101707 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:40:23.006 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314889+0000", 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314894+0000", 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3600000000000004e-06 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314906+0000", 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1963999999999999e-05 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314912+0000", 2023-09-30T09:40:23.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2939999999999996e-06 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.774633+0000", 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.45972106400000001 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.778979+0000", 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043463390000000003 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.785183+0000", 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062043089999999999 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.797548+0000", 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012364841 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.798913+0000", 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013653890000000001 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.811924+0000", 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013010661999999999 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812345+0000", 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00042060699999999998 2023-09-30T09:40:23.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.814649+0000", 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023046479999999999 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.824324+0000", 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0096749469999999997 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.012 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101711 8.c 8:342f96d5:::benchmark_data_smithi019_478679_object9263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.594122021, 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.50859565699999998, 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.013 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101711 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379667+0000", 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379676+0000", 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1120000000000002e-06 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.015 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379696+0000", 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9777999999999999e-05 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379705+0000", 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0049999999999993e-06 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.824943+0000", 2023-09-30T09:40:23.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.44523816599999999 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.830542+0000", 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055991670000000004 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.837804+0000", 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0072618020000000004 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.871495+0000", 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.033691169999999999 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.872040+0000", 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00054504200000000003 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.018 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.873778+0000", 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00173759 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.874777+0000", 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00099985400000000011 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.876989+0000", 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022113549999999999 2023-09-30T09:40:23.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.888264+0000", 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011275253000000001 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101717 8.c 8:35bdb3c9:::benchmark_data_smithi019_478679_object9269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:40:23.020 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.568874522000002, 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.49835376599999998, 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101717 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:40:23.021 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404914+0000", 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.022 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404923+0000", 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4540000000000007e-06 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404937+0000", 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4056000000000001e-05 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.023 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404944+0000", 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3300000000000001e-06 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.845543+0000", 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.44059901200000001 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.851287+0000", 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057437360000000002 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.024 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.859094+0000", 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0078075109999999996 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.872354+0000", 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013259979 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.025 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.889815+0000", 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.017460304999999999 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.891113+0000", 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001298738 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.892101+0000", 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00098792200000000002 2023-09-30T09:40:23.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.894073+0000", 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019714590000000001 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.903270+0000", 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091969100000000008 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.027 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101732 8.c 8:37cad26d:::benchmark_data_smithi019_478679_object9284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.469821007, 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.45659628400000002, 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.028 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101732 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:40:23.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503968+0000", 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503977+0000", 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1269999999999992e-06 2023-09-30T09:40:23.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503994+0000", 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.702e-05 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.504002+0000", 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0490000000000004e-06 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.903740+0000", 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.39973803200000002 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.908656+0000", 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049154869999999996 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.916127+0000", 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007471742 2023-09-30T09:40:23.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.946021+0000", 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.029893491000000001 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.947537+0000", 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001516107 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.948747+0000", 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001209768 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.949116+0000", 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000369735 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.951057+0000", 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001940499 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.960566+0000", 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0095088229999999996 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101748 8.c 8:3570d742:::benchmark_data_smithi019_478679_object9300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.035 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.435383126000001, 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.43868606199999999, 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101748 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.036 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538406+0000", 2023-09-30T09:40:23.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538413+0000", 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.25e-06 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538425+0000", 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1833e-05 2023-09-30T09:40:23.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538431+0000", 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.541e-06 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.923820+0000", 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.38538876100000002 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.929942+0000", 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0061226579999999996 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.937368+0000", 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007425783 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.961017+0000", 2023-09-30T09:40:23.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.023649196000000001 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.962281+0000", 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001264285 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.963794+0000", 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015121139999999999 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.965116+0000", 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001322611 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.967433+0000", 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023167349999999999 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.045 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.977093+0000", 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0096604380000000004 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101750 8.c 8:3710a7dd:::benchmark_data_smithi019_478679_object9302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.046 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.417921526000001, 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.47879937299999997, 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101750 2023-09-30T09:40:23.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.048 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.048 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:40:23.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555867+0000", 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555876+0000", 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9660000000000002e-06 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555894+0000", 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7373000000000001e-05 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555904+0000", 2023-09-30T09:40:23.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0555e-05 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.977641+0000", 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.42173738700000002 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.982678+0000", 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050369020000000002 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.989714+0000", 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007035683 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.018051+0000", 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.028337256000000002 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.019726+0000", 2023-09-30T09:40:23.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00167417 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.020234+0000", 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00050844099999999999 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.023410+0000", 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003176201 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.025302+0000", 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018919290000000001 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.034668+0000", 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0093661639999999997 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101764 8.c 8:3df3ae0b:::benchmark_data_smithi019_478679_object9316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.346101712999999, 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.421116035, 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101764 2023-09-30T09:40:23.055 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627687+0000", 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627696+0000", 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3260000000000002e-06 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627713+0000", 2023-09-30T09:40:23.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7232999999999999e-05 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627722+0000", 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6969999999999999e-06 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.996707+0000", 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.36898468600000001 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.001172+0000", 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044654580000000003 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.008935+0000", 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077631189999999997 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.020869+0000", 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011933208000000001 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.022114+0000", 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012456979999999999 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.036065+0000", 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013950393 2023-09-30T09:40:23.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.037310+0000", 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001245377 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.039037+0000", 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017264960000000001 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.048805+0000", 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097681729999999998 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101769 8.c 8:36635324:::benchmark_data_smithi019_478679_object9321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.241365122000001, 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.374847404, 2023-09-30T09:40:23.062 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101769 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:40:23.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732424+0000", 2023-09-30T09:40:23.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732431+0000", 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3520000000000003e-06 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732444+0000", 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3142000000000001e-05 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732450+0000", 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9399999999999999e-06 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.049283+0000", 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.31683297199999999 2023-09-30T09:40:23.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054085+0000", 2023-09-30T09:40:23.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048020700000000003 2023-09-30T09:40:23.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060789+0000", 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006704448 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.090971+0000", 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.030181990999999998 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.092186+0000", 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001215198 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.093180+0000", 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00099359499999999989 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.094984+0000", 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018042119999999999 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.097274+0000", 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022901610000000002 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.107273+0000", 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0099984359999999994 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101783 8.c 8:3098a7bb:::benchmark_data_smithi019_478679_object9335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.071 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.812408+0000", 2023-09-30T09:40:23.072 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.161382315000001, 2023-09-30T09:40:23.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.308978906, 2023-09-30T09:40:23.072 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.072 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.072 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.072 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101783 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812408+0000", 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812408+0000", 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812407+0000", 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.074 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812414+0000", 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2710000000000003e-06 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812428+0000", 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3645999999999999e-05 2023-09-30T09:40:23.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812434+0000", 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6800000000000004e-06 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.069434+0000", 2023-09-30T09:40:23.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.25699987000000002 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.075720+0000", 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062862259999999998 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.082930+0000", 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0072098930000000002 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.094535+0000", 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011605232 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.107703+0000", 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013167717000000001 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.109063+0000", 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001359444 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.110215+0000", 2023-09-30T09:40:23.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00115266 2023-09-30T09:40:23.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.080 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.080 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.111946+0000", 2023-09-30T09:40:23.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017304709999999999 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.121387+0000", 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0094412060000000006 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101809 8.c 8:387cfa92:::benchmark_data_smithi019_478679_object9361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.896480+0000", 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.077310098000002, 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.27888291900000001, 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.082 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.083 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.083 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101809 2023-09-30T09:40:23.083 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.083 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.083 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.083 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.083 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.896480+0000", 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.896480+0000", 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.896479+0000", 2023-09-30T09:40:23.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:40:23.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.896488+0000", 2023-09-30T09:40:23.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8810000000000008e-06 2023-09-30T09:40:23.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.896504+0000", 2023-09-30T09:40:23.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5800000000000001e-05 2023-09-30T09:40:23.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.896512+0000", 2023-09-30T09:40:23.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2069999999999995e-06 2023-09-30T09:40:23.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.121935+0000", 2023-09-30T09:40:23.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.22542278900000001 2023-09-30T09:40:23.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.126770+0000", 2023-09-30T09:40:23.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048345050000000002 2023-09-30T09:40:23.088 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.088 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.133200+0000", 2023-09-30T09:40:23.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064304829999999999 2023-09-30T09:40:23.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.089 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.158346+0000", 2023-09-30T09:40:23.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.025146048000000001 2023-09-30T09:40:23.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.090 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.160123+0000", 2023-09-30T09:40:23.090 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017772059999999999 2023-09-30T09:40:23.090 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.161396+0000", 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001272689 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.165020+0000", 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036240080000000002 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.166778+0000", 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001758243 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.175363+0000", 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0085848269999999997 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101824 8.c 8:3d10be2b:::benchmark_data_smithi019_478679_object9376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.924018+0000", 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.049772458, 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26138910799999998, 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.094 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.094 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.094 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101824 2023-09-30T09:40:23.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.131 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.924018+0000", 2023-09-30T09:40:23.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.924018+0000", 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.924016+0000", 2023-09-30T09:40:23.133 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.924025+0000", 2023-09-30T09:40:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0129999999999999e-06 2023-09-30T09:40:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.924043+0000", 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7830999999999999e-05 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.924051+0000", 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3839999999999999e-06 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.136 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.139978+0000", 2023-09-30T09:40:23.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21592758400000001 2023-09-30T09:40:23.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.136 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.144770+0000", 2023-09-30T09:40:23.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047914419999999999 2023-09-30T09:40:23.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.150806+0000", 2023-09-30T09:40:23.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0060369250000000003 2023-09-30T09:40:23.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.162574+0000", 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011767147 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.163610+0000", 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00103677 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.164699+0000", 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001088276 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.175683+0000", 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010984068 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.177327+0000", 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016437940000000001 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.185407+0000", 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0080804889999999997 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.140 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101838 8.c 8:39307a07:::benchmark_data_smithi019_478679_object9390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.975166+0000", 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.99862457, 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26629054299999999, 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.141 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.142 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.142 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.142 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.165 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101838 2023-09-30T09:40:23.172 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.172 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.173 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.173 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.975166+0000", 2023-09-30T09:40:23.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.173 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.173 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.173 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.173 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.975166+0000", 2023-09-30T09:40:23.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.174 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.174 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.975164+0000", 2023-09-30T09:40:23.174 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.175 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.175 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.975172+0000", 2023-09-30T09:40:23.175 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0639999999999994e-06 2023-09-30T09:40:23.175 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.175 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.198 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.198 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.975190+0000", 2023-09-30T09:40:23.198 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7787000000000001e-05 2023-09-30T09:40:23.198 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.198 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.198 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.198 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.975198+0000", 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1410000000000005e-06 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.185921+0000", 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21072300899999999 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.191095+0000", 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0051735150000000001 2023-09-30T09:40:23.199 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.198752+0000", 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076573550000000002 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.226363+0000", 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.027610821000000001 2023-09-30T09:40:23.200 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.227454+0000", 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001091308 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.227862+0000", 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00040801900000000002 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.201 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.229258+0000", 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001395339 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.231180+0000", 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019222480000000001 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.241456+0000", 2023-09-30T09:40:23.202 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010276449 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101865 8.c 8:31dcb35d:::benchmark_data_smithi019_478679_object9417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:04.054787+0000", 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.919003069999999, 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.25225735300000002, 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.203 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101865 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054787+0000", 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.204 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054787+0000", 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054786+0000", 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.205 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054794+0000", 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4069999999999993e-06 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054808+0000", 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3886000000000001e-05 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.206 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054815+0000", 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2860000000000002e-06 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.258846+0000", 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20403090700000001 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.263697+0000", 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048503740000000002 2023-09-30T09:40:23.207 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.271145+0000", 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0074488109999999996 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.294792+0000", 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.023646555 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.208 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.295991+0000", 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001198609 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.296423+0000", 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00043201900000000001 2023-09-30T09:40:23.209 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.297779+0000", 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013563399999999999 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.299351+0000", 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015718310000000001 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.210 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.211 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.211 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.307045+0000", 2023-09-30T09:40:23.211 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076939340000000004 2023-09-30T09:40:23.211 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.211 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.211 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.215 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.215 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.215 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101869 8.c 8:3a99e5d0:::benchmark_data_smithi019_478679_object9421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.215 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:04.060513+0000", 2023-09-30T09:40:23.215 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.913277741999998, 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.25999026600000003, 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101869 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060513+0000", 2023-09-30T09:40:23.216 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.217 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.217 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.217 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.217 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060513+0000", 2023-09-30T09:40:23.217 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.217 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.217 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060511+0000", 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060524+0000", 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3142000000000001e-05 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.218 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060541+0000", 2023-09-30T09:40:23.219 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7288000000000001e-05 2023-09-30T09:40:23.219 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.219 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.219 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.219 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060549+0000", 2023-09-30T09:40:23.219 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3019999999999997e-06 2023-09-30T09:40:23.219 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.219 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.277955+0000", 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21740652199999999 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.282048+0000", 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004092933 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.220 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.221 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.288142+0000", 2023-09-30T09:40:23.221 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0060942030000000003 2023-09-30T09:40:23.221 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.221 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.221 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.221 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.296811+0000", 2023-09-30T09:40:23.221 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0086683200000000002 2023-09-30T09:40:23.221 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.308306+0000", 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011495755999999999 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.309440+0000", 2023-09-30T09:40:23.222 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011336779999999999 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.310768+0000", 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001328051 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.312461+0000", 2023-09-30T09:40:23.223 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016927470000000001 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.320503+0000", 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0080419480000000002 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.224 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.225 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.225 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101873 8.c 8:325b65cc:::benchmark_data_smithi019_478679_object9425:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.225 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:04.073839+0000", 2023-09-30T09:40:23.225 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.899951519999998, 2023-09-30T09:40:23.225 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.29358393999999999, 2023-09-30T09:40:23.225 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.225 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.225 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.226 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.226 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.226 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101873 2023-09-30T09:40:23.226 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.226 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.226 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.226 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.227 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.073839+0000", 2023-09-30T09:40:23.227 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.227 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.227 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.227 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.227 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.073839+0000", 2023-09-30T09:40:23.227 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.227 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.073837+0000", 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.073846+0000", 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7529999999999996e-06 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.228 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.073860+0000", 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3579e-05 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.073867+0000", 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0319999999999998e-06 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.229 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.320870+0000", 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.24700292099999999 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.325155+0000", 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042849280000000003 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.331219+0000", 2023-09-30T09:40:23.230 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006063901 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.355152+0000", 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.023933138999999999 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.356188+0000", 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010360739999999999 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.231 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.357458+0000", 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012699020000000001 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.358955+0000", 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014966160000000001 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.232 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.360556+0000", 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016017640000000001 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.367423+0000", 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0068664829999999996 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.233 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101879 8.c 8:338767ca:::benchmark_data_smithi019_478679_object9431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:04.085393+0000", 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.888397619999999, 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.29515331500000003, 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.234 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101879 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.085393+0000", 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.235 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.085393+0000", 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.085391+0000", 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.085399+0000", 2023-09-30T09:40:23.236 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.61e-06 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.085410+0000", 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1392999999999999e-05 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.085417+0000", 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3840000000000002e-06 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.237 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.337537+0000", 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.25212015500000001 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.342181+0000", 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046440489999999999 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.238 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.348887+0000", 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0067061120000000002 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.357891+0000", 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0090039730000000002 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.239 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.368814+0000", 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010923321 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.369834+0000", 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010193159999999999 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.370823+0000", 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0009890910000000001 2023-09-30T09:40:23.240 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.241 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.246 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.246 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.372487+0000", 2023-09-30T09:40:23.246 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001664656 2023-09-30T09:40:23.246 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.246 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.246 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.246 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.380546+0000", 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.008058674 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101944 8.c 8:386ce74b:::benchmark_data_smithi019_478679_object9496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:04.247700+0000", 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.726090738, 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.25273009299999999, 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:40:23.247 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101944 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.247700+0000", 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.248 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.247700+0000", 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.247698+0000", 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.249 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:40:23.250 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.247707+0000", 2023-09-30T09:40:23.250 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4299999999999995e-06 2023-09-30T09:40:23.250 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.250 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.250 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:40:23.250 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.247724+0000", 2023-09-30T09:40:23.250 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.605e-05 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.247732+0000", 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2719999999999997e-06 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.450298+0000", 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.202565887 2023-09-30T09:40:23.251 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.455439+0000", 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0051409760000000002 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.462469+0000", 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0070298619999999996 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.252 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.253 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.253 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.488364+0000", 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.025895167 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.488731+0000", 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00036695 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.254 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.490134+0000", 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014031639999999999 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.491258+0000", 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011241630000000001 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.255 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:40:23.256 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.492831+0000", 2023-09-30T09:40:23.256 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015734900000000001 2023-09-30T09:40:23.256 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:40:23.256 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:40:23.256 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:40:23.256 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.500430+0000", 2023-09-30T09:40:23.257 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075982819999999996 2023-09-30T09:40:23.257 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.257 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.257 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.257 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:40:23.257 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:40:23.257 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:40:23.257 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:40:23.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: pgmap v1049: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 43479/236919 objects degraded (18.352%); 5042/236919 objects misplaced (2.128%); 22 MiB/s, 481 objects/s recovering 2023-09-30T09:40:23.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-30T09:40:23.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: failed to encode map e408 with expected crc 2023-09-30T09:40:23.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: failed to encode map e408 with expected crc 2023-09-30T09:40:23.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: failed to encode map e408 with expected crc 2023-09-30T09:40:23.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: failed to encode map e408 with expected crc 2023-09-30T09:40:23.260 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: failed to encode map e408 with expected crc 2023-09-30T09:40:23.260 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: failed to encode map e408 with expected crc 2023-09-30T09:40:23.260 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:22 smithi167 ceph-mon[192880]: osdmap e408: 8 total, 8 up, 5 in 2023-09-30T09:40:23.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:23.262 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:23.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:23.363 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:23.395 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:23.441 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:23.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:23.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:23.545 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:23.573 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:23.618 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:23.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:23.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:23.714 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:23.722 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:23.749 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:23.796 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:23.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:23.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:23.900 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:23.984 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:24.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: osdmap e409: 8 total, 8 up, 5 in 2023-09-30T09:40:24.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:23 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 43479/236919 objects degraded (18.352%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:40:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:40:23] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-30T09:40:24.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: osdmap e409: 8 total, 8 up, 5 in 2023-09-30T09:40:24.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 43479/236919 objects degraded (18.352%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:24.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: osdmap e409: 8 total, 8 up, 5 in 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: failed to encode map e409 with expected crc 2023-09-30T09:40:24.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:23 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 43479/236919 objects degraded (18.352%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:24.254 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:24.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:24.358 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:24.360 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:40:24.397 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:24.828 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: pgmap v1052: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 139 op/s; 43479/236919 objects degraded (18.352%); 5042/236919 objects misplaced (2.128%); 5.8 MiB/s, 163 objects/s recovering 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: osdmap e410: 8 total, 8 up, 5 in 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 407'6555 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2632093508' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: pgmap v1052: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 139 op/s; 43479/236919 objects degraded (18.352%); 5042/236919 objects misplaced (2.128%); 5.8 MiB/s, 163 objects/s recovering 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: osdmap e410: 8 total, 8 up, 5 in 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 407'6555 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2632093508' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:25.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:24 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: pgmap v1052: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 139 op/s; 43479/236919 objects degraded (18.352%); 5042/236919 objects misplaced (2.128%); 5.8 MiB/s, 163 objects/s recovering 2023-09-30T09:40:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: osdmap e410: 8 total, 8 up, 5 in 2023-09-30T09:40:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 407'6555 2023-09-30T09:40:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: failed to encode map e410 with expected crc 2023-09-30T09:40:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2632093508' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:25.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:24 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:26.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:26.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:26.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: osdmap e411: 8 total, 8 up, 5 in 2023-09-30T09:40:26.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[216815]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: osdmap e411: 8 total, 8 up, 5 in 2023-09-30T09:40:26.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:25 smithi019 ceph-mon[224043]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: osdmap e411: 8 total, 8 up, 5 in 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:25 smithi167 ceph-mon[192880]: failed to encode map e411 with expected crc 2023-09-30T09:40:26.772 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:26.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:26.875 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:26.908 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:26.975 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:26.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:26.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:27.072 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:27.081 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:27.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[224043]: pgmap v1055: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 82 op/s; 49228/235923 objects degraded (20.866%); 7332/235923 objects misplaced (3.108%); 9.2 MiB/s, 247 objects/s recovering 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[224043]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[224043]: osdmap e412: 8 total, 8 up, 5 in 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[224043]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[224043]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[224043]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[224043]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[216815]: pgmap v1055: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 82 op/s; 49228/235923 objects degraded (20.866%); 7332/235923 objects misplaced (3.108%); 9.2 MiB/s, 247 objects/s recovering 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[216815]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[216815]: osdmap e412: 8 total, 8 up, 5 in 2023-09-30T09:40:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[216815]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[216815]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[216815]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:26 smithi019 ceph-mon[216815]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.115 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:27.257 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:27.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:27.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:27.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:26 smithi167 ceph-mon[192880]: pgmap v1055: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 82 op/s; 49228/235923 objects degraded (20.866%); 7332/235923 objects misplaced (3.108%); 9.2 MiB/s, 247 objects/s recovering 2023-09-30T09:40:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:26 smithi167 ceph-mon[192880]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:26 smithi167 ceph-mon[192880]: osdmap e412: 8 total, 8 up, 5 in 2023-09-30T09:40:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:26 smithi167 ceph-mon[192880]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:26 smithi167 ceph-mon[192880]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:26 smithi167 ceph-mon[192880]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:26 smithi167 ceph-mon[192880]: failed to encode map e412 with expected crc 2023-09-30T09:40:27.362 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:27.394 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:27.504 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:27.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:27.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:27.612 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:27.617 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:40:27.638 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:40:27.643 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:27.662 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:40:27.684 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:40:27.699 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:27.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:27.704 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:40:27.725 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:40:27.746 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:40:27.804 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:27.835 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:27.893 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:27.896 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1754108609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:27.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:27.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:27.998 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:28.029 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:28.081 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:28.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1754108609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:28.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1754108609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:28.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:28.158 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:40:28.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:40:28.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:40:28.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:40:28.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:40:28.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:40:28.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:40:28.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:40:28.160 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:40:28.187 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:28.219 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:28.261 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7] out_osds: [3, 6, 0] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:40:28.262 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:40:28.262 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-09-30T09:40:28.262 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd in 0 2023-09-30T09:40:28.571 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:28.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:28.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:28.675 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:28.705 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:28.961 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:40:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:28 smithi019 ceph-mon[216815]: pgmap v1057: 129 pgs: 1 activating+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.8 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 52449/235761 objects degraded (22.247%); 14297/235761 objects misplaced (6.064%); 12 MiB/s, 274 objects/s recovering 2023-09-30T09:40:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 52449/235761 objects degraded (22.247%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:28 smithi019 ceph-mon[224043]: pgmap v1057: 129 pgs: 1 activating+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.8 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 52449/235761 objects degraded (22.247%); 14297/235761 objects misplaced (6.064%); 12 MiB/s, 274 objects/s recovering 2023-09-30T09:40:29.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 52449/235761 objects degraded (22.247%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:28 smithi167 ceph-mon[192880]: pgmap v1057: 129 pgs: 1 activating+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.8 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 52449/235761 objects degraded (22.247%); 14297/235761 objects misplaced (6.064%); 12 MiB/s, 274 objects/s recovering 2023-09-30T09:40:29.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 52449/235761 objects degraded (22.247%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:30.314 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:30.326 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:30.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:30.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:30.429 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:30.431 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:30.477 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:30.539 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:30.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:30.643 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:30.681 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:30.806 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:30.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:30.911 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:30.977 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:31.075 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:31.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:31.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:31.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:30 smithi019 ceph-mon[224043]: pgmap v1058: 129 pgs: 2 activating+undersized+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 73 op/s; 58335/235587 objects degraded (24.762%); 13687/235587 objects misplaced (5.810%); 16 MiB/s, 316 objects/s recovering 2023-09-30T09:40:31.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:30 smithi019 ceph-mon[216815]: pgmap v1058: 129 pgs: 2 activating+undersized+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 73 op/s; 58335/235587 objects degraded (24.762%); 13687/235587 objects misplaced (5.810%); 16 MiB/s, 316 objects/s recovering 2023-09-30T09:40:31.183 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:31.249 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:31.249 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:30 smithi167 ceph-mon[192880]: pgmap v1058: 129 pgs: 2 activating+undersized+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 73 op/s; 58335/235587 objects degraded (24.762%); 13687/235587 objects misplaced (5.810%); 16 MiB/s, 316 objects/s recovering 2023-09-30T09:40:31.271 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:40:31.272 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:40:31.319 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:31.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:31.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:31.429 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:31.473 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:31.541 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:31.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:31.648 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:31.689 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:32.265 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:40:32.359 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:32.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:32.466 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:32.512 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:33.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:32 smithi019 ceph-mon[216815]: pgmap v1059: 129 pgs: 2 activating+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 74 op/s; 58302/235167 objects degraded (24.792%); 13170/235167 objects misplaced (5.600%); 16 MiB/s, 306 objects/s recovering 2023-09-30T09:40:33.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:32 smithi019 ceph-mon[224043]: pgmap v1059: 129 pgs: 2 activating+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 74 op/s; 58302/235167 objects degraded (24.792%); 13170/235167 objects misplaced (5.600%); 16 MiB/s, 306 objects/s recovering 2023-09-30T09:40:33.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:32 smithi167 ceph-mon[192880]: pgmap v1059: 129 pgs: 2 activating+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 74 op/s; 58302/235167 objects degraded (24.792%); 13170/235167 objects misplaced (5.600%); 16 MiB/s, 306 objects/s recovering 2023-09-30T09:40:33.300 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:40:33.435 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:33.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:33.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:33.540 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:33.587 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:33.650 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:33.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:33.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:33.762 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:33.800 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:33.840 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:33.840 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 58302/235167 objects degraded (24.792%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:33.951 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:33.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:34.061 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:34.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:34.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 58302/235167 objects degraded (24.792%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:34.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:40:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:40:33] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2023-09-30T09:40:34.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:34.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 58302/235167 objects degraded (24.792%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:34.100 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:34.165 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:34.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:34.273 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:34.292 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:34.360 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:34.497 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:34.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:34.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:34.534 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:34.605 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:34.651 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:34.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:34 smithi019 ceph-mon[224043]: pgmap v1060: 129 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 58302/234867 objects degraded (24.823%); 13122/234867 objects misplaced (5.587%); 10 MiB/s, 159 objects/s recovering 2023-09-30T09:40:34.923 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3781479310' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:40:34.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:34 smithi019 conmon[216791]: 2023-09-30T09:40:34.823+0000 7faf52971700 -1 mon.a@0(leader).osd e412 definitely_dead 0 2023-09-30T09:40:34.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:34 smithi019 ceph-mon[216815]: pgmap v1060: 129 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 58302/234867 objects degraded (24.823%); 13122/234867 objects misplaced (5.587%); 10 MiB/s, 159 objects/s recovering 2023-09-30T09:40:34.924 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3781479310' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:40:35.205 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:35.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:35.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:34 smithi167 ceph-mon[192880]: pgmap v1060: 129 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 58302/234867 objects degraded (24.823%); 13122/234867 objects misplaced (5.587%); 10 MiB/s, 159 objects/s recovering 2023-09-30T09:40:35.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3781479310' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-30T09:40:35.312 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:35.358 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:35.581 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:40:36.182 INFO:teuthology.orchestra.run.smithi019.stderr:marked in osd.0. 2023-09-30T09:40:36.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3820916426' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3781479310' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3820916426' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[216815]: osdmap e413: 8 total, 8 up, 6 in 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3820916426' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3781479310' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3820916426' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:40:36.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:35 smithi019 ceph-mon[224043]: osdmap e413: 8 total, 8 up, 6 in 2023-09-30T09:40:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3820916426' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:40:36.985 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:35 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:36.985 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3781479310' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-30T09:40:36.986 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3820916426' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:40:36.986 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:35 smithi167 ceph-mon[192880]: osdmap e413: 8 total, 8 up, 6 in 2023-09-30T09:40:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[216815]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[216815]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[216815]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[216815]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[216815]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[216815]: pgmap v1062: 129 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.7 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 103 op/s; 58047/232884 objects degraded (24.925%); 12579/232884 objects misplaced (5.401%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:40:37.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[216815]: osdmap e414: 8 total, 8 up, 6 in 2023-09-30T09:40:37.109 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[224043]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[224043]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[224043]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[224043]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[224043]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[224043]: pgmap v1062: 129 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.7 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 103 op/s; 58047/232884 objects degraded (24.925%); 12579/232884 objects misplaced (5.401%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:40:37.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:36 smithi019 ceph-mon[224043]: osdmap e414: 8 total, 8 up, 6 in 2023-09-30T09:40:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:36 smithi167 ceph-mon[192880]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.361 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:36 smithi167 ceph-mon[192880]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.363 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:36 smithi167 ceph-mon[192880]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.363 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:36 smithi167 ceph-mon[192880]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:36 smithi167 ceph-mon[192880]: failed to encode map e413 with expected crc 2023-09-30T09:40:37.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:36 smithi167 ceph-mon[192880]: pgmap v1062: 129 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.7 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 103 op/s; 58047/232884 objects degraded (24.925%); 12579/232884 objects misplaced (5.401%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:40:37.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:36 smithi167 ceph-mon[192880]: osdmap e414: 8 total, 8 up, 6 in 2023-09-30T09:40:37.563 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:37.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:37.667 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:37.735 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:38.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:37 smithi167 ceph-mon[192880]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:37 smithi167 ceph-mon[192880]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:37 smithi167 ceph-mon[192880]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:37 smithi167 ceph-mon[192880]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:37 smithi167 ceph-mon[192880]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:37 smithi167 ceph-mon[192880]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:37 smithi167 ceph-mon[192880]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:37 smithi167 ceph-mon[192880]: osdmap e415: 8 total, 8 up, 6 in 2023-09-30T09:40:38.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[224043]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[224043]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[224043]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[224043]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[224043]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[224043]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[224043]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[224043]: osdmap e415: 8 total, 8 up, 6 in 2023-09-30T09:40:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[216815]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[216815]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[216815]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[216815]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[216815]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[216815]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[216815]: failed to encode map e414 with expected crc 2023-09-30T09:40:38.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:37 smithi019 ceph-mon[216815]: osdmap e415: 8 total, 8 up, 6 in 2023-09-30T09:40:38.624 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-09-30T09:40:38.788 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:38.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:38.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:38.882 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:40:38.882 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:40:38.893 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:38.936 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:39.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:40:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: pgmap v1065: 129 pgs: 9 peering, 2 unknown, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 183 op/s; 57868/231864 objects degraded (24.958%); 12519/231864 objects misplaced (5.399%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:40:39.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 58047/232884 objects degraded (24.925%), 26 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:39.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: osdmap e416: 8 total, 8 up, 6 in 2023-09-30T09:40:39.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:40:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: pgmap v1065: 129 pgs: 9 peering, 2 unknown, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 183 op/s; 57868/231864 objects degraded (24.958%); 12519/231864 objects misplaced (5.399%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:40:39.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 58047/232884 objects degraded (24.925%), 26 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:39.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: osdmap e416: 8 total, 8 up, 6 in 2023-09-30T09:40:39.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:38 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:40:39.090 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:40:39.128 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:40:39.153 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:40:39.157 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:39.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:39.178 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:40:39.199 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:40:39.220 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:40:39.243 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:40:39.244 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:40:39.244 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:40:39.244 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:40:39.244 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:40:39.244 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:40:39.244 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:40:39.245 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:40:39.245 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:40:39.265 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: failed to encode map e415 with expected crc 2023-09-30T09:40:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: pgmap v1065: 129 pgs: 9 peering, 2 unknown, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 183 op/s; 57868/231864 objects degraded (24.958%); 12519/231864 objects misplaced (5.399%); 8.7 MiB/s, 140 objects/s recovering 2023-09-30T09:40:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 58047/232884 objects degraded (24.925%), 26 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: osdmap e416: 8 total, 8 up, 6 in 2023-09-30T09:40:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:38 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:40:39.310 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:39.357 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:39.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:39.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:39.461 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:39.489 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:39.535 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:39.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:39.641 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:39.664 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:40:39.675 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:40.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[216815]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[216815]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[216815]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[216815]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2681379583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[216815]: osdmap e417: 8 total, 8 up, 6 in 2023-09-30T09:40:40.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[224043]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[224043]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[224043]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[224043]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2681379583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:39 smithi019 ceph-mon[224043]: osdmap e417: 8 total, 8 up, 6 in 2023-09-30T09:40:40.193 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:40.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:40.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:40.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:39 smithi167 ceph-mon[192880]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:39 smithi167 ceph-mon[192880]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:39 smithi167 ceph-mon[192880]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:39 smithi167 ceph-mon[192880]: failed to encode map e416 with expected crc 2023-09-30T09:40:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2681379583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:39 smithi167 ceph-mon[192880]: osdmap e417: 8 total, 8 up, 6 in 2023-09-30T09:40:40.291 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:40:40.297 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:40.407 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:40.435 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:40.474 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:40.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:40.584 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:40.610 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:40.657 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:40.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:40.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:40.762 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:40.794 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:40.902 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:40.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:40.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:41.006 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: pgmap v1068: 129 pgs: 12 peering, 2 unknown, 4 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 46565/231084 objects degraded (20.151%); 10535/231084 objects misplaced (4.559%); 19 MiB/s, 303 objects/s recovering 2023-09-30T09:40:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:40 smithi167 ceph-mon[192880]: failed to encode map e416 with expected crc 2023-09-30T09:40:41.040 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: pgmap v1068: 129 pgs: 12 peering, 2 unknown, 4 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 46565/231084 objects degraded (20.151%); 10535/231084 objects misplaced (4.559%); 19 MiB/s, 303 objects/s recovering 2023-09-30T09:40:41.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[224043]: failed to encode map e416 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e417 with expected crc 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: pgmap v1068: 129 pgs: 12 peering, 2 unknown, 4 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 46565/231084 objects degraded (20.151%); 10535/231084 objects misplaced (4.559%); 19 MiB/s, 303 objects/s recovering 2023-09-30T09:40:41.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:40 smithi019 ceph-mon[216815]: failed to encode map e416 with expected crc 2023-09-30T09:40:41.099 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:41.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:41.204 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:41.230 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:41.277 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:41.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:41.381 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:41.409 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:41.660 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:41.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:41.764 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:41.787 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:41.790 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:41.833 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:41.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:41.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:41.942 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:41.975 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:42.030 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:42.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:42.135 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:42.161 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:42.205 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:42.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:42.312 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:42.332 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:40:42.347 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:42.420 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:42.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:42.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:42.524 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:42.550 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:42.593 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:42.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:42.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:42.699 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:42.725 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:42.769 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:42.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:42.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:42.860 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:42 smithi167 ceph-mon[192880]: pgmap v1069: 129 pgs: 1 activating+undersized+remapped, 11 peering, 1 unknown, 4 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 48143/231060 objects degraded (20.836%); 8950/231060 objects misplaced (3.873%); 15 MiB/s, 239 objects/s recovering 2023-09-30T09:40:42.860 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:42 smithi167 ceph-mon[192880]: failed to encode map e418 with expected crc 2023-09-30T09:40:42.860 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:42 smithi167 ceph-mon[192880]: osdmap e418: 8 total, 8 up, 6 in 2023-09-30T09:40:42.860 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:42 smithi167 ceph-mon[192880]: failed to encode map e418 with expected crc 2023-09-30T09:40:42.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:42 smithi167 ceph-mon[192880]: failed to encode map e418 with expected crc 2023-09-30T09:40:42.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3883659493' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:42.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:42 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:42.874 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:42.875 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:40:42.908 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:43.209 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:43.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[224043]: pgmap v1069: 129 pgs: 1 activating+undersized+remapped, 11 peering, 1 unknown, 4 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 48143/231060 objects degraded (20.836%); 8950/231060 objects misplaced (3.873%); 15 MiB/s, 239 objects/s recovering 2023-09-30T09:40:43.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[224043]: failed to encode map e418 with expected crc 2023-09-30T09:40:43.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[224043]: osdmap e418: 8 total, 8 up, 6 in 2023-09-30T09:40:43.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[224043]: failed to encode map e418 with expected crc 2023-09-30T09:40:43.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[224043]: failed to encode map e418 with expected crc 2023-09-30T09:40:43.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3883659493' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:43.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:43.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[216815]: pgmap v1069: 129 pgs: 1 activating+undersized+remapped, 11 peering, 1 unknown, 4 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 48143/231060 objects degraded (20.836%); 8950/231060 objects misplaced (3.873%); 15 MiB/s, 239 objects/s recovering 2023-09-30T09:40:43.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[216815]: failed to encode map e418 with expected crc 2023-09-30T09:40:43.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[216815]: osdmap e418: 8 total, 8 up, 6 in 2023-09-30T09:40:43.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[216815]: failed to encode map e418 with expected crc 2023-09-30T09:40:43.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[216815]: failed to encode map e418 with expected crc 2023-09-30T09:40:43.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3883659493' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:43.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:42 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:40:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:43.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:43.314 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:43.347 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:43.389 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:43.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:43.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:43.494 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:43.522 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:43.565 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:43.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:43.625 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0] out_osds: [3, 6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:40:43.626 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:40:43.626 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-30T09:40:43.626 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:40:43.626 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:40:43.626 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-30T09:40:43.674 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:43.701 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:43.767 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:43.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:43.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:43.876 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:43.969 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:40:43.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:43.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[216815]: failed to encode map e419 with expected crc 2023-09-30T09:40:43.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[216815]: failed to encode map e419 with expected crc 2023-09-30T09:40:43.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[216815]: osdmap e419: 8 total, 8 up, 6 in 2023-09-30T09:40:43.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 48143/231060 objects degraded (20.836%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:43.970 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:40:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:40:43] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2023-09-30T09:40:43.971 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:40:43.971 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:43.971 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[224043]: failed to encode map e419 with expected crc 2023-09-30T09:40:43.971 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[224043]: failed to encode map e419 with expected crc 2023-09-30T09:40:43.971 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[224043]: osdmap e419: 8 total, 8 up, 6 in 2023-09-30T09:40:43.971 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 48143/231060 objects degraded (20.836%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:43.971 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:44.088 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:44.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:44.117 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:44.121 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:40:44.199 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:44.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:43 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:40:44.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:43 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:40:44.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:43 smithi167 ceph-mon[192880]: failed to encode map e419 with expected crc 2023-09-30T09:40:44.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:43 smithi167 ceph-mon[192880]: failed to encode map e419 with expected crc 2023-09-30T09:40:44.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:43 smithi167 ceph-mon[192880]: osdmap e419: 8 total, 8 up, 6 in 2023-09-30T09:40:44.230 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 48143/231060 objects degraded (20.836%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:44.232 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:44.302 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:44.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:44.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:44.457 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:44.505 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:44.559 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:44.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:44.566 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:128358 7.d e306/302 7:b44ac402:::benchmark_data_smithi019_404601_object15080:head v 306'7966, mlcod=286'3458)", 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:00.013599+0000", 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.54860131100003, 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.29377689000000001, 2023-09-30T09:40:44.567 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013599+0000", 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013599+0000", 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.568 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013600+0000", 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.08e-06 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013611+0000", 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1062e-05 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.569 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013633+0000", 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1801e-05 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013642+0000", 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8449999999999992e-06 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.570 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.284819+0000", 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27117700500000003 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.284845+0000", 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6444e-05 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.307320+0000", 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022475095 2023-09-30T09:40:44.571 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.307376+0000", 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5557999999999999e-05 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:128360 7.b e306/301 7:d422a517:::benchmark_data_smithi019_404601_object15082:head v 306'7873, mlcod=291'4872)", 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:00.013667+0000", 2023-09-30T09:40:44.572 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.54853348099999, 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.28383565399999999, 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013667+0000", 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013667+0000", 2023-09-30T09:40:44.573 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013667+0000", 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0199999999999998e-07 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013671+0000", 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1570000000000002e-06 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.574 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013677+0000", 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2480000000000001e-06 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013682+0000", 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8500000000000002e-06 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.575 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.284779+0000", 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27109628299999999 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.284817+0000", 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7972000000000002e-05 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.297458+0000", 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012641584000000001 2023-09-30T09:40:44.576 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.297502+0000", 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4258000000000002e-05 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:128357 7.a e306/302 7:503abca5:::benchmark_data_smithi019_404601_object15079:head v 306'8145, mlcod=286'3542)", 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:00.013842+0000", 2023-09-30T09:40:44.577 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.54835812900001, 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.28302493899999998, 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013842+0000", 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.578 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013842+0000", 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013843+0000", 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3900000000000004e-07 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.013849+0000", 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.438e-06 2023-09-30T09:40:44.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.285216+0000", 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27136657600000003 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.285223+0000", 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9009999999999997e-06 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.285444+0000", 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022070100000000001 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.285472+0000", 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8716000000000001e-05 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.296810+0000", 2023-09-30T09:40:44.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011337774 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.296867+0000", 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6993999999999997e-05 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:128356 7.5 e306/301 7:a01fc4fa:::benchmark_data_smithi019_404601_object15078:head v 306'8001, mlcod=286'3496)", 2023-09-30T09:40:44.582 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:00.015151+0000", 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.547048969, 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.292259722, 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.015151+0000", 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.583 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.015151+0000", 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.015152+0000", 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7700000000000004e-07 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.015162+0000", 2023-09-30T09:40:44.584 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0353000000000001e-05 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.287271+0000", 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.272108774 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.287287+0000", 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6019999999999999e-05 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.585 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.288039+0000", 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0007517 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.288074+0000", 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5580999999999997e-05 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.586 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.307381+0000", 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.019306092 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:00.307411+0000", 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0425000000000001e-05 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.587 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:145247 7.6 e314/308 7:6ffe3934:::benchmark_data_smithi019_404601_object31969:head v 314'9003, mlcod=284'1160)", 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:34:36.517093+0000", 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 368.04510724699998, 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25551791000000001, 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517093+0000", 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.588 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517093+0000", 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517094+0000", 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.296e-06 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.517103+0000", 2023-09-30T09:40:44.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6549999999999993e-06 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719169+0000", 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20206605999999999 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719181+0000", 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2208000000000001e-05 2023-09-30T09:40:44.590 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719584+0000", 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000402354 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.719615+0000", 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1711999999999997e-05 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.591 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.772565+0000", 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.052949323 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:34:36.772611+0000", 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6301999999999998e-05 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155902 7.d e325/302 7:b6a3d450:::benchmark_data_smithi019_404601_object42624:head v 325'9721, mlcod=286'3458)", 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.307314+0000", 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 344.25488639100001, 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.23823868200000001, 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307314+0000", 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307314+0000", 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307314+0000", 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6399999999999998e-07 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.594 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307318+0000", 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6219999999999998e-06 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307323+0000", 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6289999999999998e-06 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.595 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307328+0000", 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4399999999999998e-06 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307385+0000", 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6990999999999997e-05 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.307402+0000", 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7577000000000001e-05 2023-09-30T09:40:44.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317476+0000", 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010073268999999999 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545552+0000", 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22807689 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.597 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155897 7.2 e325/301 7:45a09b7b:::benchmark_data_smithi019_404601_object42619:head v 325'9840, mlcod=288'3633)", 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.309357+0000", 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 344.25284323599999, 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25470845600000003, 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309357+0000", 2023-09-30T09:40:44.598 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309357+0000", 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309358+0000", 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9800000000000002e-07 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.599 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309365+0000", 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9029999999999998e-06 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309376+0000", 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1236000000000001e-05 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309386+0000", 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0273e-05 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309471+0000", 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4431999999999994e-05 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.309497+0000", 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6186e-05 2023-09-30T09:40:44.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564016+0000", 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25451902799999998 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564065+0000", 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9400000000000001e-05 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.602 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155904 7.1 e325/316 7:8f01ff57:::benchmark_data_smithi019_404601_object42626:head v 325'9702, mlcod=0'0)", 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.313701+0000", 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 344.24849952900001, 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.24660228100000001, 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.603 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313701+0000", 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313701+0000", 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313702+0000", 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.850000000000001e-07 2023-09-30T09:40:44.604 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313709+0000", 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8199999999999997e-06 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313720+0000", 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0589e-05 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313729+0000", 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5590000000000006e-06 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545499+0000", 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.231770059 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545542+0000", 2023-09-30T09:40:44.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3189999999999998e-05 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.560235+0000", 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014692935000000001 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.560303+0000", 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8144000000000004e-05 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.607 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155910 7.2 e325/301 7:4c9226c2:::benchmark_data_smithi019_404601_object42632:head v 325'9841, mlcod=288'3633)", 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.313754+0000", 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 344.248446554, 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25034391900000003, 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.608 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313754+0000", 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313754+0000", 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313754+0000", 2023-09-30T09:40:44.609 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7700000000000001e-07 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313758+0000", 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7890000000000001e-06 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313764+0000", 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.083e-06 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313770+0000", 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3459999999999999e-06 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545530+0000", 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.23175974899999999 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545556+0000", 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5942e-05 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564071+0000", 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018515485000000002 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564098+0000", 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6248e-05 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.612 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155898 7.a e325/302 7:5ec05665:::benchmark_data_smithi019_404601_object42620:head v 325'9895, mlcod=286'3542)", 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.313782+0000", 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 344.24841806799998, 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25034780899999998, 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.613 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313782+0000", 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313782+0000", 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313782+0000", 2023-09-30T09:40:44.614 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6400000000000001e-07 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.313789+0000", 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9569999999999996e-06 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545496+0000", 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.23170665600000001 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.615 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545512+0000", 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6263e-05 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545610+0000", 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7787000000000004e-05 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.616 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545636+0000", 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6353e-05 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564104+0000", 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018467378999999999 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.564130+0000", 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6250000000000001e-05 2023-09-30T09:40:44.617 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155909 7.9 e325/315 7:991cfd1d:::benchmark_data_smithi019_404601_object42631:head v 325'9572, mlcod=0'0)", 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.314429+0000", 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 344.24777070099998, 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.245911667, 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.618 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.314429+0000", 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.314429+0000", 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.619 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.314430+0000", 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2799999999999995e-07 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.314442+0000", 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1884e-05 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545563+0000", 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.23112110999999999 2023-09-30T09:40:44.620 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545570+0000", 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9890000000000003e-06 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.547758+0000", 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021876830000000002 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.621 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.547783+0000", 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5232000000000002e-05 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.560308+0000", 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012524709 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.560341+0000", 2023-09-30T09:40:44.622 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3331999999999999e-05 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:155905 7.5 e325/308 7:a9bd4049:::benchmark_data_smithi019_404601_object42627:head v 325'9726, mlcod=289'4080)", 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:35:00.317398+0000", 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 344.24480218299999, 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.256421277, 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.623 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317398+0000", 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317398+0000", 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.624 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317399+0000", 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0890000000000001e-06 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.317407+0000", 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8529999999999992e-06 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545519+0000", 2023-09-30T09:40:44.625 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22811198999999999 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.545529+0000", 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0433e-05 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.553328+0000", 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077986449999999999 2023-09-30T09:40:44.626 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.553358+0000", 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0039000000000002e-05 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.573785+0000", 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020426889 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.627 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:35:00.573819+0000", 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4338999999999998e-05 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:209080 7.b e341/321 7:d0cfcec3:::benchmark_data_smithi019_404601_object95802:head v 341'12769, mlcod=292'5339)", 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:37:01.011390+0000", 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.550809896, 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31896941000000001, 2023-09-30T09:40:44.628 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011390+0000", 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011390+0000", 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.629 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011391+0000", 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5799999999999999e-07 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011398+0000", 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6760000000000001e-06 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.630 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011408+0000", 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0567000000000001e-05 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011415+0000", 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1980000000000004e-06 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.631 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.014773+0000", 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003357737 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.014799+0000", 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.635e-05 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330323+0000", 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.315523897 2023-09-30T09:40:44.632 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330360+0000", 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6427000000000002e-05 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(7.b 341/321 [PushOp(7:d3e2f893:::benchmark_data_smithi019_404601_object95815:head, version: 297'5822, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:d3e2f893:::benchmark_data_smithi019_404601_object95815:head@297'5822, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:37:01.011810+0000", 2023-09-30T09:40:44.633 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.55039009399999, 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31848859099999999, 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011810+0000", 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.634 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011810+0000", 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011811+0000", 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3e-07 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011877+0000", 2023-09-30T09:40:44.635 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6383999999999993e-05 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011896+0000", 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8743e-05 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.011912+0000", 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5883e-05 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.636 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018277+0000", 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063649379999999997 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018284+0000", 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4039999999999998e-06 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330299+0000", 2023-09-30T09:40:44.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.312014609 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:209094 7.9 e341/327 7:947e3dfc:::benchmark_data_smithi019_404601_object95816:head v 341'12891, mlcod=0'0)", 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:37:01.013705+0000", 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.548495239, 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31664726300000001, 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.638 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.013705+0000", 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.013705+0000", 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.013706+0000", 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.37e-07 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.013714+0000", 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9009999999999994e-06 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.013725+0000", 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0988999999999999e-05 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.013732+0000", 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1749999999999999e-06 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.013787+0000", 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5158000000000003e-05 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.013807+0000", 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9635000000000001e-05 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.021219+0000", 2023-09-30T09:40:44.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0074123490000000004 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330352+0000", 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30913331900000002 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(7.2 341/301 [PushOp(7:483fc758:::benchmark_data_smithi019_404601_object101404:head, version: 298'6400, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:483fc758:::benchmark_data_smithi019_404601_object101404:head@298'6400, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:40:44.643 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:37:01.016219+0000", 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.54598118000001, 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.314073934, 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016219+0000", 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016219+0000", 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016220+0000", 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4099999999999997e-07 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016257+0000", 2023-09-30T09:40:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7138000000000003e-05 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016277+0000", 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9689000000000001e-05 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016293+0000", 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6691999999999999e-05 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016389+0000", 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5340999999999994e-05 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016401+0000", 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2463000000000001e-05 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330293+0000", 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31389167000000001 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:209098 7.2 e341/301 7:4df6ffbf:::benchmark_data_smithi019_404601_object95820:head v 341'13165, mlcod=288'3633)", 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:37:01.016326+0000", 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.54587421900001, 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33181557699999997, 2023-09-30T09:40:44.648 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016326+0000", 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016326+0000", 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016326+0000", 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.89e-07 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016331+0000", 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0340000000000004e-06 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016339+0000", 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.277e-06 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.016344+0000", 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3279999999999996e-06 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330382+0000", 2023-09-30T09:40:44.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31403840399999999 2023-09-30T09:40:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330405+0000", 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2558999999999999e-05 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.348097+0000", 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017692046999999999 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.348142+0000", 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4539000000000002e-05 2023-09-30T09:40:44.653 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:209095 7.5 e341/327 7:a47d10bc:::benchmark_data_smithi019_404601_object95817:head v 341'13105, mlcod=297'6024)", 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:37:01.018363+0000", 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.543837505, 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31992390100000001, 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018363+0000", 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018363+0000", 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018364+0000", 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9400000000000004e-07 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018371+0000", 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3250000000000002e-06 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018382+0000", 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0852999999999999e-05 2023-09-30T09:40:44.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330287+0000", 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31190544999999997 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330366+0000", 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8614000000000003e-05 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330400+0000", 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3735000000000001e-05 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.338251+0000", 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0078519009999999997 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.338287+0000", 2023-09-30T09:40:44.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5129000000000002e-05 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35424.0:209096 7.e e341/327 7:7ec29afd:::benchmark_data_smithi019_404601_object95818:head v 341'13150, mlcod=0'0)", 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:37:01.018815+0000", 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.54338545799999, 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31838600700000003, 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018815+0000", 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018815+0000", 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018815+0000", 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1099999999999995e-07 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.018822+0000", 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6869999999999997e-06 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.661 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330328+0000", 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31150623599999999 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330337+0000", 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8000000000000004e-06 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330407+0000", 2023-09-30T09:40:44.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9441000000000001e-05 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330436+0000", 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9553999999999999e-05 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.337143+0000", 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0067067719999999997 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.337201+0000", 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7806000000000002e-05 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(7.9 341/327 [PushOp(7:9b112d6f:::benchmark_data_smithi019_404601_object95819:head, version: 297'5904, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:9b112d6f:::benchmark_data_smithi019_404601_object95819:head@297'5904, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:37:01.021132+0000", 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 223.54106778100001, 2023-09-30T09:40:44.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31216305700000002, 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.021132+0000", 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.021132+0000", 2023-09-30T09:40:44.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.021133+0000", 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9800000000000003e-07 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.021190+0000", 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7176999999999998e-05 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330295+0000", 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.309105148 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330332+0000", 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6523000000000003e-05 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330579+0000", 2023-09-30T09:40:44.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024699100000000001 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.330590+0000", 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1174e-05 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:37:01.333295+0000", 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002705346 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.668 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:40:44.669 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:40:44.669 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:40:44.669 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:40:44.748 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:45.184 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[216815]: pgmap v1072: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 3 peering, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 36817/230391 objects degraded (15.980%); 3710/230391 objects misplaced (1.610%); 15 MiB/s, 240 objects/s recovering 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[216815]: osdmap e420: 8 total, 8 up, 6 in 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[216815]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[216815]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[216815]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[216815]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[216815]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[224043]: pgmap v1072: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 3 peering, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 36817/230391 objects degraded (15.980%); 3710/230391 objects misplaced (1.610%); 15 MiB/s, 240 objects/s recovering 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[224043]: osdmap e420: 8 total, 8 up, 6 in 2023-09-30T09:40:45.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[224043]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.186 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[224043]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.186 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[224043]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.186 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[224043]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.186 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:44 smithi019 ceph-mon[224043]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-30T09:40:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:44 smithi167 ceph-mon[192880]: pgmap v1072: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 3 peering, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 36817/230391 objects degraded (15.980%); 3710/230391 objects misplaced (1.610%); 15 MiB/s, 240 objects/s recovering 2023-09-30T09:40:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:44 smithi167 ceph-mon[192880]: osdmap e420: 8 total, 8 up, 6 in 2023-09-30T09:40:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:44 smithi167 ceph-mon[192880]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:44 smithi167 ceph-mon[192880]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:44 smithi167 ceph-mon[192880]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:44 smithi167 ceph-mon[192880]: failed to encode map e420 with expected crc 2023-09-30T09:40:45.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:44 smithi167 ceph-mon[192880]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-30T09:40:45.647 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:40:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:45.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:40:45.753 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:45.785 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:45.975 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:45 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:40:45.977 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:45 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:40:45.998 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:40:46.026 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:40:46.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:45 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:40:46.273 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:46.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:46.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:46.400 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:46.430 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:46.865 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:46.883 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:47.016 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:40:47.043 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:40:47.074 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:40:47.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:46 smithi019 ceph-mon[216815]: pgmap v1074: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 62 op/s; 36573/229956 objects degraded (15.904%); 3494/229956 objects misplaced (1.519%); 5.0 MiB/s, 159 objects/s recovering 2023-09-30T09:40:47.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:46 smithi019 ceph-mon[224043]: pgmap v1074: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 62 op/s; 36573/229956 objects degraded (15.904%); 3494/229956 objects misplaced (1.519%); 5.0 MiB/s, 159 objects/s recovering 2023-09-30T09:40:47.110 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:40:47.138 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:40:47.163 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:40:47.196 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:40:47.228 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:40:47.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:40:47.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:40:47.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:40:47.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:40:47.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:40:47.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:40:47.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:40:47.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:40:47.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:46 smithi167 ceph-mon[192880]: pgmap v1074: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 62 op/s; 36573/229956 objects degraded (15.904%); 3494/229956 objects misplaced (1.519%); 5.0 MiB/s, 159 objects/s recovering 2023-09-30T09:40:47.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/650324703' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:48.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/650324703' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:48.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/650324703' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:48 smithi167 ceph-mon[192880]: pgmap v1075: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 36487/229404 objects degraded (15.905%); 3494/229404 objects misplaced (1.523%); 5.4 MiB/s, 164 objects/s recovering 2023-09-30T09:40:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 36487/229404 objects degraded (15.905%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:48 smithi019 ceph-mon[216815]: pgmap v1075: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 36487/229404 objects degraded (15.905%); 3494/229404 objects misplaced (1.523%); 5.4 MiB/s, 164 objects/s recovering 2023-09-30T09:40:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 36487/229404 objects degraded (15.905%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:48 smithi019 ceph-mon[224043]: pgmap v1075: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 36487/229404 objects degraded (15.905%); 3494/229404 objects misplaced (1.523%); 5.4 MiB/s, 164 objects/s recovering 2023-09-30T09:40:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:40:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 36487/229404 objects degraded (15.905%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:50.285 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:50.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:50.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:50.331 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:50 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-30T09:40:50.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:50 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-30T09:40:50.350 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:50 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-30T09:40:50.387 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:50.428 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:50.890 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:50.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:50.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:50.969 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:40:50.969 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "" 2023-09-30T09:40:50.969 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:40:50.995 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:51.022 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:51.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:51 smithi167 ceph-mon[192880]: pgmap v1076: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 33852/226146 objects degraded (14.969%); 5116/226146 objects misplaced (2.262%); 7.9 MiB/s, 194 objects/s recovering 2023-09-30T09:40:51.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:51 smithi019 ceph-mon[216815]: pgmap v1076: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 33852/226146 objects degraded (14.969%); 5116/226146 objects misplaced (2.262%); 7.9 MiB/s, 194 objects/s recovering 2023-09-30T09:40:51.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:51 smithi019 ceph-mon[224043]: pgmap v1076: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 33852/226146 objects degraded (14.969%); 5116/226146 objects misplaced (2.262%); 7.9 MiB/s, 194 objects/s recovering 2023-09-30T09:40:51.399 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:51.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:51.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:51.507 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:40:51.542 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:51.975 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:40:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:51.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:40:52.080 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:52.106 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:40:52.130 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:52.220 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:52.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:52.327 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:52.334 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:40:52.334 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:40:52.394 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:53.059 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:53 smithi019 ceph-mon[224043]: pgmap v1077: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 33852/226029 objects degraded (14.977%); 5116/226029 objects misplaced (2.263%); 7.0 MiB/s, 172 objects/s recovering 2023-09-30T09:40:53.059 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4292457560' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:53.060 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:53 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:53.080 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:40:53.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:53 smithi019 ceph-mon[216815]: pgmap v1077: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 33852/226029 objects degraded (14.977%); 5116/226029 objects misplaced (2.263%); 7.0 MiB/s, 172 objects/s recovering 2023-09-30T09:40:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4292457560' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:53 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:53 smithi167 ceph-mon[192880]: pgmap v1077: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 33852/226029 objects degraded (14.977%); 5116/226029 objects misplaced (2.263%); 7.0 MiB/s, 172 objects/s recovering 2023-09-30T09:40:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4292457560' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:53 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:40:54.066 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:40:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:40:53] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-09-30T09:40:54.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:54.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:40:54.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[216815]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[216815]: osdmap e421: 8 total, 8 up, 6 in 2023-09-30T09:40:54.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[216815]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[216815]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 33852/226029 objects degraded (14.977%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:54.197 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:40:54.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:54.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:40:54.307 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:40:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[224043]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[224043]: osdmap e421: 8 total, 8 up, 6 in 2023-09-30T09:40:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[224043]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[224043]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:54 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 33852/226029 objects degraded (14.977%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:54.348 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:54 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:40:54.348 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:54 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:40:54.348 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:54 smithi167 ceph-mon[192880]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.349 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:54 smithi167 ceph-mon[192880]: osdmap e421: 8 total, 8 up, 6 in 2023-09-30T09:40:54.349 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:54 smithi167 ceph-mon[192880]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.349 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:54 smithi167 ceph-mon[192880]: failed to encode map e421 with expected crc 2023-09-30T09:40:54.349 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:54 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 33852/226029 objects degraded (14.977%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:54.356 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:54.570 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:54.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:54.675 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:54.707 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:54.850 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:40:55.049 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:55.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:55.154 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:55.185 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:55.232 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:55.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:55.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:55.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:55 smithi019 ceph-mon[224043]: pgmap v1079: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 33677/224253 objects degraded (15.017%); 4352/224253 objects misplaced (1.941%); 9.5 MiB/s, 206 objects/s recovering 2023-09-30T09:40:55.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:55 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:55.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:55 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:55 smithi019 ceph-mon[216815]: pgmap v1079: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 33677/224253 objects degraded (15.017%); 4352/224253 objects misplaced (1.941%); 9.5 MiB/s, 206 objects/s recovering 2023-09-30T09:40:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:55 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:55 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:55.338 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:40:55.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:55 smithi167 ceph-mon[192880]: pgmap v1079: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 33677/224253 objects degraded (15.017%); 4352/224253 objects misplaced (1.941%); 9.5 MiB/s, 206 objects/s recovering 2023-09-30T09:40:55.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:55 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:55.365 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:55 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:55.366 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:55.367 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:55.415 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:40:55.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:55.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:40:55.522 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:55.557 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:55.618 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:55.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:55.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:55.657 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:55.721 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:40:55.751 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:55.811 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:40:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:55.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:40:55.918 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:55.971 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:56.086 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:56.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:56.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:56.192 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:56.233 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:56.288 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:56.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:56.396 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:56.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:56 smithi019 ceph-mon[216815]: pgmap v1080: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 33380/222378 objects degraded (15.010%); 4352/222378 objects misplaced (1.957%); 8.6 MiB/s, 143 objects/s recovering 2023-09-30T09:40:56.445 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:56 smithi019 ceph-mon[224043]: pgmap v1080: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 33380/222378 objects degraded (15.010%); 4352/222378 objects misplaced (1.957%); 8.6 MiB/s, 143 objects/s recovering 2023-09-30T09:40:56.451 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:56.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:56 smithi167 ceph-mon[192880]: pgmap v1080: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 33380/222378 objects degraded (15.010%); 4352/222378 objects misplaced (1.957%); 8.6 MiB/s, 143 objects/s recovering 2023-09-30T09:40:56.988 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0] out_osds: [3, 6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:40:56.988 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:40:56.988 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-30T09:40:56.988 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:40:57.086 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:40:57.388 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:57.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:57.493 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:40:57.534 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:58.650 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:40:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:58.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:40:58.762 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:40:58.791 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:58.826 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:58.841 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:40:58.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:58.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:40:58.885 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:40:58.947 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:40:58.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:58 smithi019 ceph-mon[224043]: pgmap v1081: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 33207/221352 objects degraded (15.002%); 4352/221352 objects misplaced (1.966%); 8.9 MiB/s, 148 objects/s recovering 2023-09-30T09:40:58.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:58.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:58.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 33207/221352 objects degraded (15.002%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:58.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:58 smithi019 ceph-mon[216815]: pgmap v1081: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 33207/221352 objects degraded (15.002%); 4352/221352 objects misplaced (1.966%); 8.9 MiB/s, 148 objects/s recovering 2023-09-30T09:40:58.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:58.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:58.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 33207/221352 objects degraded (15.002%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:58.992 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:58 smithi167 ceph-mon[192880]: pgmap v1081: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 33207/221352 objects degraded (15.002%); 4352/221352 objects misplaced (1.966%); 8.9 MiB/s, 148 objects/s recovering 2023-09-30T09:40:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 33207/221352 objects degraded (15.002%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:40:59.454 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:40:59.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:40:59.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:40:59.495 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:40:59.526 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:40:59.561 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:40:59.564 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:40:59.601 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:40:59.625 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:40:59.638 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:40:59.675 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:40:59.701 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:40:59.701 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:40:59.701 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:40:59.701 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:40:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3321943693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:40:59.707 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:40:59.745 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:40:59.762 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:40:59.763 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:40:59.763 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:40:59.763 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:40:59.763 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:40:59.763 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:40:59.763 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:40:59.763 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:40:59.936 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:40:59.936 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:40:59.936 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:40:59.936 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:40:59.936 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:41:00.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:41:00.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:41:00.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:00.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:40:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3321943693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:00.023 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:41:00.023 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:41:00.023 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:00.023 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:40:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3321943693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:00.079 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:00.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:00.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:00.190 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:00.240 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:01.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:00 smithi167 ceph-mon[192880]: pgmap v1082: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 32924/218421 objects degraded (15.074%); 3484/218421 objects misplaced (1.595%); 12 MiB/s, 184 objects/s recovering 2023-09-30T09:41:01.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:00 smithi019 ceph-mon[216815]: pgmap v1082: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 32924/218421 objects degraded (15.074%); 3484/218421 objects misplaced (1.595%); 12 MiB/s, 184 objects/s recovering 2023-09-30T09:41:01.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:00 smithi019 ceph-mon[224043]: pgmap v1082: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 32924/218421 objects degraded (15.074%); 3484/218421 objects misplaced (1.595%); 12 MiB/s, 184 objects/s recovering 2023-09-30T09:41:02.640 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:02.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:02.744 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:02.806 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:02.960 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:02.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:02.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:03.069 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:03.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:02 smithi019 ceph-mon[216815]: pgmap v1083: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 32924/218283 objects degraded (15.083%); 3484/218283 objects misplaced (1.596%); 12 MiB/s, 184 objects/s recovering 2023-09-30T09:41:03.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:02 smithi019 ceph-mon[224043]: pgmap v1083: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 32924/218283 objects degraded (15.083%); 3484/218283 objects misplaced (1.596%); 12 MiB/s, 184 objects/s recovering 2023-09-30T09:41:03.098 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:03.140 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:03.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:03.244 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:03.270 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:02 smithi167 ceph-mon[192880]: pgmap v1083: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 32924/218283 objects degraded (15.083%); 3484/218283 objects misplaced (1.596%); 12 MiB/s, 184 objects/s recovering 2023-09-30T09:41:03.273 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:03.324 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:03.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:03.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:03.431 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:03.473 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:03.616 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:03.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:03.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:03.659 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:41:03.722 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:03.752 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:03.796 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:03.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:03.901 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:03.930 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:03.981 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:03.986 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:41:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:41:03] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-09-30T09:41:03.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:03.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:03.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 32924/218283 objects degraded (15.083%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:03.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:04.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 32924/218283 objects degraded (15.083%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:04.090 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:04.119 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:04.172 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:04.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:04.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:04.280 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:04.320 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:04.320 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:04.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 32924/218283 objects degraded (15.083%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:04.332 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:04.470 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:04.502 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:41:04.503 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:04.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:04.611 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:04.695 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:04.743 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:41:04.760 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:41:05.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:04 smithi019 ceph-mon[224043]: pgmap v1084: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 32750/216204 objects degraded (15.148%); 2551/216204 objects misplaced (1.180%); 15 MiB/s, 251 objects/s recovering 2023-09-30T09:41:05.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1987081133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:05.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:04 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:05.239 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:04 smithi019 ceph-mon[216815]: pgmap v1084: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 32750/216204 objects degraded (15.148%); 2551/216204 objects misplaced (1.180%); 15 MiB/s, 251 objects/s recovering 2023-09-30T09:41:05.239 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1987081133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:05.239 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:04 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:04 smithi167 ceph-mon[192880]: pgmap v1084: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 32750/216204 objects degraded (15.148%); 2551/216204 objects misplaced (1.180%); 15 MiB/s, 251 objects/s recovering 2023-09-30T09:41:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1987081133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:04 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:05.494 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:05.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:05.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:05.586 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:41:05.599 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:05.627 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:05.671 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:05.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:05.776 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:05.805 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:05.853 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:05.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:05.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:05.960 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:05.992 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:06.047 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:06.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:06.158 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:06.194 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:05 smithi167 ceph-mon[192880]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:41:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:05 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:05 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:05 smithi167 ceph-mon[192880]: osdmap e422: 8 total, 8 up, 6 in 2023-09-30T09:41:06.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:05 smithi019 ceph-mon[224043]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:41:06.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:05 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:05 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:05 smithi019 ceph-mon[224043]: osdmap e422: 8 total, 8 up, 6 in 2023-09-30T09:41:06.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:05 smithi019 ceph-mon[216815]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:41:06.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:05 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:06.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:05 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:06.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:05 smithi019 ceph-mon[216815]: osdmap e422: 8 total, 8 up, 6 in 2023-09-30T09:41:06.602 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:06.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:06.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:06.710 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:06.736 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:06.779 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:06.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:06.883 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:06.911 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:07.000 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:06 smithi167 ceph-mon[192880]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.000 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:06 smithi167 ceph-mon[192880]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:06 smithi167 ceph-mon[192880]: pgmap v1086: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 32439/214044 objects degraded (15.155%); 2551/214044 objects misplaced (1.192%); 12 MiB/s, 205 objects/s recovering 2023-09-30T09:41:07.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:06 smithi167 ceph-mon[192880]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.001 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:06 smithi167 ceph-mon[192880]: osdmap e423: 8 total, 8 up, 6 in 2023-09-30T09:41:07.160 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:07.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:06 smithi019 ceph-mon[216815]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:06 smithi019 ceph-mon[216815]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:06 smithi019 ceph-mon[216815]: pgmap v1086: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 32439/214044 objects degraded (15.155%); 2551/214044 objects misplaced (1.192%); 12 MiB/s, 205 objects/s recovering 2023-09-30T09:41:07.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:06 smithi019 ceph-mon[216815]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.164 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:06 smithi019 ceph-mon[216815]: osdmap e423: 8 total, 8 up, 6 in 2023-09-30T09:41:07.164 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:07 smithi019 ceph-mon[224043]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:07 smithi019 ceph-mon[224043]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:07 smithi019 ceph-mon[224043]: pgmap v1086: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 32439/214044 objects degraded (15.155%); 2551/214044 objects misplaced (1.192%); 12 MiB/s, 205 objects/s recovering 2023-09-30T09:41:07.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:07 smithi019 ceph-mon[224043]: failed to encode map e422 with expected crc 2023-09-30T09:41:07.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:07 smithi019 ceph-mon[224043]: osdmap e423: 8 total, 8 up, 6 in 2023-09-30T09:41:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:07.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:07.248 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1084,"stamp":"2023-09-30T09:41:03.647117+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4725718714,"num_objects":72068,"num_object_clones":0,"num_object_copies":216204,"num_objects_missing_on_primary":2659,"num_objects_missing":2659,"num_objects_degraded":32750,"num_objects_misplaced":2551,"num_objects_unfound":0,"num_objects_dirty":72068,"num_whiteouts":0,"num_read":5989,"num_read_kb":12231,"num_write":113781,"num_write_kb":5929426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28515,"num_bytes_recovered":1761837672,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149294,"ondisk_log_size":149294,"up":387,"acting":374,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":428,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15327720,"kb_used_data":12669240,"kb_used_omap":21,"kb_used_meta":2658282,"kb_avail":547143192,"statfs":{"total":575970213888,"available":560274628608,"internally_reserved":0,"allocated":12973301760,"data_stored":12961650595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21604,"internal_metadata":2722081692},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,3,3,5,1,2,4,4,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-195821568,"num_objects":-2988,"num_object_clones":0,"num_object_copies":-8964,"num_objects_missing_on_primary":-645,"num_objects_missing":-645,"num_objects_degraded":-927,"num_objects_misplaced":-2565,"num_objects_unfound":0,"num_objects_dirty":-2988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2988,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2666,"num_bytes_recovered":170131456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.582686"},"pg_stats":[{"pgid":"8.11","version":"421'5916","reported_seq":5944,"reported_epoch":421,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:03.246180+0000","last_change":"2023-09-30T09:40:37.604830+0000","last_active":"2023-09-30T09:41:03.246180+0000","last_peered":"2023-09-30T09:41:03.246180+0000","last_clean":"2023-09-30T09:40:21.233253+0000","last_became_active":"2023-09-30T09:40:37.596780+0000","last_became_peered":"2023-09-30T09:40:37.596780+0000","last_unstale":"2023-09-30T09:41:03.246180+0000","last_undegraded":"2023-09-30T09:40:37.578459+0000","last_fullsized":"2023-09-30T09:41:03.246180+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5916,"log_dups_size":0,"ondisk_log_size":5916,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140869632,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":58,"num_objects_missing":58,"num_objects_degraded":58,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3548,"num_write_kb":182304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":2150}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":909,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904497+0000","last_change":"2023-09-30T09:38:30.746866+0000","last_active":"2023-09-30T09:40:43.904497+0000","last_peered":"2023-09-30T09:40:43.904497+0000","last_clean":"2023-09-30T09:40:43.904497+0000","last_became_active":"2023-09-30T09:38:30.746361+0000","last_became_peered":"2023-09-30T09:38:30.746361+0000","last_unstale":"2023-09-30T09:40:43.904497+0000","last_undegraded":"2023-09-30T09:40:43.904497+0000","last_fullsized":"2023-09-30T09:40:43.904497+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:06:25.130460+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":439,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.901550+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:40:43.901550+0000","last_peered":"2023-09-30T09:40:43.901550+0000","last_clean":"2023-09-30T09:40:43.901550+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:40:43.901550+0000","last_undegraded":"2023-09-30T09:40:43.901550+0000","last_fullsized":"2023-09-30T09:40:43.901550+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":344,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.910945+0000","last_change":"2023-09-30T09:39:47.872348+0000","last_active":"2023-09-30T09:40:43.910945+0000","last_peered":"2023-09-30T09:40:43.910945+0000","last_clean":"2023-09-30T09:40:43.910945+0000","last_became_active":"2023-09-30T09:39:47.870179+0000","last_became_peered":"2023-09-30T09:39:47.870179+0000","last_unstale":"2023-09-30T09:40:43.910945+0000","last_undegraded":"2023-09-30T09:40:43.910945+0000","last_fullsized":"2023-09-30T09:40:43.910945+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:33:45.842608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.12","version":"421'5749","reported_seq":5849,"reported_epoch":421,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:40:59.205686+0000","last_change":"2023-09-30T09:40:37.640460+0000","last_active":"2023-09-30T09:40:59.205686+0000","last_peered":"2023-09-30T09:40:59.205686+0000","last_clean":"2023-09-30T09:40:21.246088+0000","last_became_active":"2023-09-30T09:40:37.631437+0000","last_became_peered":"2023-09-30T09:40:37.631437+0000","last_unstale":"2023-09-30T09:40:59.205686+0000","last_undegraded":"2023-09-30T09:40:37.608497+0000","last_fullsized":"2023-09-30T09:40:59.205686+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145391616,"num_objects":2219,"num_object_clones":0,"num_object_copies":6657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":49,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3415,"num_write_kb":180256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5","objects":2219}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":905,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.902370+0000","last_change":"2023-09-30T09:40:39.587771+0000","last_active":"2023-09-30T09:40:43.902370+0000","last_peered":"2023-09-30T09:40:43.902370+0000","last_clean":"2023-09-30T09:40:43.902370+0000","last_became_active":"2023-09-30T09:40:39.587334+0000","last_became_peered":"2023-09-30T09:40:39.587334+0000","last_unstale":"2023-09-30T09:40:43.902370+0000","last_undegraded":"2023-09-30T09:40:43.902370+0000","last_fullsized":"2023-09-30T09:40:43.902370+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:43:27.541800+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":251,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903127+0000","last_change":"2023-09-30T09:39:47.869479+0000","last_active":"2023-09-30T09:40:43.903127+0000","last_peered":"2023-09-30T09:40:43.903127+0000","last_clean":"2023-09-30T09:40:43.903127+0000","last_became_active":"2023-09-30T09:39:47.869184+0000","last_became_peered":"2023-09-30T09:39:47.869184+0000","last_unstale":"2023-09-30T09:40:43.903127+0000","last_undegraded":"2023-09-30T09:40:43.903127+0000","last_fullsized":"2023-09-30T09:40:43.903127+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:03:52.322427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":16,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.907862+0000","last_change":"2023-09-30T09:40:36.599223+0000","last_active":"2023-09-30T09:40:43.907862+0000","last_peered":"2023-09-30T09:40:43.907862+0000","last_clean":"2023-09-30T09:40:43.907862+0000","last_became_active":"2023-09-30T09:40:36.597312+0000","last_became_peered":"2023-09-30T09:40:36.597312+0000","last_unstale":"2023-09-30T09:40:43.907862+0000","last_undegraded":"2023-09-30T09:40:43.907862+0000","last_fullsized":"2023-09-30T09:40:43.907862+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:29.960110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"421'5880","reported_seq":11784,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:41:01.535631+0000","last_change":"2023-09-30T09:40:20.789486+0000","last_active":"2023-09-30T09:41:01.535631+0000","last_peered":"2023-09-30T09:41:01.535631+0000","last_clean":"2023-09-30T09:41:01.535631+0000","last_became_active":"2023-09-30T09:40:20.780460+0000","last_became_peered":"2023-09-30T09:40:20.780460+0000","last_unstale":"2023-09-30T09:41:01.535631+0000","last_undegraded":"2023-09-30T09:41:01.535631+0000","last_fullsized":"2023-09-30T09:41:01.535631+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5880,"log_dups_size":0,"ondisk_log_size":5880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146505728,"num_objects":2236,"num_object_clones":0,"num_object_copies":6708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3498,"num_write_kb":183456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2903,"num_bytes_recovered":171573248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":654,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.901807+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:40:43.901807+0000","last_peered":"2023-09-30T09:40:43.901807+0000","last_clean":"2023-09-30T09:40:43.901807+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:40:43.901807+0000","last_undegraded":"2023-09-30T09:40:43.901807+0000","last_fullsized":"2023-09-30T09:40:43.901807+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":403,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.902815+0000","last_change":"2023-09-30T09:39:58.574864+0000","last_active":"2023-09-30T09:40:43.902815+0000","last_peered":"2023-09-30T09:40:43.902815+0000","last_clean":"2023-09-30T09:40:43.902815+0000","last_became_active":"2023-09-30T09:39:58.574566+0000","last_became_peered":"2023-09-30T09:39:58.574566+0000","last_unstale":"2023-09-30T09:40:43.902815+0000","last_undegraded":"2023-09-30T09:40:43.902815+0000","last_fullsized":"2023-09-30T09:40:43.902815+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:23:17.675484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":18,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.892343+0000","last_change":"2023-09-30T09:40:36.617977+0000","last_active":"2023-09-30T09:40:43.892343+0000","last_peered":"2023-09-30T09:40:43.892343+0000","last_clean":"2023-09-30T09:40:43.892343+0000","last_became_active":"2023-09-30T09:40:36.615491+0000","last_became_peered":"2023-09-30T09:40:36.615491+0000","last_unstale":"2023-09-30T09:40:43.892343+0000","last_undegraded":"2023-09-30T09:40:43.892343+0000","last_fullsized":"2023-09-30T09:40:43.892343+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:07.410092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":795,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.912903+0000","last_change":"2023-09-30T09:38:30.744203+0000","last_active":"2023-09-30T09:40:43.912903+0000","last_peered":"2023-09-30T09:40:43.912903+0000","last_clean":"2023-09-30T09:40:43.912903+0000","last_became_active":"2023-09-30T09:38:30.740607+0000","last_became_peered":"2023-09-30T09:38:30.740607+0000","last_unstale":"2023-09-30T09:40:43.912903+0000","last_undegraded":"2023-09-30T09:40:43.912903+0000","last_fullsized":"2023-09-30T09:40:43.912903+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.164029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":18,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.892264+0000","last_change":"2023-09-30T09:40:36.618009+0000","last_active":"2023-09-30T09:40:43.892264+0000","last_peered":"2023-09-30T09:40:43.892264+0000","last_clean":"2023-09-30T09:40:43.892264+0000","last_became_active":"2023-09-30T09:40:36.615053+0000","last_became_peered":"2023-09-30T09:40:36.615053+0000","last_unstale":"2023-09-30T09:40:43.892264+0000","last_undegraded":"2023-09-30T09:40:43.892264+0000","last_fullsized":"2023-09-30T09:40:43.892264+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:37:53.101972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":392,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.900929+0000","last_change":"2023-09-30T09:39:47.872093+0000","last_active":"2023-09-30T09:40:43.900929+0000","last_peered":"2023-09-30T09:40:43.900929+0000","last_clean":"2023-09-30T09:40:43.900929+0000","last_became_active":"2023-09-30T09:39:47.871723+0000","last_became_peered":"2023-09-30T09:39:47.871723+0000","last_unstale":"2023-09-30T09:40:43.900929+0000","last_undegraded":"2023-09-30T09:40:43.900929+0000","last_fullsized":"2023-09-30T09:40:43.900929+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:18:06.256339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":608,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903927+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:40:43.903927+0000","last_peered":"2023-09-30T09:40:43.903927+0000","last_clean":"2023-09-30T09:40:43.903927+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:40:43.903927+0000","last_undegraded":"2023-09-30T09:40:43.903927+0000","last_fullsized":"2023-09-30T09:40:43.903927+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":309,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903906+0000","last_change":"2023-09-30T09:38:27.687229+0000","last_active":"2023-09-30T09:40:43.903906+0000","last_peered":"2023-09-30T09:40:43.903906+0000","last_clean":"2023-09-30T09:40:43.903906+0000","last_became_active":"2023-09-30T09:38:27.686912+0000","last_became_peered":"2023-09-30T09:38:27.686912+0000","last_unstale":"2023-09-30T09:40:43.903906+0000","last_undegraded":"2023-09-30T09:40:43.903906+0000","last_fullsized":"2023-09-30T09:40:43.903906+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:05:28.924932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":385,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913441+0000","last_change":"2023-09-30T09:40:36.608363+0000","last_active":"2023-09-30T09:40:43.913441+0000","last_peered":"2023-09-30T09:40:43.913441+0000","last_clean":"2023-09-30T09:40:43.913441+0000","last_became_active":"2023-09-30T09:40:36.608028+0000","last_became_peered":"2023-09-30T09:40:36.608028+0000","last_unstale":"2023-09-30T09:40:43.913441+0000","last_undegraded":"2023-09-30T09:40:43.913441+0000","last_fullsized":"2023-09-30T09:40:43.913441+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:02.611886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":627,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904005+0000","last_change":"2023-09-30T09:40:01.615146+0000","last_active":"2023-09-30T09:40:43.904005+0000","last_peered":"2023-09-30T09:40:43.904005+0000","last_clean":"2023-09-30T09:40:43.904005+0000","last_became_active":"2023-09-30T09:40:01.614727+0000","last_became_peered":"2023-09-30T09:40:01.614727+0000","last_unstale":"2023-09-30T09:40:43.904005+0000","last_undegraded":"2023-09-30T09:40:43.904005+0000","last_fullsized":"2023-09-30T09:40:43.904005+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:46:24.281214+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":119,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913206+0000","last_change":"2023-09-30T09:40:36.618707+0000","last_active":"2023-09-30T09:40:43.913206+0000","last_peered":"2023-09-30T09:40:43.913206+0000","last_clean":"2023-09-30T09:40:43.913206+0000","last_became_active":"2023-09-30T09:40:36.597642+0000","last_became_peered":"2023-09-30T09:40:36.597642+0000","last_unstale":"2023-09-30T09:40:43.913206+0000","last_undegraded":"2023-09-30T09:40:43.913206+0000","last_fullsized":"2023-09-30T09:40:43.913206+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:09.447554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":258,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903088+0000","last_change":"2023-09-30T09:39:58.587940+0000","last_active":"2023-09-30T09:40:43.903088+0000","last_peered":"2023-09-30T09:40:43.903088+0000","last_clean":"2023-09-30T09:40:43.903088+0000","last_became_active":"2023-09-30T09:39:58.587622+0000","last_became_peered":"2023-09-30T09:39:58.587622+0000","last_unstale":"2023-09-30T09:40:43.903088+0000","last_undegraded":"2023-09-30T09:40:43.903088+0000","last_fullsized":"2023-09-30T09:40:43.903088+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:45:14.945251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":460,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.960136+0000","last_change":"2023-09-30T09:40:43.945675+0000","last_active":"2023-09-30T09:40:43.960136+0000","last_peered":"2023-09-30T09:40:43.960136+0000","last_clean":"2023-09-30T09:40:43.960136+0000","last_became_active":"2023-09-30T09:40:43.945291+0000","last_became_peered":"2023-09-30T09:40:43.945291+0000","last_unstale":"2023-09-30T09:40:43.960136+0000","last_undegraded":"2023-09-30T09:40:43.960136+0000","last_fullsized":"2023-09-30T09:40:43.960136+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:41.055314+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":445,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.902569+0000","last_change":"2023-09-30T09:39:47.882770+0000","last_active":"2023-09-30T09:40:43.902569+0000","last_peered":"2023-09-30T09:40:43.902569+0000","last_clean":"2023-09-30T09:40:43.902569+0000","last_became_active":"2023-09-30T09:39:47.864005+0000","last_became_peered":"2023-09-30T09:39:47.864005+0000","last_unstale":"2023-09-30T09:40:43.902569+0000","last_undegraded":"2023-09-30T09:40:43.902569+0000","last_fullsized":"2023-09-30T09:40:43.902569+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:09.572406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14166,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":426,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.906027+0000","last_change":"2023-09-30T09:39:47.876698+0000","last_active":"2023-09-30T09:40:43.906027+0000","last_peered":"2023-09-30T09:40:43.906027+0000","last_clean":"2023-09-30T09:40:43.906027+0000","last_became_active":"2023-09-30T09:39:47.876412+0000","last_became_peered":"2023-09-30T09:39:47.876412+0000","last_unstale":"2023-09-30T09:40:43.906027+0000","last_undegraded":"2023-09-30T09:40:43.906027+0000","last_fullsized":"2023-09-30T09:40:43.906027+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:27:37.752128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"421'6025","reported_seq":15481,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:40:59.367859+0000","last_change":"2023-09-30T09:40:42.887469+0000","last_active":"2023-09-30T09:40:59.367859+0000","last_peered":"2023-09-30T09:40:59.367859+0000","last_clean":"2023-09-30T09:40:59.367859+0000","last_became_active":"2023-09-30T09:40:42.880205+0000","last_became_peered":"2023-09-30T09:40:42.880205+0000","last_unstale":"2023-09-30T09:40:59.367859+0000","last_undegraded":"2023-09-30T09:40:59.367859+0000","last_fullsized":"2023-09-30T09:40:59.367859+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6025,"log_dups_size":0,"ondisk_log_size":6025,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147685376,"num_objects":2254,"num_object_clones":0,"num_object_copies":6762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3596,"num_write_kb":187168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4668,"num_bytes_recovered":274399232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":648,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.905465+0000","last_change":"2023-09-30T09:40:01.605467+0000","last_active":"2023-09-30T09:40:43.905465+0000","last_peered":"2023-09-30T09:40:43.905465+0000","last_clean":"2023-09-30T09:40:43.905465+0000","last_became_active":"2023-09-30T09:40:01.605059+0000","last_became_peered":"2023-09-30T09:40:01.605059+0000","last_unstale":"2023-09-30T09:40:43.905465+0000","last_undegraded":"2023-09-30T09:40:43.905465+0000","last_fullsized":"2023-09-30T09:40:43.905465+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":71,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914297+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:40:43.914297+0000","last_peered":"2023-09-30T09:40:43.914297+0000","last_clean":"2023-09-30T09:40:43.914297+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:40:43.914297+0000","last_undegraded":"2023-09-30T09:40:43.914297+0000","last_fullsized":"2023-09-30T09:40:43.914297+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":413,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914789+0000","last_change":"2023-09-30T09:40:22.811912+0000","last_active":"2023-09-30T09:40:43.914789+0000","last_peered":"2023-09-30T09:40:43.914789+0000","last_clean":"2023-09-30T09:40:43.914789+0000","last_became_active":"2023-09-30T09:40:22.810606+0000","last_became_peered":"2023-09-30T09:40:22.810606+0000","last_unstale":"2023-09-30T09:40:43.914789+0000","last_undegraded":"2023-09-30T09:40:43.914789+0000","last_fullsized":"2023-09-30T09:40:43.914789+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:46:15.913218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.19","version":"421'5923","reported_seq":7081,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:40:59.347179+0000","last_change":"2023-09-30T09:40:00.506187+0000","last_active":"2023-09-30T09:40:59.347179+0000","last_peered":"2023-09-30T09:40:59.347179+0000","last_clean":"2023-09-30T09:39:46.775307+0000","last_became_active":"2023-09-30T09:40:00.498948+0000","last_became_peered":"2023-09-30T09:40:00.498948+0000","last_unstale":"2023-09-30T09:40:59.347179+0000","last_undegraded":"2023-09-30T09:39:59.671418+0000","last_fullsized":"2023-09-30T09:39:59.584439+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5923,"log_dups_size":0,"ondisk_log_size":5923,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146964480,"num_objects":2243,"num_object_clones":0,"num_object_copies":6729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5102,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3535,"num_write_kb":184864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":353,"num_bytes_recovered":23134208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":2258},{"shards":"4,5","objects":-15}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":807,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.905169+0000","last_change":"2023-09-30T09:39:50.837500+0000","last_active":"2023-09-30T09:40:43.905169+0000","last_peered":"2023-09-30T09:40:43.905169+0000","last_clean":"2023-09-30T09:40:43.905169+0000","last_became_active":"2023-09-30T09:39:50.837185+0000","last_became_peered":"2023-09-30T09:39:50.837185+0000","last_unstale":"2023-09-30T09:40:43.905169+0000","last_undegraded":"2023-09-30T09:40:43.905169+0000","last_fullsized":"2023-09-30T09:40:43.905169+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:35:32.171780+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":319,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914100+0000","last_change":"2023-09-30T09:39:58.583120+0000","last_active":"2023-09-30T09:40:43.914100+0000","last_peered":"2023-09-30T09:40:43.914100+0000","last_clean":"2023-09-30T09:40:43.914100+0000","last_became_active":"2023-09-30T09:39:58.582799+0000","last_became_peered":"2023-09-30T09:39:58.582799+0000","last_unstale":"2023-09-30T09:40:43.914100+0000","last_undegraded":"2023-09-30T09:40:43.914100+0000","last_fullsized":"2023-09-30T09:40:43.914100+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:51:22.821149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":160,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.902596+0000","last_change":"2023-09-30T09:40:36.622997+0000","last_active":"2023-09-30T09:40:43.902596+0000","last_peered":"2023-09-30T09:40:43.902596+0000","last_clean":"2023-09-30T09:40:43.902596+0000","last_became_active":"2023-09-30T09:40:36.622694+0000","last_became_peered":"2023-09-30T09:40:36.622694+0000","last_unstale":"2023-09-30T09:40:43.902596+0000","last_undegraded":"2023-09-30T09:40:43.902596+0000","last_fullsized":"2023-09-30T09:40:43.902596+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:27:59.419339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":851,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.905753+0000","last_change":"2023-09-30T09:40:39.589288+0000","last_active":"2023-09-30T09:40:43.905753+0000","last_peered":"2023-09-30T09:40:43.905753+0000","last_clean":"2023-09-30T09:40:43.905753+0000","last_became_active":"2023-09-30T09:40:39.588929+0000","last_became_peered":"2023-09-30T09:40:39.588929+0000","last_unstale":"2023-09-30T09:40:43.905753+0000","last_undegraded":"2023-09-30T09:40:43.905753+0000","last_fullsized":"2023-09-30T09:40:43.905753+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":95,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913591+0000","last_change":"2023-09-30T09:40:36.592638+0000","last_active":"2023-09-30T09:40:43.913591+0000","last_peered":"2023-09-30T09:40:43.913591+0000","last_clean":"2023-09-30T09:40:43.913591+0000","last_became_active":"2023-09-30T09:40:36.592313+0000","last_became_peered":"2023-09-30T09:40:36.592313+0000","last_unstale":"2023-09-30T09:40:43.913591+0000","last_undegraded":"2023-09-30T09:40:43.913591+0000","last_fullsized":"2023-09-30T09:40:43.913591+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:18:12.781018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":317,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.915509+0000","last_change":"2023-09-30T09:39:58.586100+0000","last_active":"2023-09-30T09:40:43.915509+0000","last_peered":"2023-09-30T09:40:43.915509+0000","last_clean":"2023-09-30T09:40:43.915509+0000","last_became_active":"2023-09-30T09:39:58.585756+0000","last_became_peered":"2023-09-30T09:39:58.585756+0000","last_unstale":"2023-09-30T09:40:43.915509+0000","last_undegraded":"2023-09-30T09:40:43.915509+0000","last_fullsized":"2023-09-30T09:40:43.915509+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:56.123014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":906,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904307+0000","last_change":"2023-09-30T09:39:50.836014+0000","last_active":"2023-09-30T09:40:43.904307+0000","last_peered":"2023-09-30T09:40:43.904307+0000","last_clean":"2023-09-30T09:40:43.904307+0000","last_became_active":"2023-09-30T09:39:50.835632+0000","last_became_peered":"2023-09-30T09:39:50.835632+0000","last_unstale":"2023-09-30T09:40:43.904307+0000","last_undegraded":"2023-09-30T09:40:43.904307+0000","last_fullsized":"2023-09-30T09:40:43.904307+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:26.067574+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":487,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903466+0000","last_change":"2023-09-30T09:39:58.604069+0000","last_active":"2023-09-30T09:40:43.903466+0000","last_peered":"2023-09-30T09:40:43.903466+0000","last_clean":"2023-09-30T09:40:43.903466+0000","last_became_active":"2023-09-30T09:39:58.585674+0000","last_became_peered":"2023-09-30T09:39:58.585674+0000","last_unstale":"2023-09-30T09:40:43.903466+0000","last_undegraded":"2023-09-30T09:40:43.903466+0000","last_fullsized":"2023-09-30T09:40:43.903466+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:03.491449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":437,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914665+0000","last_change":"2023-09-30T09:38:45.606456+0000","last_active":"2023-09-30T09:40:43.914665+0000","last_peered":"2023-09-30T09:40:43.914665+0000","last_clean":"2023-09-30T09:40:43.914665+0000","last_became_active":"2023-09-30T09:38:45.604792+0000","last_became_peered":"2023-09-30T09:38:45.604792+0000","last_unstale":"2023-09-30T09:40:43.914665+0000","last_undegraded":"2023-09-30T09:40:43.914665+0000","last_fullsized":"2023-09-30T09:40:43.914665+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:11.533941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"421'5951","reported_seq":6033,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:40:59.149583+0000","last_change":"2023-09-30T09:39:15.603264+0000","last_active":"2023-09-30T09:40:59.149583+0000","last_peered":"2023-09-30T09:40:59.149583+0000","last_clean":"2023-09-30T09:40:59.149583+0000","last_became_active":"2023-09-30T09:39:15.600826+0000","last_became_peered":"2023-09-30T09:39:15.600826+0000","last_unstale":"2023-09-30T09:40:59.149583+0000","last_undegraded":"2023-09-30T09:40:59.149583+0000","last_fullsized":"2023-09-30T09:40:59.149583+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5951,"log_dups_size":0,"ondisk_log_size":5951,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152633344,"num_objects":2329,"num_object_clones":0,"num_object_copies":6987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3545,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":787,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913060+0000","last_change":"2023-09-30T09:38:30.737676+0000","last_active":"2023-09-30T09:40:43.913060+0000","last_peered":"2023-09-30T09:40:43.913060+0000","last_clean":"2023-09-30T09:40:43.913060+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:40:43.913060+0000","last_undegraded":"2023-09-30T09:40:43.913060+0000","last_fullsized":"2023-09-30T09:40:43.913060+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.740851+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.740851+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":83,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914952+0000","last_change":"2023-09-30T09:39:58.609474+0000","last_active":"2023-09-30T09:40:43.914952+0000","last_peered":"2023-09-30T09:40:43.914952+0000","last_clean":"2023-09-30T09:40:43.914952+0000","last_became_active":"2023-09-30T09:39:58.583107+0000","last_became_peered":"2023-09-30T09:39:58.583107+0000","last_unstale":"2023-09-30T09:40:43.914952+0000","last_undegraded":"2023-09-30T09:40:43.914952+0000","last_fullsized":"2023-09-30T09:40:43.914952+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:31.980540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":130,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903545+0000","last_change":"2023-09-30T09:39:47.869741+0000","last_active":"2023-09-30T09:40:43.903545+0000","last_peered":"2023-09-30T09:40:43.903545+0000","last_clean":"2023-09-30T09:40:43.903545+0000","last_became_active":"2023-09-30T09:39:47.869354+0000","last_became_peered":"2023-09-30T09:39:47.869354+0000","last_unstale":"2023-09-30T09:40:43.903545+0000","last_undegraded":"2023-09-30T09:40:43.903545+0000","last_fullsized":"2023-09-30T09:40:43.903545+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:28:16.966118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"421'5905","reported_seq":12522,"reported_epoch":421,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:40:59.161095+0000","last_change":"2023-09-30T09:40:37.624519+0000","last_active":"2023-09-30T09:40:59.161095+0000","last_peered":"2023-09-30T09:40:59.161095+0000","last_clean":"2023-09-30T09:40:21.254315+0000","last_became_active":"2023-09-30T09:40:37.616502+0000","last_became_peered":"2023-09-30T09:40:37.616502+0000","last_unstale":"2023-09-30T09:40:59.161095+0000","last_undegraded":"2023-09-30T09:40:37.596565+0000","last_fullsized":"2023-09-30T09:40:59.161095+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5905,"log_dups_size":0,"ondisk_log_size":5905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149356544,"num_objects":2279,"num_object_clones":0,"num_object_copies":6837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":69,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3503,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3153,"num_bytes_recovered":191037440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"0,4,7","objects":2279}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":779,"reported_epoch":421,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:40:53.070748+0000","last_change":"2023-09-30T09:40:37.595615+0000","last_active":"2023-09-30T09:40:53.070748+0000","last_peered":"2023-09-30T09:40:53.070748+0000","last_clean":"2023-09-30T09:31:49.462297+0000","last_became_active":"2023-09-30T09:40:37.595324+0000","last_became_peered":"2023-09-30T09:40:37.595324+0000","last_unstale":"2023-09-30T09:40:53.070748+0000","last_undegraded":"2023-09-30T09:40:37.578525+0000","last_fullsized":"2023-09-30T09:40:53.070748+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5","objects":-1},{"shards":"0,4,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":67,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.915324+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:40:43.915324+0000","last_peered":"2023-09-30T09:40:43.915324+0000","last_clean":"2023-09-30T09:40:43.915324+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:40:43.915324+0000","last_undegraded":"2023-09-30T09:40:43.915324+0000","last_fullsized":"2023-09-30T09:40:43.915324+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":463,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903440+0000","last_change":"2023-09-30T09:40:36.593049+0000","last_active":"2023-09-30T09:40:43.903440+0000","last_peered":"2023-09-30T09:40:43.903440+0000","last_clean":"2023-09-30T09:40:43.903440+0000","last_became_active":"2023-09-30T09:40:36.590336+0000","last_became_peered":"2023-09-30T09:40:36.590336+0000","last_unstale":"2023-09-30T09:40:43.903440+0000","last_undegraded":"2023-09-30T09:40:43.903440+0000","last_fullsized":"2023-09-30T09:40:43.903440+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:54.795919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"421'6016","reported_seq":9758,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:03.220299+0000","last_change":"2023-09-30T09:40:40.784953+0000","last_active":"2023-09-30T09:41:03.220299+0000","last_peered":"2023-09-30T09:41:03.220299+0000","last_clean":"2023-09-30T09:38:26.430308+0000","last_became_active":"2023-09-30T09:40:39.913250+0000","last_became_peered":"2023-09-30T09:40:39.913250+0000","last_unstale":"2023-09-30T09:41:03.220299+0000","last_undegraded":"2023-09-30T09:40:39.696160+0000","last_fullsized":"2023-09-30T09:40:37.581870+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6016,"log_dups_size":0,"ondisk_log_size":6016,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147521536,"num_objects":2251,"num_object_clones":0,"num_object_copies":6753,"num_objects_missing_on_primary":52,"num_objects_missing":52,"num_objects_degraded":2304,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3575,"num_write_kb":186432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":924,"num_bytes_recovered":60555264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,2],"avail_no_missing":["6"],"object_location_counts":[{"shards":"0,2","objects":268},{"shards":"0,2,6","objects":227},{"shards":"0,6,7","objects":1756}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":588,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904000+0000","last_change":"2023-09-30T09:39:56.915934+0000","last_active":"2023-09-30T09:40:43.904000+0000","last_peered":"2023-09-30T09:40:43.904000+0000","last_clean":"2023-09-30T09:40:43.904000+0000","last_became_active":"2023-09-30T09:39:56.915617+0000","last_became_peered":"2023-09-30T09:39:56.915617+0000","last_unstale":"2023-09-30T09:40:43.904000+0000","last_undegraded":"2023-09-30T09:40:43.904000+0000","last_fullsized":"2023-09-30T09:40:43.904000+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":83,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.912696+0000","last_change":"2023-09-30T09:39:47.881884+0000","last_active":"2023-09-30T09:40:43.912696+0000","last_peered":"2023-09-30T09:40:43.912696+0000","last_clean":"2023-09-30T09:40:43.912696+0000","last_became_active":"2023-09-30T09:39:47.871068+0000","last_became_peered":"2023-09-30T09:39:47.871068+0000","last_unstale":"2023-09-30T09:40:43.912696+0000","last_undegraded":"2023-09-30T09:40:43.912696+0000","last_fullsized":"2023-09-30T09:40:43.912696+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:03:57.045359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":390,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913777+0000","last_change":"2023-09-30T09:39:47.851588+0000","last_active":"2023-09-30T09:40:43.913777+0000","last_peered":"2023-09-30T09:40:43.913777+0000","last_clean":"2023-09-30T09:40:43.913777+0000","last_became_active":"2023-09-30T09:39:47.851010+0000","last_became_peered":"2023-09-30T09:39:47.851010+0000","last_unstale":"2023-09-30T09:40:43.913777+0000","last_undegraded":"2023-09-30T09:40:43.913777+0000","last_fullsized":"2023-09-30T09:40:43.913777+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:47:13.369816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"421'6012","reported_seq":7788,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:01.694836+0000","last_change":"2023-09-30T09:39:55.210294+0000","last_active":"2023-09-30T09:41:01.694836+0000","last_peered":"2023-09-30T09:41:01.694836+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:39:55.103786+0000","last_became_peered":"2023-09-30T09:39:55.103786+0000","last_unstale":"2023-09-30T09:41:01.694836+0000","last_undegraded":"2023-09-30T09:39:51.345690+0000","last_fullsized":"2023-09-30T09:39:48.814537+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6012,"log_dups_size":0,"ondisk_log_size":6012,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149585920,"num_objects":2282,"num_object_clones":0,"num_object_copies":6846,"num_objects_missing_on_primary":1073,"num_objects_missing":1073,"num_objects_degraded":3637,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3576,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":647,"num_bytes_recovered":42401792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":-11},{"shards":"1,2,3,6","objects":8},{"shards":"1,2,6","objects":1212},{"shards":"1,3,6","objects":1073}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":636,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.908398+0000","last_change":"2023-09-30T09:38:48.662225+0000","last_active":"2023-09-30T09:40:43.908398+0000","last_peered":"2023-09-30T09:40:43.908398+0000","last_clean":"2023-09-30T09:40:43.908398+0000","last_became_active":"2023-09-30T09:38:48.661883+0000","last_became_peered":"2023-09-30T09:38:48.661883+0000","last_unstale":"2023-09-30T09:40:43.908398+0000","last_undegraded":"2023-09-30T09:40:43.908398+0000","last_fullsized":"2023-09-30T09:40:43.908398+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:01.892446+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":420,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.905988+0000","last_change":"2023-09-30T09:40:36.623400+0000","last_active":"2023-09-30T09:40:43.905988+0000","last_peered":"2023-09-30T09:40:43.905988+0000","last_clean":"2023-09-30T09:40:43.905988+0000","last_became_active":"2023-09-30T09:40:36.622842+0000","last_became_peered":"2023-09-30T09:40:36.622842+0000","last_unstale":"2023-09-30T09:40:43.905988+0000","last_undegraded":"2023-09-30T09:40:43.905988+0000","last_fullsized":"2023-09-30T09:40:43.905988+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:47:52.429210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":18,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.891023+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:40:43.891023+0000","last_peered":"2023-09-30T09:40:43.891023+0000","last_clean":"2023-09-30T09:40:43.891023+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:40:43.891023+0000","last_undegraded":"2023-09-30T09:40:43.891023+0000","last_fullsized":"2023-09-30T09:40:43.891023+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"421'6070","reported_seq":7391,"reported_epoch":421,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:40:59.372491+0000","last_change":"2023-09-30T09:39:16.613964+0000","last_active":"2023-09-30T09:40:59.372491+0000","last_peered":"2023-09-30T09:40:59.372491+0000","last_clean":"2023-09-30T09:38:44.551281+0000","last_became_active":"2023-09-30T09:39:16.603809+0000","last_became_peered":"2023-09-30T09:39:16.603809+0000","last_unstale":"2023-09-30T09:40:59.372491+0000","last_undegraded":"2023-09-30T09:39:16.562159+0000","last_fullsized":"2023-09-30T09:40:59.372491+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6070,"log_dups_size":0,"ondisk_log_size":6070,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155353088,"num_objects":2370,"num_object_clones":0,"num_object_copies":7110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1117,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3594,"num_write_kb":190880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":612,"num_bytes_recovered":40140800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":1117},{"shards":"1,4,5","objects":1253}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":876,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913514+0000","last_change":"2023-09-30T09:38:45.616475+0000","last_active":"2023-09-30T09:40:43.913514+0000","last_peered":"2023-09-30T09:40:43.913514+0000","last_clean":"2023-09-30T09:40:43.913514+0000","last_became_active":"2023-09-30T09:38:45.616163+0000","last_became_peered":"2023-09-30T09:38:45.616163+0000","last_unstale":"2023-09-30T09:40:43.913514+0000","last_undegraded":"2023-09-30T09:40:43.913514+0000","last_fullsized":"2023-09-30T09:40:43.913514+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:11:18.666981+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":51,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.906208+0000","last_change":"2023-09-30T09:39:58.588073+0000","last_active":"2023-09-30T09:40:43.906208+0000","last_peered":"2023-09-30T09:40:43.906208+0000","last_clean":"2023-09-30T09:40:43.906208+0000","last_became_active":"2023-09-30T09:39:58.577667+0000","last_became_peered":"2023-09-30T09:39:58.577667+0000","last_unstale":"2023-09-30T09:40:43.906208+0000","last_undegraded":"2023-09-30T09:40:43.906208+0000","last_fullsized":"2023-09-30T09:40:43.906208+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:28:13.160407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":399,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.902648+0000","last_change":"2023-09-30T09:39:47.865003+0000","last_active":"2023-09-30T09:40:43.902648+0000","last_peered":"2023-09-30T09:40:43.902648+0000","last_clean":"2023-09-30T09:40:43.902648+0000","last_became_active":"2023-09-30T09:39:47.864581+0000","last_became_peered":"2023-09-30T09:39:47.864581+0000","last_unstale":"2023-09-30T09:40:43.902648+0000","last_undegraded":"2023-09-30T09:40:43.902648+0000","last_fullsized":"2023-09-30T09:40:43.902648+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:13:20.328082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"421'5837","reported_seq":5859,"reported_epoch":421,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:03.229733+0000","last_change":"2023-09-30T09:40:37.604868+0000","last_active":"2023-09-30T09:41:03.229733+0000","last_peered":"2023-09-30T09:41:03.229733+0000","last_clean":"2023-09-30T09:40:21.078874+0000","last_became_active":"2023-09-30T09:40:37.598926+0000","last_became_peered":"2023-09-30T09:40:37.598926+0000","last_unstale":"2023-09-30T09:41:03.229733+0000","last_undegraded":"2023-09-30T09:40:37.578588+0000","last_fullsized":"2023-09-30T09:41:03.229733+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5837,"log_dups_size":0,"ondisk_log_size":5837,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144998400,"num_objects":2212,"num_object_clones":0,"num_object_copies":6636,"num_objects_missing_on_primary":65,"num_objects_missing":65,"num_objects_degraded":65,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3468,"num_write_kb":181792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":2212}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":872,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903131+0000","last_change":"2023-09-30T09:39:50.832403+0000","last_active":"2023-09-30T09:40:43.903131+0000","last_peered":"2023-09-30T09:40:43.903131+0000","last_clean":"2023-09-30T09:40:43.903131+0000","last_became_active":"2023-09-30T09:39:50.832136+0000","last_became_peered":"2023-09-30T09:39:50.832136+0000","last_unstale":"2023-09-30T09:40:43.903131+0000","last_undegraded":"2023-09-30T09:40:43.903131+0000","last_fullsized":"2023-09-30T09:40:43.903131+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:16:37.452222+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":314,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.901352+0000","last_change":"2023-09-30T09:39:47.874554+0000","last_active":"2023-09-30T09:40:43.901352+0000","last_peered":"2023-09-30T09:40:43.901352+0000","last_clean":"2023-09-30T09:40:43.901352+0000","last_became_active":"2023-09-30T09:39:47.870194+0000","last_became_peered":"2023-09-30T09:39:47.870194+0000","last_unstale":"2023-09-30T09:40:43.901352+0000","last_undegraded":"2023-09-30T09:40:43.901352+0000","last_fullsized":"2023-09-30T09:40:43.901352+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:17.231070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.907584+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:40:43.907584+0000","last_peered":"2023-09-30T09:40:43.907584+0000","last_clean":"2023-09-30T09:40:43.907584+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:40:43.907584+0000","last_undegraded":"2023-09-30T09:40:43.907584+0000","last_fullsized":"2023-09-30T09:40:43.907584+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"421'5760","reported_seq":5861,"reported_epoch":421,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:40:59.182541+0000","last_change":"2023-09-30T09:40:37.656453+0000","last_active":"2023-09-30T09:40:59.182541+0000","last_peered":"2023-09-30T09:40:59.182541+0000","last_clean":"2023-09-30T09:40:21.766887+0000","last_became_active":"2023-09-30T09:40:37.631427+0000","last_became_peered":"2023-09-30T09:40:37.631427+0000","last_unstale":"2023-09-30T09:40:59.182541+0000","last_undegraded":"2023-09-30T09:40:37.605179+0000","last_fullsized":"2023-09-30T09:40:59.182541+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144932864,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":52,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3425,"num_write_kb":180384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5","objects":2211}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":744,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.891103+0000","last_change":"2023-09-30T09:40:39.590158+0000","last_active":"2023-09-30T09:40:43.891103+0000","last_peered":"2023-09-30T09:40:43.891103+0000","last_clean":"2023-09-30T09:40:43.891103+0000","last_became_active":"2023-09-30T09:40:39.586660+0000","last_became_peered":"2023-09-30T09:40:39.586660+0000","last_unstale":"2023-09-30T09:40:43.891103+0000","last_undegraded":"2023-09-30T09:40:43.891103+0000","last_fullsized":"2023-09-30T09:40:43.891103+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:13.120381+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":427,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903272+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:40:43.903272+0000","last_peered":"2023-09-30T09:40:43.903272+0000","last_clean":"2023-09-30T09:40:43.903272+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:40:43.903272+0000","last_undegraded":"2023-09-30T09:40:43.903272+0000","last_fullsized":"2023-09-30T09:40:43.903272+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":406,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.908662+0000","last_change":"2023-09-30T09:39:47.875703+0000","last_active":"2023-09-30T09:40:43.908662+0000","last_peered":"2023-09-30T09:40:43.908662+0000","last_clean":"2023-09-30T09:40:43.908662+0000","last_became_active":"2023-09-30T09:39:47.875450+0000","last_became_peered":"2023-09-30T09:39:47.875450+0000","last_unstale":"2023-09-30T09:40:43.908662+0000","last_undegraded":"2023-09-30T09:40:43.908662+0000","last_fullsized":"2023-09-30T09:40:43.908662+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:58:16.042133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"421'5888","reported_seq":5975,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:01.276203+0000","last_change":"2023-09-30T09:39:49.041784+0000","last_active":"2023-09-30T09:41:01.276203+0000","last_peered":"2023-09-30T09:41:01.276203+0000","last_clean":"2023-09-30T09:39:46.263479+0000","last_became_active":"2023-09-30T09:39:49.034883+0000","last_became_peered":"2023-09-30T09:39:49.034883+0000","last_unstale":"2023-09-30T09:41:01.276203+0000","last_undegraded":"2023-09-30T09:39:48.846697+0000","last_fullsized":"2023-09-30T09:39:48.814582+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148602880,"num_objects":2267,"num_object_clones":0,"num_object_copies":6801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3505,"num_write_kb":184736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[1,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5","objects":-321},{"shards":"1,5,6","objects":2231}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":594,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903180+0000","last_change":"2023-09-30T09:39:50.834859+0000","last_active":"2023-09-30T09:40:43.903180+0000","last_peered":"2023-09-30T09:40:43.903180+0000","last_clean":"2023-09-30T09:40:43.903180+0000","last_became_active":"2023-09-30T09:39:50.834483+0000","last_became_peered":"2023-09-30T09:39:50.834483+0000","last_unstale":"2023-09-30T09:40:43.903180+0000","last_undegraded":"2023-09-30T09:40:43.903180+0000","last_fullsized":"2023-09-30T09:40:43.903180+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:35.521874+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":418,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.912964+0000","last_change":"2023-09-30T09:38:45.620180+0000","last_active":"2023-09-30T09:40:43.912964+0000","last_peered":"2023-09-30T09:40:43.912964+0000","last_clean":"2023-09-30T09:40:43.912964+0000","last_became_active":"2023-09-30T09:38:45.619776+0000","last_became_peered":"2023-09-30T09:38:45.619776+0000","last_unstale":"2023-09-30T09:40:43.912964+0000","last_undegraded":"2023-09-30T09:40:43.912964+0000","last_fullsized":"2023-09-30T09:40:43.912964+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:25:08.739481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":121,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.908898+0000","last_change":"2023-09-30T09:38:45.610897+0000","last_active":"2023-09-30T09:40:43.908898+0000","last_peered":"2023-09-30T09:40:43.908898+0000","last_clean":"2023-09-30T09:40:43.908898+0000","last_became_active":"2023-09-30T09:38:45.610628+0000","last_became_peered":"2023-09-30T09:38:45.610628+0000","last_unstale":"2023-09-30T09:40:43.908898+0000","last_undegraded":"2023-09-30T09:40:43.908898+0000","last_fullsized":"2023-09-30T09:40:43.908898+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:04.928128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.e","version":"421'7024","reported_seq":7113,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:41:03.296745+0000","last_change":"2023-09-30T09:40:43.499107+0000","last_active":"2023-09-30T09:41:03.296745+0000","last_peered":"2023-09-30T09:41:03.296745+0000","last_clean":"2023-09-30T09:41:03.296745+0000","last_became_active":"2023-09-30T09:40:42.368900+0000","last_became_peered":"2023-09-30T09:40:42.368900+0000","last_unstale":"2023-09-30T09:41:03.296745+0000","last_undegraded":"2023-09-30T09:41:03.296745+0000","last_fullsized":"2023-09-30T09:41:03.296745+0000","mapping_epoch":414,"log_start":"369'2601","ondisk_log_start":"369'2601","created":358,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4423,"log_dups_size":0,"ondisk_log_size":4423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:53.392866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286261248,"num_objects":4368,"num_object_clones":0,"num_object_copies":13104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7024,"num_write_kb":364544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":718,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914586+0000","last_change":"2023-09-30T09:39:56.918064+0000","last_active":"2023-09-30T09:40:43.914586+0000","last_peered":"2023-09-30T09:40:43.914586+0000","last_clean":"2023-09-30T09:40:43.914586+0000","last_became_active":"2023-09-30T09:39:56.917689+0000","last_became_peered":"2023-09-30T09:39:56.917689+0000","last_unstale":"2023-09-30T09:40:43.914586+0000","last_undegraded":"2023-09-30T09:40:43.914586+0000","last_fullsized":"2023-09-30T09:40:43.914586+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:11.538673+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":649,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:40:53.070928+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:40:53.070928+0000","last_peered":"2023-09-30T09:40:53.070928+0000","last_clean":"2023-09-30T09:40:53.070928+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:40:53.070928+0000","last_undegraded":"2023-09-30T09:40:53.070928+0000","last_fullsized":"2023-09-30T09:40:53.070928+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":437,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.902831+0000","last_change":"2023-09-30T09:39:47.855281+0000","last_active":"2023-09-30T09:40:43.902831+0000","last_peered":"2023-09-30T09:40:43.902831+0000","last_clean":"2023-09-30T09:40:43.902831+0000","last_became_active":"2023-09-30T09:39:47.854970+0000","last_became_peered":"2023-09-30T09:39:47.854970+0000","last_unstale":"2023-09-30T09:40:43.902831+0000","last_undegraded":"2023-09-30T09:40:43.902831+0000","last_fullsized":"2023-09-30T09:40:43.902831+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:37:22.880321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904103+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:40:43.904103+0000","last_peered":"2023-09-30T09:40:43.904103+0000","last_clean":"2023-09-30T09:40:43.904103+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:40:43.904103+0000","last_undegraded":"2023-09-30T09:40:43.904103+0000","last_fullsized":"2023-09-30T09:40:43.904103+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"421'5987","reported_seq":7707,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:40:59.411648+0000","last_change":"2023-09-30T09:40:00.514475+0000","last_active":"2023-09-30T09:40:59.411648+0000","last_peered":"2023-09-30T09:40:59.411648+0000","last_clean":"2023-09-30T09:38:26.554137+0000","last_became_active":"2023-09-30T09:40:00.502041+0000","last_became_peered":"2023-09-30T09:40:00.502041+0000","last_unstale":"2023-09-30T09:40:59.411648+0000","last_undegraded":"2023-09-30T09:39:59.696250+0000","last_fullsized":"2023-09-30T09:39:59.582903+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5987,"log_dups_size":0,"ondisk_log_size":5987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150044672,"num_objects":2289,"num_object_clones":0,"num_object_copies":6867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3080,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3557,"num_write_kb":187104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":635,"num_bytes_recovered":41615360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"0,3,5","objects":507},{"shards":"0,5,7","objects":1200},{"shards":"3,5,7","objects":572},{"shards":"5,7","objects":10}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":923,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.910529+0000","last_change":"2023-09-30T09:40:39.596051+0000","last_active":"2023-09-30T09:40:43.910529+0000","last_peered":"2023-09-30T09:40:43.910529+0000","last_clean":"2023-09-30T09:40:43.910529+0000","last_became_active":"2023-09-30T09:40:39.595782+0000","last_became_peered":"2023-09-30T09:40:39.595782+0000","last_unstale":"2023-09-30T09:40:43.910529+0000","last_undegraded":"2023-09-30T09:40:43.910529+0000","last_fullsized":"2023-09-30T09:40:43.910529+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:18:09.686486+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":43,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:40:43.893779+0000","last_change":"2023-09-30T09:40:36.623450+0000","last_active":"2023-09-30T09:40:43.893779+0000","last_peered":"2023-09-30T09:40:43.893779+0000","last_clean":"2023-09-30T09:30:20.310268+0000","last_became_active":"2023-09-30T09:40:36.617588+0000","last_became_peered":"2023-09-30T09:40:36.617588+0000","last_unstale":"2023-09-30T09:40:43.893779+0000","last_undegraded":"2023-09-30T09:40:36.582089+0000","last_fullsized":"2023-09-30T09:40:43.893779+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.469417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":430,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.906104+0000","last_change":"2023-09-30T09:40:36.597929+0000","last_active":"2023-09-30T09:40:43.906104+0000","last_peered":"2023-09-30T09:40:43.906104+0000","last_clean":"2023-09-30T09:40:43.906104+0000","last_became_active":"2023-09-30T09:40:36.597614+0000","last_became_peered":"2023-09-30T09:40:36.597614+0000","last_unstale":"2023-09-30T09:40:43.906104+0000","last_undegraded":"2023-09-30T09:40:43.906104+0000","last_fullsized":"2023-09-30T09:40:43.906104+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:44:14.764059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":156,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904008+0000","last_change":"2023-09-30T09:39:47.852546+0000","last_active":"2023-09-30T09:40:43.904008+0000","last_peered":"2023-09-30T09:40:43.904008+0000","last_clean":"2023-09-30T09:40:43.904008+0000","last_became_active":"2023-09-30T09:39:47.852155+0000","last_became_peered":"2023-09-30T09:39:47.852155+0000","last_unstale":"2023-09-30T09:40:43.904008+0000","last_undegraded":"2023-09-30T09:40:43.904008+0000","last_fullsized":"2023-09-30T09:40:43.904008+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:06:46.337484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"421'6874","reported_seq":10989,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:40:59.117237+0000","last_change":"2023-09-30T09:40:22.885021+0000","last_active":"2023-09-30T09:40:59.117237+0000","last_peered":"2023-09-30T09:40:59.117237+0000","last_clean":"2023-09-30T09:38:44.556600+0000","last_became_active":"2023-09-30T09:38:46.733175+0000","last_became_peered":"2023-09-30T09:38:46.733175+0000","last_unstale":"2023-09-30T09:40:59.117237+0000","last_undegraded":"2023-09-30T09:40:22.884147+0000","last_fullsized":"2023-09-30T09:38:46.604027+0000","mapping_epoch":372,"log_start":"370'3138","ondisk_log_start":"370'3138","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3736,"log_dups_size":0,"ondisk_log_size":3736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:41.383870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294649856,"num_objects":4496,"num_object_clones":0,"num_object_copies":13488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6874,"num_write_kb":363840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2013,"num_bytes_recovered":131923968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1300},{"shards":"1,4","objects":3196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":840,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913853+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:40:43.913853+0000","last_peered":"2023-09-30T09:40:43.913853+0000","last_clean":"2023-09-30T09:40:43.913853+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:40:43.913853+0000","last_undegraded":"2023-09-30T09:40:43.913853+0000","last_fullsized":"2023-09-30T09:40:43.913853+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":651,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.906160+0000","last_change":"2023-09-30T09:40:37.590450+0000","last_active":"2023-09-30T09:40:43.906160+0000","last_peered":"2023-09-30T09:40:43.906160+0000","last_clean":"2023-09-30T09:40:43.906160+0000","last_became_active":"2023-09-30T09:40:37.589740+0000","last_became_peered":"2023-09-30T09:40:37.589740+0000","last_unstale":"2023-09-30T09:40:43.906160+0000","last_undegraded":"2023-09-30T09:40:43.906160+0000","last_fullsized":"2023-09-30T09:40:43.906160+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:24:28.314167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":172,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903036+0000","last_change":"2023-09-30T09:40:36.592442+0000","last_active":"2023-09-30T09:40:43.903036+0000","last_peered":"2023-09-30T09:40:43.903036+0000","last_clean":"2023-09-30T09:40:43.903036+0000","last_became_active":"2023-09-30T09:40:36.592119+0000","last_became_peered":"2023-09-30T09:40:36.592119+0000","last_unstale":"2023-09-30T09:40:43.903036+0000","last_undegraded":"2023-09-30T09:40:43.903036+0000","last_fullsized":"2023-09-30T09:40:43.903036+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:53:12.526007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":350,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904201+0000","last_change":"2023-09-30T09:39:58.585200+0000","last_active":"2023-09-30T09:40:43.904201+0000","last_peered":"2023-09-30T09:40:43.904201+0000","last_clean":"2023-09-30T09:40:43.904201+0000","last_became_active":"2023-09-30T09:39:58.584863+0000","last_became_peered":"2023-09-30T09:39:58.584863+0000","last_unstale":"2023-09-30T09:40:43.904201+0000","last_undegraded":"2023-09-30T09:40:43.904201+0000","last_fullsized":"2023-09-30T09:40:43.904201+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:48:09.624537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"421'5853","reported_seq":6888,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:40:59.399833+0000","last_change":"2023-09-30T09:40:00.506552+0000","last_active":"2023-09-30T09:40:59.399833+0000","last_peered":"2023-09-30T09:40:59.399833+0000","last_clean":"2023-09-30T09:39:46.286949+0000","last_became_active":"2023-09-30T09:40:00.498736+0000","last_became_peered":"2023-09-30T09:40:00.498736+0000","last_unstale":"2023-09-30T09:40:59.399833+0000","last_undegraded":"2023-09-30T09:39:59.664435+0000","last_fullsized":"2023-09-30T09:39:59.588306+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5853,"log_dups_size":0,"ondisk_log_size":5853,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149585920,"num_objects":2282,"num_object_clones":0,"num_object_copies":6846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5092,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3464,"num_write_kb":183904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20447232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":2267},{"shards":"4,5","objects":15}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1047,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.902283+0000","last_change":"2023-09-30T09:38:30.748839+0000","last_active":"2023-09-30T09:40:43.902283+0000","last_peered":"2023-09-30T09:40:43.902283+0000","last_clean":"2023-09-30T09:40:43.902283+0000","last_became_active":"2023-09-30T09:38:30.748346+0000","last_became_peered":"2023-09-30T09:38:30.748346+0000","last_unstale":"2023-09-30T09:40:43.902283+0000","last_undegraded":"2023-09-30T09:40:43.902283+0000","last_fullsized":"2023-09-30T09:40:43.902283+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:46:58.191800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":744,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913178+0000","last_change":"2023-09-30T09:40:01.598664+0000","last_active":"2023-09-30T09:40:43.913178+0000","last_peered":"2023-09-30T09:40:43.913178+0000","last_clean":"2023-09-30T09:40:43.913178+0000","last_became_active":"2023-09-30T09:40:01.598306+0000","last_became_peered":"2023-09-30T09:40:01.598306+0000","last_unstale":"2023-09-30T09:40:43.913178+0000","last_undegraded":"2023-09-30T09:40:43.913178+0000","last_fullsized":"2023-09-30T09:40:43.913178+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:19:20.361905+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":85,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904500+0000","last_change":"2023-09-30T09:39:47.869531+0000","last_active":"2023-09-30T09:40:43.904500+0000","last_peered":"2023-09-30T09:40:43.904500+0000","last_clean":"2023-09-30T09:40:43.904500+0000","last_became_active":"2023-09-30T09:39:47.869178+0000","last_became_peered":"2023-09-30T09:39:47.869178+0000","last_unstale":"2023-09-30T09:40:43.904500+0000","last_undegraded":"2023-09-30T09:40:43.904500+0000","last_fullsized":"2023-09-30T09:40:43.904500+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:33.299189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":383,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.910884+0000","last_change":"2023-09-30T09:40:36.597443+0000","last_active":"2023-09-30T09:40:43.910884+0000","last_peered":"2023-09-30T09:40:43.910884+0000","last_clean":"2023-09-30T09:40:43.910884+0000","last_became_active":"2023-09-30T09:40:36.596947+0000","last_became_peered":"2023-09-30T09:40:36.596947+0000","last_unstale":"2023-09-30T09:40:43.910884+0000","last_undegraded":"2023-09-30T09:40:43.910884+0000","last_fullsized":"2023-09-30T09:40:43.910884+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:17.799153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"421'6992","reported_seq":7070,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:41:01.617766+0000","last_change":"2023-09-30T09:37:56.539888+0000","last_active":"2023-09-30T09:41:01.617766+0000","last_peered":"2023-09-30T09:41:01.617766+0000","last_clean":"2023-09-30T09:41:01.617766+0000","last_became_active":"2023-09-30T09:37:56.539552+0000","last_became_peered":"2023-09-30T09:37:56.539552+0000","last_unstale":"2023-09-30T09:41:01.617766+0000","last_undegraded":"2023-09-30T09:41:01.617766+0000","last_fullsized":"2023-09-30T09:41:01.617766+0000","mapping_epoch":358,"log_start":"369'2924","ondisk_log_start":"369'2924","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4068,"log_dups_size":0,"ondisk_log_size":4068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:11.316452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291897344,"num_objects":4454,"num_object_clones":0,"num_object_copies":13362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6992,"num_write_kb":366272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":638,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.905896+0000","last_change":"2023-09-30T09:39:50.839273+0000","last_active":"2023-09-30T09:40:43.905896+0000","last_peered":"2023-09-30T09:40:43.905896+0000","last_clean":"2023-09-30T09:40:43.905896+0000","last_became_active":"2023-09-30T09:39:50.838892+0000","last_became_peered":"2023-09-30T09:39:50.838892+0000","last_unstale":"2023-09-30T09:40:43.905896+0000","last_undegraded":"2023-09-30T09:40:43.905896+0000","last_fullsized":"2023-09-30T09:40:43.905896+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":89449,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":830,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913107+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:40:43.913107+0000","last_peered":"2023-09-30T09:40:43.913107+0000","last_clean":"2023-09-30T09:40:43.913107+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:40:43.913107+0000","last_undegraded":"2023-09-30T09:40:43.913107+0000","last_fullsized":"2023-09-30T09:40:43.913107+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:16:49.259733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":18,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.892647+0000","last_change":"2023-09-30T09:40:36.615213+0000","last_active":"2023-09-30T09:40:43.892647+0000","last_peered":"2023-09-30T09:40:43.892647+0000","last_clean":"2023-09-30T09:40:43.892647+0000","last_became_active":"2023-09-30T09:40:36.614950+0000","last_became_peered":"2023-09-30T09:40:36.614950+0000","last_unstale":"2023-09-30T09:40:43.892647+0000","last_undegraded":"2023-09-30T09:40:43.892647+0000","last_fullsized":"2023-09-30T09:40:43.892647+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:13.113238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903946+0000","last_change":"2023-09-30T09:39:58.580826+0000","last_active":"2023-09-30T09:40:43.903946+0000","last_peered":"2023-09-30T09:40:43.903946+0000","last_clean":"2023-09-30T09:40:43.903946+0000","last_became_active":"2023-09-30T09:39:58.580493+0000","last_became_peered":"2023-09-30T09:39:58.580493+0000","last_unstale":"2023-09-30T09:40:43.903946+0000","last_undegraded":"2023-09-30T09:40:43.903946+0000","last_fullsized":"2023-09-30T09:40:43.903946+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:09:47.409633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.f","version":"421'6943","reported_seq":8796,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:01.653429+0000","last_change":"2023-09-30T09:40:36.615864+0000","last_active":"2023-09-30T09:41:01.653429+0000","last_peered":"2023-09-30T09:41:01.653429+0000","last_clean":"2023-09-30T09:38:44.567377+0000","last_became_active":"2023-09-30T09:38:46.789998+0000","last_became_peered":"2023-09-30T09:38:46.789998+0000","last_unstale":"2023-09-30T09:41:01.653429+0000","last_undegraded":"2023-09-30T09:38:46.692264+0000","last_fullsized":"2023-09-30T09:38:46.597825+0000","mapping_epoch":372,"log_start":"369'2868","ondisk_log_start":"369'2868","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4075,"log_dups_size":0,"ondisk_log_size":4075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:32:25.586705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291438592,"num_objects":4447,"num_object_clones":0,"num_object_copies":13341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6943,"num_write_kb":364480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":882,"num_bytes_recovered":57802752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":2040},{"shards":"1,2,3","objects":1584}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":696,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.907925+0000","last_change":"2023-09-30T09:40:01.612760+0000","last_active":"2023-09-30T09:40:43.907925+0000","last_peered":"2023-09-30T09:40:43.907925+0000","last_clean":"2023-09-30T09:40:43.907925+0000","last_became_active":"2023-09-30T09:40:01.612373+0000","last_became_peered":"2023-09-30T09:40:01.612373+0000","last_unstale":"2023-09-30T09:40:43.907925+0000","last_undegraded":"2023-09-30T09:40:43.907925+0000","last_fullsized":"2023-09-30T09:40:43.907925+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:30:07.640092+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":919,"reported_epoch":420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:40:43.912391+0000","last_change":"2023-09-30T09:39:59.604423+0000","last_active":"2023-09-30T09:40:43.912391+0000","last_peered":"2023-09-30T09:40:43.912391+0000","last_clean":"2023-09-30T09:39:56.898848+0000","last_became_active":"2023-09-30T09:39:59.603953+0000","last_became_peered":"2023-09-30T09:39:59.603953+0000","last_unstale":"2023-09-30T09:40:43.912391+0000","last_undegraded":"2023-09-30T09:39:59.583340+0000","last_fullsized":"2023-09-30T09:39:59.582766+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:37:14.962146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":148,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903838+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:40:43.903838+0000","last_peered":"2023-09-30T09:40:43.903838+0000","last_clean":"2023-09-30T09:40:43.903838+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:40:43.903838+0000","last_undegraded":"2023-09-30T09:40:43.903838+0000","last_fullsized":"2023-09-30T09:40:43.903838+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":18,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.891171+0000","last_change":"2023-09-30T09:40:36.615339+0000","last_active":"2023-09-30T09:40:43.891171+0000","last_peered":"2023-09-30T09:40:43.891171+0000","last_clean":"2023-09-30T09:40:43.891171+0000","last_became_active":"2023-09-30T09:40:36.615031+0000","last_became_peered":"2023-09-30T09:40:36.615031+0000","last_unstale":"2023-09-30T09:40:43.891171+0000","last_undegraded":"2023-09-30T09:40:43.891171+0000","last_fullsized":"2023-09-30T09:40:43.891171+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:58:52.570227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"421'7254","reported_seq":12026,"reported_epoch":421,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:41:03.219236+0000","last_change":"2023-09-30T09:40:36.661898+0000","last_active":"2023-09-30T09:41:03.219236+0000","last_peered":"2023-09-30T09:41:03.219236+0000","last_clean":"2023-09-30T09:38:26.563448+0000","last_became_active":"2023-09-30T09:40:03.070221+0000","last_became_peered":"2023-09-30T09:40:03.070221+0000","last_unstale":"2023-09-30T09:41:03.219236+0000","last_undegraded":"2023-09-30T09:41:03.219236+0000","last_fullsized":"2023-09-30T09:41:03.219236+0000","mapping_epoch":398,"log_start":"369'2719","ondisk_log_start":"369'2719","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4535,"log_dups_size":0,"ondisk_log_size":4535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:14:12.185089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297664512,"num_objects":4542,"num_object_clones":0,"num_object_copies":13626,"num_objects_missing_on_primary":394,"num_objects_missing":394,"num_objects_degraded":0,"num_objects_misplaced":2318,"num_objects_unfound":0,"num_objects_dirty":4542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7254,"num_write_kb":377472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2333,"num_bytes_recovered":152895488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3,5","objects":394},{"shards":"0,3,7","objects":765},{"shards":"0,5,7","objects":3383}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":913,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.910434+0000","last_change":"2023-09-30T09:39:50.836634+0000","last_active":"2023-09-30T09:40:43.910434+0000","last_peered":"2023-09-30T09:40:43.910434+0000","last_clean":"2023-09-30T09:40:43.910434+0000","last_became_active":"2023-09-30T09:39:50.836332+0000","last_became_peered":"2023-09-30T09:39:50.836332+0000","last_unstale":"2023-09-30T09:40:43.910434+0000","last_undegraded":"2023-09-30T09:40:43.910434+0000","last_fullsized":"2023-09-30T09:40:43.910434+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:13:19.526894+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":460,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914746+0000","last_change":"2023-09-30T09:40:36.629866+0000","last_active":"2023-09-30T09:40:43.914746+0000","last_peered":"2023-09-30T09:40:43.914746+0000","last_clean":"2023-09-30T09:40:43.914746+0000","last_became_active":"2023-09-30T09:40:36.606980+0000","last_became_peered":"2023-09-30T09:40:36.606980+0000","last_unstale":"2023-09-30T09:40:43.914746+0000","last_undegraded":"2023-09-30T09:40:43.914746+0000","last_fullsized":"2023-09-30T09:40:43.914746+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:14:56.024827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1076,"reported_epoch":420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:40:43.908514+0000","last_change":"2023-09-30T09:39:58.581253+0000","last_active":"2023-09-30T09:40:43.908514+0000","last_peered":"2023-09-30T09:40:43.908514+0000","last_clean":"2023-09-30T09:13:44.499731+0000","last_became_active":"2023-09-30T09:39:58.580546+0000","last_became_peered":"2023-09-30T09:39:58.580546+0000","last_unstale":"2023-09-30T09:40:43.908514+0000","last_undegraded":"2023-09-30T09:39:58.564516+0000","last_fullsized":"2023-09-30T09:39:58.563866+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:38.732897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":455,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913035+0000","last_change":"2023-09-30T09:39:58.586341+0000","last_active":"2023-09-30T09:40:43.913035+0000","last_peered":"2023-09-30T09:40:43.913035+0000","last_clean":"2023-09-30T09:40:43.913035+0000","last_became_active":"2023-09-30T09:39:58.585972+0000","last_became_peered":"2023-09-30T09:39:58.585972+0000","last_unstale":"2023-09-30T09:40:43.913035+0000","last_undegraded":"2023-09-30T09:40:43.913035+0000","last_fullsized":"2023-09-30T09:40:43.913035+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:07:15.400194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"421'7164","reported_seq":12023,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:40:59.359608+0000","last_change":"2023-09-30T09:40:21.847418+0000","last_active":"2023-09-30T09:40:59.359608+0000","last_peered":"2023-09-30T09:40:59.359608+0000","last_clean":"2023-09-30T09:40:59.359608+0000","last_became_active":"2023-09-30T09:40:21.816742+0000","last_became_peered":"2023-09-30T09:40:21.816742+0000","last_unstale":"2023-09-30T09:40:59.359608+0000","last_undegraded":"2023-09-30T09:40:59.359608+0000","last_fullsized":"2023-09-30T09:40:59.359608+0000","mapping_epoch":407,"log_start":"370'3457","ondisk_log_start":"370'3457","created":358,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3707,"log_dups_size":0,"ondisk_log_size":3707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:50:47.096303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300875800,"num_objects":4592,"num_object_clones":0,"num_object_copies":13776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4592,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7164,"num_write_kb":376129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2381,"num_bytes_recovered":129368064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":854,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.905706+0000","last_change":"2023-09-30T09:39:50.836446+0000","last_active":"2023-09-30T09:40:43.905706+0000","last_peered":"2023-09-30T09:40:43.905706+0000","last_clean":"2023-09-30T09:40:43.905706+0000","last_became_active":"2023-09-30T09:39:50.836087+0000","last_became_peered":"2023-09-30T09:39:50.836087+0000","last_unstale":"2023-09-30T09:40:43.905706+0000","last_undegraded":"2023-09-30T09:40:43.905706+0000","last_fullsized":"2023-09-30T09:40:43.905706+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:43:34.865123+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":750,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914885+0000","last_change":"2023-09-30T09:39:58.627875+0000","last_active":"2023-09-30T09:40:43.914885+0000","last_peered":"2023-09-30T09:40:43.914885+0000","last_clean":"2023-09-30T09:40:43.914885+0000","last_became_active":"2023-09-30T09:39:47.851060+0000","last_became_peered":"2023-09-30T09:39:47.851060+0000","last_unstale":"2023-09-30T09:40:43.914885+0000","last_undegraded":"2023-09-30T09:40:43.914885+0000","last_fullsized":"2023-09-30T09:40:43.914885+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:42:52.871435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":136,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.908758+0000","last_change":"2023-09-30T09:39:58.612882+0000","last_active":"2023-09-30T09:40:43.908758+0000","last_peered":"2023-09-30T09:40:43.908758+0000","last_clean":"2023-09-30T09:40:43.908758+0000","last_became_active":"2023-09-30T09:39:58.581004+0000","last_became_peered":"2023-09-30T09:39:58.581004+0000","last_unstale":"2023-09-30T09:40:43.908758+0000","last_undegraded":"2023-09-30T09:40:43.908758+0000","last_fullsized":"2023-09-30T09:40:43.908758+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:44.825201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":392,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903350+0000","last_change":"2023-09-30T09:40:36.607459+0000","last_active":"2023-09-30T09:40:43.903350+0000","last_peered":"2023-09-30T09:40:43.903350+0000","last_clean":"2023-09-30T09:40:43.903350+0000","last_became_active":"2023-09-30T09:40:36.607067+0000","last_became_peered":"2023-09-30T09:40:36.607067+0000","last_unstale":"2023-09-30T09:40:43.903350+0000","last_undegraded":"2023-09-30T09:40:43.903350+0000","last_fullsized":"2023-09-30T09:40:43.903350+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:52:17.679386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"421'6066","reported_seq":9648,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:40:59.159289+0000","last_change":"2023-09-30T09:39:53.709648+0000","last_active":"2023-09-30T09:40:59.159289+0000","last_peered":"2023-09-30T09:40:59.159289+0000","last_clean":"2023-09-30T09:40:59.159289+0000","last_became_active":"2023-09-30T09:39:15.641278+0000","last_became_peered":"2023-09-30T09:39:15.641278+0000","last_unstale":"2023-09-30T09:40:59.159289+0000","last_undegraded":"2023-09-30T09:40:59.159289+0000","last_fullsized":"2023-09-30T09:40:59.159289+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6066,"log_dups_size":0,"ondisk_log_size":6066,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149979136,"num_objects":2289,"num_object_clones":0,"num_object_copies":6867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3591,"num_write_kb":188128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1743,"num_bytes_recovered":114196480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":777,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.913322+0000","last_change":"2023-09-30T09:40:01.608312+0000","last_active":"2023-09-30T09:40:43.913322+0000","last_peered":"2023-09-30T09:40:43.913322+0000","last_clean":"2023-09-30T09:40:43.913322+0000","last_became_active":"2023-09-30T09:40:01.607950+0000","last_became_peered":"2023-09-30T09:40:01.607950+0000","last_unstale":"2023-09-30T09:40:43.913322+0000","last_undegraded":"2023-09-30T09:40:43.913322+0000","last_fullsized":"2023-09-30T09:40:43.913322+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:48.255506+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":246,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.906156+0000","last_change":"2023-09-30T09:39:58.576981+0000","last_active":"2023-09-30T09:40:43.906156+0000","last_peered":"2023-09-30T09:40:43.906156+0000","last_clean":"2023-09-30T09:40:43.906156+0000","last_became_active":"2023-09-30T09:39:58.576650+0000","last_became_peered":"2023-09-30T09:39:58.576650+0000","last_unstale":"2023-09-30T09:40:43.906156+0000","last_undegraded":"2023-09-30T09:40:43.906156+0000","last_fullsized":"2023-09-30T09:40:43.906156+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:50:03.041046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":116,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.903753+0000","last_change":"2023-09-30T09:39:58.573938+0000","last_active":"2023-09-30T09:40:43.903753+0000","last_peered":"2023-09-30T09:40:43.903753+0000","last_clean":"2023-09-30T09:40:43.903753+0000","last_became_active":"2023-09-30T09:39:58.573645+0000","last_became_peered":"2023-09-30T09:39:58.573645+0000","last_unstale":"2023-09-30T09:40:43.903753+0000","last_undegraded":"2023-09-30T09:40:43.903753+0000","last_fullsized":"2023-09-30T09:40:43.903753+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:24:06.400443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"421'6002","reported_seq":7855,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:01.671589+0000","last_change":"2023-09-30T09:39:55.208626+0000","last_active":"2023-09-30T09:41:01.671589+0000","last_peered":"2023-09-30T09:41:01.671589+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:39:53.183491+0000","last_became_peered":"2023-09-30T09:39:53.183491+0000","last_unstale":"2023-09-30T09:41:01.671589+0000","last_undegraded":"2023-09-30T09:39:51.257439+0000","last_fullsized":"2023-09-30T09:39:48.816158+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6002,"log_dups_size":0,"ondisk_log_size":6002,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145129472,"num_objects":2215,"num_object_clones":0,"num_object_copies":6645,"num_objects_missing_on_primary":1013,"num_objects_missing":1013,"num_objects_degraded":3492,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3567,"num_write_kb":184992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":676,"num_bytes_recovered":44302336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":11},{"shards":"1,2,3,6","objects":6},{"shards":"1,2,6","objects":1185},{"shards":"1,3,6","objects":1013}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":624,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.915445+0000","last_change":"2023-09-30T09:40:01.613293+0000","last_active":"2023-09-30T09:40:43.915445+0000","last_peered":"2023-09-30T09:40:43.915445+0000","last_clean":"2023-09-30T09:40:43.915445+0000","last_became_active":"2023-09-30T09:40:01.612955+0000","last_became_peered":"2023-09-30T09:40:01.612955+0000","last_unstale":"2023-09-30T09:40:43.915445+0000","last_undegraded":"2023-09-30T09:40:43.915445+0000","last_fullsized":"2023-09-30T09:40:43.915445+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:24:11.658719+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":399,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.905919+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:40:43.905919+0000","last_peered":"2023-09-30T09:40:43.905919+0000","last_clean":"2023-09-30T09:40:43.905919+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:40:43.905919+0000","last_undegraded":"2023-09-30T09:40:43.905919+0000","last_fullsized":"2023-09-30T09:40:43.905919+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.905814+0000","last_change":"2023-09-30T09:38:45.595684+0000","last_active":"2023-09-30T09:40:43.905814+0000","last_peered":"2023-09-30T09:40:43.905814+0000","last_clean":"2023-09-30T09:40:43.905814+0000","last_became_active":"2023-09-30T09:38:45.595231+0000","last_became_peered":"2023-09-30T09:38:45.595231+0000","last_unstale":"2023-09-30T09:40:43.905814+0000","last_undegraded":"2023-09-30T09:40:43.905814+0000","last_fullsized":"2023-09-30T09:40:43.905814+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:43:29.805772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"421'6065","reported_seq":13665,"reported_epoch":421,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:41:03.374582+0000","last_change":"2023-09-30T09:40:43.629441+0000","last_active":"2023-09-30T09:41:03.374582+0000","last_peered":"2023-09-30T09:41:03.374582+0000","last_clean":"2023-09-30T09:38:26.430308+0000","last_became_active":"2023-09-30T09:40:43.323944+0000","last_became_peered":"2023-09-30T09:40:43.323944+0000","last_unstale":"2023-09-30T09:41:03.374582+0000","last_undegraded":"2023-09-30T09:41:03.374582+0000","last_fullsized":"2023-09-30T09:40:37.579710+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6065,"log_dups_size":0,"ondisk_log_size":6065,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150077440,"num_objects":2290,"num_object_clones":0,"num_object_copies":6870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":233,"num_objects_unfound":0,"num_objects_dirty":2290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3624,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2854,"num_bytes_recovered":178847744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,2],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,2","objects":2057},{"shards":"0,2,6","objects":233}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":379,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.902740+0000","last_change":"2023-09-30T09:39:47.886619+0000","last_active":"2023-09-30T09:40:43.902740+0000","last_peered":"2023-09-30T09:40:43.902740+0000","last_clean":"2023-09-30T09:40:43.902740+0000","last_became_active":"2023-09-30T09:39:47.869696+0000","last_became_peered":"2023-09-30T09:39:47.869696+0000","last_unstale":"2023-09-30T09:40:43.902740+0000","last_undegraded":"2023-09-30T09:40:43.902740+0000","last_fullsized":"2023-09-30T09:40:43.902740+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:06.351925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":145,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.901447+0000","last_change":"2023-09-30T09:40:36.597938+0000","last_active":"2023-09-30T09:40:43.901447+0000","last_peered":"2023-09-30T09:40:43.901447+0000","last_clean":"2023-09-30T09:40:43.901447+0000","last_became_active":"2023-09-30T09:40:36.597684+0000","last_became_peered":"2023-09-30T09:40:36.597684+0000","last_unstale":"2023-09-30T09:40:43.901447+0000","last_undegraded":"2023-09-30T09:40:43.901447+0000","last_fullsized":"2023-09-30T09:40:43.901447+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:15:35.036544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"421'5891","reported_seq":8437,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:40:58.808680+0000","last_change":"2023-09-30T09:40:36.658459+0000","last_active":"2023-09-30T09:40:58.808680+0000","last_peered":"2023-09-30T09:40:58.808680+0000","last_clean":"2023-09-30T09:39:46.536373+0000","last_became_active":"2023-09-30T09:40:36.630807+0000","last_became_peered":"2023-09-30T09:40:36.630807+0000","last_unstale":"2023-09-30T09:40:58.808680+0000","last_undegraded":"2023-09-30T09:40:36.594542+0000","last_fullsized":"2023-09-30T09:40:36.580057+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5891,"log_dups_size":0,"ondisk_log_size":5891,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147783680,"num_objects":2255,"num_object_clones":0,"num_object_copies":6765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3489,"num_write_kb":183808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":1726},{"shards":"4,7","objects":529}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":711,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.914405+0000","last_change":"2023-09-30T09:39:47.871391+0000","last_active":"2023-09-30T09:40:43.914405+0000","last_peered":"2023-09-30T09:40:43.914405+0000","last_clean":"2023-09-30T09:40:43.914405+0000","last_became_active":"2023-09-30T09:39:47.870992+0000","last_became_peered":"2023-09-30T09:39:47.870992+0000","last_unstale":"2023-09-30T09:40:43.914405+0000","last_undegraded":"2023-09-30T09:40:43.914405+0000","last_fullsized":"2023-09-30T09:40:43.914405+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:54.087577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:54.087577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":323,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904039+0000","last_change":"2023-09-30T09:39:47.874169+0000","last_active":"2023-09-30T09:40:43.904039+0000","last_peered":"2023-09-30T09:40:43.904039+0000","last_clean":"2023-09-30T09:40:43.904039+0000","last_became_active":"2023-09-30T09:39:47.873772+0000","last_became_peered":"2023-09-30T09:39:47.873772+0000","last_unstale":"2023-09-30T09:40:43.904039+0000","last_undegraded":"2023-09-30T09:40:43.904039+0000","last_fullsized":"2023-09-30T09:40:43.904039+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:38:30.789860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":382,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.906073+0000","last_change":"2023-09-30T09:40:36.607821+0000","last_active":"2023-09-30T09:40:43.906073+0000","last_peered":"2023-09-30T09:40:43.906073+0000","last_clean":"2023-09-30T09:40:43.906073+0000","last_became_active":"2023-09-30T09:40:36.607503+0000","last_became_peered":"2023-09-30T09:40:36.607503+0000","last_unstale":"2023-09-30T09:40:43.906073+0000","last_undegraded":"2023-09-30T09:40:43.906073+0000","last_fullsized":"2023-09-30T09:40:43.906073+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:37:46.306973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"421'5932","reported_seq":6013,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-30T09:40:59.284047+0000","last_change":"2023-09-30T09:39:15.638244+0000","last_active":"2023-09-30T09:40:59.284047+0000","last_peered":"2023-09-30T09:40:59.284047+0000","last_clean":"2023-09-30T09:40:59.284047+0000","last_became_active":"2023-09-30T09:39:15.634600+0000","last_became_peered":"2023-09-30T09:39:15.634600+0000","last_unstale":"2023-09-30T09:40:59.284047+0000","last_undegraded":"2023-09-30T09:40:59.284047+0000","last_fullsized":"2023-09-30T09:40:59.284047+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145358848,"num_objects":2218,"num_object_clones":0,"num_object_copies":6654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3526,"num_write_kb":183808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":796,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.907662+0000","last_change":"2023-09-30T09:38:48.664091+0000","last_active":"2023-09-30T09:40:43.907662+0000","last_peered":"2023-09-30T09:40:43.907662+0000","last_clean":"2023-09-30T09:40:43.907662+0000","last_became_active":"2023-09-30T09:38:48.663723+0000","last_became_peered":"2023-09-30T09:38:48.663723+0000","last_unstale":"2023-09-30T09:40:43.907662+0000","last_undegraded":"2023-09-30T09:40:43.907662+0000","last_fullsized":"2023-09-30T09:40:43.907662+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:57.093254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:57.093254+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":250,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.904125+0000","last_change":"2023-09-30T09:39:58.634444+0000","last_active":"2023-09-30T09:40:43.904125+0000","last_peered":"2023-09-30T09:40:43.904125+0000","last_clean":"2023-09-30T09:40:43.904125+0000","last_became_active":"2023-09-30T09:39:58.616654+0000","last_became_peered":"2023-09-30T09:39:58.616654+0000","last_unstale":"2023-09-30T09:40:43.904125+0000","last_undegraded":"2023-09-30T09:40:43.904125+0000","last_fullsized":"2023-09-30T09:40:43.904125+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:24:26.437811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":416,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-30T09:40:43.901003+0000","last_change":"2023-09-30T09:39:47.867540+0000","last_active":"2023-09-30T09:40:43.901003+0000","last_peered":"2023-09-30T09:40:43.901003+0000","last_clean":"2023-09-30T09:40:43.901003+0000","last_became_active":"2023-09-30T09:39:47.867178+0000","last_became_peered":"2023-09-30T09:39:47.867178+0000","last_unstale":"2023-09-30T09:40:43.901003+0000","last_undegraded":"2023-09-30T09:40:43.901003+0000","last_fullsized":"2023-09-30T09:40:43.901003+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:51.232357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":16602,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":4721147928,"num_objects":72040,"num_object_clones":0,"num_object_copies":216120,"num_objects_missing_on_primary":2655,"num_objects_missing":2655,"num_objects_degraded":32718,"num_objects_misplaced":2551,"num_objects_unfound":0,"num_objects_dirty":72040,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112852,"num_write_kb":5916481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28211,"num_bytes_recovered":1748172800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15772626944,"data_stored":15772614728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143272,"ondisk_log_size":143272,"up":78,"acting":67,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":370971,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1065358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":31,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":217,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234352,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1796128,"kb_used_data":1368472,"kb_used_omap":0,"kb_used_meta":427648,"kb_avail":91949024,"statfs":{"total":95995035648,"available":94155800576,"internally_reserved":0,"allocated":1401315328,"data_stored":1399865892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":437911552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7799999999999998,"5min":5.9119999999999999,"15min":3.1339999999999999},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.224},"max":{"1min":33.237000000000002,"5min":215.416,"15min":215.416},"last":0.23999999999999999},{"interface":"front","average":{"1min":2.8969999999999998,"5min":5.9089999999999998,"15min":2.9740000000000002},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.23899999999999999},"max":{"1min":33.207000000000001,"5min":215.041,"15min":215.041},"last":0.21299999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":2.2570000000000001,"15min":2.7829999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.2879999999999998,"5min":89.308000000000007,"15min":89.308000000000007},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.4350000000000001,"15min":3.2519999999999998},"min":{"1min":0.35099999999999998,"5min":0.307,"15min":0.307},"max":{"1min":3.5139999999999998,"5min":89.325999999999993,"15min":89.325999999999993},"last":0.378}]},{"osd":2,"last update":"Sat Sep 30 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":4.3890000000000002,"15min":2.7639999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.24299999999999999},"max":{"1min":9.2560000000000002,"5min":98.698999999999998,"15min":98.698999999999998},"last":117.14},{"interface":"front","average":{"1min":1.6859999999999999,"5min":4.2530000000000001,"15min":2.71},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":10.41,"5min":99.409999999999997,"15min":99.409999999999997},"last":117.044}]},{"osd":3,"last update":"Sat Sep 30 09:40:07 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":10.452999999999999,"15min":7.835},"min":{"1min":0.39600000000000002,"5min":0.34200000000000003,"15min":0.28899999999999998},"max":{"1min":10.138,"5min":277.57799999999997,"15min":277.57799999999997},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.1419999999999999,"5min":10.784000000000001,"15min":7.2480000000000002},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.4100000000000001,"5min":278.69299999999998,"15min":278.69299999999998},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":10.101000000000001,"5min":2.778,"15min":1.663},"min":{"1min":0.308,"5min":0.23799999999999999,"15min":0.23000000000000001},"max":{"1min":175.72,"5min":175.72,"15min":175.72},"last":0.29899999999999999},{"interface":"front","average":{"1min":10.561,"5min":2.6360000000000001,"15min":1.5409999999999999},"min":{"1min":0.32900000000000001,"5min":0.252,"15min":0.251},"max":{"1min":179.048,"5min":179.048,"15min":179.048},"last":0.27900000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":3.0640000000000001,"15min":1.8620000000000001},"min":{"1min":0.315,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.1799999999999997,"5min":188.41,"15min":188.41},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.2390000000000001,"5min":3.0379999999999998,"15min":1.875},"min":{"1min":0.36399999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5609999999999999,"5min":188.43000000000001,"15min":188.43000000000001},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":8.0410000000000004,"5min":4.4509999999999996,"15min":2.371},"min":{"1min":0.42999999999999999,"5min":0.33200000000000002,"15min":0.32400000000000001},"max":{"1min":129.14500000000001,"5min":206.244,"15min":206.244},"last":0.439},{"interface":"front","average":{"1min":7.8600000000000003,"5min":4.3280000000000003,"15min":2.3010000000000002},"min":{"1min":0.44400000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":128.494,"5min":202.78800000000001,"15min":202.78800000000001},"last":0.40799999999999997}]}]},{"osd":6,"up_from":0,"seq":197568495989,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":301,"seq":1292785156187,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2450812,"kb_used_data":2018420,"kb_used_omap":9,"kb_used_meta":432374,"kb_avail":91294340,"statfs":{"total":95995035648,"available":93485404160,"internally_reserved":0,"allocated":2066862080,"data_stored":2064471628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":442750994},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":4.5650000000000004,"15min":3.254},"min":{"1min":0.23499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.4380000000000002,"5min":138.43799999999999,"15min":138.43799999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":3.9460000000000002,"15min":3.3999999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25},"max":{"1min":2.9209999999999998,"5min":138.482,"15min":138.482},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":2.149,"15min":2.552},"min":{"1min":0.27000000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.1440000000000001,"5min":13.656000000000001,"15min":16.437000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":2.6299999999999999,"15min":2.7250000000000001},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.0459999999999998,"5min":59.078000000000003,"15min":59.078000000000003},"last":0.33300000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":5.343,"15min":7.7460000000000004},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.23899999999999999},"max":{"1min":3.5230000000000001,"5min":191.07300000000001,"15min":191.07300000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.621,"5min":5.1399999999999997,"15min":8.0370000000000008},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.339,"5min":191.11500000000001,"15min":191.11500000000001},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":3.444,"15min":2.4849999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":2.4009999999999998,"5min":142.77099999999999,"15min":142.77099999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.66000000000000003,"5min":4.2919999999999998,"15min":2.3519999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.637,"5min":144.898,"15min":144.898},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":2.1720000000000002,"15min":2.1699999999999999},"min":{"1min":0.41999999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.3580000000000001,"5min":24.25,"15min":24.25},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.755,"5min":2.7930000000000001,"15min":2.0390000000000001},"min":{"1min":0.374,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.669,"5min":58.999000000000002,"15min":58.999000000000002},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":4.0179999999999998,"15min":11.82},"min":{"1min":0.45100000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":2.4980000000000002,"5min":96.719999999999999,"15min":209.221},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.79300000000000004,"5min":4.0430000000000001,"15min":11.724},"min":{"1min":0.47599999999999998,"5min":0.40000000000000002,"15min":0.377},"max":{"1min":2.4710000000000001,"5min":96.356999999999999,"15min":208.43600000000001},"last":0.68200000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":2.2959999999999998,"15min":8.3309999999999995},"min":{"1min":0.48499999999999999,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":2.3780000000000001,"5min":15.587,"15min":165.59399999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.73799999999999999,"5min":2.3100000000000001,"15min":8.4719999999999995},"min":{"1min":0.46200000000000002,"5min":0.441,"15min":0.441},"max":{"1min":2.4079999999999999,"5min":15.226000000000001,"15min":165.833},"last":0.70399999999999996}]}]},{"osd":0,"up_from":94,"seq":403726926086,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2797456,"kb_used_data":2277704,"kb_used_omap":5,"kb_used_meta":519738,"kb_avail":90947696,"statfs":{"total":95995035648,"available":93130440704,"internally_reserved":0,"allocated":2332368896,"data_stored":2329787678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":532211965},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:40:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.798,"15min":2.54},"min":{"1min":0.191,"5min":0.191,"15min":0.17799999999999999},"max":{"1min":6.5579999999999998,"5min":15.401,"15min":17.966000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.3460000000000001,"5min":2.5739999999999998,"15min":6.7910000000000004},"min":{"1min":0.23699999999999999,"5min":0.214,"15min":0.214},"max":{"1min":5.8769999999999998,"5min":25.869,"15min":138.92400000000001},"last":0.309}]},{"osd":2,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6459999999999999,"5min":2.6259999999999999,"15min":2.117},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":45.918999999999997,"5min":45.918999999999997,"15min":55.823999999999998},"last":2.3420000000000001},{"interface":"front","average":{"1min":3.8319999999999999,"5min":2.6699999999999999,"15min":2.3069999999999999},"min":{"1min":0.32100000000000001,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":45.886000000000003,"5min":45.886000000000003,"15min":55.790999999999997},"last":0.434}]},{"osd":3,"last update":"Sat Sep 30 09:41:02 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.2160000000000002,"15min":4.3929999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.221},"max":{"1min":3.6339999999999999,"5min":16.271000000000001,"15min":218.06999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.256,"5min":1.877,"15min":3.5600000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.245},"max":{"1min":5.5259999999999998,"5min":14.987,"15min":218.63200000000001},"last":2.3250000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":1.9119999999999999,"15min":1.901},"min":{"1min":0.36099999999999999,"5min":0.35299999999999998,"15min":0.29099999999999998},"max":{"1min":7.3150000000000004,"5min":10.614000000000001,"15min":121.486},"last":0.372},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.927,"15min":2.7829999999999999},"min":{"1min":0.35299999999999998,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":5.8979999999999997,"5min":17.044,"15min":136.262},"last":0.51900000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":1.97,"15min":3.1469999999999998},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":2.2320000000000002,"5min":16.995999999999999,"15min":136.34},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.5780000000000001,"15min":2.3090000000000002},"min":{"1min":0.44400000000000001,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":5.3949999999999996,"5min":11.991,"15min":135.381},"last":2.2440000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.669,"15min":3.3599999999999999},"min":{"1min":0.34899999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":6.8579999999999997,"5min":12.021000000000001,"15min":214.785},"last":2.3660000000000001},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.1840000000000002,"15min":3.4780000000000002},"min":{"1min":0.41999999999999998,"5min":0.40300000000000002,"15min":0.35399999999999998},"max":{"1min":7.3639999999999999,"5min":10.695,"15min":214.93899999999999},"last":0.40400000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.744,"15min":3.3079999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.373},"max":{"1min":5.492,"5min":12.048,"15min":150.96899999999999},"last":2.395},{"interface":"front","average":{"1min":1.444,"5min":2.1589999999999998,"15min":3.2919999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.1699999999999999,"5min":12.153,"15min":150.328},"last":2.2869999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599027,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3086364,"kb_used_data":2562388,"kb_used_omap":5,"kb_used_meta":523834,"kb_avail":90658788,"statfs":{"total":95995035648,"available":92834598912,"internally_reserved":0,"allocated":2623885312,"data_stored":2621526519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":536406669},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,3,0,1,2,4,4,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.6839999999999999,"15min":2.1840000000000002},"min":{"1min":0.218,"5min":0.20499999999999999,"15min":0.187},"max":{"1min":3.1179999999999999,"5min":15.063000000000001,"15min":77.379000000000005},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.234,"5min":1.724,"15min":2.1019999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.19700000000000001},"max":{"1min":7.0880000000000001,"5min":15.548999999999999,"15min":77.066999999999993},"last":0.26400000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":3.2810000000000001,"15min":3.5350000000000001},"min":{"1min":0.24199999999999999,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":2.1320000000000001,"5min":64.412999999999997,"15min":64.412999999999997},"last":1.6990000000000001},{"interface":"front","average":{"1min":1.028,"5min":3.036,"15min":3.1000000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":4.8099999999999996,"5min":63.247999999999998,"15min":63.247999999999998},"last":1.75}]},{"osd":3,"last update":"Sat Sep 30 09:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":4.7489999999999997,"15min":3.738},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28000000000000003},"max":{"1min":4.7619999999999996,"5min":134.078,"15min":134.078},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.0820000000000001,"5min":4.8579999999999997,"15min":3.8029999999999999},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.22500000000000001},"max":{"1min":4.7850000000000001,"5min":135.73400000000001,"15min":135.73400000000001},"last":0.39100000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":2.0390000000000001,"15min":1.659},"min":{"1min":0.308,"5min":0.308,"15min":0.28999999999999998},"max":{"1min":2.4780000000000002,"5min":83.012,"15min":83.012},"last":0.373},{"interface":"front","average":{"1min":0.83999999999999997,"5min":2.1499999999999999,"15min":1.841},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.7200000000000002,"5min":81.335999999999999,"15min":81.335999999999999},"last":0.34899999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.885,"15min":2.298},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.9780000000000002,"5min":10.41,"15min":155.477},"last":1.7350000000000001},{"interface":"front","average":{"1min":1.018,"5min":1.5029999999999999,"15min":2.032},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.448,"5min":8.0839999999999996,"15min":155.46000000000001},"last":0.47699999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.8700000000000001,"15min":3.4009999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":5.891,"5min":34.414999999999999,"15min":154.417},"last":0.437},{"interface":"front","average":{"1min":1.149,"5min":2.1699999999999999,"15min":3.6179999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.895,"5min":34.514000000000003,"15min":154.04400000000001},"last":1.7929999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.6000000000000001,"15min":3.1459999999999999},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.7709999999999999,"5min":13.304,"15min":213.404},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.681,"5min":2.0819999999999999,"15min":3.415},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40000000000000002},"max":{"1min":11.074999999999999,"5min":11.074999999999999,"15min":213.27199999999999},"last":1.7669999999999999}]}]},{"osd":3,"up_from":0,"seq":468151435500,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019262,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2231828,"kb_used_data":1858380,"kb_used_omap":0,"kb_used_meta":373440,"kb_avail":91513324,"statfs":{"total":95995035648,"available":93709643776,"internally_reserved":0,"allocated":1902981120,"data_stored":1901543428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":382402560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":3.7949999999999999,"15min":3.3660000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.5529999999999999,"5min":105.179,"15min":119.986},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":3.8719999999999999,"15min":3.4140000000000001},"min":{"1min":0.27600000000000002,"5min":0.252,"15min":0.25},"max":{"1min":2.79,"5min":104.70999999999999,"15min":119.863},"last":0.34499999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":2.532,"15min":6.7729999999999997},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.25700000000000001},"max":{"1min":3.4089999999999998,"5min":84.105999999999995,"15min":160.69999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.5499999999999998,"15min":6.7229999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":5.173,"5min":83.941000000000003,"15min":160.50700000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:40:40 2023","interfaces":[{"interface":"back","average":{"1min":4.7670000000000003,"5min":3.621,"15min":3.637},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":52.414999999999999,"5min":93.161000000000001,"15min":140.601},"last":0.497},{"interface":"front","average":{"1min":4.1200000000000001,"5min":3.6480000000000001,"15min":3.6110000000000002},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.309},"max":{"1min":52.427,"5min":93.114000000000004,"15min":140.54900000000001},"last":0.46300000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":2.036,"15min":3.5649999999999999},"min":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.26400000000000001},"max":{"1min":2.544,"5min":28.527000000000001,"15min":281.42200000000003},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.61799999999999999,"5min":2.069,"15min":3.5430000000000001},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.312},"max":{"1min":1.4239999999999999,"5min":28.486000000000001,"15min":281.44999999999999},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":3.1099999999999999,"15min":2.2879999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":2.5219999999999998,"5min":172.97800000000001,"15min":172.97800000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.79700000000000004,"5min":3.2509999999999999,"15min":2.387},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26200000000000001},"max":{"1min":7.734,"5min":172.92400000000001,"15min":172.92400000000001},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:40:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":2.9359999999999999,"15min":2.298},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.23799999999999999},"max":{"1min":8.4019999999999992,"5min":182.10300000000001,"15min":202.72499999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.1059999999999999,"5min":2.9340000000000002,"15min":2.3220000000000001},"min":{"1min":0.376,"5min":0.312,"15min":0.249},"max":{"1min":16.013999999999999,"5min":181.76300000000001,"15min":202.42400000000001},"last":0.47999999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:40:40 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":2.6499999999999999,"15min":3.3820000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.30599999999999999},"max":{"1min":8.1850000000000005,"5min":117.154,"15min":230.90100000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.3899999999999999,"5min":2.5830000000000002,"15min":3.181},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.1630000000000003,"5min":117.508,"15min":229.97},"last":0.42499999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757626,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2965132,"kb_used_data":2583876,"kb_used_omap":0,"kb_used_meta":381248,"kb_avail":90780020,"statfs":{"total":95995035648,"available":92958740480,"internally_reserved":0,"allocated":2645889024,"data_stored":2644455450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":390397952},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:39:58 2023","interfaces":[{"interface":"back","average":{"1min":4.9409999999999998,"5min":5.0949999999999998,"15min":3.5760000000000001},"min":{"1min":0.23999999999999999,"5min":0.223,"15min":0.22},"max":{"1min":64.852999999999994,"5min":98.548000000000002,"15min":148.79900000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":5.4160000000000004,"5min":5.3899999999999997,"15min":3.899},"min":{"1min":0.28100000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":64.016999999999996,"5min":98.388000000000005,"15min":148.779},"last":1.6830000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:40:46 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.786,"15min":2.0539999999999998},"min":{"1min":0.27500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":1.9399999999999999,"5min":13.363,"15min":60.799999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.47099999999999997,"5min":2.1120000000000001,"15min":2.9609999999999999},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.29,"5min":16.757000000000001,"15min":63.798999999999999},"last":1.7270000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:40:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":5.2939999999999996,"15min":4.0110000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":54.572000000000003,"5min":227.18000000000001,"15min":227.18000000000001},"last":1.756},{"interface":"front","average":{"1min":4.1790000000000003,"5min":5.46,"15min":4.4509999999999996},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":54.514000000000003,"5min":227.46600000000001,"15min":227.46600000000001},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:40:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":5.4000000000000004,"15min":4.9699999999999998},"min":{"1min":0.40699999999999997,"5min":0.30499999999999999,"15min":0.29699999999999999},"max":{"1min":9.5579999999999998,"5min":154.56700000000001,"15min":226.34700000000001},"last":1.8049999999999999},{"interface":"front","average":{"1min":1.4810000000000001,"5min":4.8819999999999997,"15min":4.7110000000000003},"min":{"1min":0.40400000000000003,"5min":0.33600000000000002,"15min":0.313},"max":{"1min":8.9320000000000004,"5min":154.58000000000001,"15min":226.36000000000001},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Sep 30 09:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.79600000000000004,"15min":0.93600000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23699999999999999},"max":{"1min":8.4220000000000006,"5min":12.634,"15min":12.634},"last":1.6100000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.94199999999999995,"15min":1.0249999999999999},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.252},"max":{"1min":1.8779999999999999,"5min":7.5410000000000004,"15min":9.9640000000000004},"last":1.6990000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:40:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.014,"15min":1.851},"min":{"1min":0.33900000000000002,"5min":0.30399999999999999,"15min":0.25900000000000001},"max":{"1min":7.4740000000000002,"5min":7.4740000000000002,"15min":155.602},"last":0.439},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.3300000000000001,"15min":2.0129999999999999},"min":{"1min":0.378,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.5839999999999996,"5min":10.880000000000001,"15min":154.00800000000001},"last":1.956}]},{"osd":7,"last update":"Sat Sep 30 09:40:17 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":1.5580000000000001,"15min":1.897},"min":{"1min":0.40100000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.617000000000001,"5min":13.617000000000001,"15min":133.40700000000001},"last":1.9950000000000001},{"interface":"front","average":{"1min":1.1599999999999999,"5min":0.86199999999999999,"15min":1.6100000000000001},"min":{"1min":0.39600000000000002,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":5.665,"5min":5.7880000000000003,"15min":132.797},"last":0.48299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2320171008,"data_stored":2320171008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2055733248,"data_stored":2055733248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2617970688,"data_stored":2617966616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1132920832,"data_stored":1132920832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1895563264,"data_stored":1895563264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2634223616,"data_stored":2634219544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1726218240,"data_stored":1726218240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1389826048,"data_stored":1389821976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:41:07.256 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:41:07.257 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-09-30T09:41:07.266 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:07.341 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:07.639 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:07.675 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:07.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:07.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:07.785 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:07.788 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:41:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:41:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:41:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:41:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36538 8.6 8:6d039046:::benchmark_data_smithi019_478679_object36537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-30T09:41:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.552182+0000", 2023-09-30T09:41:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.23207650200001, 2023-09-30T09:41:07.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2871744899999999, 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36538 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.790 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552182+0000", 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552182+0000", 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552192+0000", 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6220000000000004e-06 2023-09-30T09:41:07.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552217+0000", 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5508e-05 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552237+0000", 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9551999999999998e-05 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552247+0000", 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8870000000000004e-06 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552313+0000", 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6358000000000004e-05 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552481+0000", 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000167553 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552628+0000", 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014709400000000001 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.554856+0000", 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022282040000000001 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.555232+0000", 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00037634499999999999 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.650667+0000", 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.095434503000000004 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.650685+0000", 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7935999999999999e-05 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.660214+0000", 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.009528822 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.660228+0000", 2023-09-30T09:41:07.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4523e-05 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.839232+0000", 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1790039509999999 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.839357+0000", 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012463200000000001 2023-09-30T09:41:07.797 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36573 8.6 8:6478847f:::benchmark_data_smithi019_478679_object36572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.705432+0000", 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.07882650400001, 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1429046679999999, 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.798 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36573 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705432+0000", 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705432+0000", 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705434+0000", 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5799999999999999e-06 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705459+0000", 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5428000000000001e-05 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705476+0000", 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7261e-05 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705487+0000", 2023-09-30T09:41:07.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0342e-05 2023-09-30T09:41:07.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705691+0000", 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000204157 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705702+0000", 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0879e-05 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.663427+0000", 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95772474200000002 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.663440+0000", 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3129000000000001e-05 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841104+0000", 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1776643179999999 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841253+0000", 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000148646 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.806 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841392+0000", 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000139507 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.844591+0000", 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031987589999999998 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.845467+0000", 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00087615200000000005 2023-09-30T09:41:07.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846194+0000", 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00072731399999999996 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846271+0000", 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6915999999999993e-05 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.848337+0000", 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002065538 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36575 8.c 8:30e8fa6b:::benchmark_data_smithi019_478679_object36574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.705506+0000", 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.07875305600001, 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1391458339999998, 2023-09-30T09:41:07.809 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36575 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705506+0000", 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705506+0000", 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705506+0000", 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9599999999999999e-07 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705525+0000", 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9218000000000001e-05 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705532+0000", 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7859999999999999e-06 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705537+0000", 2023-09-30T09:41:07.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.152e-06 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705597+0000", 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9351000000000001e-05 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705606+0000", 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8360000000000004e-06 2023-09-30T09:41:07.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659107+0000", 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95350058299999996 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659121+0000", 2023-09-30T09:41:07.814 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3862e-05 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.835026+0000", 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1759052400000001 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.835210+0000", 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000184208 2023-09-30T09:41:07.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.835398+0000", 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018811199999999999 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.840805+0000", 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0054062279999999999 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.840862+0000", 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6898999999999999e-05 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.842785+0000", 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019238009999999999 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.842841+0000", 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5458000000000003e-05 2023-09-30T09:41:07.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.844651+0000", 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018106039999999999 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36589 8.c 8:398bde77:::benchmark_data_smithi019_478679_object36588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:41:07.818 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.714335+0000", 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.069923512, 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1343528040000002, 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36589 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.819 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714335+0000", 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714335+0000", 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714337+0000", 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.446e-06 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714359+0000", 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2802999999999998e-05 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714372+0000", 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2373e-05 2023-09-30T09:41:07.821 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714379+0000", 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6559999999999992e-06 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714457+0000", 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7766e-05 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714469+0000", 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1314999999999999e-05 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659147+0000", 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94467839300000001 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659155+0000", 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1939999999999998e-06 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.838893+0000", 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1797375400000001 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.839042+0000", 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014981900000000001 2023-09-30T09:41:07.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.839238+0000", 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019502300000000001 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846753+0000", 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0075152120000000003 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846796+0000", 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3318000000000001e-05 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846846+0000", 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9576000000000001e-05 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846901+0000", 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5272999999999997e-05 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.848688+0000", 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0017870970000000001 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36592 8.c 8:34c28d90:::benchmark_data_smithi019_478679_object36591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.715952+0000", 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.068306455, 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1369867139999998, 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36592 2023-09-30T09:41:07.828 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715952+0000", 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715952+0000", 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715953+0000", 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1960000000000001e-06 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715975+0000", 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1679999999999999e-05 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715985+0000", 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9299999999999998e-06 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715993+0000", 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5209999999999997e-06 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.716060+0000", 2023-09-30T09:41:07.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7007000000000004e-05 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.716067+0000", 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6899999999999992e-06 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659181+0000", 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94311415899999995 2023-09-30T09:41:07.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659196+0000", 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4797000000000001e-05 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.840909+0000", 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.181712804 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841068+0000", 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000159166 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841259+0000", 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000190676 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846810+0000", 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055510280000000004 2023-09-30T09:41:07.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.848734+0000", 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019241200000000001 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.850919+0000", 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002184685 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.850988+0000", 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9750000000000001e-05 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.852939+0000", 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019505049999999999 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.836 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36604 8.c 8:3abdb19f:::benchmark_data_smithi019_478679_object36603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.724974+0000", 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.05928484699999, 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1295574159999999, 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36604 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.837 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724974+0000", 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724974+0000", 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724975+0000", 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7000000000000004e-07 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724991+0000", 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6755000000000001e-05 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724998+0000", 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4500000000000001e-06 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.725002+0000", 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1520000000000002e-06 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.725057+0000", 2023-09-30T09:41:07.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4630999999999999e-05 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.725063+0000", 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4670000000000001e-06 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659221+0000", 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.93415796600000001 2023-09-30T09:41:07.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659229+0000", 2023-09-30T09:41:07.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4719999999999995e-06 2023-09-30T09:41:07.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.844688+0000", 2023-09-30T09:41:07.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.185458876 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.844837+0000", 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014893799999999999 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.845033+0000", 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000195634 2023-09-30T09:41:07.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.850862+0000", 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058289700000000002 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.852962+0000", 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021005300000000002 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.853019+0000", 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6668e-05 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.853086+0000", 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6956000000000001e-05 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.854531+0000", 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0014452810000000001 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:54238 8.c 8:366ec1be:::benchmark_data_smithi019_478679_object54237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 143.144435157, 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0706041239999999, 2023-09-30T09:41:07.847 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 54238 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4300000000000001e-07 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.849 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639843+0000", 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8371000000000001e-05 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639856+0000", 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3366999999999999e-05 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639862+0000", 2023-09-30T09:41:07.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.161e-06 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639930+0000", 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7676999999999996e-05 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639945+0000", 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5138e-05 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:45.597093+0000", 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95714748900000002 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:45.597106+0000", 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3617e-05 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676568+0000", 2023-09-30T09:41:07.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0794613529999999 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676752+0000", 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018476000000000001 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676942+0000", 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018961800000000001 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708135+0000", 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031193262999999999 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708162+0000", 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6557999999999999e-05 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708218+0000", 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5918000000000002e-05 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708303+0000", 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5107999999999999e-05 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.710428+0000", 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021247829999999999 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.855 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76882 8.11 8:8e91d271:::benchmark_data_smithi019_478679_object76881:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e379)", 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:14.640986+0000", 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 113.143272474, 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96239549800000002, 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.856 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76882 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.640986+0000", 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.640986+0000", 2023-09-30T09:41:07.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.640988+0000", 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.669e-06 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641048+0000", 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0260000000000002e-05 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641068+0000", 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9786999999999999e-05 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641078+0000", 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6819999999999998e-06 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.772497+0000", 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.13141914499999999 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.772505+0000", 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6440000000000006e-06 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.599955+0000", 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.82744946500000005 2023-09-30T09:41:07.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.600182+0000", 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022735299999999999 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.600387+0000", 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020526400000000001 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.602509+0000", 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021219390000000002 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.602985+0000", 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00047572400000000001 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.603318+0000", 2023-09-30T09:41:07.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033325299999999999 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.603359+0000", 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0389999999999998e-05 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.603382+0000", 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2923000000000001e-05 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.863 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76883 8.16 8:6fedded8:::benchmark_data_smithi019_478679_object76882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e379)", 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 113.143163121, 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.937896308, 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.864 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76883 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9400000000000005e-07 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641122+0000", 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5469e-05 2023-09-30T09:41:07.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641129+0000", 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7079999999999997e-06 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641135+0000", 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8420000000000001e-06 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.735104+0000", 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.093968634999999995 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.735114+0000", 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0591000000000001e-05 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.623444+0000", 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.888329232 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.623456+0000", 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2207e-05 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.570797+0000", 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94734084100000004 2023-09-30T09:41:07.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.570930+0000", 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013319100000000001 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.571052+0000", 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012170700000000001 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.574539+0000", 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034873180000000001 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576645+0000", 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021061999999999999 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578718+0000", 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020728700000000001 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578793+0000", 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4670000000000002e-05 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578992+0000", 2023-09-30T09:41:07.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019923299999999999 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76913 8.16 8:6fac3ea3:::benchmark_data_smithi019_478679_object76912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 112.151739611, 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9468952980000001, 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.873 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76913 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632521+0000", 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.545e-06 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632547+0000", 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6449999999999999e-05 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632566+0000", 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8515000000000001e-05 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632576+0000", 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0199e-05 2023-09-30T09:41:07.876 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632655+0000", 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9171999999999993e-05 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632669+0000", 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4055e-05 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.877 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624641+0000", 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99197187499999995 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624650+0000", 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9570000000000008e-06 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572117+0000", 2023-09-30T09:41:07.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94746728000000002 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572215+0000", 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8041000000000005e-05 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572312+0000", 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6371000000000003e-05 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576694+0000", 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043828060000000004 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576706+0000", 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1773999999999999e-05 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579137+0000", 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024311419999999999 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579213+0000", 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5665000000000001e-05 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579414+0000", 2023-09-30T09:41:07.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020145100000000001 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76925 8.16 8:6e562ee8:::benchmark_data_smithi019_478679_object76924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 112.127612101, 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9248296740000002, 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.882 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76925 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656648+0000", 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.443e-06 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656673+0000", 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4562e-05 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656689+0000", 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5948999999999999e-05 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656697+0000", 2023-09-30T09:41:07.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9789999999999993e-06 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656753+0000", 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6739e-05 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656760+0000", 2023-09-30T09:41:07.886 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4710000000000004e-06 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624673+0000", 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96791319200000003 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624681+0000", 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9060000000000002e-06 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572984+0000", 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94830274000000003 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573098+0000", 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000114694 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573214+0000", 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115318 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576712+0000", 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003498143 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578659+0000", 2023-09-30T09:41:07.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001947308 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581148+0000", 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002488873 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581230+0000", 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2106000000000003e-05 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581476+0000", 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024625100000000001 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76991 8.16 8:6eb44e73:::benchmark_data_smithi019_478679_object76990:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:15.785404+0000", 2023-09-30T09:41:07.891 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 111.99885420299999, 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.796369071, 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76991 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.892 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.785404+0000", 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.785404+0000", 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.785406+0000", 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.139e-06 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.785424+0000", 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8649999999999999e-05 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.785435+0000", 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0936e-05 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.894 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.785441+0000", 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5659999999999997e-06 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.785506+0000", 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5544999999999997e-05 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.785517+0000", 2023-09-30T09:41:07.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0913999999999999e-05 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624701+0000", 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.83918396200000001 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624705+0000", 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9079999999999997e-06 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573774+0000", 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94906857700000002 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573866+0000", 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2545000000000002e-05 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573972+0000", 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000106196 2023-09-30T09:41:07.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578603+0000", 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0046301399999999996 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579056+0000", 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00045361300000000001 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581571+0000", 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025152199999999999 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581628+0000", 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6811000000000002e-05 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581774+0000", 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014534899999999999 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:77082 8.16 8:6efe4c3d:::benchmark_data_smithi019_478679_object77081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:16.741550+0000", 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 111.042708324, 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.84047945000000002, 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.900 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 77082 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.741550+0000", 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.901 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.741550+0000", 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.741551+0000", 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6899999999999997e-07 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.741604+0000", 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3052e-05 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.741613+0000", 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3360000000000001e-06 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.741620+0000", 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.883e-06 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.741668+0000", 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7716000000000002e-05 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.741677+0000", 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2410000000000001e-06 2023-09-30T09:41:07.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.574590+0000", 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.83291302099999998 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.574748+0000", 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000158016 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.574941+0000", 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019279099999999999 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578608+0000", 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036665780000000002 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581010+0000", 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024019829999999999 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581854+0000", 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00084442800000000004 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581890+0000", 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5598e-05 2023-09-30T09:41:07.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582030+0000", 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014023799999999999 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:77090 8.16 8:6fda2744:::benchmark_data_smithi019_478679_object77089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:16.747685+0000", 2023-09-30T09:41:07.908 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 111.036573402, 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.83468042099999995, 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 77090 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.909 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.747685+0000", 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.747685+0000", 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.747687+0000", 2023-09-30T09:41:07.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3009999999999999e-06 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.747708+0000", 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1143999999999999e-05 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.747724+0000", 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6529999999999999e-05 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.747732+0000", 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3250000000000002e-06 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.747797+0000", 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5487999999999994e-05 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.747808+0000", 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0492999999999999e-05 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.575626+0000", 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.82781843499999996 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.575771+0000", 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014494900000000001 2023-09-30T09:41:07.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.575963+0000", 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019163200000000001 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578679+0000", 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027160079999999998 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581077+0000", 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023988799999999999 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582076+0000", 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00099863699999999992 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582145+0000", 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9079999999999996e-05 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582366+0000", 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022051899999999999 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.916 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:77108 8.16 8:6c292569:::benchmark_data_smithi019_478679_object77107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:16.769251+0000", 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 111.015007482, 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.81338393600000003, 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 77108 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.917 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.769251+0000", 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.769251+0000", 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.769252+0000", 2023-09-30T09:41:07.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1829999999999999e-06 2023-09-30T09:41:07.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.769310+0000", 2023-09-30T09:41:07.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7633999999999997e-05 2023-09-30T09:41:07.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.769323+0000", 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2733e-05 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.769329+0000", 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4829999999999996e-06 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.769390+0000", 2023-09-30T09:41:07.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0303999999999997e-05 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.769398+0000", 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8920000000000006e-06 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576753+0000", 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.80735476900000003 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576906+0000", 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000152862 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.577096+0000", 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000190414 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579080+0000", 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019835249999999999 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581521+0000", 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024413759999999999 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582427+0000", 2023-09-30T09:41:07.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00090540900000000003 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582465+0000", 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8476999999999998e-05 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582635+0000", 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000169875 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.924 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:77120 8.16 8:6bf17547:::benchmark_data_smithi019_478679_object77119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:16.780366+0000", 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 111.00389248899999, 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.80258131799999999, 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.925 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 77120 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.780366+0000", 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.780366+0000", 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.780368+0000", 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8160000000000001e-06 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.780390+0000", 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2472000000000001e-05 2023-09-30T09:41:07.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.780403+0000", 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2284e-05 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.780412+0000", 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7770000000000008e-06 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.780491+0000", 2023-09-30T09:41:07.928 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9926999999999996e-05 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.780507+0000", 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5583e-05 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.577844+0000", 2023-09-30T09:41:07.929 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.79733700100000005 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.577936+0000", 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2417999999999995e-05 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578050+0000", 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113805 2023-09-30T09:41:07.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581022+0000", 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029713750000000001 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581814+0000", 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00079236400000000005 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582707+0000", 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00089256899999999996 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582774+0000", 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7748000000000003e-05 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.582948+0000", 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000173179 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:106333 8.6 8.abc8c966 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:40:21.861178+0000", 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 45.923080466000002, 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.91904804500000004, 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.933 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 106333 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.861178+0000", 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.934 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.861178+0000", 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.861180+0000", 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1210000000000001e-06 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.861195+0000", 2023-09-30T09:41:07.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4243000000000001e-05 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.861220+0000", 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5803e-05 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.861233+0000", 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2325e-05 2023-09-30T09:41:07.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.861315+0000", 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2430999999999997e-05 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.861327+0000", 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1979e-05 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.937 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:22.780209+0000", 2023-09-30T09:41:07.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.91888196 2023-09-30T09:41:07.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:22.780226+0000", 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7183000000000001e-05 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:106351 8.16 8.dcd0a476 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:40:21.904864+0000", 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 45.879394705999999, 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.875407037, 2023-09-30T09:41:07.939 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 106351 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.904864+0000", 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.904864+0000", 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.904866+0000", 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4080000000000001e-06 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.904875+0000", 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2279999999999998e-06 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.904892+0000", 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7623999999999999e-05 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.904904+0000", 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1941e-05 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.904970+0000", 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5617999999999994e-05 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.904981+0000", 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.097e-05 2023-09-30T09:41:07.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:22.780260+0000", 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.875279048 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:22.780271+0000", 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1199999999999999e-05 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:114197 8.16 8:6ac0166d:::benchmark_data_smithi019_478679_object21749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:05.700064+0000", 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.0841944639999999, 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.92068745200000002, 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 114197 2023-09-30T09:41:07.945 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.700064+0000", 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.700064+0000", 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.700065+0000", 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2330000000000001e-06 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.700071+0000", 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1359999999999996e-06 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.947 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.700086+0000", 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4912000000000001e-05 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.700096+0000", 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0502999999999999e-05 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.700159+0000", 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3095999999999994e-05 2023-09-30T09:41:07.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.700168+0000", 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2329999999999995e-06 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.615268+0000", 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.915099471 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.949 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.615405+0000", 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013715799999999999 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.615530+0000", 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012500599999999999 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.618095+0000", 2023-09-30T09:41:07.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025650759999999999 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.620591+0000", 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024961760000000001 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.620668+0000", 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6840000000000003e-05 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.620707+0000", 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8649000000000002e-05 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.620752+0000", 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4963000000000003e-05 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:114210 8.16 8:6dbced34:::benchmark_data_smithi019_478679_object21762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:05.773647+0000", 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.0106114069999999, 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.85498821800000002, 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:41:07.953 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 114210 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.773647+0000", 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.773647+0000", 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:41:07.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.773649+0000", 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5570000000000001e-06 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.773655+0000", 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3330000000000002e-06 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.773674+0000", 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9188e-05 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.773685+0000", 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0569e-05 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.773771+0000", 2023-09-30T09:41:07.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6077999999999998e-05 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:05.773780+0000", 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0229999999999998e-06 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.618498+0000", 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.84471806299999996 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.618613+0000", 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115173 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.618710+0000", 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6966999999999995e-05 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:41:07.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.624932+0000", 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062213540000000001 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.628501+0000", 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003568959 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.628525+0000", 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4070000000000002e-05 2023-09-30T09:41:07.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.628590+0000", 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4898000000000001e-05 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:06.628636+0000", 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5986000000000003e-05 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.960 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.961 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.961 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:41:07.961 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:41:07.961 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:41:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:07.962 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:07.962 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:07.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:08.064 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[224043]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[224043]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[224043]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[224043]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[224043]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[216815]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[216815]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[216815]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[216815]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:08 smithi019 ceph-mon[216815]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.106 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:08.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:08 smithi167 ceph-mon[192880]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:08 smithi167 ceph-mon[192880]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:08 smithi167 ceph-mon[192880]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:08 smithi167 ceph-mon[192880]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:08 smithi167 ceph-mon[192880]: failed to encode map e423 with expected crc 2023-09-30T09:41:08.983 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:08.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:08.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:09.087 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:09.114 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:09.174 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:09.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:09.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:09.211 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:09 smithi167 ceph-mon[192880]: pgmap v1088: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 225 op/s; 32250/213000 objects degraded (15.141%); 2551/213000 objects misplaced (1.198%); 8.7 MiB/s, 148 objects/s recovering 2023-09-30T09:41:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 32250/213000 objects degraded (15.141%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:09.282 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:09.325 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:09 smithi019 ceph-mon[216815]: pgmap v1088: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 225 op/s; 32250/213000 objects degraded (15.141%); 2551/213000 objects misplaced (1.198%); 8.7 MiB/s, 148 objects/s recovering 2023-09-30T09:41:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 32250/213000 objects degraded (15.141%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:09 smithi019 ceph-mon[224043]: pgmap v1088: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 225 op/s; 32250/213000 objects degraded (15.141%); 2551/213000 objects misplaced (1.198%); 8.7 MiB/s, 148 objects/s recovering 2023-09-30T09:41:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 32250/213000 objects degraded (15.141%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:09.376 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:09.378 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:41:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:09.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:09.440 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:41:09.482 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:09.557 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:09.706 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:09.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:09.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:09.819 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:09.853 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:09.901 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:09.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:09.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:10.007 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:10.052 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:10.306 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:41:10.334 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:41:10.358 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:41:10.393 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:41:10.405 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:10.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:10.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:10.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:41:10.455 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:41:10.481 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:41:10.509 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:41:10.509 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:10.510 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:41:10.510 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:41:10.511 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:41:10.511 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:41:10.511 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:41:10.511 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:41:10.511 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:41:10.511 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:41:10.538 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:10.585 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:10.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:10.699 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:10.729 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:10.786 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:10.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:10.891 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:10.930 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:11.214 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:11.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:11.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:11 smithi019 ceph-mon[216815]: pgmap v1089: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 359 op/s; 31917/209649 objects degraded (15.224%); 1877/209649 objects misplaced (0.895%); 14 MiB/s, 249 objects/s recovering 2023-09-30T09:41:11.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/218694487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:11.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:11 smithi019 ceph-mon[224043]: pgmap v1089: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 359 op/s; 31917/209649 objects degraded (15.224%); 1877/209649 objects misplaced (0.895%); 14 MiB/s, 249 objects/s recovering 2023-09-30T09:41:11.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/218694487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:11.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:11 smithi167 ceph-mon[192880]: pgmap v1089: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 359 op/s; 31917/209649 objects degraded (15.224%); 1877/209649 objects misplaced (0.895%); 14 MiB/s, 249 objects/s recovering 2023-09-30T09:41:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/218694487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:11.322 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:11.354 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:11.409 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:11.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:11.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:11.517 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:11.555 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:11.691 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:11.692 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:11.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:11.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:11.794 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:11.799 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:11.849 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:11.971 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:11.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:12.078 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:12.106 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:12.164 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:12.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:12.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:12.272 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:12.302 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:12.368 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:12.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:12.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:12.474 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:12.491 INFO:teuthology.orchestra.run.smithi019.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-09-30T09:41:12.505 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:12.557 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:12.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:12.664 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:12.709 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:13.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:13 smithi167 ceph-mon[192880]: pgmap v1090: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 31917/209481 objects degraded (15.236%); 1877/209481 objects misplaced (0.896%); 7.2 MiB/s, 136 objects/s recovering 2023-09-30T09:41:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3699493592' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-30T09:41:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:13 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-30T09:41:13.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:13 smithi019 ceph-mon[216815]: pgmap v1090: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 31917/209481 objects degraded (15.236%); 1877/209481 objects misplaced (0.896%); 7.2 MiB/s, 136 objects/s recovering 2023-09-30T09:41:13.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3699493592' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-30T09:41:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:13 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-30T09:41:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:13 smithi019 ceph-mon[224043]: pgmap v1090: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 31917/209481 objects degraded (15.236%); 1877/209481 objects misplaced (0.896%); 7.2 MiB/s, 136 objects/s recovering 2023-09-30T09:41:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3699493592' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-30T09:41:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:13 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-30T09:41:13.419 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:13.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:13.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:13.524 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:13.552 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:13.617 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:13.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:13.700 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:41:13.728 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:13.796 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:14.012 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:41:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:41:13] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-09-30T09:41:14.012 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 31917/209481 objects degraded (15.236%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:14.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 31917/209481 objects degraded (15.236%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:14.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 31917/209481 objects degraded (15.236%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:14.604 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:14.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:41:14.708 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:14.751 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:15.193 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:15.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:15 smithi167 ceph-mon[192880]: pgmap v1091: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 31720/207096 objects degraded (15.317%); 904/207096 objects misplaced (0.437%); 10 MiB/s, 184 objects/s recovering 2023-09-30T09:41:15.329 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:41:15.330 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:41:15.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:15 smithi019 ceph-mon[216815]: pgmap v1091: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 31720/207096 objects degraded (15.317%); 904/207096 objects misplaced (0.437%); 10 MiB/s, 184 objects/s recovering 2023-09-30T09:41:15.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:15 smithi019 ceph-mon[224043]: pgmap v1091: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 31720/207096 objects degraded (15.317%); 904/207096 objects misplaced (0.437%); 10 MiB/s, 184 objects/s recovering 2023-09-30T09:41:15.337 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:15.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:15.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:15.445 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:15.474 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:15.547 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:15.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:15.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:15.653 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:15.706 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:16.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2347418898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:16.024 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2347418898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:16.037 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:41:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2347418898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:17.144 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:17.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:17.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:17.248 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:17 smithi167 ceph-mon[192880]: pgmap v1092: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 31390/204663 objects degraded (15.337%); 904/204663 objects misplaced (0.442%); 9.4 MiB/s, 167 objects/s recovering 2023-09-30T09:41:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:17 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2347418898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:41:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:17 smithi167 ceph-mon[192880]: osdmap e424: 8 total, 8 up, 6 in 2023-09-30T09:41:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:17 smithi167 ceph-mon[192880]: failed to encode map e424 with expected crc 2023-09-30T09:41:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:17 smithi167 ceph-mon[192880]: failed to encode map e424 with expected crc 2023-09-30T09:41:17.279 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:17.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[216815]: pgmap v1092: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 31390/204663 objects degraded (15.337%); 904/204663 objects misplaced (0.442%); 9.4 MiB/s, 167 objects/s recovering 2023-09-30T09:41:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2347418898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:41:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[216815]: osdmap e424: 8 total, 8 up, 6 in 2023-09-30T09:41:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[216815]: failed to encode map e424 with expected crc 2023-09-30T09:41:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[216815]: failed to encode map e424 with expected crc 2023-09-30T09:41:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[224043]: pgmap v1092: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 31390/204663 objects degraded (15.337%); 904/204663 objects misplaced (0.442%); 9.4 MiB/s, 167 objects/s recovering 2023-09-30T09:41:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2347418898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:41:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[224043]: osdmap e424: 8 total, 8 up, 6 in 2023-09-30T09:41:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[224043]: failed to encode map e424 with expected crc 2023-09-30T09:41:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:17 smithi019 ceph-mon[224043]: failed to encode map e424 with expected crc 2023-09-30T09:41:17.337 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:17.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:17.441 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:17.470 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:17.519 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:17.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:17.624 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:17.651 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:17.697 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:17.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:17.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:17.802 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:17.836 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:18.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:41:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:18.334 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:18.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:18.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:18.445 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:18.472 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:18.476 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:18.529 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:18.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:18.639 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:18.710 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:18.788 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:18.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:18.894 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:18.936 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:18.997 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:18.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:19.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:19.100 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:19.132 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:19 smithi019 ceph-mon[216815]: pgmap v1094: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 315 op/s; 28826/203523 objects degraded (14.164%); 2852/203523 objects misplaced (1.401%); 10 MiB/s, 183 objects/s recovering 2023-09-30T09:41:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28826/203523 objects degraded (14.164%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:19 smithi019 ceph-mon[224043]: pgmap v1094: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 315 op/s; 28826/203523 objects degraded (14.164%); 2852/203523 objects misplaced (1.401%); 10 MiB/s, 183 objects/s recovering 2023-09-30T09:41:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28826/203523 objects degraded (14.164%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:19.398 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:19.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:19.502 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:19 smithi167 ceph-mon[192880]: pgmap v1094: 129 pgs: 1 active+recovering+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 315 op/s; 28826/203523 objects degraded (14.164%); 2852/203523 objects misplaced (1.401%); 10 MiB/s, 183 objects/s recovering 2023-09-30T09:41:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28826/203523 objects degraded (14.164%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:19.529 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:19.581 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:19.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:19.603 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:19.686 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:19.702 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0] out_osds: [3, 6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:41:19.703 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:41:19.703 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-30T09:41:19.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:41:19.736 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:19.970 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:19.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:19.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:20.088 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:20.134 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:20.193 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:20.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:20.299 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:20.367 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:41:20.368 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:20.608 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:20.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:20.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:20.716 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:20.745 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:20.793 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:20.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:20.899 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:20.948 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:21.010 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:21.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:21.039 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:21 smithi019 ceph-mon[216815]: pgmap v1095: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 304 op/s; 22857/200517 objects degraded (11.399%); 10138/200517 objects misplaced (5.056%); 17 MiB/s, 266 objects/s recovering 2023-09-30T09:41:21.114 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:21.176 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:41:21.189 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:21.216 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:41:21.258 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:41:21.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:21 smithi167 ceph-mon[192880]: pgmap v1095: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 304 op/s; 22857/200517 objects degraded (11.399%); 10138/200517 objects misplaced (5.056%); 17 MiB/s, 266 objects/s recovering 2023-09-30T09:41:21.295 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:41:21.329 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:41:21.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:21 smithi019 ceph-mon[224043]: pgmap v1095: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 304 op/s; 22857/200517 objects degraded (11.399%); 10138/200517 objects misplaced (5.056%); 17 MiB/s, 266 objects/s recovering 2023-09-30T09:41:21.363 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:41:21.393 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:41:21.428 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:41:21.449 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:41:21.449 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:41:21.449 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:41:21.449 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:41:21.449 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:41:21.449 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:41:21.449 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:41:21.450 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:41:21.825 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:21.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:21.932 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:21.960 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:22.009 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:22.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:22.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2060429972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:22.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2060429972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:22.114 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:22.141 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:22.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2060429972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:22.671 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:22.680 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:22.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:22.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:22.785 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:22.844 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:22.953 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:22.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:23.063 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:23.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:23 smithi019 ceph-mon[224043]: pgmap v1096: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 303 op/s; 22857/200361 objects degraded (11.408%); 9678/200361 objects misplaced (4.830%); 20 MiB/s, 312 objects/s recovering 2023-09-30T09:41:23.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:23 smithi019 ceph-mon[216815]: pgmap v1096: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 303 op/s; 22857/200361 objects degraded (11.408%); 9678/200361 objects misplaced (4.830%); 20 MiB/s, 312 objects/s recovering 2023-09-30T09:41:23.116 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:23.122 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:23.298 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:23.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:23.393 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:41:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:41:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:41:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:41:23.394 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:41:23.406 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:23.453 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:23.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:23 smithi167 ceph-mon[192880]: pgmap v1096: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 303 op/s; 22857/200361 objects degraded (11.408%); 9678/200361 objects misplaced (4.830%); 20 MiB/s, 312 objects/s recovering 2023-09-30T09:41:24.010 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:24.011 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:41:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:41:23] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-09-30T09:41:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:24.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:24.113 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:24.154 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:24.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22857/200361 objects degraded (11.408%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:24.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22857/200361 objects degraded (11.408%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:24.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22857/200361 objects degraded (11.408%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:24.658 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:41:24.993 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:24.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:25.100 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:25.141 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:25.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:25 smithi019 ceph-mon[216815]: pgmap v1097: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 299 op/s; 22664/198102 objects degraded (11.441%); 9570/198102 objects misplaced (4.831%); 17 MiB/s, 264 objects/s recovering 2023-09-30T09:41:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:25 smithi019 ceph-mon[224043]: pgmap v1097: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 299 op/s; 22664/198102 objects degraded (11.441%); 9570/198102 objects misplaced (4.831%); 17 MiB/s, 264 objects/s recovering 2023-09-30T09:41:25.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:25 smithi167 ceph-mon[192880]: pgmap v1097: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 299 op/s; 22664/198102 objects degraded (11.441%); 9570/198102 objects misplaced (4.831%); 17 MiB/s, 264 objects/s recovering 2023-09-30T09:41:25.846 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:41:25.890 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:25.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:25.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:25.996 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:26.073 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:26.936 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:26.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:26.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:27.043 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:27.096 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:27.225 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:41:27.227 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:41:27.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:27 smithi019 ceph-mon[216815]: pgmap v1098: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 22464/195621 objects degraded (11.483%); 8704/195621 objects misplaced (4.449%); 19 MiB/s, 298 objects/s recovering 2023-09-30T09:41:27.248 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:27 smithi019 ceph-mon[224043]: pgmap v1098: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 22464/195621 objects degraded (11.483%); 8704/195621 objects misplaced (4.449%); 19 MiB/s, 298 objects/s recovering 2023-09-30T09:41:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:27 smithi167 ceph-mon[192880]: pgmap v1098: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 22464/195621 objects degraded (11.483%); 8704/195621 objects misplaced (4.449%); 19 MiB/s, 298 objects/s recovering 2023-09-30T09:41:28.240 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:28.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:28.345 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:28.406 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:28.458 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:28.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:28.565 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:28.612 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:28.649 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:28.685 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1098,"stamp":"2023-09-30T09:41:25.655871+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4276076218,"num_objects":65207,"num_object_clones":0,"num_object_copies":195621,"num_objects_missing_on_primary":1877,"num_objects_missing":1877,"num_objects_degraded":22464,"num_objects_misplaced":8704,"num_objects_unfound":0,"num_objects_dirty":65207,"num_whiteouts":0,"num_read":5989,"num_read_kb":12231,"num_write":120642,"num_write_kb":5929426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33363,"num_bytes_recovered":2068808296,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":152619,"ondisk_log_size":152619,"up":387,"acting":375,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":428,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14502488,"kb_used_data":11793384,"kb_used_omap":21,"kb_used_meta":2709034,"kb_avail":547968424,"statfs":{"total":575970213888,"available":561119666176,"internally_reserved":0,"allocated":12076425216,"data_stored":12064684416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21604,"internal_metadata":2774051740},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-197525504,"num_objects":-3014,"num_object_clones":0,"num_object_copies":-9042,"num_objects_missing_on_primary":-196,"num_objects_missing":-196,"num_objects_degraded":-8926,"num_objects_misplaced":7800,"num_objects_unfound":0,"num_objects_dirty":-3014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3014,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":196018176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004118"},"pg_stats":[{"pgid":"8.11","version":"424'6092","reported_seq":6123,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:23.303894+0000","last_change":"2023-09-30T09:40:37.604830+0000","last_active":"2023-09-30T09:41:23.303894+0000","last_peered":"2023-09-30T09:41:23.303894+0000","last_clean":"2023-09-30T09:40:21.233253+0000","last_became_active":"2023-09-30T09:40:37.596780+0000","last_became_peered":"2023-09-30T09:40:37.596780+0000","last_unstale":"2023-09-30T09:41:23.303894+0000","last_undegraded":"2023-09-30T09:40:37.578459+0000","last_fullsized":"2023-09-30T09:41:23.303894+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6092,"log_dups_size":0,"ondisk_log_size":6092,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129335296,"num_objects":1974,"num_object_clones":0,"num_object_copies":5922,"num_objects_missing_on_primary":58,"num_objects_missing":58,"num_objects_degraded":58,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3724,"num_write_kb":182304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1974}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":912,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.601957+0000","last_change":"2023-09-30T09:38:30.746866+0000","last_active":"2023-09-30T09:41:06.601957+0000","last_peered":"2023-09-30T09:41:06.601957+0000","last_clean":"2023-09-30T09:41:06.601957+0000","last_became_active":"2023-09-30T09:38:30.746361+0000","last_became_peered":"2023-09-30T09:38:30.746361+0000","last_unstale":"2023-09-30T09:41:06.601957+0000","last_undegraded":"2023-09-30T09:41:06.601957+0000","last_fullsized":"2023-09-30T09:41:06.601957+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:06:25.130460+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":442,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593809+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:41:06.593809+0000","last_peered":"2023-09-30T09:41:06.593809+0000","last_clean":"2023-09-30T09:41:06.593809+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:41:06.593809+0000","last_undegraded":"2023-09-30T09:41:06.593809+0000","last_fullsized":"2023-09-30T09:41:06.593809+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":347,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.597007+0000","last_change":"2023-09-30T09:39:47.872348+0000","last_active":"2023-09-30T09:41:06.597007+0000","last_peered":"2023-09-30T09:41:06.597007+0000","last_clean":"2023-09-30T09:41:06.597007+0000","last_became_active":"2023-09-30T09:39:47.870179+0000","last_became_peered":"2023-09-30T09:39:47.870179+0000","last_unstale":"2023-09-30T09:41:06.597007+0000","last_undegraded":"2023-09-30T09:41:06.597007+0000","last_fullsized":"2023-09-30T09:41:06.597007+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:33:45.842608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.12","version":"424'5999","reported_seq":6102,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:23.822491+0000","last_change":"2023-09-30T09:40:37.640460+0000","last_active":"2023-09-30T09:41:23.822491+0000","last_peered":"2023-09-30T09:41:23.822491+0000","last_clean":"2023-09-30T09:40:21.246088+0000","last_became_active":"2023-09-30T09:40:37.631437+0000","last_became_peered":"2023-09-30T09:40:37.631437+0000","last_unstale":"2023-09-30T09:41:23.822491+0000","last_undegraded":"2023-09-30T09:40:37.608497+0000","last_fullsized":"2023-09-30T09:41:23.822491+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5999,"log_dups_size":0,"ondisk_log_size":5999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129007616,"num_objects":1969,"num_object_clones":0,"num_object_copies":5907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":49,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3665,"num_write_kb":180256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5","objects":1969}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":908,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.601308+0000","last_change":"2023-09-30T09:40:39.587771+0000","last_active":"2023-09-30T09:41:06.601308+0000","last_peered":"2023-09-30T09:41:06.601308+0000","last_clean":"2023-09-30T09:41:06.601308+0000","last_became_active":"2023-09-30T09:40:39.587334+0000","last_became_peered":"2023-09-30T09:40:39.587334+0000","last_unstale":"2023-09-30T09:41:06.601308+0000","last_undegraded":"2023-09-30T09:41:06.601308+0000","last_fullsized":"2023-09-30T09:41:06.601308+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:43:27.541800+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":254,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593608+0000","last_change":"2023-09-30T09:39:47.869479+0000","last_active":"2023-09-30T09:41:06.593608+0000","last_peered":"2023-09-30T09:41:06.593608+0000","last_clean":"2023-09-30T09:41:06.593608+0000","last_became_active":"2023-09-30T09:39:47.869184+0000","last_became_peered":"2023-09-30T09:39:47.869184+0000","last_unstale":"2023-09-30T09:41:06.593608+0000","last_undegraded":"2023-09-30T09:41:06.593608+0000","last_fullsized":"2023-09-30T09:41:06.593608+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:03:52.322427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":19,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.590177+0000","last_change":"2023-09-30T09:40:36.599223+0000","last_active":"2023-09-30T09:41:06.590177+0000","last_peered":"2023-09-30T09:41:06.590177+0000","last_clean":"2023-09-30T09:41:06.590177+0000","last_became_active":"2023-09-30T09:40:36.597312+0000","last_became_peered":"2023-09-30T09:40:36.597312+0000","last_unstale":"2023-09-30T09:41:06.590177+0000","last_undegraded":"2023-09-30T09:41:06.590177+0000","last_fullsized":"2023-09-30T09:41:06.590177+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:29.960110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"424'6079","reported_seq":11986,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:21.653282+0000","last_change":"2023-09-30T09:40:20.789486+0000","last_active":"2023-09-30T09:41:21.653282+0000","last_peered":"2023-09-30T09:41:21.653282+0000","last_clean":"2023-09-30T09:41:21.653282+0000","last_became_active":"2023-09-30T09:40:20.780460+0000","last_became_peered":"2023-09-30T09:40:20.780460+0000","last_unstale":"2023-09-30T09:41:21.653282+0000","last_undegraded":"2023-09-30T09:41:21.653282+0000","last_fullsized":"2023-09-30T09:41:21.653282+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6079,"log_dups_size":0,"ondisk_log_size":6079,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133464064,"num_objects":2037,"num_object_clones":0,"num_object_copies":6111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3697,"num_write_kb":183456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2903,"num_bytes_recovered":171573248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":657,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593896+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:41:06.593896+0000","last_peered":"2023-09-30T09:41:06.593896+0000","last_clean":"2023-09-30T09:41:06.593896+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:41:06.593896+0000","last_undegraded":"2023-09-30T09:41:06.593896+0000","last_fullsized":"2023-09-30T09:41:06.593896+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":406,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.597444+0000","last_change":"2023-09-30T09:39:58.574864+0000","last_active":"2023-09-30T09:41:06.597444+0000","last_peered":"2023-09-30T09:41:06.597444+0000","last_clean":"2023-09-30T09:41:06.597444+0000","last_became_active":"2023-09-30T09:39:58.574566+0000","last_became_peered":"2023-09-30T09:39:58.574566+0000","last_unstale":"2023-09-30T09:41:06.597444+0000","last_undegraded":"2023-09-30T09:41:06.597444+0000","last_fullsized":"2023-09-30T09:41:06.597444+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:23:17.675484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":21,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.590339+0000","last_change":"2023-09-30T09:40:36.617977+0000","last_active":"2023-09-30T09:41:06.590339+0000","last_peered":"2023-09-30T09:41:06.590339+0000","last_clean":"2023-09-30T09:41:06.590339+0000","last_became_active":"2023-09-30T09:40:36.615491+0000","last_became_peered":"2023-09-30T09:40:36.615491+0000","last_unstale":"2023-09-30T09:41:06.590339+0000","last_undegraded":"2023-09-30T09:41:06.590339+0000","last_fullsized":"2023-09-30T09:41:06.590339+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:07.410092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":798,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596236+0000","last_change":"2023-09-30T09:38:30.744203+0000","last_active":"2023-09-30T09:41:06.596236+0000","last_peered":"2023-09-30T09:41:06.596236+0000","last_clean":"2023-09-30T09:41:06.596236+0000","last_became_active":"2023-09-30T09:38:30.740607+0000","last_became_peered":"2023-09-30T09:38:30.740607+0000","last_unstale":"2023-09-30T09:41:06.596236+0000","last_undegraded":"2023-09-30T09:41:06.596236+0000","last_fullsized":"2023-09-30T09:41:06.596236+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.164029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.590253+0000","last_change":"2023-09-30T09:40:36.618009+0000","last_active":"2023-09-30T09:41:06.590253+0000","last_peered":"2023-09-30T09:41:06.590253+0000","last_clean":"2023-09-30T09:41:06.590253+0000","last_became_active":"2023-09-30T09:40:36.615053+0000","last_became_peered":"2023-09-30T09:40:36.615053+0000","last_unstale":"2023-09-30T09:41:06.590253+0000","last_undegraded":"2023-09-30T09:41:06.590253+0000","last_fullsized":"2023-09-30T09:41:06.590253+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:37:53.101972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":395,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593381+0000","last_change":"2023-09-30T09:39:47.872093+0000","last_active":"2023-09-30T09:41:06.593381+0000","last_peered":"2023-09-30T09:41:06.593381+0000","last_clean":"2023-09-30T09:41:06.593381+0000","last_became_active":"2023-09-30T09:39:47.871723+0000","last_became_peered":"2023-09-30T09:39:47.871723+0000","last_unstale":"2023-09-30T09:41:06.593381+0000","last_undegraded":"2023-09-30T09:41:06.593381+0000","last_fullsized":"2023-09-30T09:41:06.593381+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:18:06.256339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":611,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595588+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:41:06.595588+0000","last_peered":"2023-09-30T09:41:06.595588+0000","last_clean":"2023-09-30T09:41:06.595588+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:41:06.595588+0000","last_undegraded":"2023-09-30T09:41:06.595588+0000","last_fullsized":"2023-09-30T09:41:06.595588+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":312,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.603679+0000","last_change":"2023-09-30T09:38:27.687229+0000","last_active":"2023-09-30T09:41:06.603679+0000","last_peered":"2023-09-30T09:41:06.603679+0000","last_clean":"2023-09-30T09:41:06.603679+0000","last_became_active":"2023-09-30T09:38:27.686912+0000","last_became_peered":"2023-09-30T09:38:27.686912+0000","last_unstale":"2023-09-30T09:41:06.603679+0000","last_undegraded":"2023-09-30T09:41:06.603679+0000","last_fullsized":"2023-09-30T09:41:06.603679+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:05:28.924932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":388,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595472+0000","last_change":"2023-09-30T09:40:36.608363+0000","last_active":"2023-09-30T09:41:06.595472+0000","last_peered":"2023-09-30T09:41:06.595472+0000","last_clean":"2023-09-30T09:41:06.595472+0000","last_became_active":"2023-09-30T09:40:36.608028+0000","last_became_peered":"2023-09-30T09:40:36.608028+0000","last_unstale":"2023-09-30T09:41:06.595472+0000","last_undegraded":"2023-09-30T09:41:06.595472+0000","last_fullsized":"2023-09-30T09:41:06.595472+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:02.611886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":630,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.603975+0000","last_change":"2023-09-30T09:40:01.615146+0000","last_active":"2023-09-30T09:41:06.603975+0000","last_peered":"2023-09-30T09:41:06.603975+0000","last_clean":"2023-09-30T09:41:06.603975+0000","last_became_active":"2023-09-30T09:40:01.614727+0000","last_became_peered":"2023-09-30T09:40:01.614727+0000","last_unstale":"2023-09-30T09:41:06.603975+0000","last_undegraded":"2023-09-30T09:41:06.603975+0000","last_fullsized":"2023-09-30T09:41:06.603975+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:46:24.281214+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":122,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596436+0000","last_change":"2023-09-30T09:40:36.618707+0000","last_active":"2023-09-30T09:41:06.596436+0000","last_peered":"2023-09-30T09:41:06.596436+0000","last_clean":"2023-09-30T09:41:06.596436+0000","last_became_active":"2023-09-30T09:40:36.597642+0000","last_became_peered":"2023-09-30T09:40:36.597642+0000","last_unstale":"2023-09-30T09:41:06.596436+0000","last_undegraded":"2023-09-30T09:41:06.596436+0000","last_fullsized":"2023-09-30T09:41:06.596436+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:09.447554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":261,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596429+0000","last_change":"2023-09-30T09:39:58.587940+0000","last_active":"2023-09-30T09:41:06.596429+0000","last_peered":"2023-09-30T09:41:06.596429+0000","last_clean":"2023-09-30T09:41:06.596429+0000","last_became_active":"2023-09-30T09:39:58.587622+0000","last_became_peered":"2023-09-30T09:39:58.587622+0000","last_unstale":"2023-09-30T09:41:06.596429+0000","last_undegraded":"2023-09-30T09:41:06.596429+0000","last_fullsized":"2023-09-30T09:41:06.596429+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:45:14.945251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":463,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594538+0000","last_change":"2023-09-30T09:40:43.945675+0000","last_active":"2023-09-30T09:41:06.594538+0000","last_peered":"2023-09-30T09:41:06.594538+0000","last_clean":"2023-09-30T09:41:06.594538+0000","last_became_active":"2023-09-30T09:40:43.945291+0000","last_became_peered":"2023-09-30T09:40:43.945291+0000","last_unstale":"2023-09-30T09:41:06.594538+0000","last_undegraded":"2023-09-30T09:41:06.594538+0000","last_fullsized":"2023-09-30T09:41:06.594538+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:41.055314+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":448,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594221+0000","last_change":"2023-09-30T09:39:47.882770+0000","last_active":"2023-09-30T09:41:06.594221+0000","last_peered":"2023-09-30T09:41:06.594221+0000","last_clean":"2023-09-30T09:41:06.594221+0000","last_became_active":"2023-09-30T09:39:47.864005+0000","last_became_peered":"2023-09-30T09:39:47.864005+0000","last_unstale":"2023-09-30T09:41:06.594221+0000","last_undegraded":"2023-09-30T09:41:06.594221+0000","last_fullsized":"2023-09-30T09:41:06.594221+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:09.572406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14166,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":429,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594694+0000","last_change":"2023-09-30T09:39:47.876698+0000","last_active":"2023-09-30T09:41:06.594694+0000","last_peered":"2023-09-30T09:41:06.594694+0000","last_clean":"2023-09-30T09:41:06.594694+0000","last_became_active":"2023-09-30T09:39:47.876412+0000","last_became_peered":"2023-09-30T09:39:47.876412+0000","last_unstale":"2023-09-30T09:41:06.594694+0000","last_undegraded":"2023-09-30T09:41:06.594694+0000","last_fullsized":"2023-09-30T09:41:06.594694+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:27:37.752128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"424'6271","reported_seq":15730,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:24.339141+0000","last_change":"2023-09-30T09:40:42.887469+0000","last_active":"2023-09-30T09:41:24.339141+0000","last_peered":"2023-09-30T09:41:24.339141+0000","last_clean":"2023-09-30T09:41:24.339141+0000","last_became_active":"2023-09-30T09:40:42.880205+0000","last_became_peered":"2023-09-30T09:40:42.880205+0000","last_unstale":"2023-09-30T09:41:24.339141+0000","last_undegraded":"2023-09-30T09:41:24.339141+0000","last_fullsized":"2023-09-30T09:41:24.339141+0000","mapping_epoch":418,"log_start":"363'282","ondisk_log_start":"363'282","created":379,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5989,"log_dups_size":0,"ondisk_log_size":5989,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131563520,"num_objects":2008,"num_object_clones":0,"num_object_copies":6024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3842,"num_write_kb":187168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4668,"num_bytes_recovered":274399232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":651,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.601572+0000","last_change":"2023-09-30T09:40:01.605467+0000","last_active":"2023-09-30T09:41:06.601572+0000","last_peered":"2023-09-30T09:41:06.601572+0000","last_clean":"2023-09-30T09:41:06.601572+0000","last_became_active":"2023-09-30T09:40:01.605059+0000","last_became_peered":"2023-09-30T09:40:01.605059+0000","last_unstale":"2023-09-30T09:41:06.601572+0000","last_undegraded":"2023-09-30T09:41:06.601572+0000","last_fullsized":"2023-09-30T09:41:06.601572+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":74,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596076+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:41:06.596076+0000","last_peered":"2023-09-30T09:41:06.596076+0000","last_clean":"2023-09-30T09:41:06.596076+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:41:06.596076+0000","last_undegraded":"2023-09-30T09:41:06.596076+0000","last_fullsized":"2023-09-30T09:41:06.596076+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":416,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596432+0000","last_change":"2023-09-30T09:40:22.811912+0000","last_active":"2023-09-30T09:41:06.596432+0000","last_peered":"2023-09-30T09:41:06.596432+0000","last_clean":"2023-09-30T09:41:06.596432+0000","last_became_active":"2023-09-30T09:40:22.810606+0000","last_became_peered":"2023-09-30T09:40:22.810606+0000","last_unstale":"2023-09-30T09:41:06.596432+0000","last_undegraded":"2023-09-30T09:41:06.596432+0000","last_fullsized":"2023-09-30T09:41:06.596432+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:46:15.913218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.19","version":"424'6141","reported_seq":9427,"reported_epoch":424,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:41:24.420908+0000","last_change":"2023-09-30T09:41:18.167998+0000","last_active":"2023-09-30T09:41:24.420908+0000","last_peered":"2023-09-30T09:41:24.420908+0000","last_clean":"2023-09-30T09:39:46.775307+0000","last_became_active":"2023-09-30T09:40:00.498948+0000","last_became_peered":"2023-09-30T09:40:00.498948+0000","last_unstale":"2023-09-30T09:41:24.420908+0000","last_undegraded":"2023-09-30T09:41:24.420908+0000","last_fullsized":"2023-09-30T09:39:59.584439+0000","mapping_epoch":398,"log_start":"363'133","ondisk_log_start":"363'133","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6008,"log_dups_size":0,"ondisk_log_size":6008,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132677632,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3100,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3753,"num_write_kb":184864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69533696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":1550},{"shards":"4,5","objects":475}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":810,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.602553+0000","last_change":"2023-09-30T09:39:50.837500+0000","last_active":"2023-09-30T09:41:06.602553+0000","last_peered":"2023-09-30T09:41:06.602553+0000","last_clean":"2023-09-30T09:41:06.602553+0000","last_became_active":"2023-09-30T09:39:50.837185+0000","last_became_peered":"2023-09-30T09:39:50.837185+0000","last_unstale":"2023-09-30T09:41:06.602553+0000","last_undegraded":"2023-09-30T09:41:06.602553+0000","last_fullsized":"2023-09-30T09:41:06.602553+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:35:32.171780+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":322,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595937+0000","last_change":"2023-09-30T09:39:58.583120+0000","last_active":"2023-09-30T09:41:06.595937+0000","last_peered":"2023-09-30T09:41:06.595937+0000","last_clean":"2023-09-30T09:41:06.595937+0000","last_became_active":"2023-09-30T09:39:58.582799+0000","last_became_peered":"2023-09-30T09:39:58.582799+0000","last_unstale":"2023-09-30T09:41:06.595937+0000","last_undegraded":"2023-09-30T09:41:06.595937+0000","last_fullsized":"2023-09-30T09:41:06.595937+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:51:22.821149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":163,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.604531+0000","last_change":"2023-09-30T09:40:36.622997+0000","last_active":"2023-09-30T09:41:06.604531+0000","last_peered":"2023-09-30T09:41:06.604531+0000","last_clean":"2023-09-30T09:41:06.604531+0000","last_became_active":"2023-09-30T09:40:36.622694+0000","last_became_peered":"2023-09-30T09:40:36.622694+0000","last_unstale":"2023-09-30T09:41:06.604531+0000","last_undegraded":"2023-09-30T09:41:06.604531+0000","last_fullsized":"2023-09-30T09:41:06.604531+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:27:59.419339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":854,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594460+0000","last_change":"2023-09-30T09:40:39.589288+0000","last_active":"2023-09-30T09:41:06.594460+0000","last_peered":"2023-09-30T09:41:06.594460+0000","last_clean":"2023-09-30T09:41:06.594460+0000","last_became_active":"2023-09-30T09:40:39.588929+0000","last_became_peered":"2023-09-30T09:40:39.588929+0000","last_unstale":"2023-09-30T09:41:06.594460+0000","last_undegraded":"2023-09-30T09:41:06.594460+0000","last_fullsized":"2023-09-30T09:41:06.594460+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":98,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595583+0000","last_change":"2023-09-30T09:40:36.592638+0000","last_active":"2023-09-30T09:41:06.595583+0000","last_peered":"2023-09-30T09:41:06.595583+0000","last_clean":"2023-09-30T09:41:06.595583+0000","last_became_active":"2023-09-30T09:40:36.592313+0000","last_became_peered":"2023-09-30T09:40:36.592313+0000","last_unstale":"2023-09-30T09:41:06.595583+0000","last_undegraded":"2023-09-30T09:41:06.595583+0000","last_fullsized":"2023-09-30T09:41:06.595583+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:18:12.781018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":320,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596846+0000","last_change":"2023-09-30T09:39:58.586100+0000","last_active":"2023-09-30T09:41:06.596846+0000","last_peered":"2023-09-30T09:41:06.596846+0000","last_clean":"2023-09-30T09:41:06.596846+0000","last_became_active":"2023-09-30T09:39:58.585756+0000","last_became_peered":"2023-09-30T09:39:58.585756+0000","last_unstale":"2023-09-30T09:41:06.596846+0000","last_undegraded":"2023-09-30T09:41:06.596846+0000","last_fullsized":"2023-09-30T09:41:06.596846+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:56.123014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":909,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.601337+0000","last_change":"2023-09-30T09:39:50.836014+0000","last_active":"2023-09-30T09:41:06.601337+0000","last_peered":"2023-09-30T09:41:06.601337+0000","last_clean":"2023-09-30T09:41:06.601337+0000","last_became_active":"2023-09-30T09:39:50.835632+0000","last_became_peered":"2023-09-30T09:39:50.835632+0000","last_unstale":"2023-09-30T09:41:06.601337+0000","last_undegraded":"2023-09-30T09:41:06.601337+0000","last_fullsized":"2023-09-30T09:41:06.601337+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:26.067574+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":490,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.602463+0000","last_change":"2023-09-30T09:39:58.604069+0000","last_active":"2023-09-30T09:41:06.602463+0000","last_peered":"2023-09-30T09:41:06.602463+0000","last_clean":"2023-09-30T09:41:06.602463+0000","last_became_active":"2023-09-30T09:39:58.585674+0000","last_became_peered":"2023-09-30T09:39:58.585674+0000","last_unstale":"2023-09-30T09:41:06.602463+0000","last_undegraded":"2023-09-30T09:41:06.602463+0000","last_fullsized":"2023-09-30T09:41:06.602463+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:03.491449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":440,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596693+0000","last_change":"2023-09-30T09:38:45.606456+0000","last_active":"2023-09-30T09:41:06.596693+0000","last_peered":"2023-09-30T09:41:06.596693+0000","last_clean":"2023-09-30T09:41:06.596693+0000","last_became_active":"2023-09-30T09:38:45.604792+0000","last_became_peered":"2023-09-30T09:38:45.604792+0000","last_unstale":"2023-09-30T09:41:06.596693+0000","last_undegraded":"2023-09-30T09:41:06.596693+0000","last_fullsized":"2023-09-30T09:41:06.596693+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:11.533941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"424'6188","reported_seq":6273,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:24.307527+0000","last_change":"2023-09-30T09:39:15.603264+0000","last_active":"2023-09-30T09:41:24.307527+0000","last_peered":"2023-09-30T09:41:24.307527+0000","last_clean":"2023-09-30T09:41:24.307527+0000","last_became_active":"2023-09-30T09:39:15.600826+0000","last_became_peered":"2023-09-30T09:39:15.600826+0000","last_unstale":"2023-09-30T09:41:24.307527+0000","last_undegraded":"2023-09-30T09:41:24.307527+0000","last_fullsized":"2023-09-30T09:41:24.307527+0000","mapping_epoch":358,"log_start":"363'195","ondisk_log_start":"363'195","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5993,"log_dups_size":0,"ondisk_log_size":5993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3782,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":790,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596330+0000","last_change":"2023-09-30T09:38:30.737676+0000","last_active":"2023-09-30T09:41:06.596330+0000","last_peered":"2023-09-30T09:41:06.596330+0000","last_clean":"2023-09-30T09:41:06.596330+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:41:06.596330+0000","last_undegraded":"2023-09-30T09:41:06.596330+0000","last_fullsized":"2023-09-30T09:41:06.596330+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.740851+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.740851+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":86,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596993+0000","last_change":"2023-09-30T09:39:58.609474+0000","last_active":"2023-09-30T09:41:06.596993+0000","last_peered":"2023-09-30T09:41:06.596993+0000","last_clean":"2023-09-30T09:41:06.596993+0000","last_became_active":"2023-09-30T09:39:58.583107+0000","last_became_peered":"2023-09-30T09:39:58.583107+0000","last_unstale":"2023-09-30T09:41:06.596993+0000","last_undegraded":"2023-09-30T09:41:06.596993+0000","last_fullsized":"2023-09-30T09:41:06.596993+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:31.980540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":133,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.600771+0000","last_change":"2023-09-30T09:39:47.869741+0000","last_active":"2023-09-30T09:41:06.600771+0000","last_peered":"2023-09-30T09:41:06.600771+0000","last_clean":"2023-09-30T09:41:06.600771+0000","last_became_active":"2023-09-30T09:39:47.869354+0000","last_became_peered":"2023-09-30T09:39:47.869354+0000","last_unstale":"2023-09-30T09:41:06.600771+0000","last_undegraded":"2023-09-30T09:41:06.600771+0000","last_fullsized":"2023-09-30T09:41:06.600771+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:28:16.966118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"424'6160","reported_seq":12780,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:23.835488+0000","last_change":"2023-09-30T09:40:37.624519+0000","last_active":"2023-09-30T09:41:23.835488+0000","last_peered":"2023-09-30T09:41:23.835488+0000","last_clean":"2023-09-30T09:40:21.254315+0000","last_became_active":"2023-09-30T09:40:37.616502+0000","last_became_peered":"2023-09-30T09:40:37.616502+0000","last_unstale":"2023-09-30T09:41:23.835488+0000","last_undegraded":"2023-09-30T09:40:37.596565+0000","last_fullsized":"2023-09-30T09:41:23.835488+0000","mapping_epoch":414,"log_start":"363'154","ondisk_log_start":"363'154","created":358,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132644864,"num_objects":2024,"num_object_clones":0,"num_object_copies":6072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":69,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3758,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3153,"num_bytes_recovered":191037440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"0,4,7","objects":2024}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":782,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:06.590677+0000","last_change":"2023-09-30T09:40:37.595615+0000","last_active":"2023-09-30T09:41:06.590677+0000","last_peered":"2023-09-30T09:41:06.590677+0000","last_clean":"2023-09-30T09:31:49.462297+0000","last_became_active":"2023-09-30T09:40:37.595324+0000","last_became_peered":"2023-09-30T09:40:37.595324+0000","last_unstale":"2023-09-30T09:41:06.590677+0000","last_undegraded":"2023-09-30T09:40:37.578525+0000","last_fullsized":"2023-09-30T09:41:06.590677+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5","objects":-1},{"shards":"0,4,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":70,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596608+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:41:06.596608+0000","last_peered":"2023-09-30T09:41:06.596608+0000","last_clean":"2023-09-30T09:41:06.596608+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:41:06.596608+0000","last_undegraded":"2023-09-30T09:41:06.596608+0000","last_fullsized":"2023-09-30T09:41:06.596608+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":466,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593774+0000","last_change":"2023-09-30T09:40:36.593049+0000","last_active":"2023-09-30T09:41:06.593774+0000","last_peered":"2023-09-30T09:41:06.593774+0000","last_clean":"2023-09-30T09:41:06.593774+0000","last_became_active":"2023-09-30T09:40:36.590336+0000","last_became_peered":"2023-09-30T09:40:36.590336+0000","last_unstale":"2023-09-30T09:41:06.593774+0000","last_undegraded":"2023-09-30T09:41:06.593774+0000","last_fullsized":"2023-09-30T09:41:06.593774+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:54.795919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"424'6206","reported_seq":9951,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:23.258333+0000","last_change":"2023-09-30T09:40:40.784953+0000","last_active":"2023-09-30T09:41:23.258333+0000","last_peered":"2023-09-30T09:41:23.258333+0000","last_clean":"2023-09-30T09:38:26.430308+0000","last_became_active":"2023-09-30T09:40:39.913250+0000","last_became_peered":"2023-09-30T09:40:39.913250+0000","last_unstale":"2023-09-30T09:41:23.258333+0000","last_undegraded":"2023-09-30T09:40:39.696160+0000","last_fullsized":"2023-09-30T09:40:37.581870+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6206,"log_dups_size":0,"ondisk_log_size":6206,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135069696,"num_objects":2061,"num_object_clones":0,"num_object_copies":6183,"num_objects_missing_on_primary":52,"num_objects_missing":52,"num_objects_degraded":2304,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3765,"num_write_kb":186432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":924,"num_bytes_recovered":60555264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,2],"avail_no_missing":["6"],"object_location_counts":[{"shards":"0,2","objects":78},{"shards":"0,2,6","objects":227},{"shards":"0,6,7","objects":1756}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":591,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.603887+0000","last_change":"2023-09-30T09:39:56.915934+0000","last_active":"2023-09-30T09:41:06.603887+0000","last_peered":"2023-09-30T09:41:06.603887+0000","last_clean":"2023-09-30T09:41:06.603887+0000","last_became_active":"2023-09-30T09:39:56.915617+0000","last_became_peered":"2023-09-30T09:39:56.915617+0000","last_unstale":"2023-09-30T09:41:06.603887+0000","last_undegraded":"2023-09-30T09:41:06.603887+0000","last_fullsized":"2023-09-30T09:41:06.603887+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":86,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596106+0000","last_change":"2023-09-30T09:39:47.881884+0000","last_active":"2023-09-30T09:41:06.596106+0000","last_peered":"2023-09-30T09:41:06.596106+0000","last_clean":"2023-09-30T09:41:06.596106+0000","last_became_active":"2023-09-30T09:39:47.871068+0000","last_became_peered":"2023-09-30T09:39:47.871068+0000","last_unstale":"2023-09-30T09:41:06.596106+0000","last_undegraded":"2023-09-30T09:41:06.596106+0000","last_fullsized":"2023-09-30T09:41:06.596106+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:03:57.045359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":393,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595845+0000","last_change":"2023-09-30T09:39:47.851588+0000","last_active":"2023-09-30T09:41:06.595845+0000","last_peered":"2023-09-30T09:41:06.595845+0000","last_clean":"2023-09-30T09:41:06.595845+0000","last_became_active":"2023-09-30T09:39:47.851010+0000","last_became_peered":"2023-09-30T09:39:47.851010+0000","last_unstale":"2023-09-30T09:41:06.595845+0000","last_undegraded":"2023-09-30T09:41:06.595845+0000","last_fullsized":"2023-09-30T09:41:06.595845+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:47:13.369816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"424'6209","reported_seq":8578,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:21.952743+0000","last_change":"2023-09-30T09:39:55.210294+0000","last_active":"2023-09-30T09:41:21.952743+0000","last_peered":"2023-09-30T09:41:21.952743+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:39:55.103786+0000","last_became_peered":"2023-09-30T09:39:55.103786+0000","last_unstale":"2023-09-30T09:41:21.952743+0000","last_undegraded":"2023-09-30T09:39:51.345690+0000","last_fullsized":"2023-09-30T09:39:48.814537+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6209,"log_dups_size":0,"ondisk_log_size":6209,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136675328,"num_objects":2085,"num_object_clones":0,"num_object_copies":6255,"num_objects_missing_on_primary":877,"num_objects_missing":877,"num_objects_degraded":3244,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3773,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":844,"num_bytes_recovered":55312384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":-11},{"shards":"1,2,3,6","objects":7},{"shards":"1,2,6","objects":1212},{"shards":"1,3,6","objects":877}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":639,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.597122+0000","last_change":"2023-09-30T09:38:48.662225+0000","last_active":"2023-09-30T09:41:06.597122+0000","last_peered":"2023-09-30T09:41:06.597122+0000","last_clean":"2023-09-30T09:41:06.597122+0000","last_became_active":"2023-09-30T09:38:48.661883+0000","last_became_peered":"2023-09-30T09:38:48.661883+0000","last_unstale":"2023-09-30T09:41:06.597122+0000","last_undegraded":"2023-09-30T09:41:06.597122+0000","last_fullsized":"2023-09-30T09:41:06.597122+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:01.892446+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":423,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.602245+0000","last_change":"2023-09-30T09:40:36.623400+0000","last_active":"2023-09-30T09:41:06.602245+0000","last_peered":"2023-09-30T09:41:06.602245+0000","last_clean":"2023-09-30T09:41:06.602245+0000","last_became_active":"2023-09-30T09:40:36.622842+0000","last_became_peered":"2023-09-30T09:40:36.622842+0000","last_unstale":"2023-09-30T09:41:06.602245+0000","last_undegraded":"2023-09-30T09:41:06.602245+0000","last_fullsized":"2023-09-30T09:41:06.602245+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:47:52.429210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":21,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.588767+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:41:06.588767+0000","last_peered":"2023-09-30T09:41:06.588767+0000","last_clean":"2023-09-30T09:41:06.588767+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:41:06.588767+0000","last_undegraded":"2023-09-30T09:41:06.588767+0000","last_fullsized":"2023-09-30T09:41:06.588767+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"424'6306","reported_seq":9868,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:24.128190+0000","last_change":"2023-09-30T09:41:18.158568+0000","last_active":"2023-09-30T09:41:24.128190+0000","last_peered":"2023-09-30T09:41:24.128190+0000","last_clean":"2023-09-30T09:41:24.128190+0000","last_became_active":"2023-09-30T09:39:16.603809+0000","last_became_peered":"2023-09-30T09:39:16.603809+0000","last_unstale":"2023-09-30T09:41:24.128190+0000","last_undegraded":"2023-09-30T09:41:24.128190+0000","last_fullsized":"2023-09-30T09:41:24.128190+0000","mapping_epoch":380,"log_start":"363'236","ondisk_log_start":"363'236","created":358,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6070,"log_dups_size":0,"ondisk_log_size":6070,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139886592,"num_objects":2134,"num_object_clones":0,"num_object_copies":6402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3830,"num_write_kb":190880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1729,"num_bytes_recovered":113344512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":879,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595517+0000","last_change":"2023-09-30T09:38:45.616475+0000","last_active":"2023-09-30T09:41:06.595517+0000","last_peered":"2023-09-30T09:41:06.595517+0000","last_clean":"2023-09-30T09:41:06.595517+0000","last_became_active":"2023-09-30T09:38:45.616163+0000","last_became_peered":"2023-09-30T09:38:45.616163+0000","last_unstale":"2023-09-30T09:41:06.595517+0000","last_undegraded":"2023-09-30T09:41:06.595517+0000","last_fullsized":"2023-09-30T09:41:06.595517+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:11:18.666981+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":54,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594923+0000","last_change":"2023-09-30T09:39:58.588073+0000","last_active":"2023-09-30T09:41:06.594923+0000","last_peered":"2023-09-30T09:41:06.594923+0000","last_clean":"2023-09-30T09:41:06.594923+0000","last_became_active":"2023-09-30T09:39:58.577667+0000","last_became_peered":"2023-09-30T09:39:58.577667+0000","last_unstale":"2023-09-30T09:41:06.594923+0000","last_undegraded":"2023-09-30T09:41:06.594923+0000","last_fullsized":"2023-09-30T09:41:06.594923+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:28:13.160407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":402,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594298+0000","last_change":"2023-09-30T09:39:47.865003+0000","last_active":"2023-09-30T09:41:06.594298+0000","last_peered":"2023-09-30T09:41:06.594298+0000","last_clean":"2023-09-30T09:41:06.594298+0000","last_became_active":"2023-09-30T09:39:47.864581+0000","last_became_peered":"2023-09-30T09:39:47.864581+0000","last_unstale":"2023-09-30T09:41:06.594298+0000","last_undegraded":"2023-09-30T09:41:06.594298+0000","last_fullsized":"2023-09-30T09:41:06.594298+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:13:20.328082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"424'6026","reported_seq":6051,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:23.045003+0000","last_change":"2023-09-30T09:40:37.604868+0000","last_active":"2023-09-30T09:41:23.045003+0000","last_peered":"2023-09-30T09:41:23.045003+0000","last_clean":"2023-09-30T09:40:21.078874+0000","last_became_active":"2023-09-30T09:40:37.598926+0000","last_became_peered":"2023-09-30T09:40:37.598926+0000","last_unstale":"2023-09-30T09:41:23.045003+0000","last_undegraded":"2023-09-30T09:40:37.578588+0000","last_fullsized":"2023-09-30T09:41:23.045003+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132612096,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":65,"num_objects_missing":65,"num_objects_degraded":65,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3657,"num_write_kb":181792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":2023}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":875,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.597740+0000","last_change":"2023-09-30T09:39:50.832403+0000","last_active":"2023-09-30T09:41:06.597740+0000","last_peered":"2023-09-30T09:41:06.597740+0000","last_clean":"2023-09-30T09:41:06.597740+0000","last_became_active":"2023-09-30T09:39:50.832136+0000","last_became_peered":"2023-09-30T09:39:50.832136+0000","last_unstale":"2023-09-30T09:41:06.597740+0000","last_undegraded":"2023-09-30T09:41:06.597740+0000","last_fullsized":"2023-09-30T09:41:06.597740+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:16:37.452222+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":317,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593721+0000","last_change":"2023-09-30T09:39:47.874554+0000","last_active":"2023-09-30T09:41:06.593721+0000","last_peered":"2023-09-30T09:41:06.593721+0000","last_clean":"2023-09-30T09:41:06.593721+0000","last_became_active":"2023-09-30T09:39:47.870194+0000","last_became_peered":"2023-09-30T09:39:47.870194+0000","last_unstale":"2023-09-30T09:41:06.593721+0000","last_undegraded":"2023-09-30T09:41:06.593721+0000","last_fullsized":"2023-09-30T09:41:06.593721+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:17.231070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":118,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.597293+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:41:06.597293+0000","last_peered":"2023-09-30T09:41:06.597293+0000","last_clean":"2023-09-30T09:41:06.597293+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:41:06.597293+0000","last_undegraded":"2023-09-30T09:41:06.597293+0000","last_fullsized":"2023-09-30T09:41:06.597293+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"424'5993","reported_seq":6097,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:24.207393+0000","last_change":"2023-09-30T09:40:37.656453+0000","last_active":"2023-09-30T09:41:24.207393+0000","last_peered":"2023-09-30T09:41:24.207393+0000","last_clean":"2023-09-30T09:40:21.766887+0000","last_became_active":"2023-09-30T09:40:37.631427+0000","last_became_peered":"2023-09-30T09:40:37.631427+0000","last_unstale":"2023-09-30T09:41:24.207393+0000","last_undegraded":"2023-09-30T09:40:37.605179+0000","last_fullsized":"2023-09-30T09:41:24.207393+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5993,"log_dups_size":0,"ondisk_log_size":5993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129662976,"num_objects":1978,"num_object_clones":0,"num_object_copies":5934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":52,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3658,"num_write_kb":180384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5","objects":1978}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":747,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.588844+0000","last_change":"2023-09-30T09:40:39.590158+0000","last_active":"2023-09-30T09:41:06.588844+0000","last_peered":"2023-09-30T09:41:06.588844+0000","last_clean":"2023-09-30T09:41:06.588844+0000","last_became_active":"2023-09-30T09:40:39.586660+0000","last_became_peered":"2023-09-30T09:40:39.586660+0000","last_unstale":"2023-09-30T09:41:06.588844+0000","last_undegraded":"2023-09-30T09:41:06.588844+0000","last_fullsized":"2023-09-30T09:41:06.588844+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:13.120381+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":430,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593685+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:41:06.593685+0000","last_peered":"2023-09-30T09:41:06.593685+0000","last_clean":"2023-09-30T09:41:06.593685+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:41:06.593685+0000","last_undegraded":"2023-09-30T09:41:06.593685+0000","last_fullsized":"2023-09-30T09:41:06.593685+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":409,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.597186+0000","last_change":"2023-09-30T09:39:47.875703+0000","last_active":"2023-09-30T09:41:06.597186+0000","last_peered":"2023-09-30T09:41:06.597186+0000","last_clean":"2023-09-30T09:41:06.597186+0000","last_became_active":"2023-09-30T09:39:47.875450+0000","last_became_peered":"2023-09-30T09:39:47.875450+0000","last_unstale":"2023-09-30T09:41:06.597186+0000","last_undegraded":"2023-09-30T09:41:06.597186+0000","last_fullsized":"2023-09-30T09:41:06.597186+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:58:16.042133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"424'6088","reported_seq":7407,"reported_epoch":424,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:21.538468+0000","last_change":"2023-09-30T09:41:14.789820+0000","last_active":"2023-09-30T09:41:21.538468+0000","last_peered":"2023-09-30T09:41:21.538468+0000","last_clean":"2023-09-30T09:39:46.263479+0000","last_became_active":"2023-09-30T09:39:49.034883+0000","last_became_peered":"2023-09-30T09:39:49.034883+0000","last_unstale":"2023-09-30T09:41:21.538468+0000","last_undegraded":"2023-09-30T09:39:48.846697+0000","last_fullsized":"2023-09-30T09:39:48.814582+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6088,"log_dups_size":0,"ondisk_log_size":6088,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135495680,"num_objects":2067,"num_object_clones":0,"num_object_copies":6201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":486,"num_objects_misplaced":1488,"num_objects_unfound":0,"num_objects_dirty":2067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3705,"num_write_kb":184736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":614,"num_bytes_recovered":40239104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[1,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5","objects":93},{"shards":"1,5,6","objects":1488}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":597,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596480+0000","last_change":"2023-09-30T09:39:50.834859+0000","last_active":"2023-09-30T09:41:06.596480+0000","last_peered":"2023-09-30T09:41:06.596480+0000","last_clean":"2023-09-30T09:41:06.596480+0000","last_became_active":"2023-09-30T09:39:50.834483+0000","last_became_peered":"2023-09-30T09:39:50.834483+0000","last_unstale":"2023-09-30T09:41:06.596480+0000","last_undegraded":"2023-09-30T09:41:06.596480+0000","last_fullsized":"2023-09-30T09:41:06.596480+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:58:35.521874+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":421,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595515+0000","last_change":"2023-09-30T09:38:45.620180+0000","last_active":"2023-09-30T09:41:06.595515+0000","last_peered":"2023-09-30T09:41:06.595515+0000","last_clean":"2023-09-30T09:41:06.595515+0000","last_became_active":"2023-09-30T09:38:45.619776+0000","last_became_peered":"2023-09-30T09:38:45.619776+0000","last_unstale":"2023-09-30T09:41:06.595515+0000","last_undegraded":"2023-09-30T09:41:06.595515+0000","last_fullsized":"2023-09-30T09:41:06.595515+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:25:08.739481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":124,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596995+0000","last_change":"2023-09-30T09:38:45.610897+0000","last_active":"2023-09-30T09:41:06.596995+0000","last_peered":"2023-09-30T09:41:06.596995+0000","last_clean":"2023-09-30T09:41:06.596995+0000","last_became_active":"2023-09-30T09:38:45.610628+0000","last_became_peered":"2023-09-30T09:38:45.610628+0000","last_unstale":"2023-09-30T09:41:06.596995+0000","last_undegraded":"2023-09-30T09:41:06.596995+0000","last_fullsized":"2023-09-30T09:41:06.596995+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:04.928128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.e","version":"424'7393","reported_seq":7485,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:23.220618+0000","last_change":"2023-09-30T09:40:43.499107+0000","last_active":"2023-09-30T09:41:23.220618+0000","last_peered":"2023-09-30T09:41:23.220618+0000","last_clean":"2023-09-30T09:41:23.220618+0000","last_became_active":"2023-09-30T09:40:42.368900+0000","last_became_peered":"2023-09-30T09:40:42.368900+0000","last_unstale":"2023-09-30T09:41:23.220618+0000","last_undegraded":"2023-09-30T09:41:23.220618+0000","last_fullsized":"2023-09-30T09:41:23.220618+0000","mapping_epoch":414,"log_start":"369'2601","ondisk_log_start":"369'2601","created":358,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4792,"log_dups_size":0,"ondisk_log_size":4792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:53.392866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262078464,"num_objects":3999,"num_object_clones":0,"num_object_copies":11997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7393,"num_write_kb":364544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":721,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596624+0000","last_change":"2023-09-30T09:39:56.918064+0000","last_active":"2023-09-30T09:41:06.596624+0000","last_peered":"2023-09-30T09:41:06.596624+0000","last_clean":"2023-09-30T09:41:06.596624+0000","last_became_active":"2023-09-30T09:39:56.917689+0000","last_became_peered":"2023-09-30T09:39:56.917689+0000","last_unstale":"2023-09-30T09:41:06.596624+0000","last_undegraded":"2023-09-30T09:41:06.596624+0000","last_fullsized":"2023-09-30T09:41:06.596624+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:11.538673+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":652,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.589600+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:41:06.589600+0000","last_peered":"2023-09-30T09:41:06.589600+0000","last_clean":"2023-09-30T09:41:06.589600+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:41:06.589600+0000","last_undegraded":"2023-09-30T09:41:06.589600+0000","last_fullsized":"2023-09-30T09:41:06.589600+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":440,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.604602+0000","last_change":"2023-09-30T09:39:47.855281+0000","last_active":"2023-09-30T09:41:06.604602+0000","last_peered":"2023-09-30T09:41:06.604602+0000","last_clean":"2023-09-30T09:41:06.604602+0000","last_became_active":"2023-09-30T09:39:47.854970+0000","last_became_peered":"2023-09-30T09:39:47.854970+0000","last_unstale":"2023-09-30T09:41:06.604602+0000","last_undegraded":"2023-09-30T09:41:06.604602+0000","last_fullsized":"2023-09-30T09:41:06.604602+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:37:22.880321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":113,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.601137+0000","last_change":"2023-09-30T09:34:47.539844+0000","last_active":"2023-09-30T09:41:06.601137+0000","last_peered":"2023-09-30T09:41:06.601137+0000","last_clean":"2023-09-30T09:41:06.601137+0000","last_became_active":"2023-09-30T09:34:47.539557+0000","last_became_peered":"2023-09-30T09:34:47.539557+0000","last_unstale":"2023-09-30T09:41:06.601137+0000","last_undegraded":"2023-09-30T09:41:06.601137+0000","last_fullsized":"2023-09-30T09:41:06.601137+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:30:49.442285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"424'6222","reported_seq":8647,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:24.213361+0000","last_change":"2023-09-30T09:40:00.514475+0000","last_active":"2023-09-30T09:41:24.213361+0000","last_peered":"2023-09-30T09:41:24.213361+0000","last_clean":"2023-09-30T09:38:26.554137+0000","last_became_active":"2023-09-30T09:40:00.502041+0000","last_became_peered":"2023-09-30T09:40:00.502041+0000","last_unstale":"2023-09-30T09:41:24.213361+0000","last_undegraded":"2023-09-30T09:39:59.696250+0000","last_fullsized":"2023-09-30T09:39:59.582903+0000","mapping_epoch":398,"log_start":"363'168","ondisk_log_start":"363'168","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134643712,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2612,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3792,"num_write_kb":187104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":869,"num_bytes_recovered":56950784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"0,3,5","objects":273},{"shards":"0,5,7","objects":1200},{"shards":"3,5,7","objects":572},{"shards":"5,7","objects":9}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":926,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.603341+0000","last_change":"2023-09-30T09:40:39.596051+0000","last_active":"2023-09-30T09:41:06.603341+0000","last_peered":"2023-09-30T09:41:06.603341+0000","last_clean":"2023-09-30T09:41:06.603341+0000","last_became_active":"2023-09-30T09:40:39.595782+0000","last_became_peered":"2023-09-30T09:40:39.595782+0000","last_unstale":"2023-09-30T09:41:06.603341+0000","last_undegraded":"2023-09-30T09:41:06.603341+0000","last_fullsized":"2023-09-30T09:41:06.603341+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:18:09.686486+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":46,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:06.589305+0000","last_change":"2023-09-30T09:40:36.623450+0000","last_active":"2023-09-30T09:41:06.589305+0000","last_peered":"2023-09-30T09:41:06.589305+0000","last_clean":"2023-09-30T09:30:20.310268+0000","last_became_active":"2023-09-30T09:40:36.617588+0000","last_became_peered":"2023-09-30T09:40:36.617588+0000","last_unstale":"2023-09-30T09:41:06.589305+0000","last_undegraded":"2023-09-30T09:40:36.582089+0000","last_fullsized":"2023-09-30T09:41:06.589305+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.469417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":433,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594756+0000","last_change":"2023-09-30T09:40:36.597929+0000","last_active":"2023-09-30T09:41:06.594756+0000","last_peered":"2023-09-30T09:41:06.594756+0000","last_clean":"2023-09-30T09:41:06.594756+0000","last_became_active":"2023-09-30T09:40:36.597614+0000","last_became_peered":"2023-09-30T09:40:36.597614+0000","last_unstale":"2023-09-30T09:41:06.594756+0000","last_undegraded":"2023-09-30T09:41:06.594756+0000","last_fullsized":"2023-09-30T09:41:06.594756+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:44:14.764059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":159,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595675+0000","last_change":"2023-09-30T09:39:47.852546+0000","last_active":"2023-09-30T09:41:06.595675+0000","last_peered":"2023-09-30T09:41:06.595675+0000","last_clean":"2023-09-30T09:41:06.595675+0000","last_became_active":"2023-09-30T09:39:47.852155+0000","last_became_peered":"2023-09-30T09:39:47.852155+0000","last_unstale":"2023-09-30T09:41:06.595675+0000","last_undegraded":"2023-09-30T09:41:06.595675+0000","last_fullsized":"2023-09-30T09:41:06.595675+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:06:46.337484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"424'7300","reported_seq":11418,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:23.826682+0000","last_change":"2023-09-30T09:40:22.885021+0000","last_active":"2023-09-30T09:41:23.826682+0000","last_peered":"2023-09-30T09:41:23.826682+0000","last_clean":"2023-09-30T09:38:44.556600+0000","last_became_active":"2023-09-30T09:38:46.733175+0000","last_became_peered":"2023-09-30T09:38:46.733175+0000","last_unstale":"2023-09-30T09:41:23.826682+0000","last_undegraded":"2023-09-30T09:40:22.884147+0000","last_fullsized":"2023-09-30T09:38:46.604027+0000","mapping_epoch":372,"log_start":"375'3538","ondisk_log_start":"375'3538","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3762,"log_dups_size":0,"ondisk_log_size":3762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:41.383870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266731520,"num_objects":4070,"num_object_clones":0,"num_object_copies":12210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7300,"num_write_kb":363840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2013,"num_bytes_recovered":131923968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1300},{"shards":"1,4","objects":2770}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":843,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595855+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:41:06.595855+0000","last_peered":"2023-09-30T09:41:06.595855+0000","last_clean":"2023-09-30T09:41:06.595855+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:41:06.595855+0000","last_undegraded":"2023-09-30T09:41:06.595855+0000","last_fullsized":"2023-09-30T09:41:06.595855+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":654,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596570+0000","last_change":"2023-09-30T09:40:37.590450+0000","last_active":"2023-09-30T09:41:06.596570+0000","last_peered":"2023-09-30T09:41:06.596570+0000","last_clean":"2023-09-30T09:41:06.596570+0000","last_became_active":"2023-09-30T09:40:37.589740+0000","last_became_peered":"2023-09-30T09:40:37.589740+0000","last_unstale":"2023-09-30T09:41:06.596570+0000","last_undegraded":"2023-09-30T09:41:06.596570+0000","last_fullsized":"2023-09-30T09:41:06.596570+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:24:28.314167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":176,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:11.897529+0000","last_change":"2023-09-30T09:40:36.592442+0000","last_active":"2023-09-30T09:41:11.897529+0000","last_peered":"2023-09-30T09:41:11.897529+0000","last_clean":"2023-09-30T09:41:11.897529+0000","last_became_active":"2023-09-30T09:40:36.592119+0000","last_became_peered":"2023-09-30T09:40:36.592119+0000","last_unstale":"2023-09-30T09:41:11.897529+0000","last_undegraded":"2023-09-30T09:41:11.897529+0000","last_fullsized":"2023-09-30T09:41:11.897529+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:53:12.526007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":353,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.604252+0000","last_change":"2023-09-30T09:39:58.585200+0000","last_active":"2023-09-30T09:41:06.604252+0000","last_peered":"2023-09-30T09:41:06.604252+0000","last_clean":"2023-09-30T09:41:06.604252+0000","last_became_active":"2023-09-30T09:39:58.584863+0000","last_became_peered":"2023-09-30T09:39:58.584863+0000","last_unstale":"2023-09-30T09:41:06.604252+0000","last_undegraded":"2023-09-30T09:41:06.604252+0000","last_fullsized":"2023-09-30T09:41:06.604252+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:48:09.624537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"424'6081","reported_seq":7803,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:24.399662+0000","last_change":"2023-09-30T09:40:00.506552+0000","last_active":"2023-09-30T09:41:24.399662+0000","last_peered":"2023-09-30T09:41:24.399662+0000","last_clean":"2023-09-30T09:39:46.286949+0000","last_became_active":"2023-09-30T09:40:00.498736+0000","last_became_peered":"2023-09-30T09:40:00.498736+0000","last_unstale":"2023-09-30T09:41:24.399662+0000","last_undegraded":"2023-09-30T09:39:59.664435+0000","last_fullsized":"2023-09-30T09:39:59.588306+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6081,"log_dups_size":0,"ondisk_log_size":6081,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134643712,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4636,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3692,"num_write_kb":183904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":540,"num_bytes_recovered":35389440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":2039},{"shards":"4,5","objects":15}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1050,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.601226+0000","last_change":"2023-09-30T09:38:30.748839+0000","last_active":"2023-09-30T09:41:06.601226+0000","last_peered":"2023-09-30T09:41:06.601226+0000","last_clean":"2023-09-30T09:41:06.601226+0000","last_became_active":"2023-09-30T09:38:30.748346+0000","last_became_peered":"2023-09-30T09:38:30.748346+0000","last_unstale":"2023-09-30T09:41:06.601226+0000","last_undegraded":"2023-09-30T09:41:06.601226+0000","last_fullsized":"2023-09-30T09:41:06.601226+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:46:58.191800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":747,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595691+0000","last_change":"2023-09-30T09:40:01.598664+0000","last_active":"2023-09-30T09:41:06.595691+0000","last_peered":"2023-09-30T09:41:06.595691+0000","last_clean":"2023-09-30T09:41:06.595691+0000","last_became_active":"2023-09-30T09:40:01.598306+0000","last_became_peered":"2023-09-30T09:40:01.598306+0000","last_unstale":"2023-09-30T09:41:06.595691+0000","last_undegraded":"2023-09-30T09:41:06.595691+0000","last_fullsized":"2023-09-30T09:41:06.595691+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:19:20.361905+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":88,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.601546+0000","last_change":"2023-09-30T09:39:47.869531+0000","last_active":"2023-09-30T09:41:06.601546+0000","last_peered":"2023-09-30T09:41:06.601546+0000","last_clean":"2023-09-30T09:41:06.601546+0000","last_became_active":"2023-09-30T09:39:47.869178+0000","last_became_peered":"2023-09-30T09:39:47.869178+0000","last_unstale":"2023-09-30T09:41:06.601546+0000","last_undegraded":"2023-09-30T09:41:06.601546+0000","last_fullsized":"2023-09-30T09:41:06.601546+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:33.299189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":386,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596955+0000","last_change":"2023-09-30T09:40:36.597443+0000","last_active":"2023-09-30T09:41:06.596955+0000","last_peered":"2023-09-30T09:41:06.596955+0000","last_clean":"2023-09-30T09:41:06.596955+0000","last_became_active":"2023-09-30T09:40:36.596947+0000","last_became_peered":"2023-09-30T09:40:36.596947+0000","last_unstale":"2023-09-30T09:41:06.596955+0000","last_undegraded":"2023-09-30T09:41:06.596955+0000","last_fullsized":"2023-09-30T09:41:06.596955+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:17.799153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"424'7404","reported_seq":7485,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:21.915987+0000","last_change":"2023-09-30T09:37:56.539888+0000","last_active":"2023-09-30T09:41:21.915987+0000","last_peered":"2023-09-30T09:41:21.915987+0000","last_clean":"2023-09-30T09:41:21.915987+0000","last_became_active":"2023-09-30T09:37:56.539552+0000","last_became_peered":"2023-09-30T09:37:56.539552+0000","last_unstale":"2023-09-30T09:41:21.915987+0000","last_undegraded":"2023-09-30T09:41:21.915987+0000","last_fullsized":"2023-09-30T09:41:21.915987+0000","mapping_epoch":358,"log_start":"370'3324","ondisk_log_start":"370'3324","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4080,"log_dups_size":0,"ondisk_log_size":4080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:11.316452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264896512,"num_objects":4042,"num_object_clones":0,"num_object_copies":12126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7404,"num_write_kb":366272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":641,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594584+0000","last_change":"2023-09-30T09:39:50.839273+0000","last_active":"2023-09-30T09:41:06.594584+0000","last_peered":"2023-09-30T09:41:06.594584+0000","last_clean":"2023-09-30T09:41:06.594584+0000","last_became_active":"2023-09-30T09:39:50.838892+0000","last_became_peered":"2023-09-30T09:39:50.838892+0000","last_unstale":"2023-09-30T09:41:06.594584+0000","last_undegraded":"2023-09-30T09:41:06.594584+0000","last_fullsized":"2023-09-30T09:41:06.594584+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":89449,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":833,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595672+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:41:06.595672+0000","last_peered":"2023-09-30T09:41:06.595672+0000","last_clean":"2023-09-30T09:41:06.595672+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:41:06.595672+0000","last_undegraded":"2023-09-30T09:41:06.595672+0000","last_fullsized":"2023-09-30T09:41:06.595672+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:16:49.259733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.589218+0000","last_change":"2023-09-30T09:40:36.615213+0000","last_active":"2023-09-30T09:41:06.589218+0000","last_peered":"2023-09-30T09:41:06.589218+0000","last_clean":"2023-09-30T09:41:06.589218+0000","last_became_active":"2023-09-30T09:40:36.614950+0000","last_became_peered":"2023-09-30T09:40:36.614950+0000","last_unstale":"2023-09-30T09:41:06.589218+0000","last_undegraded":"2023-09-30T09:41:06.589218+0000","last_fullsized":"2023-09-30T09:41:06.589218+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:13.113238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":111,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593811+0000","last_change":"2023-09-30T09:39:58.580826+0000","last_active":"2023-09-30T09:41:06.593811+0000","last_peered":"2023-09-30T09:41:06.593811+0000","last_clean":"2023-09-30T09:41:06.593811+0000","last_became_active":"2023-09-30T09:39:58.580493+0000","last_became_peered":"2023-09-30T09:39:58.580493+0000","last_unstale":"2023-09-30T09:41:06.593811+0000","last_undegraded":"2023-09-30T09:41:06.593811+0000","last_fullsized":"2023-09-30T09:41:06.593811+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:09:47.409633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.f","version":"424'7297","reported_seq":9153,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:21.900699+0000","last_change":"2023-09-30T09:40:36.615864+0000","last_active":"2023-09-30T09:41:21.900699+0000","last_peered":"2023-09-30T09:41:21.900699+0000","last_clean":"2023-09-30T09:38:44.567377+0000","last_became_active":"2023-09-30T09:38:46.789998+0000","last_became_peered":"2023-09-30T09:38:46.789998+0000","last_unstale":"2023-09-30T09:41:21.900699+0000","last_undegraded":"2023-09-30T09:38:46.692264+0000","last_fullsized":"2023-09-30T09:38:46.597825+0000","mapping_epoch":372,"log_start":"370'3168","ondisk_log_start":"370'3168","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4129,"log_dups_size":0,"ondisk_log_size":4129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:32:25.586705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268238848,"num_objects":4093,"num_object_clones":0,"num_object_copies":12279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7297,"num_write_kb":364480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":882,"num_bytes_recovered":57802752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":1686},{"shards":"1,2,3","objects":1584}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":699,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596793+0000","last_change":"2023-09-30T09:40:01.612760+0000","last_active":"2023-09-30T09:41:06.596793+0000","last_peered":"2023-09-30T09:41:06.596793+0000","last_clean":"2023-09-30T09:41:06.596793+0000","last_became_active":"2023-09-30T09:40:01.612373+0000","last_became_peered":"2023-09-30T09:40:01.612373+0000","last_unstale":"2023-09-30T09:41:06.596793+0000","last_undegraded":"2023-09-30T09:41:06.596793+0000","last_fullsized":"2023-09-30T09:41:06.596793+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:30:07.640092+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":922,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:06.595126+0000","last_change":"2023-09-30T09:39:59.604423+0000","last_active":"2023-09-30T09:41:06.595126+0000","last_peered":"2023-09-30T09:41:06.595126+0000","last_clean":"2023-09-30T09:39:56.898848+0000","last_became_active":"2023-09-30T09:39:59.603953+0000","last_became_peered":"2023-09-30T09:39:59.603953+0000","last_unstale":"2023-09-30T09:41:06.595126+0000","last_undegraded":"2023-09-30T09:39:59.583340+0000","last_fullsized":"2023-09-30T09:39:59.582766+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:37:14.962146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":151,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593930+0000","last_change":"2023-09-30T09:33:33.594963+0000","last_active":"2023-09-30T09:41:06.593930+0000","last_peered":"2023-09-30T09:41:06.593930+0000","last_clean":"2023-09-30T09:41:06.593930+0000","last_became_active":"2023-09-30T09:33:33.594536+0000","last_became_peered":"2023-09-30T09:33:33.594536+0000","last_unstale":"2023-09-30T09:41:06.593930+0000","last_undegraded":"2023-09-30T09:41:06.593930+0000","last_fullsized":"2023-09-30T09:41:06.593930+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:35:54.508855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":21,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.588913+0000","last_change":"2023-09-30T09:40:36.615339+0000","last_active":"2023-09-30T09:41:06.588913+0000","last_peered":"2023-09-30T09:41:06.588913+0000","last_clean":"2023-09-30T09:41:06.588913+0000","last_became_active":"2023-09-30T09:40:36.615031+0000","last_became_peered":"2023-09-30T09:40:36.615031+0000","last_unstale":"2023-09-30T09:41:06.588913+0000","last_undegraded":"2023-09-30T09:41:06.588913+0000","last_fullsized":"2023-09-30T09:41:06.588913+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:58:52.570227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"424'7680","reported_seq":14776,"reported_epoch":424,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:41:23.180246+0000","last_change":"2023-09-30T09:41:14.789418+0000","last_active":"2023-09-30T09:41:23.180246+0000","last_peered":"2023-09-30T09:41:23.180246+0000","last_clean":"2023-09-30T09:38:26.563448+0000","last_became_active":"2023-09-30T09:40:03.070221+0000","last_became_peered":"2023-09-30T09:40:03.070221+0000","last_unstale":"2023-09-30T09:41:23.180246+0000","last_undegraded":"2023-09-30T09:41:23.180246+0000","last_fullsized":"2023-09-30T09:41:23.180246+0000","mapping_epoch":398,"log_start":"369'3119","ondisk_log_start":"369'3119","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4561,"log_dups_size":0,"ondisk_log_size":4561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:14:12.185089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269746176,"num_objects":4116,"num_object_clones":0,"num_object_copies":12348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4116,"num_objects_unfound":0,"num_objects_dirty":4116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7680,"num_write_kb":377472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3492,"num_bytes_recovered":228851712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,0],"avail_no_missing":["7","0","3","5"],"object_location_counts":[{"shards":"0,5,7","objects":4116}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":916,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596048+0000","last_change":"2023-09-30T09:39:50.836634+0000","last_active":"2023-09-30T09:41:06.596048+0000","last_peered":"2023-09-30T09:41:06.596048+0000","last_clean":"2023-09-30T09:41:06.596048+0000","last_became_active":"2023-09-30T09:39:50.836332+0000","last_became_peered":"2023-09-30T09:39:50.836332+0000","last_unstale":"2023-09-30T09:41:06.596048+0000","last_undegraded":"2023-09-30T09:41:06.596048+0000","last_fullsized":"2023-09-30T09:41:06.596048+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:13:19.526894+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":463,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596788+0000","last_change":"2023-09-30T09:40:36.629866+0000","last_active":"2023-09-30T09:41:06.596788+0000","last_peered":"2023-09-30T09:41:06.596788+0000","last_clean":"2023-09-30T09:41:06.596788+0000","last_became_active":"2023-09-30T09:40:36.606980+0000","last_became_peered":"2023-09-30T09:40:36.606980+0000","last_unstale":"2023-09-30T09:41:06.596788+0000","last_undegraded":"2023-09-30T09:41:06.596788+0000","last_fullsized":"2023-09-30T09:41:06.596788+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:14:56.024827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1079,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:06.593305+0000","last_change":"2023-09-30T09:39:58.581253+0000","last_active":"2023-09-30T09:41:06.593305+0000","last_peered":"2023-09-30T09:41:06.593305+0000","last_clean":"2023-09-30T09:13:44.499731+0000","last_became_active":"2023-09-30T09:39:58.580546+0000","last_became_peered":"2023-09-30T09:39:58.580546+0000","last_unstale":"2023-09-30T09:41:06.593305+0000","last_undegraded":"2023-09-30T09:39:58.564516+0000","last_fullsized":"2023-09-30T09:39:58.563866+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:38.732897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":458,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595600+0000","last_change":"2023-09-30T09:39:58.586341+0000","last_active":"2023-09-30T09:41:06.595600+0000","last_peered":"2023-09-30T09:41:06.595600+0000","last_clean":"2023-09-30T09:41:06.595600+0000","last_became_active":"2023-09-30T09:39:58.585972+0000","last_became_peered":"2023-09-30T09:39:58.585972+0000","last_unstale":"2023-09-30T09:41:06.595600+0000","last_undegraded":"2023-09-30T09:41:06.595600+0000","last_fullsized":"2023-09-30T09:41:06.595600+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:07:15.400194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"424'7675","reported_seq":12537,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:24.207144+0000","last_change":"2023-09-30T09:40:21.847418+0000","last_active":"2023-09-30T09:41:24.207144+0000","last_peered":"2023-09-30T09:41:24.207144+0000","last_clean":"2023-09-30T09:41:24.207144+0000","last_became_active":"2023-09-30T09:40:21.816742+0000","last_became_peered":"2023-09-30T09:40:21.816742+0000","last_unstale":"2023-09-30T09:41:24.207144+0000","last_undegraded":"2023-09-30T09:41:24.207144+0000","last_fullsized":"2023-09-30T09:41:24.207144+0000","mapping_epoch":407,"log_start":"376'3957","ondisk_log_start":"376'3957","created":358,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3718,"log_dups_size":0,"ondisk_log_size":3718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:50:47.096303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267386904,"num_objects":4081,"num_object_clones":0,"num_object_copies":12243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4081,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7675,"num_write_kb":376129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2381,"num_bytes_recovered":129368064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":857,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.601936+0000","last_change":"2023-09-30T09:39:50.836446+0000","last_active":"2023-09-30T09:41:06.601936+0000","last_peered":"2023-09-30T09:41:06.601936+0000","last_clean":"2023-09-30T09:41:06.601936+0000","last_became_active":"2023-09-30T09:39:50.836087+0000","last_became_peered":"2023-09-30T09:39:50.836087+0000","last_unstale":"2023-09-30T09:41:06.601936+0000","last_undegraded":"2023-09-30T09:41:06.601936+0000","last_fullsized":"2023-09-30T09:41:06.601936+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:43:34.865123+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":753,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596921+0000","last_change":"2023-09-30T09:39:58.627875+0000","last_active":"2023-09-30T09:41:06.596921+0000","last_peered":"2023-09-30T09:41:06.596921+0000","last_clean":"2023-09-30T09:41:06.596921+0000","last_became_active":"2023-09-30T09:39:47.851060+0000","last_became_peered":"2023-09-30T09:39:47.851060+0000","last_unstale":"2023-09-30T09:41:06.596921+0000","last_undegraded":"2023-09-30T09:41:06.596921+0000","last_fullsized":"2023-09-30T09:41:06.596921+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:42:52.871435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":139,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.597236+0000","last_change":"2023-09-30T09:39:58.612882+0000","last_active":"2023-09-30T09:41:06.597236+0000","last_peered":"2023-09-30T09:41:06.597236+0000","last_clean":"2023-09-30T09:41:06.597236+0000","last_became_active":"2023-09-30T09:39:58.581004+0000","last_became_peered":"2023-09-30T09:39:58.581004+0000","last_unstale":"2023-09-30T09:41:06.597236+0000","last_undegraded":"2023-09-30T09:41:06.597236+0000","last_fullsized":"2023-09-30T09:41:06.597236+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:44.825201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":395,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593698+0000","last_change":"2023-09-30T09:40:36.607459+0000","last_active":"2023-09-30T09:41:06.593698+0000","last_peered":"2023-09-30T09:41:06.593698+0000","last_clean":"2023-09-30T09:41:06.593698+0000","last_became_active":"2023-09-30T09:40:36.607067+0000","last_became_peered":"2023-09-30T09:40:36.607067+0000","last_unstale":"2023-09-30T09:41:06.593698+0000","last_undegraded":"2023-09-30T09:41:06.593698+0000","last_fullsized":"2023-09-30T09:41:06.593698+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:52:17.679386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"424'6311","reported_seq":9896,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:24.204382+0000","last_change":"2023-09-30T09:39:53.709648+0000","last_active":"2023-09-30T09:41:24.204382+0000","last_peered":"2023-09-30T09:41:24.204382+0000","last_clean":"2023-09-30T09:41:24.204382+0000","last_became_active":"2023-09-30T09:39:15.641278+0000","last_became_peered":"2023-09-30T09:39:15.641278+0000","last_unstale":"2023-09-30T09:41:24.204382+0000","last_undegraded":"2023-09-30T09:41:24.204382+0000","last_fullsized":"2023-09-30T09:41:24.204382+0000","mapping_epoch":379,"log_start":"363'278","ondisk_log_start":"363'278","created":379,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6033,"log_dups_size":0,"ondisk_log_size":6033,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133922816,"num_objects":2044,"num_object_clones":0,"num_object_copies":6132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3836,"num_write_kb":188128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1743,"num_bytes_recovered":114196480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":780,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.595373+0000","last_change":"2023-09-30T09:40:01.608312+0000","last_active":"2023-09-30T09:41:06.595373+0000","last_peered":"2023-09-30T09:41:06.595373+0000","last_clean":"2023-09-30T09:41:06.595373+0000","last_became_active":"2023-09-30T09:40:01.607950+0000","last_became_peered":"2023-09-30T09:40:01.607950+0000","last_unstale":"2023-09-30T09:41:06.595373+0000","last_undegraded":"2023-09-30T09:41:06.595373+0000","last_fullsized":"2023-09-30T09:41:06.595373+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:48.255506+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":249,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594841+0000","last_change":"2023-09-30T09:39:58.576981+0000","last_active":"2023-09-30T09:41:06.594841+0000","last_peered":"2023-09-30T09:41:06.594841+0000","last_clean":"2023-09-30T09:41:06.594841+0000","last_became_active":"2023-09-30T09:39:58.576650+0000","last_became_peered":"2023-09-30T09:39:58.576650+0000","last_unstale":"2023-09-30T09:41:06.594841+0000","last_undegraded":"2023-09-30T09:41:06.594841+0000","last_fullsized":"2023-09-30T09:41:06.594841+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:50:03.041046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":119,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593843+0000","last_change":"2023-09-30T09:39:58.573938+0000","last_active":"2023-09-30T09:41:06.593843+0000","last_peered":"2023-09-30T09:41:06.593843+0000","last_clean":"2023-09-30T09:41:06.593843+0000","last_became_active":"2023-09-30T09:39:58.573645+0000","last_became_peered":"2023-09-30T09:39:58.573645+0000","last_unstale":"2023-09-30T09:41:06.593843+0000","last_undegraded":"2023-09-30T09:41:06.593843+0000","last_fullsized":"2023-09-30T09:41:06.593843+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:24:06.400443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"424'6196","reported_seq":8632,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:21.902917+0000","last_change":"2023-09-30T09:39:55.208626+0000","last_active":"2023-09-30T09:41:21.902917+0000","last_peered":"2023-09-30T09:41:21.902917+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:39:53.183491+0000","last_became_peered":"2023-09-30T09:39:53.183491+0000","last_unstale":"2023-09-30T09:41:21.902917+0000","last_undegraded":"2023-09-30T09:39:51.257439+0000","last_fullsized":"2023-09-30T09:39:48.816158+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6196,"log_dups_size":0,"ondisk_log_size":6196,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132415488,"num_objects":2021,"num_object_clones":0,"num_object_copies":6063,"num_objects_missing_on_primary":821,"num_objects_missing":821,"num_objects_degraded":3106,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3761,"num_write_kb":184992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":870,"num_bytes_recovered":57016320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":11},{"shards":"1,2,3,6","objects":4},{"shards":"1,2,6","objects":1185},{"shards":"1,3,6","objects":821}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":627,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596763+0000","last_change":"2023-09-30T09:40:01.613293+0000","last_active":"2023-09-30T09:41:06.596763+0000","last_peered":"2023-09-30T09:41:06.596763+0000","last_clean":"2023-09-30T09:41:06.596763+0000","last_became_active":"2023-09-30T09:40:01.612955+0000","last_became_peered":"2023-09-30T09:40:01.612955+0000","last_unstale":"2023-09-30T09:41:06.596763+0000","last_undegraded":"2023-09-30T09:41:06.596763+0000","last_fullsized":"2023-09-30T09:41:06.596763+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:24:11.658719+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":402,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.602166+0000","last_change":"2023-09-30T09:33:32.657220+0000","last_active":"2023-09-30T09:41:06.602166+0000","last_peered":"2023-09-30T09:41:06.602166+0000","last_clean":"2023-09-30T09:41:06.602166+0000","last_became_active":"2023-09-30T09:33:32.656941+0000","last_became_peered":"2023-09-30T09:33:32.656941+0000","last_unstale":"2023-09-30T09:41:06.602166+0000","last_undegraded":"2023-09-30T09:41:06.602166+0000","last_fullsized":"2023-09-30T09:41:06.602166+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:42:38.385155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":128,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594509+0000","last_change":"2023-09-30T09:38:45.595684+0000","last_active":"2023-09-30T09:41:06.594509+0000","last_peered":"2023-09-30T09:41:06.594509+0000","last_clean":"2023-09-30T09:41:06.594509+0000","last_became_active":"2023-09-30T09:38:45.595231+0000","last_became_peered":"2023-09-30T09:38:45.595231+0000","last_unstale":"2023-09-30T09:41:06.594509+0000","last_undegraded":"2023-09-30T09:41:06.594509+0000","last_fullsized":"2023-09-30T09:41:06.594509+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:43:29.805772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"424'6249","reported_seq":14655,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:23.282207+0000","last_change":"2023-09-30T09:41:06.618361+0000","last_active":"2023-09-30T09:41:23.282207+0000","last_peered":"2023-09-30T09:41:23.282207+0000","last_clean":"2023-09-30T09:41:23.282207+0000","last_became_active":"2023-09-30T09:41:06.607479+0000","last_became_peered":"2023-09-30T09:41:06.607479+0000","last_unstale":"2023-09-30T09:41:23.282207+0000","last_undegraded":"2023-09-30T09:41:23.282207+0000","last_fullsized":"2023-09-30T09:41:23.282207+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6249,"log_dups_size":0,"ondisk_log_size":6249,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138018816,"num_objects":2106,"num_object_clones":0,"num_object_copies":6318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3808,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3251,"num_bytes_recovered":194117632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":382,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.604604+0000","last_change":"2023-09-30T09:39:47.886619+0000","last_active":"2023-09-30T09:41:06.604604+0000","last_peered":"2023-09-30T09:41:06.604604+0000","last_clean":"2023-09-30T09:41:06.604604+0000","last_became_active":"2023-09-30T09:39:47.869696+0000","last_became_peered":"2023-09-30T09:39:47.869696+0000","last_unstale":"2023-09-30T09:41:06.604604+0000","last_undegraded":"2023-09-30T09:41:06.604604+0000","last_fullsized":"2023-09-30T09:41:06.604604+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:06.351925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":148,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593744+0000","last_change":"2023-09-30T09:40:36.597938+0000","last_active":"2023-09-30T09:41:06.593744+0000","last_peered":"2023-09-30T09:41:06.593744+0000","last_clean":"2023-09-30T09:41:06.593744+0000","last_became_active":"2023-09-30T09:40:36.597684+0000","last_became_peered":"2023-09-30T09:40:36.597684+0000","last_unstale":"2023-09-30T09:41:06.593744+0000","last_undegraded":"2023-09-30T09:41:06.593744+0000","last_fullsized":"2023-09-30T09:41:06.593744+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:15:35.036544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"424'6098","reported_seq":8647,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:23.593543+0000","last_change":"2023-09-30T09:40:36.658459+0000","last_active":"2023-09-30T09:41:23.593543+0000","last_peered":"2023-09-30T09:41:23.593543+0000","last_clean":"2023-09-30T09:39:46.536373+0000","last_became_active":"2023-09-30T09:40:36.630807+0000","last_became_peered":"2023-09-30T09:40:36.630807+0000","last_unstale":"2023-09-30T09:41:23.593543+0000","last_undegraded":"2023-09-30T09:40:36.594542+0000","last_fullsized":"2023-09-30T09:40:36.580057+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6098,"log_dups_size":0,"ondisk_log_size":6098,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134217728,"num_objects":2048,"num_object_clones":0,"num_object_copies":6144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3696,"num_write_kb":183808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":1726},{"shards":"4,7","objects":322}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":714,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.596345+0000","last_change":"2023-09-30T09:39:47.871391+0000","last_active":"2023-09-30T09:41:06.596345+0000","last_peered":"2023-09-30T09:41:06.596345+0000","last_clean":"2023-09-30T09:41:06.596345+0000","last_became_active":"2023-09-30T09:39:47.870992+0000","last_became_peered":"2023-09-30T09:39:47.870992+0000","last_unstale":"2023-09-30T09:41:06.596345+0000","last_undegraded":"2023-09-30T09:41:06.596345+0000","last_fullsized":"2023-09-30T09:41:06.596345+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:54.087577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:54.087577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":326,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593931+0000","last_change":"2023-09-30T09:39:47.874169+0000","last_active":"2023-09-30T09:41:06.593931+0000","last_peered":"2023-09-30T09:41:06.593931+0000","last_clean":"2023-09-30T09:41:06.593931+0000","last_became_active":"2023-09-30T09:39:47.873772+0000","last_became_peered":"2023-09-30T09:39:47.873772+0000","last_unstale":"2023-09-30T09:41:06.593931+0000","last_undegraded":"2023-09-30T09:41:06.593931+0000","last_fullsized":"2023-09-30T09:41:06.593931+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:38:30.789860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":386,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:11.897559+0000","last_change":"2023-09-30T09:40:36.607821+0000","last_active":"2023-09-30T09:41:11.897559+0000","last_peered":"2023-09-30T09:41:11.897559+0000","last_clean":"2023-09-30T09:41:11.897559+0000","last_became_active":"2023-09-30T09:40:36.607503+0000","last_became_peered":"2023-09-30T09:40:36.607503+0000","last_unstale":"2023-09-30T09:41:11.897559+0000","last_undegraded":"2023-09-30T09:41:11.897559+0000","last_fullsized":"2023-09-30T09:41:11.897559+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:37:46.306973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"424'6176","reported_seq":6260,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-30T09:41:24.200509+0000","last_change":"2023-09-30T09:39:15.638244+0000","last_active":"2023-09-30T09:41:24.200509+0000","last_peered":"2023-09-30T09:41:24.200509+0000","last_clean":"2023-09-30T09:41:24.200509+0000","last_became_active":"2023-09-30T09:39:15.634600+0000","last_became_peered":"2023-09-30T09:39:15.634600+0000","last_unstale":"2023-09-30T09:41:24.200509+0000","last_undegraded":"2023-09-30T09:41:24.200509+0000","last_fullsized":"2023-09-30T09:41:24.200509+0000","mapping_epoch":379,"log_start":"363'90","ondisk_log_start":"363'90","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6086,"log_dups_size":0,"ondisk_log_size":6086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129368064,"num_objects":1974,"num_object_clones":0,"num_object_copies":5922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3770,"num_write_kb":183808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":799,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.594022+0000","last_change":"2023-09-30T09:38:48.664091+0000","last_active":"2023-09-30T09:41:06.594022+0000","last_peered":"2023-09-30T09:41:06.594022+0000","last_clean":"2023-09-30T09:41:06.594022+0000","last_became_active":"2023-09-30T09:38:48.663723+0000","last_became_peered":"2023-09-30T09:38:48.663723+0000","last_unstale":"2023-09-30T09:41:06.594022+0000","last_undegraded":"2023-09-30T09:41:06.594022+0000","last_fullsized":"2023-09-30T09:41:06.594022+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:57.093254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:57.093254+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":253,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.604120+0000","last_change":"2023-09-30T09:39:58.634444+0000","last_active":"2023-09-30T09:41:06.604120+0000","last_peered":"2023-09-30T09:41:06.604120+0000","last_clean":"2023-09-30T09:41:06.604120+0000","last_became_active":"2023-09-30T09:39:58.616654+0000","last_became_peered":"2023-09-30T09:39:58.616654+0000","last_unstale":"2023-09-30T09:41:06.604120+0000","last_undegraded":"2023-09-30T09:41:06.604120+0000","last_fullsized":"2023-09-30T09:41:06.604120+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:24:26.437811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":419,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-30T09:41:06.593400+0000","last_change":"2023-09-30T09:39:47.867540+0000","last_active":"2023-09-30T09:41:06.593400+0000","last_peered":"2023-09-30T09:41:06.593400+0000","last_clean":"2023-09-30T09:41:06.593400+0000","last_became_active":"2023-09-30T09:39:47.867178+0000","last_became_peered":"2023-09-30T09:39:47.867178+0000","last_unstale":"2023-09-30T09:41:06.593400+0000","last_undegraded":"2023-09-30T09:41:06.593400+0000","last_fullsized":"2023-09-30T09:41:06.593400+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:51.232357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":16602,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":4271505432,"num_objects":65179,"num_object_clones":0,"num_object_copies":195537,"num_objects_missing_on_primary":1873,"num_objects_missing":1873,"num_objects_degraded":22432,"num_objects_misplaced":8704,"num_objects_unfound":0,"num_objects_dirty":65179,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":119713,"num_write_kb":5916481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33059,"num_bytes_recovered":2055143424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14689316864,"data_stored":14689304648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146597,"ondisk_log_size":146597,"up":78,"acting":68,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":370971,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1065358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":31,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":217,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234356,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1735812,"kb_used_data":1296124,"kb_used_omap":0,"kb_used_meta":439680,"kb_avail":92009340,"statfs":{"total":95995035648,"available":94217564160,"internally_reserved":0,"allocated":1327230976,"data_stored":1325770299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":450232320},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7799999999999998,"5min":5.9119999999999999,"15min":3.1339999999999999},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.224},"max":{"1min":33.237000000000002,"5min":215.416,"15min":215.416},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.8969999999999998,"5min":5.9089999999999998,"15min":2.9740000000000002},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.23899999999999999},"max":{"1min":33.207000000000001,"5min":215.041,"15min":215.041},"last":0.30199999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":2.2570000000000001,"15min":2.7829999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.2879999999999998,"5min":89.308000000000007,"15min":89.308000000000007},"last":2.593},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.4350000000000001,"15min":3.2519999999999998},"min":{"1min":0.35099999999999998,"5min":0.307,"15min":0.307},"max":{"1min":3.5139999999999998,"5min":89.325999999999993,"15min":89.325999999999993},"last":2.6099999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":4.3890000000000002,"15min":2.7639999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.24299999999999999},"max":{"1min":9.2560000000000002,"5min":98.698999999999998,"15min":98.698999999999998},"last":2.0110000000000001},{"interface":"front","average":{"1min":1.6859999999999999,"5min":4.2530000000000001,"15min":2.71},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":10.41,"5min":99.409999999999997,"15min":99.409999999999997},"last":2.7229999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":9.0980000000000008,"15min":7.8339999999999996},"min":{"1min":0.34499999999999997,"5min":0.34399999999999997,"15min":0.28899999999999998},"max":{"1min":6.8289999999999997,"5min":277.57799999999997,"15min":277.57799999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":9.2609999999999992,"15min":7.2789999999999999},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.6840000000000002,"5min":278.69299999999998,"15min":278.69299999999998},"last":1.8999999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":10.101000000000001,"5min":2.778,"15min":1.663},"min":{"1min":0.308,"5min":0.23799999999999999,"15min":0.23000000000000001},"max":{"1min":175.72,"5min":175.72,"15min":175.72},"last":0.31900000000000001},{"interface":"front","average":{"1min":10.561,"5min":2.6360000000000001,"15min":1.5409999999999999},"min":{"1min":0.32900000000000001,"5min":0.252,"15min":0.251},"max":{"1min":179.048,"5min":179.048,"15min":179.048},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":3.0640000000000001,"15min":1.8620000000000001},"min":{"1min":0.315,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.1799999999999997,"5min":188.41,"15min":188.41},"last":0.504},{"interface":"front","average":{"1min":1.2390000000000001,"5min":3.0379999999999998,"15min":1.875},"min":{"1min":0.36399999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5609999999999999,"5min":188.43000000000001,"15min":188.43000000000001},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":8.0410000000000004,"5min":4.4509999999999996,"15min":2.371},"min":{"1min":0.42999999999999999,"5min":0.33200000000000002,"15min":0.32400000000000001},"max":{"1min":129.14500000000001,"5min":206.244,"15min":206.244},"last":0.48399999999999999},{"interface":"front","average":{"1min":7.8600000000000003,"5min":4.3280000000000003,"15min":2.3010000000000002},"min":{"1min":0.44400000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":128.494,"5min":202.78800000000001,"15min":202.78800000000001},"last":0.53000000000000003}]}]},{"osd":6,"up_from":0,"seq":197568495994,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":301,"seq":1292785156191,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2289192,"kb_used_data":1860960,"kb_used_omap":9,"kb_used_meta":428214,"kb_avail":91455960,"statfs":{"total":95995035648,"available":93650903040,"internally_reserved":0,"allocated":1905623040,"data_stored":1903213941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":438491154},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":4.5650000000000004,"15min":3.254},"min":{"1min":0.23499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.4380000000000002,"5min":138.43799999999999,"15min":138.43799999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":3.9460000000000002,"15min":3.3999999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25},"max":{"1min":2.9209999999999998,"5min":138.482,"15min":138.482},"last":0.34699999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":2.149,"15min":2.552},"min":{"1min":0.27000000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.1440000000000001,"5min":13.656000000000001,"15min":16.437000000000001},"last":1.6659999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":2.6299999999999999,"15min":2.7250000000000001},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.0459999999999998,"5min":59.078000000000003,"15min":59.078000000000003},"last":0.75800000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":5.343,"15min":7.7460000000000004},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.23899999999999999},"max":{"1min":3.5230000000000001,"5min":191.07300000000001,"15min":191.07300000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.621,"5min":5.1399999999999997,"15min":8.0370000000000008},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.339,"5min":191.11500000000001,"15min":191.11500000000001},"last":1.5389999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":3.444,"15min":2.4849999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":2.4009999999999998,"5min":142.77099999999999,"15min":142.77099999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.66000000000000003,"5min":4.2919999999999998,"15min":2.3519999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.637,"5min":144.898,"15min":144.898},"last":0.93300000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":2.1720000000000002,"15min":2.1699999999999999},"min":{"1min":0.41999999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.3580000000000001,"5min":24.25,"15min":24.25},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.755,"5min":2.7930000000000001,"15min":2.0390000000000001},"min":{"1min":0.374,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.669,"5min":58.999000000000002,"15min":58.999000000000002},"last":0.89600000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":4.0179999999999998,"15min":11.82},"min":{"1min":0.45100000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":2.4980000000000002,"5min":96.719999999999999,"15min":209.221},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.79300000000000004,"5min":4.0430000000000001,"15min":11.724},"min":{"1min":0.47599999999999998,"5min":0.40000000000000002,"15min":0.377},"max":{"1min":2.4710000000000001,"5min":96.356999999999999,"15min":208.43600000000001},"last":0.81100000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":2.2959999999999998,"15min":8.3309999999999995},"min":{"1min":0.48499999999999999,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":2.3780000000000001,"5min":15.587,"15min":165.59399999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.73799999999999999,"5min":2.3100000000000001,"15min":8.4719999999999995},"min":{"1min":0.46200000000000002,"5min":0.441,"15min":0.441},"max":{"1min":2.4079999999999999,"5min":15.226000000000001,"15min":165.833},"last":0.79100000000000004}]}]},{"osd":0,"up_from":94,"seq":403726926090,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2651392,"kb_used_data":2135800,"kb_used_omap":5,"kb_used_meta":515578,"kb_avail":91093760,"statfs":{"total":95995035648,"available":93280010240,"internally_reserved":0,"allocated":2187059200,"data_stored":2184455657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":527952125},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:40:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.798,"15min":2.54},"min":{"1min":0.191,"5min":0.191,"15min":0.17799999999999999},"max":{"1min":6.5579999999999998,"5min":15.401,"15min":17.966000000000001},"last":2.1720000000000002},{"interface":"front","average":{"1min":1.3460000000000001,"5min":2.5739999999999998,"15min":6.7910000000000004},"min":{"1min":0.23699999999999999,"5min":0.214,"15min":0.214},"max":{"1min":5.8769999999999998,"5min":25.869,"15min":138.92400000000001},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6459999999999999,"5min":2.6259999999999999,"15min":2.117},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":45.918999999999997,"5min":45.918999999999997,"15min":55.823999999999998},"last":0.41099999999999998},{"interface":"front","average":{"1min":3.8319999999999999,"5min":2.6699999999999999,"15min":2.3069999999999999},"min":{"1min":0.32100000000000001,"5min":0.26200000000000001,"15min":0.245},"max":{"1min":45.886000000000003,"5min":45.886000000000003,"15min":55.790999999999997},"last":0.47399999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:41:02 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.2160000000000002,"15min":4.3929999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.221},"max":{"1min":3.6339999999999999,"5min":16.271000000000001,"15min":218.06999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.256,"5min":1.877,"15min":3.5600000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.245},"max":{"1min":5.5259999999999998,"5min":14.987,"15min":218.63200000000001},"last":1.361}]},{"osd":4,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":1.9119999999999999,"15min":1.901},"min":{"1min":0.36099999999999999,"5min":0.35299999999999998,"15min":0.29099999999999998},"max":{"1min":7.3150000000000004,"5min":10.614000000000001,"15min":121.486},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.927,"15min":2.7829999999999999},"min":{"1min":0.35299999999999998,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":5.8979999999999997,"5min":17.044,"15min":136.262},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":1.97,"15min":3.1469999999999998},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":2.2320000000000002,"5min":16.995999999999999,"15min":136.34},"last":0.497},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.5780000000000001,"15min":2.3090000000000002},"min":{"1min":0.44400000000000001,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":5.3949999999999996,"5min":11.991,"15min":135.381},"last":1.28}]},{"osd":6,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.669,"15min":3.3599999999999999},"min":{"1min":0.34899999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":6.8579999999999997,"5min":12.021000000000001,"15min":214.785},"last":1.399},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.1840000000000002,"15min":3.4780000000000002},"min":{"1min":0.41999999999999998,"5min":0.40300000000000002,"15min":0.35399999999999998},"max":{"1min":7.3639999999999999,"5min":10.695,"15min":214.93899999999999},"last":0.67400000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.744,"15min":3.3079999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.373},"max":{"1min":5.492,"5min":12.048,"15min":150.96899999999999},"last":1.323},{"interface":"front","average":{"1min":1.444,"5min":2.1589999999999998,"15min":3.2919999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.1699999999999999,"5min":12.153,"15min":150.328},"last":0.69799999999999995}]}]},{"osd":2,"up_from":104,"seq":446676599031,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2953092,"kb_used_data":2398716,"kb_used_omap":5,"kb_used_meta":554362,"kb_avail":90792060,"statfs":{"total":95995035648,"available":92971069440,"internally_reserved":0,"allocated":2456285184,"data_stored":2453911451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":567667341},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.6839999999999999,"15min":2.1840000000000002},"min":{"1min":0.218,"5min":0.20499999999999999,"15min":0.187},"max":{"1min":3.1179999999999999,"5min":15.063000000000001,"15min":77.379000000000005},"last":7.7389999999999999},{"interface":"front","average":{"1min":1.234,"5min":1.724,"15min":2.1019999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.19700000000000001},"max":{"1min":7.0880000000000001,"5min":15.548999999999999,"15min":77.066999999999993},"last":2.8279999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":3.2810000000000001,"15min":3.5350000000000001},"min":{"1min":0.24199999999999999,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":2.1320000000000001,"5min":64.412999999999997,"15min":64.412999999999997},"last":1.3500000000000001},{"interface":"front","average":{"1min":1.028,"5min":3.036,"15min":3.1000000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":4.8099999999999996,"5min":63.247999999999998,"15min":63.247999999999998},"last":1.169}]},{"osd":3,"last update":"Sat Sep 30 09:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":4.7489999999999997,"15min":3.738},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28000000000000003},"max":{"1min":4.7619999999999996,"5min":134.078,"15min":134.078},"last":1.504},{"interface":"front","average":{"1min":1.0820000000000001,"5min":4.8579999999999997,"15min":3.8029999999999999},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.22500000000000001},"max":{"1min":4.7850000000000001,"5min":135.73400000000001,"15min":135.73400000000001},"last":7.7089999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":2.0390000000000001,"15min":1.659},"min":{"1min":0.308,"5min":0.308,"15min":0.28999999999999998},"max":{"1min":2.4780000000000002,"5min":83.012,"15min":83.012},"last":1.087},{"interface":"front","average":{"1min":0.83999999999999997,"5min":2.1499999999999999,"15min":1.841},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.7200000000000002,"5min":81.335999999999999,"15min":81.335999999999999},"last":7.5899999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.885,"15min":2.298},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.9780000000000002,"5min":10.41,"15min":155.477},"last":1.4490000000000001},{"interface":"front","average":{"1min":1.018,"5min":1.5029999999999999,"15min":2.032},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.448,"5min":8.0839999999999996,"15min":155.46000000000001},"last":1.1379999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.8700000000000001,"15min":3.4009999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":5.891,"5min":34.414999999999999,"15min":154.417},"last":7.6459999999999999},{"interface":"front","average":{"1min":1.149,"5min":2.1699999999999999,"15min":3.6179999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.895,"5min":34.514000000000003,"15min":154.04400000000001},"last":1.4199999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.6000000000000001,"15min":3.1459999999999999},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.7709999999999999,"5min":13.304,"15min":213.404},"last":7.6779999999999999},{"interface":"front","average":{"1min":1.681,"5min":2.0819999999999999,"15min":3.415},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40000000000000002},"max":{"1min":11.074999999999999,"5min":11.074999999999999,"15min":213.27199999999999},"last":1.3919999999999999}]}]},{"osd":3,"up_from":0,"seq":468151435504,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019267,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2128440,"kb_used_data":1748784,"kb_used_omap":0,"kb_used_meta":379648,"kb_avail":91616712,"statfs":{"total":95995035648,"available":93815513088,"internally_reserved":0,"allocated":1790754816,"data_stored":1789305883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":388759552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":3.7949999999999999,"15min":3.3660000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.5529999999999999,"5min":105.179,"15min":119.986},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":3.8719999999999999,"15min":3.4140000000000001},"min":{"1min":0.27600000000000002,"5min":0.252,"15min":0.25},"max":{"1min":2.79,"5min":104.70999999999999,"15min":119.863},"last":0.46700000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":2.532,"15min":6.7729999999999997},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.25700000000000001},"max":{"1min":3.4089999999999998,"5min":84.105999999999995,"15min":160.69999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.5499999999999998,"15min":6.7229999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":5.173,"5min":83.941000000000003,"15min":160.50700000000001},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:40:40 2023","interfaces":[{"interface":"back","average":{"1min":4.7670000000000003,"5min":3.621,"15min":3.637},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":52.414999999999999,"5min":93.161000000000001,"15min":140.601},"last":0.53600000000000003},{"interface":"front","average":{"1min":4.1200000000000001,"5min":3.6480000000000001,"15min":3.6110000000000002},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.309},"max":{"1min":52.427,"5min":93.114000000000004,"15min":140.54900000000001},"last":0.52700000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":2.036,"15min":3.5649999999999999},"min":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.26400000000000001},"max":{"1min":2.544,"5min":28.527000000000001,"15min":281.42200000000003},"last":0.629},{"interface":"front","average":{"1min":0.61799999999999999,"5min":2.069,"15min":3.5430000000000001},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.312},"max":{"1min":1.4239999999999999,"5min":28.486000000000001,"15min":281.44999999999999},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":3.1099999999999999,"15min":2.2879999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":2.5219999999999998,"5min":172.97800000000001,"15min":172.97800000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.79700000000000004,"5min":3.2509999999999999,"15min":2.387},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26200000000000001},"max":{"1min":7.734,"5min":172.92400000000001,"15min":172.92400000000001},"last":0.503}]},{"osd":6,"last update":"Sat Sep 30 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":2.8900000000000001,"15min":2.2989999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":0.66300000000000003,"5min":182.10300000000001,"15min":202.72499999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":2.8479999999999999,"15min":2.3290000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.249},"max":{"1min":2.6070000000000002,"5min":181.76300000000001,"15min":202.42400000000001},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:40:40 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":2.6499999999999999,"15min":3.3820000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.30599999999999999},"max":{"1min":8.1850000000000005,"5min":117.154,"15min":230.90100000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.3899999999999999,"5min":2.5830000000000002,"15min":3.181},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.1630000000000003,"5min":117.508,"15min":229.97},"last":0.39500000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757631,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2744560,"kb_used_data":2353000,"kb_used_omap":0,"kb_used_meta":391552,"kb_avail":91000592,"statfs":{"total":95995035648,"available":93184606208,"internally_reserved":0,"allocated":2409472000,"data_stored":2408027185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":400949248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":4.4400000000000004,"15min":3.4860000000000002},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.22},"max":{"1min":3.3679999999999999,"5min":98.548000000000002,"15min":148.79900000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":4.4980000000000002,"15min":3.786},"min":{"1min":0.26500000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.6830000000000001,"5min":98.388000000000005,"15min":148.779},"last":0.318}]},{"osd":1,"last update":"Sat Sep 30 09:40:46 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.786,"15min":2.0539999999999998},"min":{"1min":0.27500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":1.9399999999999999,"5min":13.363,"15min":60.799999999999997},"last":1.036},{"interface":"front","average":{"1min":0.47099999999999997,"5min":2.1120000000000001,"15min":2.9609999999999999},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.29,"5min":16.757000000000001,"15min":63.798999999999999},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:40:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":5.2939999999999996,"15min":4.0110000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":54.572000000000003,"5min":227.18000000000001,"15min":227.18000000000001},"last":0.497},{"interface":"front","average":{"1min":4.1790000000000003,"5min":5.46,"15min":4.4509999999999996},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":54.514000000000003,"5min":227.46600000000001,"15min":227.46600000000001},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":4.5220000000000002,"15min":4.8860000000000001},"min":{"1min":0.41499999999999998,"5min":0.30499999999999999,"15min":0.29699999999999999},"max":{"1min":2.5150000000000001,"5min":154.56700000000001,"15min":226.34700000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":4.0209999999999999,"15min":4.6180000000000003},"min":{"1min":0.40600000000000003,"5min":0.33600000000000002,"15min":0.313},"max":{"1min":1.534,"5min":154.58000000000001,"15min":226.36000000000001},"last":0.77400000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.79600000000000004,"15min":0.93600000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23699999999999999},"max":{"1min":8.4220000000000006,"5min":12.634,"15min":12.634},"last":0.374},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.94199999999999995,"15min":1.0249999999999999},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.252},"max":{"1min":1.8779999999999999,"5min":7.5410000000000004,"15min":9.9640000000000004},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.93999999999999995,"15min":1.855},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25900000000000001},"max":{"1min":1.3600000000000001,"5min":7.4740000000000002,"15min":155.602},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.262,"15min":2.0249999999999999},"min":{"1min":0.39500000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.956,"5min":10.880000000000001,"15min":154.00800000000001},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":1.544,"15min":1.901},"min":{"1min":0.38600000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.9950000000000001,"5min":13.617000000000001,"15min":133.40700000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.86099999999999999,"15min":1.607},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.307},"max":{"1min":1.0409999999999999,"5min":5.7880000000000003,"15min":132.797},"last":0.443}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2170093568,"data_stored":2170093568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1884422144,"data_stored":1884422144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2443644928,"data_stored":2443640856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1101791232,"data_stored":1101791232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1787035648,"data_stored":1787035648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2396131328,"data_stored":2396127256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1592590336,"data_stored":1592590336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1313607680,"data_stored":1313603608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:41:28.693 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:41:28.693 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 61 2023-09-30T09:41:28.719 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:28.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:28.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:28.826 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:28.860 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:28.913 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:28.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:29.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:29.049 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:29.104 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:29.109 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:29 smithi167 ceph-mon[192880]: from='client.55864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:41:29.109 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:29 smithi167 ceph-mon[192880]: pgmap v1099: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 290 op/s; 22280/194517 objects degraded (11.454%); 8195/194517 objects misplaced (4.213%); 19 MiB/s, 308 objects/s recovering 2023-09-30T09:41:29.109 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:29 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22280/194517 objects degraded (11.454%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:29.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:29.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:29 smithi019 ceph-mon[216815]: from='client.55864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:41:29.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:29 smithi019 ceph-mon[216815]: pgmap v1099: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 290 op/s; 22280/194517 objects degraded (11.454%); 8195/194517 objects misplaced (4.213%); 19 MiB/s, 308 objects/s recovering 2023-09-30T09:41:29.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:29 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22280/194517 objects degraded (11.454%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:29.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:29 smithi019 ceph-mon[224043]: from='client.55864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:41:29.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:29 smithi019 ceph-mon[224043]: pgmap v1099: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 290 op/s; 22280/194517 objects degraded (11.454%); 8195/194517 objects misplaced (4.213%); 19 MiB/s, 308 objects/s recovering 2023-09-30T09:41:29.154 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:29 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22280/194517 objects degraded (11.454%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:29.211 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:29.218 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:29.239 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:29.290 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:29.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:29.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:29.395 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:29.422 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:29.475 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:29.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:29.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:29.581 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:29.650 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:29.743 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:29.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:29.850 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:29.909 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:29.996 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:30.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:30.082 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:41:30.103 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:30.131 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:30.175 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:30.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2535434375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:30.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2535434375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:30.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:30.279 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:30.306 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:30.360 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:30.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2535434375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:30.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:30.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:30.466 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:30.498 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:30.555 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:30.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:30.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:30.663 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:30.717 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:31.077 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[216815]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 354 op/s; 22070/190743 objects degraded (11.571%); 7154/190743 objects misplaced (3.751%); 20 MiB/s, 325 objects/s recovering 2023-09-30T09:41:31.292 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:31.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:31.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[224043]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 354 op/s; 22070/190743 objects degraded (11.571%); 7154/190743 objects misplaced (3.751%); 20 MiB/s, 325 objects/s recovering 2023-09-30T09:41:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:41:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2535434375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[224043]: osdmap e425: 8 total, 8 up, 6 in 2023-09-30T09:41:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[224043]: failed to encode map e425 with expected crc 2023-09-30T09:41:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[224043]: failed to encode map e425 with expected crc 2023-09-30T09:41:31.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:41:31.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2535434375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:31.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[216815]: osdmap e425: 8 total, 8 up, 6 in 2023-09-30T09:41:31.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[216815]: failed to encode map e425 with expected crc 2023-09-30T09:41:31.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:31 smithi019 ceph-mon[216815]: failed to encode map e425 with expected crc 2023-09-30T09:41:31.399 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:31.458 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:31 smithi167 ceph-mon[192880]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 354 op/s; 22070/190743 objects degraded (11.571%); 7154/190743 objects misplaced (3.751%); 20 MiB/s, 325 objects/s recovering 2023-09-30T09:41:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:31 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:41:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:31 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2535434375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:31 smithi167 ceph-mon[192880]: osdmap e425: 8 total, 8 up, 6 in 2023-09-30T09:41:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:31 smithi167 ceph-mon[192880]: failed to encode map e425 with expected crc 2023-09-30T09:41:31.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:31 smithi167 ceph-mon[192880]: failed to encode map e425 with expected crc 2023-09-30T09:41:31.703 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:31.777 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:31.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:31.887 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:31.944 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:32.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:32 smithi019 ceph-mon[216815]: failed to encode map e425 with expected crc 2023-09-30T09:41:32.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:32 smithi019 ceph-mon[224043]: failed to encode map e425 with expected crc 2023-09-30T09:41:32.519 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:32.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:32 smithi167 ceph-mon[192880]: failed to encode map e425 with expected crc 2023-09-30T09:41:32.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:32.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:32.624 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:32.694 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:32.902 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:32.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:32.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:33.008 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:33.039 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:33.098 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:33.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:33.205 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:33.284 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:33 smithi019 ceph-mon[216815]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 22070/190605 objects degraded (11.579%); 6614/190605 objects misplaced (3.470%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:41:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:33 smithi019 ceph-mon[224043]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 22070/190605 objects degraded (11.579%); 6614/190605 objects misplaced (3.470%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:41:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:33.448 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:33.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:33.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:33.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:33 smithi167 ceph-mon[192880]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 22070/190605 objects degraded (11.579%); 6614/190605 objects misplaced (3.470%); 15 MiB/s, 234 objects/s recovering 2023-09-30T09:41:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:33.553 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:33.615 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:33.671 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:41:33.690 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:33.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:33.759 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:41:33.790 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:41:33.795 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:33.830 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:41:33.849 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:33.860 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:41:33.895 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:41:33.920 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:33.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:33.927 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:41:33.955 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:41:33.980 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:41:33.986 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:41:33.986 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:41:33.986 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:41:33.986 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:41:33.986 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:41:33.987 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:41:33.987 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:41:33.987 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:41:34.014 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:41:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:41:33] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-09-30T09:41:34.026 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:34.055 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:34.105 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:34.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:34.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:34.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:34.262 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:34 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22070/190605 objects degraded (11.579%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1070800859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:34.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:34 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22070/190605 objects degraded (11.579%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:34.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1070800859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:34.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:34 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22070/190605 objects degraded (11.579%), 18 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1070800859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:35 smithi167 ceph-mon[192880]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 21899/188259 objects degraded (11.632%); 6503/188259 objects misplaced (3.454%); 15 MiB/s, 233 objects/s recovering 2023-09-30T09:41:35.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:35 smithi019 ceph-mon[224043]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 21899/188259 objects degraded (11.632%); 6503/188259 objects misplaced (3.454%); 15 MiB/s, 233 objects/s recovering 2023-09-30T09:41:35.524 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:35 smithi019 ceph-mon[216815]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 21899/188259 objects degraded (11.632%); 6503/188259 objects misplaced (3.454%); 15 MiB/s, 233 objects/s recovering 2023-09-30T09:41:35.878 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:35.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:35.942 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:41:35.942 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:41:35.983 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:41:35.987 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:36.047 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:36.116 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:41:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:36.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:41:36.222 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:36.277 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:37.110 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:37.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:37.215 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:37.265 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:37.350 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:41:37.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:37 smithi019 ceph-mon[216815]: pgmap v1104: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 21693/185931 objects degraded (11.667%); 5325/185931 objects misplaced (2.864%); 16 MiB/s, 248 objects/s recovering 2023-09-30T09:41:37.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:37 smithi019 ceph-mon[224043]: pgmap v1104: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 21693/185931 objects degraded (11.667%); 5325/185931 objects misplaced (2.864%); 16 MiB/s, 248 objects/s recovering 2023-09-30T09:41:37.508 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:37.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:37.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:37.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:37 smithi167 ceph-mon[192880]: pgmap v1104: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 21693/185931 objects degraded (11.667%); 5325/185931 objects misplaced (2.864%); 16 MiB/s, 248 objects/s recovering 2023-09-30T09:41:37.596 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:41:37.612 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:37.747 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:37.934 INFO:teuthology.orchestra.run.smithi019.stderr:Error EINVAL: specified pg_num 61 < pg_num_max 32 2023-09-30T09:41:38.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4037039490' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-30T09:41:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:38 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-30T09:41:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4037039490' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-30T09:41:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:38 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-30T09:41:38.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4037039490' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-30T09:41:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:38 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-30T09:41:38.687 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:38.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:38.791 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:38.818 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:38.880 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:38.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:38.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:38.985 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:39.017 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:39.306 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:39.312 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:39 smithi019 ceph-mon[216815]: pgmap v1105: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 326 op/s; 21475/184710 objects degraded (11.626%); 4886/184710 objects misplaced (2.645%); 15 MiB/s, 270 objects/s recovering 2023-09-30T09:41:39.312 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:39 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 21475/184710 objects degraded (11.626%), 18 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:39.312 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:39 smithi019 ceph-mon[224043]: pgmap v1105: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 326 op/s; 21475/184710 objects degraded (11.626%); 4886/184710 objects misplaced (2.645%); 15 MiB/s, 270 objects/s recovering 2023-09-30T09:41:39.312 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:39 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 21475/184710 objects degraded (11.626%), 18 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:39.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:39.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:39.335 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:41:39.414 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:39.421 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:39.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:39 smithi167 ceph-mon[192880]: pgmap v1105: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 326 op/s; 21475/184710 objects degraded (11.626%); 4886/184710 objects misplaced (2.645%); 15 MiB/s, 270 objects/s recovering 2023-09-30T09:41:39.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:39 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 21475/184710 objects degraded (11.626%), 18 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:39.444 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:39.461 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:39.498 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:39.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:39.603 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:39.653 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:39.983 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:39.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:39.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:40.087 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:40.121 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:40.462 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:40.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:40.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:40.566 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:40.596 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:40.702 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:41:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[224043]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[224043]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[224043]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[224043]: osdmap e426: 8 total, 8 up, 6 in 2023-09-30T09:41:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[224043]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[224043]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[224043]: pgmap v1107: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 16913/180858 objects degraded (9.352%); 7449/180858 objects misplaced (4.119%); 16 MiB/s, 323 objects/s recovering 2023-09-30T09:41:40.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1963340086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:41:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[216815]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[216815]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[216815]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[216815]: osdmap e426: 8 total, 8 up, 6 in 2023-09-30T09:41:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[216815]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[216815]: failed to encode map e426 with expected crc 2023-09-30T09:41:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[216815]: pgmap v1107: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 16913/180858 objects degraded (9.352%); 7449/180858 objects misplaced (4.119%); 16 MiB/s, 323 objects/s recovering 2023-09-30T09:41:40.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1963340086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:41:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:40 smithi167 ceph-mon[192880]: failed to encode map e426 with expected crc 2023-09-30T09:41:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:40 smithi167 ceph-mon[192880]: failed to encode map e426 with expected crc 2023-09-30T09:41:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:40 smithi167 ceph-mon[192880]: failed to encode map e426 with expected crc 2023-09-30T09:41:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:40 smithi167 ceph-mon[192880]: osdmap e426: 8 total, 8 up, 6 in 2023-09-30T09:41:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:40 smithi167 ceph-mon[192880]: failed to encode map e426 with expected crc 2023-09-30T09:41:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:40 smithi167 ceph-mon[192880]: failed to encode map e426 with expected crc 2023-09-30T09:41:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:40 smithi167 ceph-mon[192880]: pgmap v1107: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 16913/180858 objects degraded (9.352%); 7449/180858 objects misplaced (4.119%); 16 MiB/s, 323 objects/s recovering 2023-09-30T09:41:41.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1963340086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:41:41.224 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:41.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:41.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:41.329 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:41.804 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:41.852 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:41.856 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:41 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:41.856 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1963340086' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:41:41.856 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:41 smithi167 ceph-mon[192880]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.856 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:41 smithi167 ceph-mon[192880]: osdmap e427: 8 total, 8 up, 6 in 2023-09-30T09:41:41.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:41 smithi167 ceph-mon[192880]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:41 smithi167 ceph-mon[192880]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.857 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:41 smithi167 ceph-mon[192880]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:41.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:41.958 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:41.985 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:41.985 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1963340086' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:41:41.985 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[224043]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.985 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[224043]: osdmap e427: 8 total, 8 up, 6 in 2023-09-30T09:41:41.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[224043]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[224043]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[224043]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.986 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:41.986 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1963340086' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:41:41.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[216815]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[216815]: osdmap e427: 8 total, 8 up, 6 in 2023-09-30T09:41:41.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[216815]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[216815]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:41 smithi019 ceph-mon[216815]: failed to encode map e427 with expected crc 2023-09-30T09:41:41.997 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:42.704 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:42.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:42.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:42.808 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:42.872 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:42 smithi019 ceph-mon[216815]: pgmap v1109: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 308 op/s; 16913/180858 objects degraded (9.352%); 7449/180858 objects misplaced (4.119%); 15 MiB/s, 308 objects/s recovering 2023-09-30T09:41:42.872 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:42 smithi019 ceph-mon[224043]: pgmap v1109: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 308 op/s; 16913/180858 objects degraded (9.352%); 7449/180858 objects misplaced (4.119%); 15 MiB/s, 308 objects/s recovering 2023-09-30T09:41:42.881 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:43.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:42 smithi167 ceph-mon[192880]: pgmap v1109: 129 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 308 op/s; 16913/180858 objects degraded (9.352%); 7449/180858 objects misplaced (4.119%); 15 MiB/s, 308 objects/s recovering 2023-09-30T09:41:43.103 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:43.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:43.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:43.212 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:43.226 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:41:43.226 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:41:43.270 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:43.835 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:43.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:43.939 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:43.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 16913/180858 objects degraded (9.352%), 17 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:43.970 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:41:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:41:43] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-09-30T09:41:43.970 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 16913/180858 objects degraded (9.352%), 17 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:43.973 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:44.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 16913/180858 objects degraded (9.352%), 17 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:44.083 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:41:44.083 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:41:44.083 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:41:44.087 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:41:44.114 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:41:44.146 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:41:44.184 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:41:44.207 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:41:44.209 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:41:44.209 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:41:44.209 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:41:44.210 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:41:44.210 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:41:44.210 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:41:44.210 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:41:44.210 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:41:44.212 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:44.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:44.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:44.314 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:41:44.316 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:44.338 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0] out_osds: [3, 6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:41:44.339 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:41:44.339 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-30T09:41:44.339 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-30T09:41:44.339 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.5 2023-09-30T09:41:44.346 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:44.409 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:44.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:44.518 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:44.545 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:44.595 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:44.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:44.619 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:41:44 smithi167 systemd[1]: Stopping Ceph osd.5 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:41:44.671 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:41:44.700 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:44.746 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:44.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:44 smithi019 ceph-mon[216815]: pgmap v1110: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 314 op/s; 16234/178383 objects degraded (9.101%); 7361/178383 objects misplaced (4.127%); 10 MiB/s, 295 objects/s recovering 2023-09-30T09:41:44.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3916762074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:44.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:44 smithi019 ceph-mon[224043]: pgmap v1110: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 314 op/s; 16234/178383 objects degraded (9.101%); 7361/178383 objects misplaced (4.127%); 10 MiB/s, 295 objects/s recovering 2023-09-30T09:41:44.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3916762074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:45.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:44 smithi167 ceph-mon[192880]: pgmap v1110: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 314 op/s; 16234/178383 objects degraded (9.101%); 7361/178383 objects misplaced (4.127%); 10 MiB/s, 295 objects/s recovering 2023-09-30T09:41:45.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3916762074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:45.064 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:45.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:45.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:45.176 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:45.187 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:45.205 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:45.217 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:45.258 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:45.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:45.363 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:45.390 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:41:45 smithi167 conmon[121532]: 2023-09-30T09:41:45.034+0000 7ff9aad0c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:41:45.390 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:41:45 smithi167 conmon[121532]: 2023-09-30T09:41:45.034+0000 7ff9aad0c700 -1 osd.5 427 *** Got signal Terminated *** 2023-09-30T09:41:45.390 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:41:45 smithi167 conmon[121532]: 2023-09-30T09:41:45.034+0000 7ff9aad0c700 -1 osd.5 427 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:41:45.392 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:45.439 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:45.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:45.543 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:45.591 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:45 smithi167 ceph-mon[192880]: osd.5 marked itself down and dead 2023-09-30T09:41:45.779 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:45 smithi019 ceph-mon[224043]: osd.5 marked itself down and dead 2023-09-30T09:41:45.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:45 smithi019 ceph-mon[216815]: osd.5 marked itself down and dead 2023-09-30T09:41:45.832 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:45.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:45.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:45.937 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:45.965 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:46.010 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:46.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:46.148 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:46.177 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:46.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:41:46 smithi167 bash[327170]: 30ccc3c97db6ae39cf39623797407d4cd822c2433c251e2781dab6407016d57e 2023-09-30T09:41:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:46 smithi167 ceph-mon[192880]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:41:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:46 smithi167 ceph-mon[192880]: failed to encode map e428 with expected crc 2023-09-30T09:41:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:46 smithi167 ceph-mon[192880]: failed to encode map e428 with expected crc 2023-09-30T09:41:46.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:46 smithi167 ceph-mon[192880]: failed to encode map e428 with expected crc 2023-09-30T09:41:46.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:46 smithi167 ceph-mon[192880]: osdmap e428: 8 total, 7 up, 6 in 2023-09-30T09:41:46.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:46 smithi167 ceph-mon[192880]: pgmap v1112: 129 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 29 stale+active+clean, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 16162/175884 objects degraded (9.189%); 6419/175884 objects misplaced (3.650%); 6.3 MiB/s, 179 objects/s recovering 2023-09-30T09:41:46.792 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:46.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:46.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:46.896 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:46.925 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:46.970 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:46.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:46.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:47.073 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:47.081 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:41:47.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[216815]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:41:47.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[216815]: failed to encode map e428 with expected crc 2023-09-30T09:41:47.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[216815]: failed to encode map e428 with expected crc 2023-09-30T09:41:47.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[216815]: failed to encode map e428 with expected crc 2023-09-30T09:41:47.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[216815]: osdmap e428: 8 total, 7 up, 6 in 2023-09-30T09:41:47.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[216815]: pgmap v1112: 129 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 29 stale+active+clean, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 16162/175884 objects degraded (9.189%); 6419/175884 objects misplaced (3.650%); 6.3 MiB/s, 179 objects/s recovering 2023-09-30T09:41:47.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[224043]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:41:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[224043]: failed to encode map e428 with expected crc 2023-09-30T09:41:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[224043]: failed to encode map e428 with expected crc 2023-09-30T09:41:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[224043]: failed to encode map e428 with expected crc 2023-09-30T09:41:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[224043]: osdmap e428: 8 total, 7 up, 6 in 2023-09-30T09:41:47.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:46 smithi019 ceph-mon[224043]: pgmap v1112: 129 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 29 stale+active+clean, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 16162/175884 objects degraded (9.189%); 6419/175884 objects misplaced (3.650%); 6.3 MiB/s, 179 objects/s recovering 2023-09-30T09:41:47.099 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:47.141 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:47.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:47.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:47.245 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:47.275 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:47.381 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:47.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:47.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:47.487 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:47.516 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:47.559 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:47.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:47.639 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[216815]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.663 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:47.744 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:47.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:47 smithi167 ceph-mon[192880]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:47 smithi167 ceph-mon[192880]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:47 smithi167 ceph-mon[192880]: osdmap e429: 8 total, 7 up, 6 in 2023-09-30T09:41:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:47 smithi167 ceph-mon[192880]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:47 smithi167 ceph-mon[192880]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:47 smithi167 ceph-mon[192880]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:47 smithi167 ceph-mon[192880]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:47 smithi167 ceph-mon[192880]: 8.c continuing backfill to osd.4 from (375'3619,427'8127] MIN to 427'8127 2023-09-30T09:41:47.833 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:47.835 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:41:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:41:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:41:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:41:47.835 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:41:47.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:47.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:47.938 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:47.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[216815]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[216815]: osdmap e429: 8 total, 7 up, 6 in 2023-09-30T09:41:47.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[216815]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[216815]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[216815]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[216815]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[216815]: 8.c continuing backfill to osd.4 from (375'3619,427'8127] MIN to 427'8127 2023-09-30T09:41:47.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[224043]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[224043]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[224043]: osdmap e429: 8 total, 7 up, 6 in 2023-09-30T09:41:47.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[224043]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[224043]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[224043]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[224043]: failed to encode map e429 with expected crc 2023-09-30T09:41:47.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:47 smithi019 ceph-mon[224043]: 8.c continuing backfill to osd.4 from (375'3619,427'8127] MIN to 427'8127 2023-09-30T09:41:47.992 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:48.105 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:48.124 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:48.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:48.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:48.229 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:48.265 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:48.376 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:48.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:48.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:48.481 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:48.515 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:48.569 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:48.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:48.675 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:48.706 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:48.757 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:48.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:48.862 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:48.895 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:48.993 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:41:48.994 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: failed to encode map e430 with expected crc 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: failed to encode map e430 with expected crc 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: failed to encode map e430 with expected crc 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: osdmap e430: 8 total, 7 up, 6 in 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: failed to encode map e430 with expected crc 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: failed to encode map e430 with expected crc 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: failed to encode map e430 with expected crc 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: pgmap v1115: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 7 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 23 stale+active+clean, 6 active+recovery_wait+degraded, 8 active+undersized+degraded, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 20560/174969 objects degraded (11.751%); 5978/174969 objects misplaced (3.417%); 8.7 MiB/s, 211 objects/s recovering 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/560695158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:48.995 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 16162/175884 objects degraded (9.189%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:49.017 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.017 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: osdmap e430: 8 total, 7 up, 6 in 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: pgmap v1115: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 7 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 23 stale+active+clean, 6 active+recovery_wait+degraded, 8 active+undersized+degraded, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 20560/174969 objects degraded (11.751%); 5978/174969 objects misplaced (3.417%); 8.7 MiB/s, 211 objects/s recovering 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/560695158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:49.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 16162/175884 objects degraded (9.189%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: osdmap e430: 8 total, 7 up, 6 in 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: failed to encode map e430 with expected crc 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: pgmap v1115: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 7 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 23 stale+active+clean, 6 active+recovery_wait+degraded, 8 active+undersized+degraded, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 20560/174969 objects degraded (11.751%); 5978/174969 objects misplaced (3.417%); 8.7 MiB/s, 211 objects/s recovering 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:41:49.019 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:41:49.020 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/560695158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:49.020 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:41:49.020 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 16162/175884 objects degraded (9.189%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:49.150 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:49.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:49.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:49.254 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:49.284 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:49.333 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:49.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:49.437 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:49.464 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:49.511 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:49.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:49.614 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:49.644 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:49.690 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:49.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:49.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:49.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:49.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:49.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[216815]: failed to encode map e431 with expected crc 2023-09-30T09:41:49.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[216815]: osdmap e431: 8 total, 7 up, 6 in 2023-09-30T09:41:49.739 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[216815]: failed to encode map e431 with expected crc 2023-09-30T09:41:49.739 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[216815]: failed to encode map e431 with expected crc 2023-09-30T09:41:49.739 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[216815]: failed to encode map e431 with expected crc 2023-09-30T09:41:49.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:49.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:49.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[224043]: failed to encode map e431 with expected crc 2023-09-30T09:41:49.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[224043]: osdmap e431: 8 total, 7 up, 6 in 2023-09-30T09:41:49.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[224043]: failed to encode map e431 with expected crc 2023-09-30T09:41:49.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[224043]: failed to encode map e431 with expected crc 2023-09-30T09:41:49.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:49 smithi019 ceph-mon[224043]: failed to encode map e431 with expected crc 2023-09-30T09:41:49.793 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:49.827 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:49 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:41:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:49 smithi167 ceph-mon[192880]: failed to encode map e431 with expected crc 2023-09-30T09:41:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:49 smithi167 ceph-mon[192880]: osdmap e431: 8 total, 7 up, 6 in 2023-09-30T09:41:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:49 smithi167 ceph-mon[192880]: failed to encode map e431 with expected crc 2023-09-30T09:41:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:49 smithi167 ceph-mon[192880]: failed to encode map e431 with expected crc 2023-09-30T09:41:50.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:49 smithi167 ceph-mon[192880]: failed to encode map e431 with expected crc 2023-09-30T09:41:50.248 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-30T09:41:50.300 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:41:50.300 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-30T09:41:50.533 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:50.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:50.606 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:41:50.637 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:50.680 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:50.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:50 smithi019 ceph-mon[216815]: pgmap v1117: 129 pgs: 1 activating+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 26 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 10 stale+active+clean, 3 active+recovery_wait+degraded, 14 active+undersized+degraded, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 62 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 143 op/s; 32110/174183 objects degraded (18.435%); 7050/174183 objects misplaced (4.047%); 5.2 MiB/s, 89 objects/s recovering 2023-09-30T09:41:50.813 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:50 smithi019 ceph-mon[224043]: pgmap v1117: 129 pgs: 1 activating+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 26 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 10 stale+active+clean, 3 active+recovery_wait+degraded, 14 active+undersized+degraded, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 62 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 143 op/s; 32110/174183 objects degraded (18.435%); 7050/174183 objects misplaced (4.047%); 5.2 MiB/s, 89 objects/s recovering 2023-09-30T09:41:50.830 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:50.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:50.934 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:50.964 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:51.021 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:51.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:50 smithi167 ceph-mon[192880]: pgmap v1117: 129 pgs: 1 activating+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 26 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 10 stale+active+clean, 3 active+recovery_wait+degraded, 14 active+undersized+degraded, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 62 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 143 op/s; 32110/174183 objects degraded (18.435%); 7050/174183 objects misplaced (4.047%); 5.2 MiB/s, 89 objects/s recovering 2023-09-30T09:41:51.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:51.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:51.056 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:41:51.073 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:51.078 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:41:51.102 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:41:51.125 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:41:51.125 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:51.148 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:41:51.159 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:51.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:41:51.189 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:41:51.189 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:41:51.189 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:41:51.189 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:41:51.189 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:41:51.190 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:41:51.190 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:41:51.190 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:41:51.190 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:41:51.217 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:51.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:51.323 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:51.356 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:51.795 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:51 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-30T09:41:51.795 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3723604359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:51.796 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:51 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-30T09:41:51.796 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3723604359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:51 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-30T09:41:52.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3723604359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:41:52.272 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:41:52.320 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:52.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:52.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:52.424 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:52.448 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:41:52.462 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:52.786 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:52 smithi019 ceph-mon[216815]: pgmap v1118: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 activating+degraded+remapped, 2 activating+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 2.3 KiB/s rd, 6.2 KiB/s wr, 112 op/s; 40839/173895 objects degraded (23.485%); 8742/173895 objects misplaced (5.027%); 8.0 MiB/s, 132 objects/s recovering 2023-09-30T09:41:52.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:52 smithi019 ceph-mon[224043]: pgmap v1118: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 activating+degraded+remapped, 2 activating+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 2.3 KiB/s rd, 6.2 KiB/s wr, 112 op/s; 40839/173895 objects degraded (23.485%); 8742/173895 objects misplaced (5.027%); 8.0 MiB/s, 132 objects/s recovering 2023-09-30T09:41:52.811 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:52.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:52.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:52.934 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:52.983 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:53.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:52 smithi167 ceph-mon[192880]: pgmap v1118: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 activating+degraded+remapped, 2 activating+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 2.3 KiB/s rd, 6.2 KiB/s wr, 112 op/s; 40839/173895 objects degraded (23.485%); 8742/173895 objects misplaced (5.027%); 8.0 MiB/s, 132 objects/s recovering 2023-09-30T09:41:53.059 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:53.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:53.164 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:53.225 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:53.355 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:53.423 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:53.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:53.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:53.478 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:41:53.478 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:41:53.535 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:53.617 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:53.685 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 40839/173895 objects degraded (23.485%), 31 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:53.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 40839/173895 objects degraded (23.485%), 31 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:54.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 40839/173895 objects degraded (23.485%), 31 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:54.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:41:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:41:53] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-09-30T09:41:54.273 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:41:54.436 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:54.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:54.541 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:54.619 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:54.625 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:41:54.975 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:54 smithi019 ceph-mon[224043]: pgmap v1119: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 activating+degraded+remapped, 1 activating+remapped, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 2.0 KiB/s rd, 5.3 KiB/s wr, 116 op/s; 46981/173466 objects degraded (27.084%); 7555/173466 objects misplaced (4.355%); 7.6 MiB/s, 124 objects/s recovering 2023-09-30T09:41:54.975 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:54 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-30T09:41:54.975 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:54 smithi019 ceph-mon[216815]: pgmap v1119: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 activating+degraded+remapped, 1 activating+remapped, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 2.0 KiB/s rd, 5.3 KiB/s wr, 116 op/s; 46981/173466 objects degraded (27.084%); 7555/173466 objects misplaced (4.355%); 7.6 MiB/s, 124 objects/s recovering 2023-09-30T09:41:54.975 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:54 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-30T09:41:54.986 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:54.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:54.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:54 smithi167 ceph-mon[192880]: pgmap v1119: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 activating+degraded+remapped, 1 activating+remapped, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 2.0 KiB/s rd, 5.3 KiB/s wr, 116 op/s; 46981/173466 objects degraded (27.084%); 7555/173466 objects misplaced (4.355%); 7.6 MiB/s, 124 objects/s recovering 2023-09-30T09:41:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:54 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-30T09:41:55.090 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:55.106 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:41:55.119 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:55.173 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:55.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:55.277 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:41:55.301 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0] out_osds: [3, 6] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:41:55.301 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:41:55.301 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-09-30T09:41:55.302 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:41:55.307 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:55.413 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:41:55.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:55.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:41:55.520 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:55.557 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:55.765 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:55.765 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:55.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:55.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:55.869 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:41:55.936 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:41:55.936 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:56.263 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:41:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:56.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:41:56.370 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:56.447 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:56.593 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:56.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:56.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:56.701 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:56.732 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:56.792 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:56.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:56.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:56 smithi019 ceph-mon[224043]: pgmap v1120: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 132 op/s; 45750/171822 objects degraded (26.626%); 7355/171822 objects misplaced (4.281%); 17 MiB/s, 274 objects/s recovering 2023-09-30T09:41:56.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:56 smithi019 ceph-mon[216815]: pgmap v1120: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 132 op/s; 45750/171822 objects degraded (26.626%); 7355/171822 objects misplaced (4.281%); 17 MiB/s, 274 objects/s recovering 2023-09-30T09:41:56.897 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:56.930 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:56.977 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:56.979 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:56 smithi167 ceph-mon[192880]: pgmap v1120: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 132 op/s; 45750/171822 objects degraded (26.626%); 7355/171822 objects misplaced (4.281%); 17 MiB/s, 274 objects/s recovering 2023-09-30T09:41:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:56.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:57.081 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:57.109 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:57.519 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:57.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:57.624 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:57.654 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:57.699 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:41:57.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:57.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:41:57.750 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:41:57.802 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:41:57.878 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:58.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:57 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2487657367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:58.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:57 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:58.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:57 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2487657367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:58.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:57 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:58.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:57 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2487657367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:58.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:57 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:41:58.430 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:41:58.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:58.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:41:58.539 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:41:58.585 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:58.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[224043]: pgmap v1121: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 1.4 KiB/s rd, 3.7 KiB/s wr, 155 op/s; 45417/170340 objects degraded (26.663%); 7283/170340 objects misplaced (4.276%); 14 MiB/s, 225 objects/s recovering 2023-09-30T09:41:58.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:58.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:41:58.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[224043]: osdmap e432: 8 total, 7 up, 6 in 2023-09-30T09:41:58.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[224043]: failed to encode map e432 with expected crc 2023-09-30T09:41:58.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[224043]: failed to encode map e432 with expected crc 2023-09-30T09:41:58.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[224043]: failed to encode map e432 with expected crc 2023-09-30T09:41:58.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 45417/170340 objects degraded (26.663%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:59.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:58 smithi167 ceph-mon[192880]: pgmap v1121: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 1.4 KiB/s rd, 3.7 KiB/s wr, 155 op/s; 45417/170340 objects degraded (26.663%); 7283/170340 objects misplaced (4.276%); 14 MiB/s, 225 objects/s recovering 2023-09-30T09:41:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:58 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:58 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:41:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:58 smithi167 ceph-mon[192880]: osdmap e432: 8 total, 7 up, 6 in 2023-09-30T09:41:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:58 smithi167 ceph-mon[192880]: failed to encode map e432 with expected crc 2023-09-30T09:41:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:58 smithi167 ceph-mon[192880]: failed to encode map e432 with expected crc 2023-09-30T09:41:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:58 smithi167 ceph-mon[192880]: failed to encode map e432 with expected crc 2023-09-30T09:41:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 45417/170340 objects degraded (26.663%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:59.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[216815]: pgmap v1121: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 1.4 KiB/s rd, 3.7 KiB/s wr, 155 op/s; 45417/170340 objects degraded (26.663%); 7283/170340 objects misplaced (4.276%); 14 MiB/s, 225 objects/s recovering 2023-09-30T09:41:59.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:41:59.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:41:59.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[216815]: osdmap e432: 8 total, 7 up, 6 in 2023-09-30T09:41:59.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[216815]: failed to encode map e432 with expected crc 2023-09-30T09:41:59.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[216815]: failed to encode map e432 with expected crc 2023-09-30T09:41:59.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[216815]: failed to encode map e432 with expected crc 2023-09-30T09:41:59.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 45417/170340 objects degraded (26.663%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:41:59.149 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:41:59.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:59.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:41:59.267 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:41:59.311 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:59.380 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:41:59.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:59.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:41:59.488 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:41:59.539 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:59.625 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:41:59.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:41:59.683 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:41:59.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:41:59.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:41:59.712 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:41:59.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:41:59.760 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:41:59.760 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:41:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:41:59.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:41:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1752808269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:59.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:41:59.795 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:41:59.802 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:41:59.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:41:59.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:41:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1752808269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:59.837 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:41:59.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:41:59.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:41:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1752808269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:41:59.845 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:41:59.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:41:59.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:41:59.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:41:59.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:41:59.845 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:41:59.846 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:41:59.846 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:41:59.846 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:42:00.275 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:42:00.279 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:42:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:00 smithi167 ceph-mon[192880]: pgmap v1123: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 45079/167466 objects degraded (26.918%); 7222/167466 objects misplaced (4.313%); 17 MiB/s, 271 objects/s recovering 2023-09-30T09:42:01.077 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:01.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:01.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:01.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:00 smithi019 ceph-mon[216815]: pgmap v1123: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 45079/167466 objects degraded (26.918%); 7222/167466 objects misplaced (4.313%); 17 MiB/s, 271 objects/s recovering 2023-09-30T09:42:01.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:00 smithi019 ceph-mon[224043]: pgmap v1123: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 45079/167466 objects degraded (26.918%); 7222/167466 objects misplaced (4.313%); 17 MiB/s, 271 objects/s recovering 2023-09-30T09:42:01.131 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:42:01.177 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:42:01.181 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:01.216 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:01.252 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:42:01.252 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:42:01.334 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:01.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:01.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:01.437 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:01.467 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:01.518 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1123,"stamp":"2023-09-30T09:41:59.678242+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3661029803,"num_objects":55822,"num_object_clones":0,"num_object_copies":167466,"num_objects_missing_on_primary":2774,"num_objects_missing":2774,"num_objects_degraded":45079,"num_objects_misplaced":7222,"num_objects_unfound":0,"num_objects_dirty":55822,"num_whiteouts":0,"num_read":5997,"num_read_kb":12245,"num_write":130031,"num_write_kb":5929463,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41798,"num_bytes_recovered":2551195795,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":152711,"ondisk_log_size":152711,"up":317,"acting":318,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":424,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13559628,"kb_used_data":10753496,"kb_used_omap":21,"kb_used_meta":2806058,"kb_avail":548911284,"statfs":{"total":575970213888,"available":562085154816,"internally_reserved":0,"allocated":11011579904,"data_stored":10999619523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21604,"internal_metadata":2873404316},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-146735104,"num_objects":-2239,"num_object_clones":0,"num_object_copies":-6717,"num_objects_missing_on_primary":1386,"num_objects_missing":-132,"num_objects_degraded":12969,"num_objects_misplaced":172,"num_objects_unfound":0,"num_objects_dirty":-2239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2239,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2715,"num_bytes_recovered":177930240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009999"},"pg_stats":[{"pgid":"8.11","version":"432'6372","reported_seq":6412,"reported_epoch":432,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:58.318360+0000","last_change":"2023-09-30T09:40:37.604830+0000","last_active":"2023-09-30T09:41:58.318360+0000","last_peered":"2023-09-30T09:41:58.318360+0000","last_clean":"2023-09-30T09:40:21.233253+0000","last_became_active":"2023-09-30T09:40:37.596780+0000","last_became_peered":"2023-09-30T09:40:37.596780+0000","last_unstale":"2023-09-30T09:41:58.318360+0000","last_undegraded":"2023-09-30T09:40:37.578459+0000","last_fullsized":"2023-09-30T09:41:58.318360+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6372,"log_dups_size":0,"ondisk_log_size":6372,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110985216,"num_objects":1694,"num_object_clones":0,"num_object_copies":5082,"num_objects_missing_on_primary":58,"num_objects_missing":58,"num_objects_degraded":58,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4004,"num_write_kb":182304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1694}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":726,"reported_epoch":430,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:47.654567+0000","last_change":"2023-09-30T09:41:46.672271+0000","last_active":"2023-09-30T09:41:47.654567+0000","last_peered":"2023-09-30T09:41:47.654567+0000","last_clean":"2023-09-30T09:32:28.427026+0000","last_became_active":"2023-09-30T09:41:46.672271+0000","last_became_peered":"2023-09-30T09:41:46.672271+0000","last_unstale":"2023-09-30T09:41:47.654567+0000","last_undegraded":"2023-09-30T09:41:46.653159+0000","last_fullsized":"2023-09-30T09:41:46.653022+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:59.526793+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":450,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.991358+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:41:48.991358+0000","last_peered":"2023-09-30T09:41:48.991358+0000","last_clean":"2023-09-30T09:41:48.991358+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:41:48.991358+0000","last_undegraded":"2023-09-30T09:41:48.991358+0000","last_fullsized":"2023-09-30T09:41:48.991358+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":362,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.992244+0000","last_change":"2023-09-30T09:41:46.665647+0000","last_active":"2023-09-30T09:41:48.992244+0000","last_peered":"2023-09-30T09:41:48.992244+0000","last_clean":"2023-09-30T09:41:40.613181+0000","last_became_active":"2023-09-30T09:41:46.665647+0000","last_became_peered":"2023-09-30T09:41:46.665647+0000","last_unstale":"2023-09-30T09:41:48.992244+0000","last_undegraded":"2023-09-30T09:41:48.992244+0000","last_fullsized":"2023-09-30T09:41:46.647419+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:29:20.680704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.12","version":"432'6301","reported_seq":6577,"reported_epoch":432,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:41:59.080187+0000","last_change":"2023-09-30T09:41:47.771843+0000","last_active":"2023-09-30T09:41:59.080187+0000","last_peered":"2023-09-30T09:41:59.080187+0000","last_clean":"2023-09-30T09:40:21.246088+0000","last_became_active":"2023-09-30T09:41:47.758227+0000","last_became_peered":"2023-09-30T09:41:47.758227+0000","last_unstale":"2023-09-30T09:41:59.080187+0000","last_undegraded":"2023-09-30T09:41:47.753542+0000","last_fullsized":"2023-09-30T09:41:59.080187+0000","mapping_epoch":429,"log_start":"363'537","ondisk_log_start":"363'537","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5764,"log_dups_size":0,"ondisk_log_size":5764,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109215744,"num_objects":1667,"num_object_clones":0,"num_object_copies":5001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1716,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3967,"num_write_kb":180256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":5046272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,2],"avail_no_missing":["4"],"object_location_counts":[{"shards":"0,2,4","objects":9},{"shards":"0,4","objects":1658}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":597,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.989487+0000","last_change":"2023-09-30T09:41:46.649236+0000","last_active":"2023-09-30T09:41:48.989487+0000","last_peered":"2023-09-30T09:41:48.989487+0000","last_clean":"2023-09-30T09:26:47.306005+0000","last_became_active":"2023-09-30T09:41:46.649236+0000","last_became_peered":"2023-09-30T09:41:46.649236+0000","last_unstale":"2023-09-30T09:41:48.989487+0000","last_undegraded":"2023-09-30T09:41:48.989487+0000","last_fullsized":"2023-09-30T09:41:46.644754+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":44725,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":262,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.988932+0000","last_change":"2023-09-30T09:39:47.869479+0000","last_active":"2023-09-30T09:41:48.988932+0000","last_peered":"2023-09-30T09:41:48.988932+0000","last_clean":"2023-09-30T09:41:48.988932+0000","last_became_active":"2023-09-30T09:39:47.869184+0000","last_became_peered":"2023-09-30T09:39:47.869184+0000","last_unstale":"2023-09-30T09:41:48.988932+0000","last_undegraded":"2023-09-30T09:41:48.988932+0000","last_fullsized":"2023-09-30T09:41:48.988932+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:03:52.322427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":27,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.986715+0000","last_change":"2023-09-30T09:40:36.599223+0000","last_active":"2023-09-30T09:41:48.986715+0000","last_peered":"2023-09-30T09:41:48.986715+0000","last_clean":"2023-09-30T09:41:48.986715+0000","last_became_active":"2023-09-30T09:40:36.597312+0000","last_became_peered":"2023-09-30T09:40:36.597312+0000","last_unstale":"2023-09-30T09:41:48.986715+0000","last_undegraded":"2023-09-30T09:41:48.986715+0000","last_fullsized":"2023-09-30T09:41:48.986715+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:29.960110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"431'6398","reported_seq":12317,"reported_epoch":431,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:56.761463+0000","last_change":"2023-09-30T09:41:46.671928+0000","last_active":"2023-09-30T09:41:56.761463+0000","last_peered":"2023-09-30T09:41:56.761463+0000","last_clean":"2023-09-30T09:41:45.607800+0000","last_became_active":"2023-09-30T09:41:46.671928+0000","last_became_peered":"2023-09-30T09:41:46.671928+0000","last_unstale":"2023-09-30T09:41:56.761463+0000","last_undegraded":"2023-09-30T09:41:46.650565+0000","last_fullsized":"2023-09-30T09:41:46.650431+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112558080,"num_objects":1718,"num_object_clones":0,"num_object_copies":5154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4016,"num_write_kb":183456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2903,"num_bytes_recovered":171573248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2","objects":1718}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":665,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.993736+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:41:48.993736+0000","last_peered":"2023-09-30T09:41:48.993736+0000","last_clean":"2023-09-30T09:41:48.993736+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:41:48.993736+0000","last_undegraded":"2023-09-30T09:41:48.993736+0000","last_fullsized":"2023-09-30T09:41:48.993736+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":413,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.654481+0000","last_change":"2023-09-30T09:39:58.574864+0000","last_active":"2023-09-30T09:41:47.654481+0000","last_peered":"2023-09-30T09:41:47.654481+0000","last_clean":"2023-09-30T09:41:47.654481+0000","last_became_active":"2023-09-30T09:39:58.574566+0000","last_became_peered":"2023-09-30T09:39:58.574566+0000","last_unstale":"2023-09-30T09:41:47.654481+0000","last_undegraded":"2023-09-30T09:41:47.654481+0000","last_fullsized":"2023-09-30T09:41:47.654481+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:23:17.675484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":34,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.990487+0000","last_change":"2023-09-30T09:41:46.656049+0000","last_active":"2023-09-30T09:41:48.990487+0000","last_peered":"2023-09-30T09:41:48.990487+0000","last_clean":"2023-09-30T09:41:40.602030+0000","last_became_active":"2023-09-30T09:41:46.656049+0000","last_became_peered":"2023-09-30T09:41:46.656049+0000","last_unstale":"2023-09-30T09:41:48.990487+0000","last_undegraded":"2023-09-30T09:41:48.990487+0000","last_fullsized":"2023-09-30T09:41:46.646005+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:38.368751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":812,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.661451+0000","last_change":"2023-09-30T09:41:46.663847+0000","last_active":"2023-09-30T09:41:47.661451+0000","last_peered":"2023-09-30T09:41:47.661451+0000","last_clean":"2023-09-30T09:41:40.608317+0000","last_became_active":"2023-09-30T09:41:46.663847+0000","last_became_peered":"2023-09-30T09:41:46.663847+0000","last_unstale":"2023-09-30T09:41:47.661451+0000","last_undegraded":"2023-09-30T09:41:47.661451+0000","last_fullsized":"2023-09-30T09:41:46.651545+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.164029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":34,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.990338+0000","last_change":"2023-09-30T09:41:46.655658+0000","last_active":"2023-09-30T09:41:48.990338+0000","last_peered":"2023-09-30T09:41:48.990338+0000","last_clean":"2023-09-30T09:41:40.601978+0000","last_became_active":"2023-09-30T09:41:46.655658+0000","last_became_peered":"2023-09-30T09:41:46.655658+0000","last_unstale":"2023-09-30T09:41:48.990338+0000","last_undegraded":"2023-09-30T09:41:48.990338+0000","last_fullsized":"2023-09-30T09:41:46.645974+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:07:49.116454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":410,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.988336+0000","last_change":"2023-09-30T09:41:46.661656+0000","last_active":"2023-09-30T09:41:48.988336+0000","last_peered":"2023-09-30T09:41:48.988336+0000","last_clean":"2023-09-30T09:41:40.610292+0000","last_became_active":"2023-09-30T09:41:46.661656+0000","last_became_peered":"2023-09-30T09:41:46.661656+0000","last_unstale":"2023-09-30T09:41:48.988336+0000","last_undegraded":"2023-09-30T09:41:48.988336+0000","last_fullsized":"2023-09-30T09:41:46.645729+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:50:10.026464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":618,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.657836+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:41:47.657836+0000","last_peered":"2023-09-30T09:41:47.657836+0000","last_clean":"2023-09-30T09:41:47.657836+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:41:47.657836+0000","last_undegraded":"2023-09-30T09:41:47.657836+0000","last_fullsized":"2023-09-30T09:41:47.657836+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.659570+0000","last_change":"2023-09-30T09:41:46.663917+0000","last_active":"2023-09-30T09:41:47.659570+0000","last_peered":"2023-09-30T09:41:47.659570+0000","last_clean":"2023-09-30T09:38:16.766845+0000","last_became_active":"2023-09-30T09:41:46.663917+0000","last_became_peered":"2023-09-30T09:41:46.663917+0000","last_unstale":"2023-09-30T09:41:47.659570+0000","last_undegraded":"2023-09-30T09:41:47.659570+0000","last_fullsized":"2023-09-30T09:41:46.654938+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:34:17.108900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":402,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.657405+0000","last_change":"2023-09-30T09:41:46.660523+0000","last_active":"2023-09-30T09:41:47.657405+0000","last_peered":"2023-09-30T09:41:47.657405+0000","last_clean":"2023-09-30T09:41:40.606114+0000","last_became_active":"2023-09-30T09:41:46.660523+0000","last_became_peered":"2023-09-30T09:41:46.660523+0000","last_unstale":"2023-09-30T09:41:47.657405+0000","last_undegraded":"2023-09-30T09:41:47.657405+0000","last_fullsized":"2023-09-30T09:41:46.649858+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:45:26.377243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":572,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.661347+0000","last_change":"2023-09-30T09:41:46.661857+0000","last_active":"2023-09-30T09:41:47.661347+0000","last_peered":"2023-09-30T09:41:47.661347+0000","last_clean":"2023-09-30T09:39:43.878280+0000","last_became_active":"2023-09-30T09:41:46.661857+0000","last_became_peered":"2023-09-30T09:41:46.661857+0000","last_unstale":"2023-09-30T09:41:47.661347+0000","last_undegraded":"2023-09-30T09:41:47.661347+0000","last_fullsized":"2023-09-30T09:41:46.650216+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:46.835984+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":129,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.661711+0000","last_change":"2023-09-30T09:40:36.618707+0000","last_active":"2023-09-30T09:41:47.661711+0000","last_peered":"2023-09-30T09:41:47.661711+0000","last_clean":"2023-09-30T09:41:47.661711+0000","last_became_active":"2023-09-30T09:40:36.597642+0000","last_became_peered":"2023-09-30T09:40:36.597642+0000","last_unstale":"2023-09-30T09:41:47.661711+0000","last_undegraded":"2023-09-30T09:41:47.661711+0000","last_fullsized":"2023-09-30T09:41:47.661711+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:29:09.447554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":273,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.655089+0000","last_change":"2023-09-30T09:41:46.696257+0000","last_active":"2023-09-30T09:41:47.655089+0000","last_peered":"2023-09-30T09:41:47.655089+0000","last_clean":"2023-09-30T09:41:40.708410+0000","last_became_active":"2023-09-30T09:41:46.696257+0000","last_became_peered":"2023-09-30T09:41:46.696257+0000","last_unstale":"2023-09-30T09:41:47.655089+0000","last_undegraded":"2023-09-30T09:41:47.655089+0000","last_fullsized":"2023-09-30T09:41:46.645275+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:18.075674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":471,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.994311+0000","last_change":"2023-09-30T09:40:43.945675+0000","last_active":"2023-09-30T09:41:48.994311+0000","last_peered":"2023-09-30T09:41:48.994311+0000","last_clean":"2023-09-30T09:41:48.994311+0000","last_became_active":"2023-09-30T09:40:43.945291+0000","last_became_peered":"2023-09-30T09:40:43.945291+0000","last_unstale":"2023-09-30T09:41:48.994311+0000","last_undegraded":"2023-09-30T09:41:48.994311+0000","last_fullsized":"2023-09-30T09:41:48.994311+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:41.055314+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":456,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.989984+0000","last_change":"2023-09-30T09:39:47.882770+0000","last_active":"2023-09-30T09:41:48.989984+0000","last_peered":"2023-09-30T09:41:48.989984+0000","last_clean":"2023-09-30T09:41:48.989984+0000","last_became_active":"2023-09-30T09:39:47.864005+0000","last_became_peered":"2023-09-30T09:39:47.864005+0000","last_unstale":"2023-09-30T09:41:48.989984+0000","last_undegraded":"2023-09-30T09:41:48.989984+0000","last_fullsized":"2023-09-30T09:41:48.989984+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:09.572406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14166,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":444,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.989906+0000","last_change":"2023-09-30T09:41:46.671025+0000","last_active":"2023-09-30T09:41:48.989906+0000","last_peered":"2023-09-30T09:41:48.989906+0000","last_clean":"2023-09-30T09:41:40.611889+0000","last_became_active":"2023-09-30T09:41:46.671025+0000","last_became_peered":"2023-09-30T09:41:46.671025+0000","last_unstale":"2023-09-30T09:41:48.989906+0000","last_undegraded":"2023-09-30T09:41:48.989906+0000","last_fullsized":"2023-09-30T09:41:46.648520+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:19:24.780098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"431'6536","reported_seq":16011,"reported_epoch":431,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:41:55.411736+0000","last_change":"2023-09-30T09:41:52.094214+0000","last_active":"2023-09-30T09:41:55.411736+0000","last_peered":"2023-09-30T09:41:55.411736+0000","last_clean":"2023-09-30T09:41:45.126495+0000","last_became_active":"2023-09-30T09:41:51.656307+0000","last_became_peered":"2023-09-30T09:41:51.656307+0000","last_unstale":"2023-09-30T09:41:55.411736+0000","last_undegraded":"2023-09-30T09:41:50.573895+0000","last_fullsized":"2023-09-30T09:41:55.411736+0000","mapping_epoch":429,"log_start":"363'702","ondisk_log_start":"363'702","created":379,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5834,"log_dups_size":0,"ondisk_log_size":5834,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114196480,"num_objects":1743,"num_object_clones":0,"num_object_copies":5229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":815,"num_objects_misplaced":928,"num_objects_unfound":0,"num_objects_dirty":1743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4107,"num_write_kb":187168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4668,"num_bytes_recovered":274399232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,0],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"0,2,7","objects":1743}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":620,"reported_epoch":430,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:47.654823+0000","last_change":"2023-09-30T09:41:46.670490+0000","last_active":"2023-09-30T09:41:47.654823+0000","last_peered":"2023-09-30T09:41:47.654823+0000","last_clean":"2023-09-30T09:35:02.568886+0000","last_became_active":"2023-09-30T09:41:46.670490+0000","last_became_peered":"2023-09-30T09:41:46.670490+0000","last_unstale":"2023-09-30T09:41:47.654823+0000","last_undegraded":"2023-09-30T09:41:46.643706+0000","last_fullsized":"2023-09-30T09:41:46.643525+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134175,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":81,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.659309+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:41:47.659309+0000","last_peered":"2023-09-30T09:41:47.659309+0000","last_clean":"2023-09-30T09:41:47.659309+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:41:47.659309+0000","last_undegraded":"2023-09-30T09:41:47.659309+0000","last_fullsized":"2023-09-30T09:41:47.659309+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":423,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.659613+0000","last_change":"2023-09-30T09:40:22.811912+0000","last_active":"2023-09-30T09:41:47.659613+0000","last_peered":"2023-09-30T09:41:47.659613+0000","last_clean":"2023-09-30T09:41:47.659613+0000","last_became_active":"2023-09-30T09:40:22.810606+0000","last_became_peered":"2023-09-30T09:40:22.810606+0000","last_unstale":"2023-09-30T09:41:47.659613+0000","last_undegraded":"2023-09-30T09:41:47.659613+0000","last_fullsized":"2023-09-30T09:41:47.659613+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:46:15.913218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.19","version":"432'6424","reported_seq":15268,"reported_epoch":432,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:41:59.104537+0000","last_change":"2023-09-30T09:41:48.986169+0000","last_active":"2023-09-30T09:41:59.104537+0000","last_peered":"2023-09-30T09:41:59.104537+0000","last_clean":"2023-09-30T09:41:45.160618+0000","last_became_active":"2023-09-30T09:41:48.978303+0000","last_became_peered":"2023-09-30T09:41:48.978303+0000","last_unstale":"2023-09-30T09:41:59.104537+0000","last_undegraded":"2023-09-30T09:41:47.666414+0000","last_fullsized":"2023-09-30T09:41:59.104537+0000","mapping_epoch":429,"log_start":"363'669","ondisk_log_start":"363'669","created":379,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114130944,"num_objects":1742,"num_object_clones":0,"num_object_copies":5226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":764,"num_objects_misplaced":978,"num_objects_unfound":0,"num_objects_dirty":1742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4036,"num_write_kb":184864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2904,"num_bytes_recovered":171114496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2,0],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"0,2,4","objects":1742}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":727,"reported_epoch":429,"state":"active+undersized","last_fresh":"2023-09-30T09:41:46.665236+0000","last_change":"2023-09-30T09:41:46.664552+0000","last_active":"2023-09-30T09:41:46.665236+0000","last_peered":"2023-09-30T09:41:46.665236+0000","last_clean":"2023-09-30T09:38:16.766366+0000","last_became_active":"2023-09-30T09:41:46.664552+0000","last_became_peered":"2023-09-30T09:41:46.664552+0000","last_unstale":"2023-09-30T09:41:46.665236+0000","last_undegraded":"2023-09-30T09:41:46.665236+0000","last_fullsized":"2023-09-30T09:41:46.651339+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:02:43.796837+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":329,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.659177+0000","last_change":"2023-09-30T09:39:58.583120+0000","last_active":"2023-09-30T09:41:47.659177+0000","last_peered":"2023-09-30T09:41:47.659177+0000","last_clean":"2023-09-30T09:41:47.659177+0000","last_became_active":"2023-09-30T09:39:58.582799+0000","last_became_peered":"2023-09-30T09:39:58.582799+0000","last_unstale":"2023-09-30T09:41:47.659177+0000","last_undegraded":"2023-09-30T09:41:47.659177+0000","last_fullsized":"2023-09-30T09:41:47.659177+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:51:22.821149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":156,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.659986+0000","last_change":"2023-09-30T09:41:46.661145+0000","last_active":"2023-09-30T09:41:47.659986+0000","last_peered":"2023-09-30T09:41:47.659986+0000","last_clean":"2023-09-30T09:40:25.845565+0000","last_became_active":"2023-09-30T09:41:46.661145+0000","last_became_peered":"2023-09-30T09:41:46.661145+0000","last_unstale":"2023-09-30T09:41:47.659986+0000","last_undegraded":"2023-09-30T09:41:47.659986+0000","last_fullsized":"2023-09-30T09:41:46.652226+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:39:29.092580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":862,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.989291+0000","last_change":"2023-09-30T09:40:39.589288+0000","last_active":"2023-09-30T09:41:48.989291+0000","last_peered":"2023-09-30T09:41:48.989291+0000","last_clean":"2023-09-30T09:41:48.989291+0000","last_became_active":"2023-09-30T09:40:39.588929+0000","last_became_peered":"2023-09-30T09:40:39.588929+0000","last_unstale":"2023-09-30T09:41:48.989291+0000","last_undegraded":"2023-09-30T09:41:48.989291+0000","last_fullsized":"2023-09-30T09:41:48.989291+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.362667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.362667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.006589625,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":105,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.657722+0000","last_change":"2023-09-30T09:40:36.592638+0000","last_active":"2023-09-30T09:41:47.657722+0000","last_peered":"2023-09-30T09:41:47.657722+0000","last_clean":"2023-09-30T09:41:47.657722+0000","last_became_active":"2023-09-30T09:40:36.592313+0000","last_became_peered":"2023-09-30T09:40:36.592313+0000","last_unstale":"2023-09-30T09:41:47.657722+0000","last_undegraded":"2023-09-30T09:41:47.657722+0000","last_fullsized":"2023-09-30T09:41:47.657722+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:18:12.781018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":334,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.661146+0000","last_change":"2023-09-30T09:41:46.664623+0000","last_active":"2023-09-30T09:41:47.661146+0000","last_peered":"2023-09-30T09:41:47.661146+0000","last_clean":"2023-09-30T09:41:40.607227+0000","last_became_active":"2023-09-30T09:41:46.664623+0000","last_became_peered":"2023-09-30T09:41:46.664623+0000","last_unstale":"2023-09-30T09:41:47.661146+0000","last_undegraded":"2023-09-30T09:41:47.661146+0000","last_fullsized":"2023-09-30T09:41:46.655258+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:46:39.973173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":871,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.658616+0000","last_change":"2023-09-30T09:41:46.664504+0000","last_active":"2023-09-30T09:41:47.658616+0000","last_peered":"2023-09-30T09:41:47.658616+0000","last_clean":"2023-09-30T09:39:43.878770+0000","last_became_active":"2023-09-30T09:41:46.664504+0000","last_became_peered":"2023-09-30T09:41:46.664504+0000","last_unstale":"2023-09-30T09:41:47.658616+0000","last_undegraded":"2023-09-30T09:41:47.658616+0000","last_fullsized":"2023-09-30T09:41:46.652023+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:57.465029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:57.465029+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:24:20.832102+0000","scrub_duration":0.003103181,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":481,"reported_epoch":430,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:47.654339+0000","last_change":"2023-09-30T09:41:46.671805+0000","last_active":"2023-09-30T09:41:47.654339+0000","last_peered":"2023-09-30T09:41:47.654339+0000","last_clean":"2023-09-30T09:35:02.561233+0000","last_became_active":"2023-09-30T09:41:46.671805+0000","last_became_peered":"2023-09-30T09:41:46.671805+0000","last_unstale":"2023-09-30T09:41:47.654339+0000","last_undegraded":"2023-09-30T09:41:46.652656+0000","last_fullsized":"2023-09-30T09:41:46.652520+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:15.591296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":4}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":447,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.661174+0000","last_change":"2023-09-30T09:38:45.606456+0000","last_active":"2023-09-30T09:41:47.661174+0000","last_peered":"2023-09-30T09:41:47.661174+0000","last_clean":"2023-09-30T09:41:47.661174+0000","last_became_active":"2023-09-30T09:38:45.604792+0000","last_became_peered":"2023-09-30T09:38:45.604792+0000","last_unstale":"2023-09-30T09:41:47.661174+0000","last_undegraded":"2023-09-30T09:41:47.661174+0000","last_fullsized":"2023-09-30T09:41:47.661174+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:11.533941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"432'6498","reported_seq":6597,"reported_epoch":432,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:59.229689+0000","last_change":"2023-09-30T09:41:46.674662+0000","last_active":"2023-09-30T09:41:59.229689+0000","last_peered":"2023-09-30T09:41:59.229689+0000","last_clean":"2023-09-30T09:41:45.338551+0000","last_became_active":"2023-09-30T09:41:46.674662+0000","last_became_peered":"2023-09-30T09:41:46.674662+0000","last_unstale":"2023-09-30T09:41:59.229689+0000","last_undegraded":"2023-09-30T09:41:46.647846+0000","last_fullsized":"2023-09-30T09:41:46.647766+0000","mapping_epoch":428,"log_start":"363'842","ondisk_log_start":"363'842","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5656,"log_dups_size":0,"ondisk_log_size":5656,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116785152,"num_objects":1782,"num_object_clones":0,"num_object_copies":5346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1782,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4092,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":1782}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":797,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.661621+0000","last_change":"2023-09-30T09:38:30.737676+0000","last_active":"2023-09-30T09:41:47.661621+0000","last_peered":"2023-09-30T09:41:47.661621+0000","last_clean":"2023-09-30T09:41:47.661621+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:41:47.661621+0000","last_undegraded":"2023-09-30T09:41:47.661621+0000","last_fullsized":"2023-09-30T09:41:47.661621+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.740851+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.740851+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":93,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.661553+0000","last_change":"2023-09-30T09:39:58.609474+0000","last_active":"2023-09-30T09:41:47.661553+0000","last_peered":"2023-09-30T09:41:47.661553+0000","last_clean":"2023-09-30T09:41:47.661553+0000","last_became_active":"2023-09-30T09:39:58.583107+0000","last_became_peered":"2023-09-30T09:39:58.583107+0000","last_unstale":"2023-09-30T09:41:47.661553+0000","last_undegraded":"2023-09-30T09:41:47.661553+0000","last_fullsized":"2023-09-30T09:41:47.661553+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:31.980540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":101,"reported_epoch":429,"state":"active+undersized","last_fresh":"2023-09-30T09:41:46.665238+0000","last_change":"2023-09-30T09:41:46.664887+0000","last_active":"2023-09-30T09:41:46.665238+0000","last_peered":"2023-09-30T09:41:46.665238+0000","last_clean":"2023-09-30T09:39:43.881875+0000","last_became_active":"2023-09-30T09:41:46.664887+0000","last_became_peered":"2023-09-30T09:41:46.664887+0000","last_unstale":"2023-09-30T09:41:46.665238+0000","last_undegraded":"2023-09-30T09:41:46.665238+0000","last_fullsized":"2023-09-30T09:41:46.651665+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:29:16.038152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"432'6461","reported_seq":13089,"reported_epoch":432,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:59.100655+0000","last_change":"2023-09-30T09:40:37.624519+0000","last_active":"2023-09-30T09:41:59.100655+0000","last_peered":"2023-09-30T09:41:59.100655+0000","last_clean":"2023-09-30T09:40:21.254315+0000","last_became_active":"2023-09-30T09:40:37.616502+0000","last_became_peered":"2023-09-30T09:40:37.616502+0000","last_unstale":"2023-09-30T09:41:59.100655+0000","last_undegraded":"2023-09-30T09:40:37.596565+0000","last_fullsized":"2023-09-30T09:41:59.100655+0000","mapping_epoch":414,"log_start":"363'748","ondisk_log_start":"363'748","created":358,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5713,"log_dups_size":0,"ondisk_log_size":5713,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112918528,"num_objects":1723,"num_object_clones":0,"num_object_copies":5169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":69,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4059,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3153,"num_bytes_recovered":191037440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"0,4,7","objects":1723}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":801,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.987603+0000","last_change":"2023-09-30T09:41:46.657904+0000","last_active":"2023-09-30T09:41:48.987603+0000","last_peered":"2023-09-30T09:41:48.987603+0000","last_clean":"2023-09-30T09:41:40.703040+0000","last_became_active":"2023-09-30T09:41:46.657904+0000","last_became_peered":"2023-09-30T09:41:46.657904+0000","last_unstale":"2023-09-30T09:41:48.987603+0000","last_undegraded":"2023-09-30T09:41:48.987603+0000","last_fullsized":"2023-09-30T09:41:46.646285+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:31.861801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:31.861801+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042476720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":77,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.659854+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:41:47.659854+0000","last_peered":"2023-09-30T09:41:47.659854+0000","last_clean":"2023-09-30T09:41:47.659854+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:41:47.659854+0000","last_undegraded":"2023-09-30T09:41:47.659854+0000","last_fullsized":"2023-09-30T09:41:47.659854+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":474,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.989173+0000","last_change":"2023-09-30T09:40:36.593049+0000","last_active":"2023-09-30T09:41:48.989173+0000","last_peered":"2023-09-30T09:41:48.989173+0000","last_clean":"2023-09-30T09:41:48.989173+0000","last_became_active":"2023-09-30T09:40:36.590336+0000","last_became_peered":"2023-09-30T09:40:36.590336+0000","last_unstale":"2023-09-30T09:41:48.989173+0000","last_undegraded":"2023-09-30T09:41:48.989173+0000","last_fullsized":"2023-09-30T09:41:48.989173+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:54.795919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"432'6522","reported_seq":10285,"reported_epoch":432,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:58.216260+0000","last_change":"2023-09-30T09:40:40.784953+0000","last_active":"2023-09-30T09:41:58.216260+0000","last_peered":"2023-09-30T09:41:58.216260+0000","last_clean":"2023-09-30T09:38:26.430308+0000","last_became_active":"2023-09-30T09:40:39.913250+0000","last_became_peered":"2023-09-30T09:40:39.913250+0000","last_unstale":"2023-09-30T09:41:58.216260+0000","last_undegraded":"2023-09-30T09:40:39.696160+0000","last_fullsized":"2023-09-30T09:40:37.581870+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114360320,"num_objects":1745,"num_object_clones":0,"num_object_copies":5235,"num_objects_missing_on_primary":52,"num_objects_missing":52,"num_objects_degraded":2299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4081,"num_write_kb":186432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":929,"num_bytes_recovered":60882944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,2],"avail_no_missing":["6"],"object_location_counts":[{"shards":"0,2","objects":-233},{"shards":"0,6,7","objects":1751}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":547,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.990621+0000","last_change":"2023-09-30T09:41:46.671929+0000","last_active":"2023-09-30T09:41:48.990621+0000","last_peered":"2023-09-30T09:41:48.990621+0000","last_clean":"2023-09-30T09:31:50.080073+0000","last_became_active":"2023-09-30T09:41:46.671929+0000","last_became_peered":"2023-09-30T09:41:46.671929+0000","last_unstale":"2023-09-30T09:41:48.990621+0000","last_undegraded":"2023-09-30T09:41:48.990621+0000","last_fullsized":"2023-09-30T09:41:46.653535+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:05.450004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:05.450004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00350746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":100,"reported_epoch":430,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:47.659834+0000","last_change":"2023-09-30T09:41:46.658359+0000","last_active":"2023-09-30T09:41:47.659834+0000","last_peered":"2023-09-30T09:41:47.659834+0000","last_clean":"2023-09-30T09:41:40.608094+0000","last_became_active":"2023-09-30T09:41:46.658359+0000","last_became_peered":"2023-09-30T09:41:46.658359+0000","last_unstale":"2023-09-30T09:41:47.659834+0000","last_undegraded":"2023-09-30T09:41:46.652022+0000","last_fullsized":"2023-09-30T09:41:46.651915+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:43:02.738072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":407,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.658859+0000","last_change":"2023-09-30T09:41:46.664572+0000","last_active":"2023-09-30T09:41:47.658859+0000","last_peered":"2023-09-30T09:41:47.658859+0000","last_clean":"2023-09-30T09:41:40.606257+0000","last_became_active":"2023-09-30T09:41:46.664572+0000","last_became_peered":"2023-09-30T09:41:46.664572+0000","last_unstale":"2023-09-30T09:41:47.658859+0000","last_undegraded":"2023-09-30T09:41:47.658859+0000","last_fullsized":"2023-09-30T09:41:46.652171+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:24.584069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"431'6490","reported_seq":9712,"reported_epoch":431,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:57.304151+0000","last_change":"2023-09-30T09:39:55.210294+0000","last_active":"2023-09-30T09:41:57.304151+0000","last_peered":"2023-09-30T09:41:57.304151+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:39:55.103786+0000","last_became_peered":"2023-09-30T09:39:55.103786+0000","last_unstale":"2023-09-30T09:41:57.304151+0000","last_undegraded":"2023-09-30T09:39:51.345690+0000","last_fullsized":"2023-09-30T09:39:48.814537+0000","mapping_epoch":390,"log_start":"363'16","ondisk_log_start":"363'16","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6474,"log_dups_size":0,"ondisk_log_size":6474,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118259712,"num_objects":1804,"num_object_clones":0,"num_object_copies":5412,"num_objects_missing_on_primary":592,"num_objects_missing":592,"num_objects_degraded":2678,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4054,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1125,"num_bytes_recovered":73728000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":-11},{"shards":"1,2,3,6","objects":11},{"shards":"1,2,6","objects":1212},{"shards":"1,3,6","objects":592}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":647,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990194+0000","last_change":"2023-09-30T09:38:48.662225+0000","last_active":"2023-09-30T09:41:48.990194+0000","last_peered":"2023-09-30T09:41:48.990194+0000","last_clean":"2023-09-30T09:41:48.990194+0000","last_became_active":"2023-09-30T09:38:48.661883+0000","last_became_peered":"2023-09-30T09:38:48.661883+0000","last_unstale":"2023-09-30T09:41:48.990194+0000","last_undegraded":"2023-09-30T09:41:48.990194+0000","last_fullsized":"2023-09-30T09:41:48.990194+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:01.892446+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":416,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.987318+0000","last_change":"2023-09-30T09:41:46.661140+0000","last_active":"2023-09-30T09:41:48.987318+0000","last_peered":"2023-09-30T09:41:48.987318+0000","last_clean":"2023-09-30T09:40:25.845703+0000","last_became_active":"2023-09-30T09:41:46.661140+0000","last_became_peered":"2023-09-30T09:41:46.661140+0000","last_unstale":"2023-09-30T09:41:48.987318+0000","last_undegraded":"2023-09-30T09:41:48.987318+0000","last_fullsized":"2023-09-30T09:41:46.642179+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:55:38.650247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":29,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.986922+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:41:48.986922+0000","last_peered":"2023-09-30T09:41:48.986922+0000","last_clean":"2023-09-30T09:41:48.986922+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:41:48.986922+0000","last_undegraded":"2023-09-30T09:41:48.986922+0000","last_fullsized":"2023-09-30T09:41:48.986922+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"432'6615","reported_seq":10191,"reported_epoch":432,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:59.003500+0000","last_change":"2023-09-30T09:41:46.661538+0000","last_active":"2023-09-30T09:41:59.003500+0000","last_peered":"2023-09-30T09:41:59.003500+0000","last_clean":"2023-09-30T09:41:45.553743+0000","last_became_active":"2023-09-30T09:41:46.661538+0000","last_became_peered":"2023-09-30T09:41:46.661538+0000","last_unstale":"2023-09-30T09:41:59.003500+0000","last_undegraded":"2023-09-30T09:41:46.644452+0000","last_fullsized":"2023-09-30T09:41:46.644307+0000","mapping_epoch":428,"log_start":"363'845","ondisk_log_start":"363'845","created":358,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119635968,"num_objects":1825,"num_object_clones":0,"num_object_copies":5475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1825,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4139,"num_write_kb":190880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1729,"num_bytes_recovered":113344512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1825}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":893,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.657671+0000","last_change":"2023-09-30T09:41:46.661885+0000","last_active":"2023-09-30T09:41:47.657671+0000","last_peered":"2023-09-30T09:41:47.657671+0000","last_clean":"2023-09-30T09:41:40.606202+0000","last_became_active":"2023-09-30T09:41:46.661885+0000","last_became_peered":"2023-09-30T09:41:46.661885+0000","last_unstale":"2023-09-30T09:41:47.657671+0000","last_undegraded":"2023-09-30T09:41:47.657671+0000","last_fullsized":"2023-09-30T09:41:46.649468+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:34:52.259993+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":62,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990223+0000","last_change":"2023-09-30T09:39:58.588073+0000","last_active":"2023-09-30T09:41:48.990223+0000","last_peered":"2023-09-30T09:41:48.990223+0000","last_clean":"2023-09-30T09:41:48.990223+0000","last_became_active":"2023-09-30T09:39:58.577667+0000","last_became_peered":"2023-09-30T09:39:58.577667+0000","last_unstale":"2023-09-30T09:41:48.990223+0000","last_undegraded":"2023-09-30T09:41:48.990223+0000","last_fullsized":"2023-09-30T09:41:48.990223+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:28:13.160407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":410,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990095+0000","last_change":"2023-09-30T09:39:47.865003+0000","last_active":"2023-09-30T09:41:48.990095+0000","last_peered":"2023-09-30T09:41:48.990095+0000","last_clean":"2023-09-30T09:41:48.990095+0000","last_became_active":"2023-09-30T09:39:47.864581+0000","last_became_peered":"2023-09-30T09:39:47.864581+0000","last_unstale":"2023-09-30T09:41:48.990095+0000","last_undegraded":"2023-09-30T09:41:48.990095+0000","last_fullsized":"2023-09-30T09:41:48.990095+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:13:20.328082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"432'6334","reported_seq":6367,"reported_epoch":432,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:41:57.992415+0000","last_change":"2023-09-30T09:40:37.604868+0000","last_active":"2023-09-30T09:41:57.992415+0000","last_peered":"2023-09-30T09:41:57.992415+0000","last_clean":"2023-09-30T09:40:21.078874+0000","last_became_active":"2023-09-30T09:40:37.598926+0000","last_became_peered":"2023-09-30T09:40:37.598926+0000","last_unstale":"2023-09-30T09:41:57.992415+0000","last_undegraded":"2023-09-30T09:40:37.578588+0000","last_fullsized":"2023-09-30T09:41:57.992415+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6334,"log_dups_size":0,"ondisk_log_size":6334,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112427008,"num_objects":1715,"num_object_clones":0,"num_object_copies":5145,"num_objects_missing_on_primary":65,"num_objects_missing":65,"num_objects_degraded":65,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3965,"num_write_kb":181792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1715}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":888,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.990300+0000","last_change":"2023-09-30T09:41:46.662200+0000","last_active":"2023-09-30T09:41:48.990300+0000","last_peered":"2023-09-30T09:41:48.990300+0000","last_clean":"2023-09-30T09:41:40.703802+0000","last_became_active":"2023-09-30T09:41:46.662200+0000","last_became_peered":"2023-09-30T09:41:46.662200+0000","last_unstale":"2023-09-30T09:41:48.990300+0000","last_undegraded":"2023-09-30T09:41:48.990300+0000","last_fullsized":"2023-09-30T09:41:46.653844+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:18:45.370166+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":332,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.991180+0000","last_change":"2023-09-30T09:41:46.671069+0000","last_active":"2023-09-30T09:41:48.991180+0000","last_peered":"2023-09-30T09:41:48.991180+0000","last_clean":"2023-09-30T09:41:40.610465+0000","last_became_active":"2023-09-30T09:41:46.671069+0000","last_became_peered":"2023-09-30T09:41:46.671069+0000","last_unstale":"2023-09-30T09:41:48.991180+0000","last_undegraded":"2023-09-30T09:41:48.991180+0000","last_fullsized":"2023-09-30T09:41:46.646170+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:22:54.243258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":126,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.991602+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:41:48.991602+0000","last_peered":"2023-09-30T09:41:48.991602+0000","last_clean":"2023-09-30T09:41:48.991602+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:41:48.991602+0000","last_undegraded":"2023-09-30T09:41:48.991602+0000","last_fullsized":"2023-09-30T09:41:48.991602+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"432'6284","reported_seq":9142,"reported_epoch":432,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-30T09:41:59.234624+0000","last_change":"2023-09-30T09:41:47.781373+0000","last_active":"2023-09-30T09:41:59.234624+0000","last_peered":"2023-09-30T09:41:59.234624+0000","last_clean":"2023-09-30T09:40:21.766887+0000","last_became_active":"2023-09-30T09:41:47.757135+0000","last_became_peered":"2023-09-30T09:41:47.757135+0000","last_unstale":"2023-09-30T09:41:59.234624+0000","last_undegraded":"2023-09-30T09:41:47.710237+0000","last_fullsized":"2023-09-30T09:41:59.234624+0000","mapping_epoch":429,"log_start":"363'479","ondisk_log_start":"363'479","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110592000,"num_objects":1687,"num_object_clones":0,"num_object_copies":5061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1687,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3949,"num_write_kb":180384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1341,"num_bytes_recovered":84475904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,2],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,2,4","objects":1189},{"shards":"0,4","objects":498}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":760,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.987039+0000","last_change":"2023-09-30T09:41:46.661170+0000","last_active":"2023-09-30T09:41:48.987039+0000","last_peered":"2023-09-30T09:41:48.987039+0000","last_clean":"2023-09-30T09:41:40.702734+0000","last_became_active":"2023-09-30T09:41:46.661170+0000","last_became_peered":"2023-09-30T09:41:46.661170+0000","last_unstale":"2023-09-30T09:41:48.987039+0000","last_undegraded":"2023-09-30T09:41:48.987039+0000","last_fullsized":"2023-09-30T09:41:46.642758+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:24:52.689126+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":438,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.989099+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:41:48.989099+0000","last_peered":"2023-09-30T09:41:48.989099+0000","last_clean":"2023-09-30T09:41:48.989099+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:41:48.989099+0000","last_undegraded":"2023-09-30T09:41:48.989099+0000","last_fullsized":"2023-09-30T09:41:48.989099+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":424,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.990691+0000","last_change":"2023-09-30T09:41:46.670621+0000","last_active":"2023-09-30T09:41:48.990691+0000","last_peered":"2023-09-30T09:41:48.990691+0000","last_clean":"2023-09-30T09:41:40.612439+0000","last_became_active":"2023-09-30T09:41:46.670621+0000","last_became_peered":"2023-09-30T09:41:46.670621+0000","last_unstale":"2023-09-30T09:41:48.990691+0000","last_undegraded":"2023-09-30T09:41:48.990691+0000","last_fullsized":"2023-09-30T09:41:46.648657+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:15:23.484364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"431'6373","reported_seq":12187,"reported_epoch":431,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:56.942886+0000","last_change":"2023-09-30T09:41:46.665909+0000","last_active":"2023-09-30T09:41:56.942886+0000","last_peered":"2023-09-30T09:41:56.942886+0000","last_clean":"2023-09-30T09:41:45.359152+0000","last_became_active":"2023-09-30T09:41:46.665909+0000","last_became_peered":"2023-09-30T09:41:46.665909+0000","last_unstale":"2023-09-30T09:41:56.942886+0000","last_undegraded":"2023-09-30T09:41:46.650118+0000","last_fullsized":"2023-09-30T09:41:46.649982+0000","mapping_epoch":428,"log_start":"363'81","ondisk_log_start":"363'81","created":358,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6292,"log_dups_size":0,"ondisk_log_size":6292,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116817920,"num_objects":1782,"num_object_clones":0,"num_object_copies":5346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1782,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3990,"num_write_kb":184736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":137756672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2","objects":1782}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":610,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.990065+0000","last_change":"2023-09-30T09:41:46.700936+0000","last_active":"2023-09-30T09:41:48.990065+0000","last_peered":"2023-09-30T09:41:48.990065+0000","last_clean":"2023-09-30T09:41:40.708334+0000","last_became_active":"2023-09-30T09:41:46.700936+0000","last_became_peered":"2023-09-30T09:41:46.700936+0000","last_unstale":"2023-09-30T09:41:48.990065+0000","last_undegraded":"2023-09-30T09:41:48.990065+0000","last_fullsized":"2023-09-30T09:41:46.645546+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:55:09.620944+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":427,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-30T09:41:46.663780+0000","last_change":"2023-09-30T09:38:45.620180+0000","last_active":"2023-09-30T09:41:46.663780+0000","last_peered":"2023-09-30T09:41:46.663780+0000","last_clean":"2023-09-30T09:41:46.663780+0000","last_became_active":"2023-09-30T09:38:45.619776+0000","last_became_peered":"2023-09-30T09:38:45.619776+0000","last_unstale":"2023-09-30T09:41:46.663780+0000","last_undegraded":"2023-09-30T09:41:46.663780+0000","last_fullsized":"2023-09-30T09:41:46.663780+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:25:08.739481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":132,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990870+0000","last_change":"2023-09-30T09:38:45.610897+0000","last_active":"2023-09-30T09:41:48.990870+0000","last_peered":"2023-09-30T09:41:48.990870+0000","last_clean":"2023-09-30T09:41:48.990870+0000","last_became_active":"2023-09-30T09:38:45.610628+0000","last_became_peered":"2023-09-30T09:38:45.610628+0000","last_unstale":"2023-09-30T09:41:48.990870+0000","last_undegraded":"2023-09-30T09:41:48.990870+0000","last_fullsized":"2023-09-30T09:41:48.990870+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:33:04.928128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.e","version":"432'7952","reported_seq":8052,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-30T09:41:58.216124+0000","last_change":"2023-09-30T09:40:43.499107+0000","last_active":"2023-09-30T09:41:58.216124+0000","last_peered":"2023-09-30T09:41:58.216124+0000","last_clean":"2023-09-30T09:41:58.216124+0000","last_became_active":"2023-09-30T09:40:42.368900+0000","last_became_peered":"2023-09-30T09:40:42.368900+0000","last_unstale":"2023-09-30T09:41:58.216124+0000","last_undegraded":"2023-09-30T09:41:58.216124+0000","last_fullsized":"2023-09-30T09:41:58.216124+0000","mapping_epoch":414,"log_start":"369'2601","ondisk_log_start":"369'2601","created":358,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5351,"log_dups_size":0,"ondisk_log_size":5351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:58:53.392866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225443840,"num_objects":3440,"num_object_clones":0,"num_object_copies":10320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7952,"num_write_kb":364544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":735,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.661107+0000","last_change":"2023-09-30T09:41:46.662269+0000","last_active":"2023-09-30T09:41:47.661107+0000","last_peered":"2023-09-30T09:41:47.661107+0000","last_clean":"2023-09-30T09:41:40.606712+0000","last_became_active":"2023-09-30T09:41:46.662269+0000","last_became_peered":"2023-09-30T09:41:46.662269+0000","last_unstale":"2023-09-30T09:41:47.661107+0000","last_undegraded":"2023-09-30T09:41:47.661107+0000","last_fullsized":"2023-09-30T09:41:46.650849+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:48:00.974816+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":660,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.987627+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:41:48.987627+0000","last_peered":"2023-09-30T09:41:48.987627+0000","last_clean":"2023-09-30T09:41:48.987627+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:41:48.987627+0000","last_undegraded":"2023-09-30T09:41:48.987627+0000","last_fullsized":"2023-09-30T09:41:48.987627+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":408,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.657637+0000","last_change":"2023-09-30T09:41:46.666247+0000","last_active":"2023-09-30T09:41:47.657637+0000","last_peered":"2023-09-30T09:41:47.657637+0000","last_clean":"2023-09-30T09:39:43.884172+0000","last_became_active":"2023-09-30T09:41:46.666247+0000","last_became_peered":"2023-09-30T09:41:46.666247+0000","last_unstale":"2023-09-30T09:41:47.657637+0000","last_undegraded":"2023-09-30T09:41:47.657637+0000","last_fullsized":"2023-09-30T09:41:46.649083+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:06.309333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":25,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.993806+0000","last_change":"2023-09-30T09:41:46.671917+0000","last_active":"2023-09-30T09:41:48.993806+0000","last_peered":"2023-09-30T09:41:48.993806+0000","last_clean":"2023-09-30T09:34:30.838310+0000","last_became_active":"2023-09-30T09:41:46.671917+0000","last_became_peered":"2023-09-30T09:41:46.671917+0000","last_unstale":"2023-09-30T09:41:48.993806+0000","last_undegraded":"2023-09-30T09:41:48.993806+0000","last_fullsized":"2023-09-30T09:41:46.651258+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:47:52.096033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"431'6476","reported_seq":9685,"reported_epoch":431,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:41:55.557082+0000","last_change":"2023-09-30T09:41:52.096995+0000","last_active":"2023-09-30T09:41:55.557082+0000","last_peered":"2023-09-30T09:41:55.557082+0000","last_clean":"2023-09-30T09:38:26.554137+0000","last_became_active":"2023-09-30T09:41:51.664835+0000","last_became_peered":"2023-09-30T09:41:51.664835+0000","last_unstale":"2023-09-30T09:41:55.557082+0000","last_undegraded":"2023-09-30T09:41:50.561050+0000","last_fullsized":"2023-09-30T09:41:55.557082+0000","mapping_epoch":429,"log_start":"363'597","ondisk_log_start":"363'597","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5879,"log_dups_size":0,"ondisk_log_size":5879,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117997568,"num_objects":1800,"num_object_clones":0,"num_object_copies":5400,"num_objects_missing_on_primary":16,"num_objects_missing":16,"num_objects_degraded":3173,"num_objects_misplaced":727,"num_objects_unfound":0,"num_objects_dirty":1800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4046,"num_write_kb":187104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1124,"num_bytes_recovered":73662464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,0],"avail_no_missing":["3"],"object_location_counts":[{"shards":"0,2,7","objects":10},{"shards":"0,3","objects":16},{"shards":"0,3,7","objects":2},{"shards":"0,7","objects":1200},{"shards":"3,7","objects":572}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":915,"reported_epoch":431,"state":"active+clean+remapped","last_fresh":"2023-09-30T09:41:48.988365+0000","last_change":"2023-09-30T09:41:48.975029+0000","last_active":"2023-09-30T09:41:48.988365+0000","last_peered":"2023-09-30T09:41:48.988365+0000","last_clean":"2023-09-30T09:41:48.988365+0000","last_became_active":"2023-09-30T09:41:48.974434+0000","last_became_peered":"2023-09-30T09:41:48.974434+0000","last_unstale":"2023-09-30T09:41:48.988365+0000","last_undegraded":"2023-09-30T09:41:48.988365+0000","last_fullsized":"2023-09-30T09:41:48.988365+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:00:48.154091+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":62,"reported_epoch":431,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:41:53.407740+0000","last_change":"2023-09-30T09:41:47.662535+0000","last_active":"2023-09-30T09:41:53.407740+0000","last_peered":"2023-09-30T09:41:53.407740+0000","last_clean":"2023-09-30T09:30:20.310268+0000","last_became_active":"2023-09-30T09:41:47.662157+0000","last_became_peered":"2023-09-30T09:41:47.662157+0000","last_unstale":"2023-09-30T09:41:53.407740+0000","last_undegraded":"2023-09-30T09:41:47.651314+0000","last_fullsized":"2023-09-30T09:41:53.407740+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:12:16.670916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":1,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":441,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990032+0000","last_change":"2023-09-30T09:40:36.597929+0000","last_active":"2023-09-30T09:41:48.990032+0000","last_peered":"2023-09-30T09:41:48.990032+0000","last_clean":"2023-09-30T09:41:48.990032+0000","last_became_active":"2023-09-30T09:40:36.597614+0000","last_became_peered":"2023-09-30T09:40:36.597614+0000","last_unstale":"2023-09-30T09:41:48.990032+0000","last_undegraded":"2023-09-30T09:41:48.990032+0000","last_fullsized":"2023-09-30T09:41:48.990032+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:44:14.764059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":172,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.657935+0000","last_change":"2023-09-30T09:41:46.664903+0000","last_active":"2023-09-30T09:41:47.657935+0000","last_peered":"2023-09-30T09:41:47.657935+0000","last_clean":"2023-09-30T09:41:40.708402+0000","last_became_active":"2023-09-30T09:41:46.664903+0000","last_became_peered":"2023-09-30T09:41:46.664903+0000","last_unstale":"2023-09-30T09:41:47.657935+0000","last_undegraded":"2023-09-30T09:41:47.657935+0000","last_fullsized":"2023-09-30T09:41:46.652060+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:26:42.090950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"432'7899","reported_seq":12025,"reported_epoch":432,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:59.098313+0000","last_change":"2023-09-30T09:40:22.885021+0000","last_active":"2023-09-30T09:41:59.098313+0000","last_peered":"2023-09-30T09:41:59.098313+0000","last_clean":"2023-09-30T09:38:44.556600+0000","last_became_active":"2023-09-30T09:38:46.733175+0000","last_became_peered":"2023-09-30T09:38:46.733175+0000","last_unstale":"2023-09-30T09:41:59.098313+0000","last_undegraded":"2023-09-30T09:40:22.884147+0000","last_fullsized":"2023-09-30T09:38:46.604027+0000","mapping_epoch":372,"log_start":"376'4138","ondisk_log_start":"376'4138","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3761,"log_dups_size":0,"ondisk_log_size":3761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:41:41.383870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227475456,"num_objects":3471,"num_object_clones":0,"num_object_copies":10413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7899,"num_write_kb":363840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2013,"num_bytes_recovered":131923968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1099},{"shards":"1,4","objects":2171}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":850,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.658957+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:41:47.658957+0000","last_peered":"2023-09-30T09:41:47.658957+0000","last_clean":"2023-09-30T09:41:47.658957+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:41:47.658957+0000","last_undegraded":"2023-09-30T09:41:47.658957+0000","last_fullsized":"2023-09-30T09:41:47.658957+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":662,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.660396+0000","last_change":"2023-09-30T09:40:37.590450+0000","last_active":"2023-09-30T09:41:47.660396+0000","last_peered":"2023-09-30T09:41:47.660396+0000","last_clean":"2023-09-30T09:41:47.660396+0000","last_became_active":"2023-09-30T09:40:37.589740+0000","last_became_peered":"2023-09-30T09:40:37.589740+0000","last_unstale":"2023-09-30T09:41:47.660396+0000","last_undegraded":"2023-09-30T09:41:47.660396+0000","last_fullsized":"2023-09-30T09:41:47.660396+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:24:28.314167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":183,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.654573+0000","last_change":"2023-09-30T09:40:36.592442+0000","last_active":"2023-09-30T09:41:47.654573+0000","last_peered":"2023-09-30T09:41:47.654573+0000","last_clean":"2023-09-30T09:41:47.654573+0000","last_became_active":"2023-09-30T09:40:36.592119+0000","last_became_peered":"2023-09-30T09:40:36.592119+0000","last_unstale":"2023-09-30T09:41:47.654573+0000","last_undegraded":"2023-09-30T09:41:47.654573+0000","last_fullsized":"2023-09-30T09:41:47.654573+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:53:12.526007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":330,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.660944+0000","last_change":"2023-09-30T09:41:46.664596+0000","last_active":"2023-09-30T09:41:47.660944+0000","last_peered":"2023-09-30T09:41:47.660944+0000","last_clean":"2023-09-30T09:39:56.895037+0000","last_became_active":"2023-09-30T09:41:46.664596+0000","last_became_peered":"2023-09-30T09:41:46.664596+0000","last_unstale":"2023-09-30T09:41:47.660944+0000","last_undegraded":"2023-09-30T09:41:47.660944+0000","last_fullsized":"2023-09-30T09:41:46.655018+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:30:14.598214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.9","version":"431'6369","reported_seq":10488,"reported_epoch":431,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:57.285235+0000","last_change":"2023-09-30T09:41:49.740351+0000","last_active":"2023-09-30T09:41:57.285235+0000","last_peered":"2023-09-30T09:41:57.285235+0000","last_clean":"2023-09-30T09:39:46.286949+0000","last_became_active":"2023-09-30T09:41:48.983584+0000","last_became_peered":"2023-09-30T09:41:48.983584+0000","last_unstale":"2023-09-30T09:41:57.285235+0000","last_undegraded":"2023-09-30T09:41:48.961738+0000","last_fullsized":"2023-09-30T09:41:47.655058+0000","mapping_epoch":429,"log_start":"363'392","ondisk_log_start":"363'392","created":358,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115769344,"num_objects":1766,"num_object_clones":0,"num_object_copies":5298,"num_objects_missing_on_primary":1518,"num_objects_missing":1518,"num_objects_degraded":3790,"num_objects_misplaced":1012,"num_objects_unfound":0,"num_objects_dirty":1766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3980,"num_write_kb":183904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1334,"num_bytes_recovered":87425024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[2,0],"avail_no_missing":["6"],"object_location_counts":[{"shards":"0,2","objects":527},{"shards":"0,6","objects":1239}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":771,"reported_epoch":430,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:47.655656+0000","last_change":"2023-09-30T09:41:46.664815+0000","last_active":"2023-09-30T09:41:47.655656+0000","last_peered":"2023-09-30T09:41:47.655656+0000","last_clean":"2023-09-30T09:33:08.289591+0000","last_became_active":"2023-09-30T09:41:46.664815+0000","last_became_peered":"2023-09-30T09:41:46.664815+0000","last_unstale":"2023-09-30T09:41:47.655656+0000","last_undegraded":"2023-09-30T09:41:46.649634+0000","last_fullsized":"2023-09-30T09:41:46.649492+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:22.696659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":753,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-30T09:41:46.664259+0000","last_change":"2023-09-30T09:40:01.598664+0000","last_active":"2023-09-30T09:41:46.664259+0000","last_peered":"2023-09-30T09:41:46.664259+0000","last_clean":"2023-09-30T09:41:46.664259+0000","last_became_active":"2023-09-30T09:40:01.598306+0000","last_became_peered":"2023-09-30T09:40:01.598306+0000","last_unstale":"2023-09-30T09:41:46.664259+0000","last_undegraded":"2023-09-30T09:41:46.664259+0000","last_fullsized":"2023-09-30T09:41:46.664259+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:19:20.361905+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":58,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.990999+0000","last_change":"2023-09-30T09:41:46.671397+0000","last_active":"2023-09-30T09:41:48.990999+0000","last_peered":"2023-09-30T09:41:48.990999+0000","last_clean":"2023-09-30T09:39:43.878859+0000","last_became_active":"2023-09-30T09:41:46.671397+0000","last_became_peered":"2023-09-30T09:41:46.671397+0000","last_unstale":"2023-09-30T09:41:48.990999+0000","last_undegraded":"2023-09-30T09:41:48.990999+0000","last_fullsized":"2023-09-30T09:41:46.646145+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:24.500229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":394,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.992174+0000","last_change":"2023-09-30T09:40:36.597443+0000","last_active":"2023-09-30T09:41:48.992174+0000","last_peered":"2023-09-30T09:41:48.992174+0000","last_clean":"2023-09-30T09:41:48.992174+0000","last_became_active":"2023-09-30T09:40:36.596947+0000","last_became_peered":"2023-09-30T09:40:36.596947+0000","last_unstale":"2023-09-30T09:41:48.992174+0000","last_undegraded":"2023-09-30T09:41:48.992174+0000","last_fullsized":"2023-09-30T09:41:48.992174+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:27:17.799153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"431'8015","reported_seq":8108,"reported_epoch":431,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:57.288620+0000","last_change":"2023-09-30T09:41:46.673960+0000","last_active":"2023-09-30T09:41:57.288620+0000","last_peered":"2023-09-30T09:41:57.288620+0000","last_clean":"2023-09-30T09:41:45.592511+0000","last_became_active":"2023-09-30T09:41:46.673960+0000","last_became_peered":"2023-09-30T09:41:46.673960+0000","last_unstale":"2023-09-30T09:41:57.288620+0000","last_undegraded":"2023-09-30T09:41:46.644993+0000","last_fullsized":"2023-09-30T09:41:46.644854+0000","mapping_epoch":428,"log_start":"376'3924","ondisk_log_start":"376'3924","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4091,"log_dups_size":0,"ondisk_log_size":4091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:34:07.162603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224854016,"num_objects":3431,"num_object_clones":0,"num_object_copies":10293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3431,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8015,"num_write_kb":366272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":3431}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":656,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.989646+0000","last_change":"2023-09-30T09:41:46.670639+0000","last_active":"2023-09-30T09:41:48.989646+0000","last_peered":"2023-09-30T09:41:48.989646+0000","last_clean":"2023-09-30T09:41:40.611666+0000","last_became_active":"2023-09-30T09:41:46.670639+0000","last_became_peered":"2023-09-30T09:41:46.670639+0000","last_unstale":"2023-09-30T09:41:48.989646+0000","last_undegraded":"2023-09-30T09:41:48.989646+0000","last_fullsized":"2023-09-30T09:41:46.648088+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":89449,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":839,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-30T09:41:46.664169+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:41:46.664169+0000","last_peered":"2023-09-30T09:41:46.664169+0000","last_clean":"2023-09-30T09:41:46.664169+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:41:46.664169+0000","last_undegraded":"2023-09-30T09:41:46.664169+0000","last_fullsized":"2023-09-30T09:41:46.664169+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:16:49.259733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":34,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.987780+0000","last_change":"2023-09-30T09:41:46.647353+0000","last_active":"2023-09-30T09:41:48.987780+0000","last_peered":"2023-09-30T09:41:48.987780+0000","last_clean":"2023-09-30T09:41:40.602050+0000","last_became_active":"2023-09-30T09:41:46.647353+0000","last_became_peered":"2023-09-30T09:41:46.647353+0000","last_unstale":"2023-09-30T09:41:48.987780+0000","last_undegraded":"2023-09-30T09:41:48.987780+0000","last_fullsized":"2023-09-30T09:41:46.642022+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:49:17.738913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":118,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.658650+0000","last_change":"2023-09-30T09:39:58.580826+0000","last_active":"2023-09-30T09:41:47.658650+0000","last_peered":"2023-09-30T09:41:47.658650+0000","last_clean":"2023-09-30T09:41:47.658650+0000","last_became_active":"2023-09-30T09:39:58.580493+0000","last_became_peered":"2023-09-30T09:39:58.580493+0000","last_unstale":"2023-09-30T09:41:47.658650+0000","last_undegraded":"2023-09-30T09:41:47.658650+0000","last_fullsized":"2023-09-30T09:41:47.658650+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:09:47.409633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.f","version":"431'7907","reported_seq":9770,"reported_epoch":431,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:56.944812+0000","last_change":"2023-09-30T09:40:36.615864+0000","last_active":"2023-09-30T09:41:56.944812+0000","last_peered":"2023-09-30T09:41:56.944812+0000","last_clean":"2023-09-30T09:38:44.567377+0000","last_became_active":"2023-09-30T09:38:46.789998+0000","last_became_peered":"2023-09-30T09:38:46.789998+0000","last_unstale":"2023-09-30T09:41:56.944812+0000","last_undegraded":"2023-09-30T09:38:46.692264+0000","last_fullsized":"2023-09-30T09:38:46.597825+0000","mapping_epoch":372,"log_start":"376'3768","ondisk_log_start":"376'3768","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4139,"log_dups_size":0,"ondisk_log_size":4139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:32:25.586705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7907,"num_write_kb":364480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":882,"num_bytes_recovered":57802752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":1076},{"shards":"1,2,3","objects":1076}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":707,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.989876+0000","last_change":"2023-09-30T09:40:01.612760+0000","last_active":"2023-09-30T09:41:48.989876+0000","last_peered":"2023-09-30T09:41:48.989876+0000","last_clean":"2023-09-30T09:41:48.989876+0000","last_became_active":"2023-09-30T09:40:01.612373+0000","last_became_peered":"2023-09-30T09:40:01.612373+0000","last_unstale":"2023-09-30T09:41:48.989876+0000","last_undegraded":"2023-09-30T09:41:48.989876+0000","last_fullsized":"2023-09-30T09:41:48.989876+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:30:07.640092+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":969,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.661787+0000","last_change":"2023-09-30T09:41:40.705877+0000","last_active":"2023-09-30T09:41:47.661787+0000","last_peered":"2023-09-30T09:41:47.661787+0000","last_clean":"2023-09-30T09:41:47.661787+0000","last_became_active":"2023-09-30T09:41:40.705251+0000","last_became_peered":"2023-09-30T09:41:40.705251+0000","last_unstale":"2023-09-30T09:41:47.661787+0000","last_undegraded":"2023-09-30T09:41:47.661787+0000","last_fullsized":"2023-09-30T09:41:47.661787+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:14:59.140482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":164,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.657644+0000","last_change":"2023-09-30T09:41:46.666760+0000","last_active":"2023-09-30T09:41:47.657644+0000","last_peered":"2023-09-30T09:41:47.657644+0000","last_clean":"2023-09-30T09:41:40.708228+0000","last_became_active":"2023-09-30T09:41:46.666760+0000","last_became_peered":"2023-09-30T09:41:46.666760+0000","last_unstale":"2023-09-30T09:41:47.657644+0000","last_undegraded":"2023-09-30T09:41:47.657644+0000","last_fullsized":"2023-09-30T09:41:46.653531+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:27:45.514731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":34,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.987208+0000","last_change":"2023-09-30T09:41:46.660464+0000","last_active":"2023-09-30T09:41:48.987208+0000","last_peered":"2023-09-30T09:41:48.987208+0000","last_clean":"2023-09-30T09:41:40.702643+0000","last_became_active":"2023-09-30T09:41:46.660464+0000","last_became_peered":"2023-09-30T09:41:46.660464+0000","last_unstale":"2023-09-30T09:41:48.987208+0000","last_undegraded":"2023-09-30T09:41:48.987208+0000","last_fullsized":"2023-09-30T09:41:46.643791+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:46:13.966613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"431'8223","reported_seq":17882,"reported_epoch":431,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-30T09:41:55.493942+0000","last_change":"2023-09-30T09:41:52.094609+0000","last_active":"2023-09-30T09:41:55.493942+0000","last_peered":"2023-09-30T09:41:55.493942+0000","last_clean":"2023-09-30T09:38:26.563448+0000","last_became_active":"2023-09-30T09:41:46.663061+0000","last_became_peered":"2023-09-30T09:41:46.663061+0000","last_unstale":"2023-09-30T09:41:55.493942+0000","last_undegraded":"2023-09-30T09:41:46.649546+0000","last_fullsized":"2023-09-30T09:41:46.649385+0000","mapping_epoch":428,"log_start":"375'3719","ondisk_log_start":"375'3719","created":358,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":4504,"log_dups_size":0,"ondisk_log_size":4504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:45:07.463276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234160128,"num_objects":3573,"num_object_clones":0,"num_object_copies":10719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2326,"num_objects_misplaced":3573,"num_objects_unfound":0,"num_objects_dirty":3573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8223,"num_write_kb":377472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4764,"num_bytes_recovered":312213504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":3573}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":931,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.991825+0000","last_change":"2023-09-30T09:41:46.670957+0000","last_active":"2023-09-30T09:41:48.991825+0000","last_peered":"2023-09-30T09:41:48.991825+0000","last_clean":"2023-09-30T09:41:40.612705+0000","last_became_active":"2023-09-30T09:41:46.670957+0000","last_became_peered":"2023-09-30T09:41:46.670957+0000","last_unstale":"2023-09-30T09:41:48.991825+0000","last_undegraded":"2023-09-30T09:41:48.991825+0000","last_fullsized":"2023-09-30T09:41:46.646545+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:32:35.538952+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":477,"reported_epoch":430,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:47.661236+0000","last_change":"2023-09-30T09:41:46.661262+0000","last_active":"2023-09-30T09:41:47.661236+0000","last_peered":"2023-09-30T09:41:47.661236+0000","last_clean":"2023-09-30T09:41:40.606869+0000","last_became_active":"2023-09-30T09:41:46.661262+0000","last_became_peered":"2023-09-30T09:41:46.661262+0000","last_unstale":"2023-09-30T09:41:47.661236+0000","last_undegraded":"2023-09-30T09:41:46.650632+0000","last_fullsized":"2023-09-30T09:41:46.650532+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:57:30.278438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1125,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990602+0000","last_change":"2023-09-30T09:41:47.676784+0000","last_active":"2023-09-30T09:41:48.990602+0000","last_peered":"2023-09-30T09:41:48.990602+0000","last_clean":"2023-09-30T09:41:48.990602+0000","last_became_active":"2023-09-30T09:41:47.676217+0000","last_became_peered":"2023-09-30T09:41:47.676217+0000","last_unstale":"2023-09-30T09:41:48.990602+0000","last_undegraded":"2023-09-30T09:41:48.990602+0000","last_fullsized":"2023-09-30T09:41:48.990602+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:45:06.021179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":471,"reported_epoch":429,"state":"active+undersized","last_fresh":"2023-09-30T09:41:46.677626+0000","last_change":"2023-09-30T09:41:46.677277+0000","last_active":"2023-09-30T09:41:46.677626+0000","last_peered":"2023-09-30T09:41:46.677626+0000","last_clean":"2023-09-30T09:41:40.606047+0000","last_became_active":"2023-09-30T09:41:46.677277+0000","last_became_peered":"2023-09-30T09:41:46.677277+0000","last_unstale":"2023-09-30T09:41:46.677626+0000","last_undegraded":"2023-09-30T09:41:46.677626+0000","last_fullsized":"2023-09-30T09:41:46.663869+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:19:56.826372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"431'8221","reported_seq":13096,"reported_epoch":431,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:55.540139+0000","last_change":"2023-09-30T09:41:46.673038+0000","last_active":"2023-09-30T09:41:55.540139+0000","last_peered":"2023-09-30T09:41:55.540139+0000","last_clean":"2023-09-30T09:41:45.566023+0000","last_became_active":"2023-09-30T09:41:46.673038+0000","last_became_peered":"2023-09-30T09:41:46.673038+0000","last_unstale":"2023-09-30T09:41:55.540139+0000","last_undegraded":"2023-09-30T09:41:46.658147+0000","last_fullsized":"2023-09-30T09:41:46.658034+0000","mapping_epoch":428,"log_start":"376'4357","ondisk_log_start":"376'4357","created":358,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":3864,"log_dups_size":0,"ondisk_log_size":3864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:55:47.386879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231604248,"num_objects":3535,"num_object_clones":0,"num_object_copies":10605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3535,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3535,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8221,"num_write_kb":376129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2381,"num_bytes_recovered":129368064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":3535}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":844,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.659482+0000","last_change":"2023-09-30T09:41:46.663035+0000","last_active":"2023-09-30T09:41:47.659482+0000","last_peered":"2023-09-30T09:41:47.659482+0000","last_clean":"2023-09-30T09:38:16.770132+0000","last_became_active":"2023-09-30T09:41:46.663035+0000","last_became_peered":"2023-09-30T09:41:46.663035+0000","last_unstale":"2023-09-30T09:41:47.659482+0000","last_undegraded":"2023-09-30T09:41:47.659482+0000","last_fullsized":"2023-09-30T09:41:46.649999+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:01:18.331712+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":760,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.661468+0000","last_change":"2023-09-30T09:39:58.627875+0000","last_active":"2023-09-30T09:41:47.661468+0000","last_peered":"2023-09-30T09:41:47.661468+0000","last_clean":"2023-09-30T09:41:47.661468+0000","last_became_active":"2023-09-30T09:39:47.851060+0000","last_became_peered":"2023-09-30T09:39:47.851060+0000","last_unstale":"2023-09-30T09:41:47.661468+0000","last_undegraded":"2023-09-30T09:41:47.661468+0000","last_fullsized":"2023-09-30T09:41:47.661468+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:42:52.871435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":147,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990814+0000","last_change":"2023-09-30T09:39:58.612882+0000","last_active":"2023-09-30T09:41:48.990814+0000","last_peered":"2023-09-30T09:41:48.990814+0000","last_clean":"2023-09-30T09:41:48.990814+0000","last_became_active":"2023-09-30T09:39:58.581004+0000","last_became_peered":"2023-09-30T09:39:58.581004+0000","last_unstale":"2023-09-30T09:41:48.990814+0000","last_undegraded":"2023-09-30T09:41:48.990814+0000","last_fullsized":"2023-09-30T09:41:48.990814+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:44.825201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":410,"reported_epoch":431,"state":"active+undersized","last_fresh":"2023-09-30T09:41:48.989131+0000","last_change":"2023-09-30T09:41:46.654066+0000","last_active":"2023-09-30T09:41:48.989131+0000","last_peered":"2023-09-30T09:41:48.989131+0000","last_clean":"2023-09-30T09:41:40.610455+0000","last_became_active":"2023-09-30T09:41:46.654066+0000","last_became_peered":"2023-09-30T09:41:46.654066+0000","last_unstale":"2023-09-30T09:41:48.989131+0000","last_undegraded":"2023-09-30T09:41:48.989131+0000","last_fullsized":"2023-09-30T09:41:46.644312+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:17:42.095570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"432'6619","reported_seq":10218,"reported_epoch":432,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:58.891974+0000","last_change":"2023-09-30T09:41:46.674662+0000","last_active":"2023-09-30T09:41:58.891974+0000","last_peered":"2023-09-30T09:41:58.891974+0000","last_clean":"2023-09-30T09:41:44.914904+0000","last_became_active":"2023-09-30T09:41:46.674662+0000","last_became_peered":"2023-09-30T09:41:46.674662+0000","last_unstale":"2023-09-30T09:41:58.891974+0000","last_undegraded":"2023-09-30T09:41:46.650328+0000","last_fullsized":"2023-09-30T09:41:46.650216+0000","mapping_epoch":428,"log_start":"363'900","ondisk_log_start":"363'900","created":379,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113737728,"num_objects":1736,"num_object_clones":0,"num_object_copies":5208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1736,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4144,"num_write_kb":188128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1743,"num_bytes_recovered":114196480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1736}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":794,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.657576+0000","last_change":"2023-09-30T09:41:46.657853+0000","last_active":"2023-09-30T09:41:47.657576+0000","last_peered":"2023-09-30T09:41:47.657576+0000","last_clean":"2023-09-30T09:41:40.605966+0000","last_became_active":"2023-09-30T09:41:46.657853+0000","last_became_peered":"2023-09-30T09:41:46.657853+0000","last_unstale":"2023-09-30T09:41:47.657576+0000","last_undegraded":"2023-09-30T09:41:47.657576+0000","last_fullsized":"2023-09-30T09:41:46.651290+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:36:39.421111+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":257,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990116+0000","last_change":"2023-09-30T09:39:58.576981+0000","last_active":"2023-09-30T09:41:48.990116+0000","last_peered":"2023-09-30T09:41:48.990116+0000","last_clean":"2023-09-30T09:41:48.990116+0000","last_became_active":"2023-09-30T09:39:58.576650+0000","last_became_peered":"2023-09-30T09:39:58.576650+0000","last_unstale":"2023-09-30T09:41:48.990116+0000","last_undegraded":"2023-09-30T09:41:48.990116+0000","last_fullsized":"2023-09-30T09:41:48.990116+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:50:03.041046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":126,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-30T09:41:47.657501+0000","last_change":"2023-09-30T09:39:58.573938+0000","last_active":"2023-09-30T09:41:47.657501+0000","last_peered":"2023-09-30T09:41:47.657501+0000","last_clean":"2023-09-30T09:41:47.657501+0000","last_became_active":"2023-09-30T09:39:58.573645+0000","last_became_peered":"2023-09-30T09:39:58.573645+0000","last_unstale":"2023-09-30T09:41:47.657501+0000","last_undegraded":"2023-09-30T09:41:47.657501+0000","last_fullsized":"2023-09-30T09:41:47.657501+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:24:06.400443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"431'6490","reported_seq":10000,"reported_epoch":431,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:56.842906+0000","last_change":"2023-09-30T09:41:49.756678+0000","last_active":"2023-09-30T09:41:56.842906+0000","last_peered":"2023-09-30T09:41:56.842906+0000","last_clean":"2023-09-30T09:38:44.567728+0000","last_became_active":"2023-09-30T09:39:53.183491+0000","last_became_peered":"2023-09-30T09:39:53.183491+0000","last_unstale":"2023-09-30T09:41:56.842906+0000","last_undegraded":"2023-09-30T09:41:49.748228+0000","last_fullsized":"2023-09-30T09:39:48.816158+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6490,"log_dups_size":0,"ondisk_log_size":6490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113147904,"num_objects":1727,"num_object_clones":0,"num_object_copies":5181,"num_objects_missing_on_primary":469,"num_objects_missing":469,"num_objects_degraded":2398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4055,"num_write_kb":184992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1226,"num_bytes_recovered":80347136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":73},{"shards":"1,2,6","objects":1185},{"shards":"1,3,6","objects":469}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":641,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.659973+0000","last_change":"2023-09-30T09:41:46.674581+0000","last_active":"2023-09-30T09:41:47.659973+0000","last_peered":"2023-09-30T09:41:47.659973+0000","last_clean":"2023-09-30T09:41:40.607157+0000","last_became_active":"2023-09-30T09:41:46.674581+0000","last_became_peered":"2023-09-30T09:41:46.674581+0000","last_unstale":"2023-09-30T09:41:47.659973+0000","last_undegraded":"2023-09-30T09:41:47.659973+0000","last_fullsized":"2023-09-30T09:41:46.650948+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:30.845794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:30.845794+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:41:14.323526+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":24,"reported_epoch":431,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:48.990215+0000","last_change":"2023-09-30T09:41:46.670883+0000","last_active":"2023-09-30T09:41:48.990215+0000","last_peered":"2023-09-30T09:41:48.990215+0000","last_clean":"2023-09-30T09:12:09.951985+0000","last_became_active":"2023-09-30T09:41:46.670883+0000","last_became_peered":"2023-09-30T09:41:46.670883+0000","last_unstale":"2023-09-30T09:41:48.990215+0000","last_undegraded":"2023-09-30T09:41:46.648365+0000","last_fullsized":"2023-09-30T09:41:46.648261+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:27:03.780167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":136,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.989388+0000","last_change":"2023-09-30T09:38:45.595684+0000","last_active":"2023-09-30T09:41:48.989388+0000","last_peered":"2023-09-30T09:41:48.989388+0000","last_clean":"2023-09-30T09:41:48.989388+0000","last_became_active":"2023-09-30T09:38:45.595231+0000","last_became_peered":"2023-09-30T09:38:45.595231+0000","last_unstale":"2023-09-30T09:41:48.989388+0000","last_undegraded":"2023-09-30T09:41:48.989388+0000","last_fullsized":"2023-09-30T09:41:48.989388+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:43:29.805772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"432'6549","reported_seq":14964,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-30T09:41:57.982635+0000","last_change":"2023-09-30T09:41:06.618361+0000","last_active":"2023-09-30T09:41:57.982635+0000","last_peered":"2023-09-30T09:41:57.982635+0000","last_clean":"2023-09-30T09:41:57.982635+0000","last_became_active":"2023-09-30T09:41:06.607479+0000","last_became_peered":"2023-09-30T09:41:06.607479+0000","last_unstale":"2023-09-30T09:41:57.982635+0000","last_undegraded":"2023-09-30T09:41:57.982635+0000","last_fullsized":"2023-09-30T09:41:57.982635+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":379,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118358016,"num_objects":1806,"num_object_clones":0,"num_object_copies":5418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4108,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3251,"num_bytes_recovered":194117632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":24,"reported_epoch":431,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:48.991963+0000","last_change":"2023-09-30T09:41:46.661514+0000","last_active":"2023-09-30T09:41:48.991963+0000","last_peered":"2023-09-30T09:41:48.991963+0000","last_clean":"2023-09-30T09:12:09.952081+0000","last_became_active":"2023-09-30T09:41:46.661514+0000","last_became_peered":"2023-09-30T09:41:46.661514+0000","last_unstale":"2023-09-30T09:41:48.991963+0000","last_undegraded":"2023-09-30T09:41:46.647100+0000","last_fullsized":"2023-09-30T09:41:46.646992+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:35:33.003714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":156,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.991283+0000","last_change":"2023-09-30T09:40:36.597938+0000","last_active":"2023-09-30T09:41:48.991283+0000","last_peered":"2023-09-30T09:41:48.991283+0000","last_clean":"2023-09-30T09:41:48.991283+0000","last_became_active":"2023-09-30T09:40:36.597684+0000","last_became_peered":"2023-09-30T09:40:36.597684+0000","last_unstale":"2023-09-30T09:41:48.991283+0000","last_undegraded":"2023-09-30T09:41:48.991283+0000","last_fullsized":"2023-09-30T09:41:48.991283+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:15:35.036544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"432'6414","reported_seq":8971,"reported_epoch":432,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:41:59.130771+0000","last_change":"2023-09-30T09:40:36.658459+0000","last_active":"2023-09-30T09:41:59.130771+0000","last_peered":"2023-09-30T09:41:59.130771+0000","last_clean":"2023-09-30T09:39:46.536373+0000","last_became_active":"2023-09-30T09:40:36.630807+0000","last_became_peered":"2023-09-30T09:40:36.630807+0000","last_unstale":"2023-09-30T09:41:59.130771+0000","last_undegraded":"2023-09-30T09:40:36.594542+0000","last_fullsized":"2023-09-30T09:40:36.580057+0000","mapping_epoch":413,"log_start":"363'541","ondisk_log_start":"363'541","created":379,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5873,"log_dups_size":0,"ondisk_log_size":5873,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113508352,"num_objects":1732,"num_object_clones":0,"num_object_copies":5196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4012,"num_write_kb":183808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":1707},{"shards":"4,7","objects":6}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":728,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.659523+0000","last_change":"2023-09-30T09:41:46.663884+0000","last_active":"2023-09-30T09:41:47.659523+0000","last_peered":"2023-09-30T09:41:47.659523+0000","last_clean":"2023-09-30T09:41:40.607524+0000","last_became_active":"2023-09-30T09:41:46.663884+0000","last_became_peered":"2023-09-30T09:41:46.663884+0000","last_unstale":"2023-09-30T09:41:47.659523+0000","last_undegraded":"2023-09-30T09:41:47.659523+0000","last_fullsized":"2023-09-30T09:41:46.654705+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:54.087577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:54.087577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":334,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.990052+0000","last_change":"2023-09-30T09:39:47.874169+0000","last_active":"2023-09-30T09:41:48.990052+0000","last_peered":"2023-09-30T09:41:48.990052+0000","last_clean":"2023-09-30T09:41:48.990052+0000","last_became_active":"2023-09-30T09:39:47.873772+0000","last_became_peered":"2023-09-30T09:39:47.873772+0000","last_unstale":"2023-09-30T09:41:48.990052+0000","last_undegraded":"2023-09-30T09:41:48.990052+0000","last_fullsized":"2023-09-30T09:41:48.990052+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:38:30.789860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":398,"reported_epoch":430,"state":"active+undersized","last_fresh":"2023-09-30T09:41:47.660258+0000","last_change":"2023-09-30T09:41:46.664058+0000","last_active":"2023-09-30T09:41:47.660258+0000","last_peered":"2023-09-30T09:41:47.660258+0000","last_clean":"2023-09-30T09:41:40.703070+0000","last_became_active":"2023-09-30T09:41:46.664058+0000","last_became_peered":"2023-09-30T09:41:46.664058+0000","last_unstale":"2023-09-30T09:41:47.660258+0000","last_undegraded":"2023-09-30T09:41:47.660258+0000","last_fullsized":"2023-09-30T09:41:46.645141+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:46:47.143250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"432'6483","reported_seq":6581,"reported_epoch":432,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:59.237058+0000","last_change":"2023-09-30T09:41:46.671242+0000","last_active":"2023-09-30T09:41:59.237058+0000","last_peered":"2023-09-30T09:41:59.237058+0000","last_clean":"2023-09-30T09:41:45.571328+0000","last_became_active":"2023-09-30T09:41:46.671242+0000","last_became_peered":"2023-09-30T09:41:46.671242+0000","last_unstale":"2023-09-30T09:41:59.237058+0000","last_undegraded":"2023-09-30T09:41:46.647746+0000","last_fullsized":"2023-09-30T09:41:46.647630+0000","mapping_epoch":428,"log_start":"363'680","ondisk_log_start":"363'680","created":379,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:37:55.517114+0000","last_clean_scrub_stamp":"2023-09-30T09:37:55.517114+0000","objects_scrubbed":0,"log_size":5803,"log_dups_size":0,"ondisk_log_size":5803,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109248512,"num_objects":1667,"num_object_clones":0,"num_object_copies":5001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1667,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4077,"num_write_kb":183808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":1667}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":807,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.991676+0000","last_change":"2023-09-30T09:38:48.664091+0000","last_active":"2023-09-30T09:41:48.991676+0000","last_peered":"2023-09-30T09:41:48.991676+0000","last_clean":"2023-09-30T09:41:48.991676+0000","last_became_active":"2023-09-30T09:38:48.663723+0000","last_became_peered":"2023-09-30T09:38:48.663723+0000","last_unstale":"2023-09-30T09:41:48.991676+0000","last_undegraded":"2023-09-30T09:41:48.991676+0000","last_fullsized":"2023-09-30T09:41:48.991676+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:57.093254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:57.093254+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023927179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":33,"reported_epoch":430,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:41:47.661330+0000","last_change":"2023-09-30T09:41:46.659556+0000","last_active":"2023-09-30T09:41:47.661330+0000","last_peered":"2023-09-30T09:41:47.661330+0000","last_clean":"2023-09-30T09:35:16.784726+0000","last_became_active":"2023-09-30T09:41:46.659556+0000","last_became_peered":"2023-09-30T09:41:46.659556+0000","last_unstale":"2023-09-30T09:41:47.661330+0000","last_undegraded":"2023-09-30T09:41:46.643945+0000","last_fullsized":"2023-09-30T09:41:46.643799+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:17.629308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":427,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-30T09:41:48.988399+0000","last_change":"2023-09-30T09:39:47.867540+0000","last_active":"2023-09-30T09:41:48.988399+0000","last_peered":"2023-09-30T09:41:48.988399+0000","last_clean":"2023-09-30T09:41:48.988399+0000","last_became_active":"2023-09-30T09:39:47.867178+0000","last_became_peered":"2023-09-30T09:39:47.867178+0000","last_unstale":"2023-09-30T09:41:48.988399+0000","last_undegraded":"2023-09-30T09:41:48.988399+0000","last_fullsized":"2023-09-30T09:41:48.988399+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:34:51.232357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":78,"acting":78,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":79,"num_bytes_recovered":16048,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21159,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":82,"acting":82,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":3656450072,"num_objects":55794,"num_object_clones":0,"num_object_copies":167382,"num_objects_missing_on_primary":2770,"num_objects_missing":2770,"num_objects_degraded":45065,"num_objects_misplaced":7218,"num_objects_unfound":0,"num_objects_dirty":55794,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129098,"num_write_kb":5916481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41519,"num_bytes_recovered":2537947136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13438365696,"data_stored":13438353480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146689,"ondisk_log_size":146689,"up":63,"acting":62,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":379916,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":54,"num_read_kb":98,"num_write":26,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":72,"acting":73,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":2,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":196,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234362,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1588504,"kb_used_data":1144592,"kb_used_omap":0,"kb_used_meta":443904,"kb_avail":92156648,"statfs":{"total":95995035648,"available":94368407552,"internally_reserved":0,"allocated":1172062208,"data_stored":1170575299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":454557696},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":5.6319999999999997,"15min":2.9809999999999999},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9350000000000001,"5min":215.416,"15min":215.416},"last":1.115},{"interface":"front","average":{"1min":0.80200000000000005,"5min":5.5490000000000004,"15min":2.855},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.8289999999999997,"5min":215.041,"15min":215.041},"last":1.1259999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":2.032,"15min":2.6480000000000001},"min":{"1min":0.29099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.2070000000000007,"5min":89.308000000000007,"15min":89.308000000000007},"last":12.225},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.1360000000000001,"15min":3.0760000000000001},"min":{"1min":0.34699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":8.2319999999999993,"5min":89.325999999999993,"15min":89.325999999999993},"last":0.34599999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:41:31 2023","interfaces":[{"interface":"back","average":{"1min":9.423,"5min":5.8360000000000003,"15min":3.2320000000000002},"min":{"1min":0.33700000000000002,"5min":0.311,"15min":0.24299999999999999},"max":{"1min":117.14,"5min":117.14,"15min":117.14},"last":0.501},{"interface":"front","average":{"1min":9.0540000000000003,"5min":5.5970000000000004,"15min":3.137},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":117.044,"5min":117.044,"15min":117.044},"last":1.0369999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":9.0980000000000008,"15min":7.8339999999999996},"min":{"1min":0.34499999999999997,"5min":0.34399999999999997,"15min":0.28899999999999998},"max":{"1min":6.8289999999999997,"5min":277.57799999999997,"15min":277.57799999999997},"last":1.171},{"interface":"front","average":{"1min":1.0840000000000001,"5min":9.2609999999999992,"15min":7.2789999999999999},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.6840000000000002,"5min":278.69299999999998,"15min":278.69299999999998},"last":1.0089999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:41:31 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":2.6909999999999998,"15min":1.651},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.23000000000000001},"max":{"1min":1.091,"5min":175.72,"15min":175.72},"last":1.0880000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":2.573,"15min":1.5149999999999999},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.251},"max":{"1min":2.1349999999999998,"5min":179.048,"15min":179.048},"last":0.95199999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:41:31 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":3.0529999999999999,"15min":1.8520000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.24099999999999999},"max":{"1min":1.1899999999999999,"5min":188.41,"15min":188.41},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.48999999999999999,"5min":2.976,"15min":1.8480000000000001},"min":{"1min":0.38600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.63900000000000001,"5min":188.43000000000001,"15min":188.43000000000001},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:41:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":4.4160000000000004,"15min":2.3610000000000002},"min":{"1min":0.439,"5min":0.33200000000000002,"15min":0.32400000000000001},"max":{"1min":0.98999999999999999,"5min":206.244,"15min":206.244},"last":1.054},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.2350000000000003,"15min":2.278},"min":{"1min":0.375,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":2.2919999999999998,"5min":202.78800000000001,"15min":202.78800000000001},"last":0.98399999999999999}]}]},{"osd":6,"up_from":0,"seq":197568496001,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":301,"seq":1292785156198,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2009492,"kb_used_data":1581260,"kb_used_omap":9,"kb_used_meta":428214,"kb_avail":91735660,"statfs":{"total":95995035648,"available":93937315840,"internally_reserved":0,"allocated":1619210240,"data_stored":1616754655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":438491154},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:41:48 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.778,"15min":3.1389999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":3.9900000000000002,"5min":59.115000000000002,"15min":138.43799999999999},"last":11.297000000000001},{"interface":"front","average":{"1min":0.91200000000000003,"5min":2.242,"15min":3.2599999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.0339999999999998,"5min":51.432000000000002,"15min":138.482},"last":1.3500000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:41:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":2.0270000000000001,"15min":2.544},"min":{"1min":0.26700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":32.960000000000001,"5min":32.960000000000001,"15min":32.960000000000001},"last":12.308999999999999},{"interface":"front","average":{"1min":3.016,"5min":2.4409999999999998,"15min":2.7450000000000001},"min":{"1min":0.33300000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":33.244999999999997,"5min":59.078000000000003,"15min":59.078000000000003},"last":0.51400000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:41:48 2023","interfaces":[{"interface":"back","average":{"1min":1.99,"5min":4.3639999999999999,"15min":7.2489999999999997},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.032,"5min":191.07300000000001,"15min":191.07300000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.264,"5min":3.8839999999999999,"15min":7.4429999999999996},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":5.6280000000000001,"5min":191.11500000000001,"15min":191.11500000000001},"last":12.417999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:41:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":3.1960000000000002,"15min":2.4289999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.4660000000000002,"5min":142.77099999999999,"15min":142.77099999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.081,"5min":4.0099999999999998,"15min":2.3380000000000001},"min":{"1min":0.35499999999999998,"5min":0.311,"15min":0.311},"max":{"1min":6.415,"5min":144.898,"15min":144.898},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":2.1720000000000002,"15min":2.1699999999999999},"min":{"1min":0.41999999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.3580000000000001,"5min":24.25,"15min":24.25},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.755,"5min":2.7930000000000001,"15min":2.0390000000000001},"min":{"1min":0.374,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.669,"5min":58.999000000000002,"15min":58.999000000000002},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 30 09:41:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":3.633,"15min":10.75},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.1360000000000001,"5min":96.719999999999999,"15min":209.221},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.242,"5min":3.7959999999999998,"15min":10.673},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.1639999999999997,"5min":96.356999999999999,"15min":208.43600000000001},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:41:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.9410000000000001,"15min":7.5830000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.9219999999999997,"5min":15.587,"15min":165.59399999999999},"last":9.298},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.871,"15min":7.7060000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":4.976,"5min":15.226000000000001,"15min":165.833},"last":9.3399999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926097,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2414404,"kb_used_data":1880760,"kb_used_omap":5,"kb_used_meta":533626,"kb_avail":91330748,"statfs":{"total":95995035648,"available":93522685952,"internally_reserved":0,"allocated":1925898240,"data_stored":1923237160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":546433277},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:41:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8929999999999998,"5min":1.538,"15min":2.5329999999999999},"min":{"1min":0.20599999999999999,"5min":0.191,"15min":0.17799999999999999},"max":{"1min":34.872999999999998,"5min":34.872999999999998,"15min":34.872999999999998},"last":0.28100000000000003},{"interface":"front","average":{"1min":3.4009999999999998,"5min":1.988,"15min":6.3890000000000002},"min":{"1min":0.23300000000000001,"5min":0.214,"15min":0.214},"max":{"1min":34.968000000000004,"5min":34.968000000000004,"15min":138.92400000000001},"last":0.318}]},{"osd":2,"last update":"Sat Sep 30 09:41:35 2023","interfaces":[{"interface":"back","average":{"1min":4.4299999999999997,"5min":2.9180000000000001,"15min":2.3380000000000001},"min":{"1min":0.27800000000000002,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":58.802,"5min":58.802,"15min":58.802},"last":127.73399999999999},{"interface":"front","average":{"1min":4.7389999999999999,"5min":2.976,"15min":2.5350000000000001},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":58.843000000000004,"5min":58.843000000000004,"15min":58.843000000000004},"last":127.69}]},{"osd":3,"last update":"Sat Sep 30 09:41:02 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.2160000000000002,"15min":4.3929999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.221},"max":{"1min":3.6339999999999999,"5min":16.271000000000001,"15min":218.06999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.256,"5min":1.877,"15min":3.5600000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.245},"max":{"1min":5.5259999999999998,"5min":14.987,"15min":218.63200000000001},"last":0.38}]},{"osd":4,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":1.9119999999999999,"15min":1.901},"min":{"1min":0.36099999999999999,"5min":0.35299999999999998,"15min":0.29099999999999998},"max":{"1min":7.3150000000000004,"5min":10.614000000000001,"15min":121.486},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.927,"15min":2.7829999999999999},"min":{"1min":0.35299999999999998,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":5.8979999999999997,"5min":17.044,"15min":136.262},"last":0.39300000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":1.97,"15min":3.1469999999999998},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":2.2320000000000002,"5min":16.995999999999999,"15min":136.34},"last":2.5859999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.5780000000000001,"15min":2.3090000000000002},"min":{"1min":0.44400000000000001,"5min":0.377,"15min":0.33700000000000002},"max":{"1min":5.3949999999999996,"5min":11.991,"15min":135.381},"last":2.968}]},{"osd":6,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.669,"15min":3.3599999999999999},"min":{"1min":0.34899999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":6.8579999999999997,"5min":12.021000000000001,"15min":214.785},"last":0.432},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.1840000000000002,"15min":3.4780000000000002},"min":{"1min":0.41999999999999998,"5min":0.40300000000000002,"15min":0.35399999999999998},"max":{"1min":7.3639999999999999,"5min":10.695,"15min":214.93899999999999},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.744,"15min":3.3079999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.373},"max":{"1min":5.492,"5min":12.048,"15min":150.96899999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.444,"5min":2.1589999999999998,"15min":3.2919999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.1699999999999999,"5min":12.153,"15min":150.328},"last":0.41799999999999998}]}]},{"osd":2,"up_from":104,"seq":446676599038,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2927472,"kb_used_data":2320296,"kb_used_omap":5,"kb_used_meta":607162,"kb_avail":90817680,"statfs":{"total":95995035648,"available":92997304320,"internally_reserved":0,"allocated":2375983104,"data_stored":2373561353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":621734541},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:41:38 2023","interfaces":[{"interface":"back","average":{"1min":4.6699999999999999,"5min":1.7949999999999999,"15min":2.343},"min":{"1min":0.217,"5min":0.20499999999999999,"15min":0.187},"max":{"1min":76.647000000000006,"5min":76.647000000000006,"15min":77.379000000000005},"last":1.8560000000000001},{"interface":"front","average":{"1min":4.4809999999999999,"5min":2.0699999999999998,"15min":2.3140000000000001},"min":{"1min":0.26400000000000001,"5min":0.217,"15min":0.19700000000000001},"max":{"1min":77.444000000000003,"5min":77.444000000000003,"15min":77.444000000000003},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:41:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.6970000000000001,"15min":3.3679999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.8559999999999999,"5min":37.701999999999998,"15min":64.412999999999997},"last":4.3150000000000004},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.6990000000000001,"15min":2.9609999999999999},"min":{"1min":0.30399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.3280000000000003,"5min":37.728999999999999,"15min":63.247999999999998},"last":0.52200000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":4.7489999999999997,"15min":3.738},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28000000000000003},"max":{"1min":4.7619999999999996,"5min":134.078,"15min":134.078},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.0820000000000001,"5min":4.8579999999999997,"15min":3.8029999999999999},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.22500000000000001},"max":{"1min":4.7850000000000001,"5min":135.73400000000001,"15min":135.73400000000001},"last":1.9079999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":1.794,"15min":1.6459999999999999},"min":{"1min":0.34599999999999997,"5min":0.308,"15min":0.28999999999999998},"max":{"1min":2.1440000000000001,"5min":83.012,"15min":83.012},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.8540000000000001,"15min":1.7929999999999999},"min":{"1min":0.34499999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.5899999999999999,"5min":81.335999999999999,"15min":81.335999999999999},"last":1.9379999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:41:38 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.5189999999999999,"15min":2.2330000000000001},"min":{"1min":0.36499999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.9209999999999994,"5min":8.9209999999999994,"15min":155.477},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.79800000000000004,"5min":1.2729999999999999,"15min":1.9770000000000001},"min":{"1min":0.47699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.2010000000000001,"5min":5.7839999999999998,"15min":155.46000000000001},"last":2.1059999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:41:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.655,"15min":3.3380000000000001},"min":{"1min":0.39800000000000002,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":7.6459999999999999,"5min":34.414999999999999,"15min":154.417},"last":1.956},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.8280000000000001,"15min":3.5409999999999999},"min":{"1min":0.47899999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.9649999999999999,"5min":34.514000000000003,"15min":154.04400000000001},"last":0.76700000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:41:38 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.238,"15min":3.0910000000000002},"min":{"1min":0.432,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.6779999999999999,"5min":7.6779999999999999,"15min":213.404},"last":1.9750000000000001},{"interface":"front","average":{"1min":1.091,"5min":1.712,"15min":3.3530000000000002},"min":{"1min":0.42299999999999999,"5min":0.40999999999999998,"15min":0.40000000000000002},"max":{"1min":8.9939999999999998,"5min":11.074999999999999,"15min":213.27199999999999},"last":0.72499999999999998}]}]},{"osd":3,"up_from":0,"seq":468151435511,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019274,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2108440,"kb_used_data":1711440,"kb_used_omap":0,"kb_used_meta":396992,"kb_avail":91636712,"statfs":{"total":95995035648,"available":93835993088,"internally_reserved":0,"allocated":1752514560,"data_stored":1751035734,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":406519808},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":3.544,"15min":2.5590000000000002},"min":{"1min":0.23899999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.363,"5min":105.179,"15min":105.179},"last":2.9060000000000001},{"interface":"front","average":{"1min":1.133,"5min":3.6200000000000001,"15min":2.6339999999999999},"min":{"1min":0.29599999999999999,"5min":0.252,"15min":0.25},"max":{"1min":5.6449999999999996,"5min":104.70999999999999,"15min":104.70999999999999},"last":2.827}]},{"osd":1,"last update":"Sat Sep 30 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":2.1669999999999998,"15min":6.1790000000000003},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.25700000000000001},"max":{"1min":2.25,"5min":84.105999999999995,"15min":160.69999999999999},"last":3.0299999999999998},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.0379999999999998,"15min":6.133},"min":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":2.6909999999999998,"5min":83.941000000000003,"15min":160.50700000000001},"last":0.56200000000000006}]},{"osd":2,"last update":"Sat Sep 30 09:41:40 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":3.492,"15min":2.9159999999999999},"min":{"1min":0.34599999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.085,"5min":93.161000000000001,"15min":93.161000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.5469999999999999,"5min":3.4900000000000002,"15min":2.9060000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.309},"max":{"1min":6.0739999999999998,"5min":93.114000000000004,"15min":93.114000000000004},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:41:53 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.5389999999999999,"15min":3.476},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.26400000000000001},"max":{"1min":2.8330000000000002,"5min":28.527000000000001,"15min":281.42200000000003},"last":3.21},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.5649999999999999,"15min":3.4239999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.863,"5min":28.486000000000001,"15min":281.44999999999999},"last":3.2429999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":3.1099999999999999,"15min":2.2879999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":2.5219999999999998,"5min":172.97800000000001,"15min":172.97800000000001},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.79700000000000004,"5min":3.2509999999999999,"15min":2.387},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26200000000000001},"max":{"1min":7.734,"5min":172.92400000000001,"15min":172.92400000000001},"last":0.56000000000000005}]},{"osd":6,"last update":"Sat Sep 30 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":2.8900000000000001,"15min":2.2989999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":0.66300000000000003,"5min":182.10300000000001,"15min":202.72499999999999},"last":0.437},{"interface":"front","average":{"1min":0.58499999999999996,"5min":2.8479999999999999,"15min":2.3290000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.249},"max":{"1min":2.6070000000000002,"5min":181.76300000000001,"15min":202.42400000000001},"last":2.0750000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":2.657,"15min":3.3860000000000001},"min":{"1min":0.34000000000000002,"5min":0.309,"15min":0.30599999999999999},"max":{"1min":2.319,"5min":117.154,"15min":230.90100000000001},"last":0.375},{"interface":"front","average":{"1min":0.61099999999999999,"5min":2.5590000000000002,"15min":3.1779999999999999},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.6830000000000001,"5min":117.508,"15min":229.97},"last":0.39900000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757635,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2511316,"kb_used_data":2115148,"kb_used_omap":0,"kb_used_meta":396160,"kb_avail":91233836,"statfs":{"total":95995035648,"available":93423448064,"internally_reserved":0,"allocated":2165911552,"data_stored":2164455322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405667840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":4.4400000000000004,"15min":3.4860000000000002},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.22},"max":{"1min":3.3679999999999999,"5min":98.548000000000002,"15min":148.79900000000001},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":4.4980000000000002,"15min":3.786},"min":{"1min":0.26500000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.6830000000000001,"5min":98.388000000000005,"15min":148.779},"last":0.373}]},{"osd":1,"last update":"Sat Sep 30 09:40:46 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.786,"15min":2.0539999999999998},"min":{"1min":0.27500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":1.9399999999999999,"5min":13.363,"15min":60.799999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.47099999999999997,"5min":2.1120000000000001,"15min":2.9609999999999999},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.29,"5min":16.757000000000001,"15min":63.798999999999999},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:41:29 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":5.4880000000000004,"15min":3.544},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":53.347999999999999,"5min":227.18000000000001,"15min":227.18000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":3.5059999999999998,"5min":5.5839999999999996,"15min":3.4729999999999999},"min":{"1min":0.33300000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":54.408000000000001,"5min":227.46600000000001,"15min":227.46600000000001},"last":0.30199999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":4.5220000000000002,"15min":4.8860000000000001},"min":{"1min":0.41499999999999998,"5min":0.30499999999999999,"15min":0.29699999999999999},"max":{"1min":2.5150000000000001,"5min":154.56700000000001,"15min":226.34700000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.63300000000000001,"5min":4.0209999999999999,"15min":4.6180000000000003},"min":{"1min":0.40600000000000003,"5min":0.33600000000000002,"15min":0.313},"max":{"1min":1.534,"5min":154.58000000000001,"15min":226.36000000000001},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:41:42 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.81200000000000006,"15min":0.90300000000000002},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.24399999999999999},"max":{"1min":1.6100000000000001,"5min":12.634,"15min":12.634},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.91300000000000003,"15min":1.0009999999999999},"min":{"1min":0.31900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.6990000000000001,"5min":7.5410000000000004,"15min":9.9640000000000004},"last":0.33600000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.93999999999999995,"15min":1.855},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25900000000000001},"max":{"1min":1.3600000000000001,"5min":7.4740000000000002,"15min":155.602},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.262,"15min":2.0249999999999999},"min":{"1min":0.39500000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.956,"5min":10.880000000000001,"15min":154.00800000000001},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":1.544,"15min":1.901},"min":{"1min":0.38600000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.9950000000000001,"5min":13.617000000000001,"15min":133.40700000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.86099999999999999,"15min":1.607},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.307},"max":{"1min":1.0409999999999999,"5min":5.7880000000000003,"15min":132.797},"last":0.441}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1913651200,"data_stored":1913651200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1608843264,"data_stored":1608843264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2369851392,"data_stored":2369847320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1056571392,"data_stored":1056571392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1746010112,"data_stored":1746010112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2150240256,"data_stored":2150236184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1430716416,"data_stored":1430716416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1162481664,"data_stored":1162477592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:42:01.526 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:42:01.526 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 36 2023-09-30T09:42:01.528 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:01.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:01.634 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:01.668 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:01.718 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:01.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:01.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:01.736 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:42:01.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:01 smithi019 ceph-mon[216815]: from='client.55915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:42:01.821 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:01.850 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:01.910 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:02.008 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:01 smithi167 ceph-mon[192880]: from='client.55915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:42:02.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:01 smithi019 ceph-mon[224043]: from='client.55915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:42:02.149 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:02.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:02.253 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:02.285 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:02.396 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:02.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:02.500 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:02.539 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:02.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:02 smithi019 ceph-mon[224043]: pgmap v1124: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 248 op/s; 44780/166434 objects degraded (26.906%); 7025/166434 objects misplaced (4.221%); 15 MiB/s, 233 objects/s recovering 2023-09-30T09:42:02.839 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:02 smithi019 ceph-mon[216815]: pgmap v1124: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 248 op/s; 44780/166434 objects degraded (26.906%); 7025/166434 objects misplaced (4.221%); 15 MiB/s, 233 objects/s recovering 2023-09-30T09:42:03.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:02 smithi167 ceph-mon[192880]: pgmap v1124: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 248 op/s; 44780/166434 objects degraded (26.906%); 7025/166434 objects misplaced (4.221%); 15 MiB/s, 233 objects/s recovering 2023-09-30T09:42:03.253 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:42:03.254 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:03.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:03.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:03.357 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:03.410 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:03.534 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:03.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:03.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:03.604 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:03.644 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:03.674 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:03.782 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:03.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:03.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:03.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:42:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:42:03] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.43.0" 2023-09-30T09:42:03.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:03.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 44780/166434 objects degraded (26.906%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:03.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:03.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 44780/166434 objects degraded (26.906%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:03.887 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:42:03.887 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:42:03.887 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:42:03.887 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:42:03.887 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:42:03.891 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:03.913 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:03.913 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 44780/166434 objects degraded (26.906%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:03.923 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:04.130 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:04.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:04.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:04.234 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:04.263 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:04.776 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:04.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:04.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:04.881 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:04.934 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:42:04.945 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:04.990 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:05.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:04 smithi019 ceph-mon[224043]: pgmap v1125: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 44377/163860 objects degraded (27.082%); 6974/163860 objects misplaced (4.256%); 15 MiB/s, 238 objects/s recovering 2023-09-30T09:42:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:04 smithi019 ceph-mon[216815]: pgmap v1125: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 44377/163860 objects degraded (27.082%); 6974/163860 objects misplaced (4.256%); 15 MiB/s, 238 objects/s recovering 2023-09-30T09:42:05.092 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:05.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:05.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:05.197 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:05.227 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:04 smithi167 ceph-mon[192880]: pgmap v1125: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 44377/163860 objects degraded (27.082%); 6974/163860 objects misplaced (4.256%); 15 MiB/s, 238 objects/s recovering 2023-09-30T09:42:05.231 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:05.291 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:05.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:05.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:05.356 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:42:05.398 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:05.469 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:05.730 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:05.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:05.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:05.818 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:42:05.838 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:05.868 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:05.925 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:05.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:05.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:06.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1119389955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:06.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:05 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:06.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1119389955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:06.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:05 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:06.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1119389955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:06.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:05 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:06.029 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:06.060 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:06.109 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:06.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:06.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:06.311 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:06.807 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 8 pg_num to 36 2023-09-30T09:42:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: pgmap v1126: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 43640/160530 objects degraded (27.185%); 6705/160530 objects misplaced (4.177%); 9.4 MiB/s, 168 objects/s recovering 2023-09-30T09:42:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:42:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: osdmap e433: 8 total, 7 up, 6 in 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1680057169' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: pgmap v1126: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 43640/160530 objects degraded (27.185%); 6705/160530 objects misplaced (4.177%); 9.4 MiB/s, 168 objects/s recovering 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: osdmap e433: 8 total, 7 up, 6 in 2023-09-30T09:42:07.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1680057169' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:42:07.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:42:07.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:07.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: pgmap v1126: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 43640/160530 objects degraded (27.185%); 6705/160530 objects misplaced (4.177%); 9.4 MiB/s, 168 objects/s recovering 2023-09-30T09:42:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:42:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: failed to encode map e433 with expected crc 2023-09-30T09:42:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: osdmap e433: 8 total, 7 up, 6 in 2023-09-30T09:42:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1680057169' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:42:07.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-09-30T09:42:07.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:07.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:07.783 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:42:07.783 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:42:07.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:42:07.783 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:42:07.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.783 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54227 8.b 8:d5842c94:::benchmark_data_smithi019_478679_object54226:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:42:07.783 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.625464+0000", 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 203.15332989199999, 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1320590269999999, 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54227 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625464+0000", 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625464+0000", 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625464+0000", 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625491+0000", 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7118e-05 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625499+0000", 2023-09-30T09:42:07.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5089999999999996e-06 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625504+0000", 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0019999999999997e-06 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625559+0000", 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5186000000000003e-05 2023-09-30T09:42:07.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625568+0000", 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9740000000000008e-06 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591222+0000", 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96565369599999995 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591242+0000", 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0449e-05 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.739019+0000", 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1477772209999999 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.740584+0000", 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015645189999999999 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.740738+0000", 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015377199999999999 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.746857+0000", 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006119162 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.749614+0000", 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027570709999999998 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.755509+0000", 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0058951810000000002 2023-09-30T09:42:07.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.755578+0000", 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9058000000000002e-05 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.757523+0000", 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019454800000000001 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54233 8.b 8:d451d48a:::benchmark_data_smithi019_478679_object54232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.632114+0000", 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 203.14668061, 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1284132279999999, 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.793 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54233 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632114+0000", 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632114+0000", 2023-09-30T09:42:07.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632112+0000", 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632160+0000", 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8038999999999999e-05 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632174+0000", 2023-09-30T09:42:07.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3993e-05 2023-09-30T09:42:07.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632182+0000", 2023-09-30T09:42:07.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8359999999999992e-06 2023-09-30T09:42:07.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632248+0000", 2023-09-30T09:42:07.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6484999999999998e-05 2023-09-30T09:42:07.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632258+0000", 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4259999999999992e-06 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591274+0000", 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95901665000000003 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591282+0000", 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.464e-06 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.742560+0000", 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.151277726 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.742653+0000", 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2881999999999996e-05 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.742799+0000", 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000146193 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.746864+0000", 2023-09-30T09:42:07.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0040655650000000002 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.751138+0000", 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042732810000000003 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.759067+0000", 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0079289270000000005 2023-09-30T09:42:07.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.759128+0000", 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1440999999999997e-05 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.760527+0000", 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001398973 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.803 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54236 8.b 8:d2fdb527:::benchmark_data_smithi019_478679_object54235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.636336+0000", 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 203.142458538, 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1255129159999999, 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.804 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54236 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636336+0000", 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636336+0000", 2023-09-30T09:42:07.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636335+0000", 2023-09-30T09:42:07.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636398+0000", 2023-09-30T09:42:07.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3142e-05 2023-09-30T09:42:07.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636412+0000", 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4188e-05 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636417+0000", 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3900000000000001e-06 2023-09-30T09:42:07.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636476+0000", 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8291000000000001e-05 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636486+0000", 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0176999999999999e-05 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591303+0000", 2023-09-30T09:42:07.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95481728200000004 2023-09-30T09:42:07.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591311+0000", 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7249999999999997e-06 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.743981+0000", 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1526703010000001 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.744125+0000", 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000143428 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.744320+0000", 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019499900000000001 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.747871+0000", 2023-09-30T09:42:07.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035515690000000001 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.752812+0000", 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049407640000000003 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.760586+0000", 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077744880000000004 2023-09-30T09:42:07.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.760652+0000", 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5429e-05 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.761849+0000", 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011968269999999999 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.814 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.815 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.815 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54239 8.b 8:db6bec44:::benchmark_data_smithi019_478679_object54238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:42:07.815 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.641227+0000", 2023-09-30T09:42:07.815 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 203.13756781699999, 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.127019604, 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54239 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.816 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641227+0000", 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641227+0000", 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641225+0000", 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641257+0000", 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1529999999999998e-05 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641276+0000", 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.897e-05 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641284+0000", 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7499999999999992e-06 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641338+0000", 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4008000000000002e-05 2023-09-30T09:42:07.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641349+0000", 2023-09-30T09:42:07.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0942e-05 2023-09-30T09:42:07.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591333+0000", 2023-09-30T09:42:07.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94998327800000004 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591339+0000", 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8900000000000001e-06 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.745603+0000", 2023-09-30T09:42:07.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1542637259999999 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.745764+0000", 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000160922 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.745927+0000", 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016264400000000001 2023-09-30T09:42:07.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.749621+0000", 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036943129999999999 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.752865+0000", 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032436539999999999 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.766700+0000", 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013835024 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.766762+0000", 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2544000000000003e-05 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.768246+0000", 2023-09-30T09:42:07.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00148379 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54244 8.b 8:d4a2aa15:::benchmark_data_smithi019_478679_object54243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.649603+0000", 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 203.12919157300001, 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1205748990000002, 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.827 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54244 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649603+0000", 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649603+0000", 2023-09-30T09:42:07.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649602+0000", 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649652+0000", 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0466000000000001e-05 2023-09-30T09:42:07.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649663+0000", 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0319999999999999e-05 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649667+0000", 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3699999999999997e-06 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649709+0000", 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2330000000000003e-05 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649716+0000", 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7580000000000003e-06 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591360+0000", 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94164374699999998 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591367+0000", 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.469e-06 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.746894+0000", 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1555268329999999 2023-09-30T09:42:07.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.746982+0000", 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8103000000000007e-05 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.747072+0000", 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0148000000000005e-05 2023-09-30T09:42:07.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.751012+0000", 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0039390950000000001 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.752941+0000", 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019299709999999999 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.768320+0000", 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01537876 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.768401+0000", 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0655000000000006e-05 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.770178+0000", 2023-09-30T09:42:07.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001776749 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106313 8.12 8:4aa4e2a6:::benchmark_data_smithi019_478679_object13865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.811295+0000", 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 105.967499558, 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3249428500000002, 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.837 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106313 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811295+0000", 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811295+0000", 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811294+0000", 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811298+0000", 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1439999999999996e-06 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811305+0000", 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.396e-06 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811311+0000", 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5620000000000003e-06 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811402+0000", 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0676999999999999e-05 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811414+0000", 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2673e-05 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783552+0000", 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97213803499999996 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783562+0000", 2023-09-30T09:42:07.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3579999999999992e-06 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.126079+0000", 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3425171389999999 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.126224+0000", 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000144888 2023-09-30T09:42:07.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:42:07.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.126559+0000", 2023-09-30T09:42:07.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00033534299999999999 2023-09-30T09:42:07.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.132747+0000", 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0061879200000000004 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.135026+0000", 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022791180000000001 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.135032+0000", 2023-09-30T09:42:07.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1120000000000001e-06 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.135063+0000", 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0518999999999997e-05 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.136238+0000", 2023-09-30T09:42:07.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001174869 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106316 8.12 8:4991c4c6:::benchmark_data_smithi019_478679_object13868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.848040+0000", 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 105.930754045, 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.293266698, 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.847 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106316 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848040+0000", 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848040+0000", 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848039+0000", 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848045+0000", 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2419999999999997e-06 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848059+0000", 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3841e-05 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848066+0000", 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9530000000000002e-06 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848117+0000", 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0683000000000002e-05 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848126+0000", 2023-09-30T09:42:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6229999999999992e-06 2023-09-30T09:42:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783607+0000", 2023-09-30T09:42:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.93548119100000005 2023-09-30T09:42:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783616+0000", 2023-09-30T09:42:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6619999999999994e-06 2023-09-30T09:42:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128058+0000", 2023-09-30T09:42:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3444416809999999 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128208+0000", 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015018099999999999 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128492+0000", 2023-09-30T09:42:07.854 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00028447699999999999 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.136246+0000", 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077533029999999996 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.855 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.136367+0000", 2023-09-30T09:42:07.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000121748 2023-09-30T09:42:07.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.856 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.139541+0000", 2023-09-30T09:42:07.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031732449999999999 2023-09-30T09:42:07.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.857 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.139600+0000", 2023-09-30T09:42:07.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9573e-05 2023-09-30T09:42:07.857 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.857 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.857 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.141307+0000", 2023-09-30T09:42:07.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017067 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106319 8.2 8:444e3951:::benchmark_data_smithi019_478679_object13871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.848569+0000", 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 105.930225726, 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2840958430000002, 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.858 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.859 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.859 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.859 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.859 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106319 2023-09-30T09:42:07.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.859 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.859 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.859 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848569+0000", 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848569+0000", 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.860 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.861 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848568+0000", 2023-09-30T09:42:07.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.861 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.861 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.861 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.861 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848574+0000", 2023-09-30T09:42:07.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6999999999999996e-06 2023-09-30T09:42:07.861 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848589+0000", 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5568000000000001e-05 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848596+0000", 2023-09-30T09:42:07.862 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.708e-06 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.850432+0000", 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001836435 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.850439+0000", 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4960000000000001e-06 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.782674+0000", 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.932235386 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.782685+0000", 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1199000000000001e-05 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.122402+0000", 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.339716664 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.122531+0000", 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128484 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.122847+0000", 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031693100000000001 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128033+0000", 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0051852039999999997 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.130479+0000", 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024462569999999999 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.130542+0000", 2023-09-30T09:42:07.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2697e-05 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.130605+0000", 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3553000000000006e-05 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.132664+0000", 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020592990000000001 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.868 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106325 8.15 8:ac1e17e7:::benchmark_data_smithi019_478679_object13877:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.851020+0000", 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 105.927774208, 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0861625150000001, 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.869 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106325 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.851020+0000", 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.870 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.851020+0000", 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.851019+0000", 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.851025+0000", 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4779999999999999e-06 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.851036+0000", 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0886999999999999e-05 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.851042+0000", 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5189999999999998e-06 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.851096+0000", 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4462000000000001e-05 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.851105+0000", 2023-09-30T09:42:07.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9709999999999993e-06 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.918997+0000", 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0678916199999999 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.919216+0000", 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000218829 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.919579+0000", 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00036339100000000002 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.923854+0000", 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042751300000000002 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.925786+0000", 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001931692 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.935017+0000", 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0092312179999999994 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.935085+0000", 2023-09-30T09:42:07.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7782999999999998e-05 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.937183+0000", 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020975099999999999 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.877 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106340 8.5 8:a1cdf9f5:::benchmark_data_smithi019_478679_object13892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.872284+0000", 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 105.90651044099999, 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2115014529999999, 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106340 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.878 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872284+0000", 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872284+0000", 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.879 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872283+0000", 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872289+0000", 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1029999999999998e-06 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872300+0000", 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1249e-05 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872305+0000", 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.31e-06 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872362+0000", 2023-09-30T09:42:07.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7157999999999997e-05 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872389+0000", 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6332000000000001e-05 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.782740+0000", 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.91035162199999997 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.782752+0000", 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2064e-05 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.070538+0000", 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.287785765 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.070689+0000", 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150889 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.071063+0000", 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00037390200000000001 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.075881+0000", 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048179319999999996 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.078655+0000", 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027745109999999999 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.082385+0000", 2023-09-30T09:42:07.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037298819999999999 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.082477+0000", 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1236999999999994e-05 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.083785+0000", 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001308802 2023-09-30T09:42:07.886 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106342 8.15 8:af2a7556:::benchmark_data_smithi019_478679_object13894:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.872953+0000", 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 105.905841307, 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0663671800000001, 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.887 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106342 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872953+0000", 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872953+0000", 2023-09-30T09:42:07.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872952+0000", 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872957+0000", 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5170000000000001e-06 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872968+0000", 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1087000000000001e-05 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872972+0000", 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4819999999999996e-06 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.873029+0000", 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7107000000000001e-05 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.873041+0000", 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1093e-05 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.920774+0000", 2023-09-30T09:42:07.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.047733555 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.920945+0000", 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017127399999999999 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.921252+0000", 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00030628200000000001 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.925825+0000", 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045731299999999999 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.925832+0000", 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.559e-06 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.893 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.937269+0000", 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011436775 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.937347+0000", 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7463e-05 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.939320+0000", 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001973589 2023-09-30T09:42:07.894 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106345 8.2 8:40cb369a:::benchmark_data_smithi019_478679_object13897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.881246+0000", 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 105.897548245, 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2537290940000001, 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.895 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106345 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881246+0000", 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881246+0000", 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881245+0000", 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881252+0000", 2023-09-30T09:42:07.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8360000000000003e-06 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881267+0000", 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5738000000000001e-05 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881274+0000", 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2920000000000001e-06 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881331+0000", 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7493e-05 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881342+0000", 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1202999999999999e-05 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783591+0000", 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90224879199999997 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783596+0000", 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3610000000000003e-06 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.124236+0000", 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.340640659 2023-09-30T09:42:07.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.124372+0000", 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000135401 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.124649+0000", 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027731700000000001 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.130512+0000", 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0058629190000000003 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.132711+0000", 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021989370000000002 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.132994+0000", 2023-09-30T09:42:07.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00028354200000000002 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.133053+0000", 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9083000000000003e-05 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.134975+0000", 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001921732 2023-09-30T09:42:07.903 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106348 8.15 8:af4bb09a:::benchmark_data_smithi019_478679_object13900:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.889970+0000", 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 105.888824064, 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0509143510000001, 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.904 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106348 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.889970+0000", 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.889970+0000", 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.889969+0000", 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.889976+0000", 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.622e-06 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.889993+0000", 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6146000000000001e-05 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.890000+0000", 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1620000000000002e-06 2023-09-30T09:42:07.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.890064+0000", 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3967000000000006e-05 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.890074+0000", 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0219e-05 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.922387+0000", 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0323133360000001 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.922551+0000", 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016336500000000001 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.922804+0000", 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00025293000000000001 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.925874+0000", 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030699740000000001 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.925883+0000", 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2799999999999992e-06 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.939394+0000", 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013510823999999999 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.939459+0000", 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5646000000000001e-05 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.940885+0000", 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014253359999999999 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.912 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106359 8.2 8:40c2f93b:::benchmark_data_smithi019_478679_object13911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:22.972504+0000", 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 104.806289924, 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.168931054, 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106359 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.913 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.972504+0000", 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.972504+0000", 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.972503+0000", 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.972510+0000", 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8240000000000002e-06 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.972524+0000", 2023-09-30T09:42:07.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3919e-05 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.972530+0000", 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4500000000000001e-06 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.972596+0000", 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5857000000000003e-05 2023-09-30T09:42:07.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.972608+0000", 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1466e-05 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128486+0000", 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1558785030000001 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128621+0000", 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013542599999999999 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128971+0000", 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00034971900000000002 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.136252+0000", 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0072805190000000001 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.136409+0000", 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000156804 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.139571+0000", 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031628229999999999 2023-09-30T09:42:07.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.139619+0000", 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.778e-05 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.141435+0000", 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001816327 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.920 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126837 8.10 8:0e821f62:::benchmark_data_smithi019_478679_object34389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.674462+0000", 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 22.104332497000001, 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.032874155, 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.921 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126837 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.674462+0000", 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.674462+0000", 2023-09-30T09:42:07.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.674461+0000", 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.674466+0000", 2023-09-30T09:42:07.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1390000000000003e-06 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.674475+0000", 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4800000000000007e-06 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.674481+0000", 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7660000000000004e-06 2023-09-30T09:42:07.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.674511+0000", 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0324999999999998e-05 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.674516+0000", 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9849999999999997e-06 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.689151+0000", 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0146346420000001 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.689271+0000", 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012006999999999999 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.689629+0000", 2023-09-30T09:42:07.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00035750799999999998 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.697311+0000", 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076825189999999996 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707217+0000", 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0099055359999999995 2023-09-30T09:42:07.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707255+0000", 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8464999999999998e-05 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707336+0000", 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0801000000000001e-05 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.928 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126839 8.12 8:4fd025f5:::benchmark_data_smithi019_478679_object34391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.691976+0000", 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 22.086818326, 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1084521220000001, 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.929 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126839 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.691976+0000", 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.691976+0000", 2023-09-30T09:42:07.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.691974+0000", 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.691984+0000", 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7200000000000001e-06 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.692031+0000", 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6649e-05 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.692047+0000", 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5835999999999999e-05 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.932 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.692148+0000", 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010145300000000001 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.692157+0000", 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6340000000000007e-06 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.645316+0000", 2023-09-30T09:42:07.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95315957200000001 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.645327+0000", 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0403e-05 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.771021+0000", 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1256942649999999 2023-09-30T09:42:07.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.771647+0000", 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00062577200000000003 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.781336+0000", 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0096893259999999998 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.781399+0000", 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2848000000000005e-05 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2", 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.781710+0000", 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031120099999999999 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.787637+0000", 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0059273160000000002 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.787689+0000", 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2036999999999997e-05 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.799390+0000", 2023-09-30T09:42:07.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011700782999999999 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.799446+0000", 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6263e-05 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.800428+0000", 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00098174400000000011 2023-09-30T09:42:07.938 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126849 8.12 8:4f788868:::benchmark_data_smithi019_478679_object34401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.835790+0000", 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 21.943004641000002, 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9694733129999999, 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.939 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126849 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835790+0000", 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.940 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835790+0000", 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835789+0000", 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.941 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835793+0000", 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4889999999999997e-06 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835823+0000", 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9592999999999999e-05 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835829+0000", 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6370000000000004e-06 2023-09-30T09:42:07.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835878+0000", 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9682000000000003e-05 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835887+0000", 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6279999999999994e-06 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.651662+0000", 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81577498599999998 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.651671+0000", 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2869999999999993e-06 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.772815+0000", 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1211436830000001 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.773518+0000", 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00070358300000000003 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.784213+0000", 2023-09-30T09:42:07.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010694082000000001 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.784273+0000", 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0921000000000002e-05 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2", 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.784456+0000", 2023-09-30T09:42:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018218600000000001 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.788906+0000", 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044500310000000001 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.791070+0000", 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002164666 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.804459+0000", 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013388954 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.804514+0000", 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4344e-05 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.948 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805263+0000", 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00074934700000000003 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126863 8.2 8:43927707:::benchmark_data_smithi019_478679_object34415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:46.049126+0000", 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 21.729668389, 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.757116868, 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.949 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126863 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049126+0000", 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049126+0000", 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049125+0000", 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049130+0000", 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6219999999999997e-06 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049144+0000", 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3954999999999999e-05 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049167+0000", 2023-09-30T09:42:07.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2149000000000001e-05 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049208+0000", 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1764000000000002e-05 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049215+0000", 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6250000000000001e-06 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.646071+0000", 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.59685646699999995 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.646079+0000", 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6159999999999996e-06 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.954 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.774139+0000", 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1280599739999999 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.774740+0000", 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00060072900000000004 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.789386+0000", 2023-09-30T09:42:07.955 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014646632 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.789447+0000", 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1136000000000006e-05 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2", 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.789762+0000", 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031450800000000002 2023-09-30T09:42:07.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.797360+0000", 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075981970000000001 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.800485+0000", 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031251069999999998 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805314+0000", 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004828494 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805367+0000", 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3581000000000001e-05 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.806243+0000", 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00087546499999999997 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126881 8.12 8:48e526dd:::benchmark_data_smithi019_478679_object34433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e429)", 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:46.720884+0000", 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 21.057909924000001, 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0850710859999999, 2023-09-30T09:42:07.959 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126881 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.720884+0000", 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.720884+0000", 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.720883+0000", 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.720889+0000", 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9630000000000003e-06 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.720904+0000", 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5126999999999999e-05 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.720911+0000", 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.28e-06 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.720960+0000", 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9252999999999998e-05 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.720970+0000", 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0132999999999999e-05 2023-09-30T09:42:07.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.773561+0000", 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.052590532 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.774104+0000", 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00054330399999999999 2023-09-30T09:42:07.964 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.786576+0000", 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012471629999999999 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.786638+0000", 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2675000000000006e-05 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.965 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2", 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.786806+0000", 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000167226 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.791032+0000", 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042266209999999999 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.800467+0000", 2023-09-30T09:42:07.966 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0094347170000000005 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805313+0000", 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048460020000000003 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805347+0000", 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4004999999999997e-05 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.967 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805955+0000", 2023-09-30T09:42:07.968 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00060861599999999997 2023-09-30T09:42:07.968 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.968 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.969 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.969 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126895 8.2 8:472a301e:::benchmark_data_smithi019_478679_object34447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e429)", 2023-09-30T09:42:07.969 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:46.736023+0000", 2023-09-30T09:42:07.969 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 21.042771338000001, 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0710260549999999, 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126895 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.736023+0000", 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.736023+0000", 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.736022+0000", 2023-09-30T09:42:07.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.736027+0000", 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4319999999999998e-06 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.736038+0000", 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0567000000000001e-05 2023-09-30T09:42:07.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.736044+0000", 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3110000000000002e-06 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.739936+0000", 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0038920780000000002 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:42:07.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.739943+0000", 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6279999999999999e-06 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.774768+0000", 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0348253160000001 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.775307+0000", 2023-09-30T09:42:07.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00053877199999999997 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.792436+0000", 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.017129367999999999 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.792531+0000", 2023-09-30T09:42:07.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4351999999999994e-05 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2", 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.792848+0000", 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000316886 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.797372+0000", 2023-09-30T09:42:07.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045241320000000002 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.800506+0000", 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031347469999999998 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.806287+0000", 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057803400000000001 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.806343+0000", 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6283000000000003e-05 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.807049+0000", 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00070597200000000002 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.978 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.979 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:42:07.979 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:42:07.979 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:42:08.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:07 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-09-30T09:42:08.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:07 smithi019 ceph-mon[224043]: osdmap e434: 8 total, 7 up, 6 in 2023-09-30T09:42:08.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:07 smithi019 ceph-mon[224043]: failed to encode map e434 with expected crc 2023-09-30T09:42:08.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-30T09:42:08.046 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:07 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-09-30T09:42:08.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:07 smithi019 ceph-mon[216815]: osdmap e434: 8 total, 7 up, 6 in 2023-09-30T09:42:08.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:07 smithi019 ceph-mon[216815]: failed to encode map e434 with expected crc 2023-09-30T09:42:08.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-30T09:42:08.150 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:08.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:08.261 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:08.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:07 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-09-30T09:42:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:07 smithi167 ceph-mon[192880]: osdmap e434: 8 total, 7 up, 6 in 2023-09-30T09:42:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:07 smithi167 ceph-mon[192880]: failed to encode map e434 with expected crc 2023-09-30T09:42:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-30T09:42:08.291 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:08.336 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:08.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:08.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:08.477 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:08.725 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:08.729 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:42:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:08.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:08.830 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:08.875 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:08.968 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:08.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:08.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:08.997 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:42:09.020 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:42:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:08 smithi167 ceph-mon[192880]: pgmap v1129: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 347 op/s; 43250/159123 objects degraded (27.180%); 6644/159123 objects misplaced (4.175%); 7.0 MiB/s, 134 objects/s recovering 2023-09-30T09:42:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:08 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-30T09:42:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:08 smithi167 ceph-mon[192880]: failed to encode map e435 with expected crc 2023-09-30T09:42:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:08 smithi167 ceph-mon[192880]: osdmap e435: 8 total, 7 up, 6 in 2023-09-30T09:42:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 43250/159123 objects degraded (27.180%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:09.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:08 smithi167 ceph-mon[192880]: osdmap e436: 8 total, 7 up, 6 in 2023-09-30T09:42:09.045 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:42:09.069 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:42:09.073 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:09.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[224043]: pgmap v1129: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 347 op/s; 43250/159123 objects degraded (27.180%); 6644/159123 objects misplaced (4.175%); 7.0 MiB/s, 134 objects/s recovering 2023-09-30T09:42:09.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-30T09:42:09.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[224043]: failed to encode map e435 with expected crc 2023-09-30T09:42:09.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[224043]: osdmap e435: 8 total, 7 up, 6 in 2023-09-30T09:42:09.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 43250/159123 objects degraded (27.180%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:09.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[224043]: osdmap e436: 8 total, 7 up, 6 in 2023-09-30T09:42:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[216815]: pgmap v1129: 129 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 3 active+recovery_wait+degraded, 17 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 347 op/s; 43250/159123 objects degraded (27.180%); 6644/159123 objects misplaced (4.175%); 7.0 MiB/s, 134 objects/s recovering 2023-09-30T09:42:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-30T09:42:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[216815]: failed to encode map e435 with expected crc 2023-09-30T09:42:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[216815]: osdmap e435: 8 total, 7 up, 6 in 2023-09-30T09:42:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 43250/159123 objects degraded (27.180%), 34 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:09.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:08 smithi019 ceph-mon[216815]: osdmap e436: 8 total, 7 up, 6 in 2023-09-30T09:42:09.091 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:42:09.102 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:42:09.126 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:09.142 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:42:09.170 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:42:09.171 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:42:09.171 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:42:09.171 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:42:09.171 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:42:09.171 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:42:09.171 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:42:09.171 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:42:09.171 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:42:09.185 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:09.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:09.288 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:09.316 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:09.362 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:09.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:09.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:09.466 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:09.492 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:09.587 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:09.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:09.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:09.690 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:09.718 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:09.765 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:09.767 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:42:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:09.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:09.869 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:09.892 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: failed to encode map e436 with expected crc 2023-09-30T09:42:09.892 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: failed to encode map e436 with expected crc 2023-09-30T09:42:09.892 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: failed to encode map e436 with expected crc 2023-09-30T09:42:09.892 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: failed to encode map e436 with expected crc 2023-09-30T09:42:09.892 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: failed to encode map e436 with expected crc 2023-09-30T09:42:09.892 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: 8.c continuing backfill to osd.4 from (376'4019,434'8510] MIN to 434'8510 2023-09-30T09:42:09.892 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: failed to encode map e436 with expected crc 2023-09-30T09:42:09.893 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2871183276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:09.893 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:09 smithi167 ceph-mon[192880]: osdmap e437: 8 total, 7 up, 6 in 2023-09-30T09:42:09.894 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:09.939 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:09.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:10.042 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:10.068 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:10.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: 8.c continuing backfill to osd.4 from (376'4019,434'8510] MIN to 434'8510 2023-09-30T09:42:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2871183276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:10.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[216815]: osdmap e437: 8 total, 7 up, 6 in 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: 8.c continuing backfill to osd.4 from (376'4019,434'8510] MIN to 434'8510 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: failed to encode map e436 with expected crc 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2871183276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:10.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:09 smithi019 ceph-mon[224043]: osdmap e437: 8 total, 7 up, 6 in 2023-09-30T09:42:10.302 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:10.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:10.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:10.410 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:10.450 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:10.505 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:42:10.505 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:42:10.530 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:10.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:10.617 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:10.638 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:10.675 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:10.734 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:10.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:10.837 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:42:10.837 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:42:10.840 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:10.860 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:10 smithi167 ceph-mon[192880]: failed to encode map e437 with expected crc 2023-09-30T09:42:10.860 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:10 smithi167 ceph-mon[192880]: failed to encode map e437 with expected crc 2023-09-30T09:42:10.860 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:10 smithi167 ceph-mon[192880]: failed to encode map e437 with expected crc 2023-09-30T09:42:10.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:10 smithi167 ceph-mon[192880]: failed to encode map e437 with expected crc 2023-09-30T09:42:10.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:10 smithi167 ceph-mon[192880]: pgmap v1133: 139 pgs: 2 active+recovery_wait+degraded, 4 peering, 1 active+clean+remapped, 7 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 38711/157419 objects degraded (24.591%); 9005/157419 objects misplaced (5.720%); 2.3 MiB/s, 239 objects/s recovering 2023-09-30T09:42:10.861 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:10 smithi167 ceph-mon[192880]: osdmap e438: 8 total, 7 up, 6 in 2023-09-30T09:42:10.924 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:11.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[216815]: failed to encode map e437 with expected crc 2023-09-30T09:42:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[216815]: failed to encode map e437 with expected crc 2023-09-30T09:42:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[216815]: failed to encode map e437 with expected crc 2023-09-30T09:42:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[216815]: failed to encode map e437 with expected crc 2023-09-30T09:42:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[216815]: pgmap v1133: 139 pgs: 2 active+recovery_wait+degraded, 4 peering, 1 active+clean+remapped, 7 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 38711/157419 objects degraded (24.591%); 9005/157419 objects misplaced (5.720%); 2.3 MiB/s, 239 objects/s recovering 2023-09-30T09:42:11.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[216815]: osdmap e438: 8 total, 7 up, 6 in 2023-09-30T09:42:11.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[224043]: failed to encode map e437 with expected crc 2023-09-30T09:42:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[224043]: failed to encode map e437 with expected crc 2023-09-30T09:42:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[224043]: failed to encode map e437 with expected crc 2023-09-30T09:42:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[224043]: failed to encode map e437 with expected crc 2023-09-30T09:42:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[224043]: pgmap v1133: 139 pgs: 2 active+recovery_wait+degraded, 4 peering, 1 active+clean+remapped, 7 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 38711/157419 objects degraded (24.591%); 9005/157419 objects misplaced (5.720%); 2.3 MiB/s, 239 objects/s recovering 2023-09-30T09:42:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:10 smithi019 ceph-mon[224043]: osdmap e438: 8 total, 7 up, 6 in 2023-09-30T09:42:11.206 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:11.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:11.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:11.310 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:11.384 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:11.572 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:42:11.613 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:11.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:11.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:11.717 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:11.747 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:11.800 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:11.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:11.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:11.907 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:11.940 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:11.978 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:42:11.995 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:11.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:11.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:12.100 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:12.138 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:12.506 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:42:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:12 smithi167 ceph-mon[192880]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:12 smithi167 ceph-mon[192880]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:12 smithi167 ceph-mon[192880]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:12 smithi167 ceph-mon[192880]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:12 smithi167 ceph-mon[192880]: 8.1c continuing backfill to osd.4 from (376'4019,434'8510] MIN to 434'8510 2023-09-30T09:42:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:12 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:12 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:12.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:12 smithi167 ceph-mon[192880]: pgmap v1135: 139 pgs: 1 remapped+peering, 2 active+recovery_wait+degraded, 4 peering, 1 active+clean+remapped, 5 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 37512/156999 objects degraded (23.893%); 7293/156999 objects misplaced (4.645%); 1.3 MiB/s, 219 objects/s recovering 2023-09-30T09:42:12.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[224043]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[224043]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[224043]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[224043]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.556 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[224043]: 8.1c continuing backfill to osd.4 from (376'4019,434'8510] MIN to 434'8510 2023-09-30T09:42:12.556 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:12.556 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:12.556 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[224043]: pgmap v1135: 139 pgs: 1 remapped+peering, 2 active+recovery_wait+degraded, 4 peering, 1 active+clean+remapped, 5 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 37512/156999 objects degraded (23.893%); 7293/156999 objects misplaced (4.645%); 1.3 MiB/s, 219 objects/s recovering 2023-09-30T09:42:12.556 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[216815]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.556 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[216815]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.556 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[216815]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.557 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[216815]: failed to encode map e438 with expected crc 2023-09-30T09:42:12.557 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[216815]: 8.1c continuing backfill to osd.4 from (376'4019,434'8510] MIN to 434'8510 2023-09-30T09:42:12.557 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:12.557 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:12.557 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:12 smithi019 ceph-mon[216815]: pgmap v1135: 139 pgs: 1 remapped+peering, 2 active+recovery_wait+degraded, 4 peering, 1 active+clean+remapped, 5 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 37512/156999 objects degraded (23.893%); 7293/156999 objects misplaced (4.645%); 1.3 MiB/s, 219 objects/s recovering 2023-09-30T09:42:12.903 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:12.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:12.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:13.009 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:13.039 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:13.154 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0] out_osds: [3, 6] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:42:13.155 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:42:13.155 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.8044449851977984 2023-09-30T09:42:13.155 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 0.8044449851977984 2023-09-30T09:42:13.187 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:13.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:13.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:13.229 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:13.292 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:13.322 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:13 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:42:13.323 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:13 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:42:13.323 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:13 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:13.325 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:13.482 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:13.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:13.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:13.554 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:13 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:42:13.554 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:13 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:42:13.554 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:13 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:13.555 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:13 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:42:13.555 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:13 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:42:13.555 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:13 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:13.586 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:13.661 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:13.671 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:42:13.672 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:42:13.815 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:42:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:42:13] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.43.0" 2023-09-30T09:42:13.894 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:13.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:13.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:14.003 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:14.031 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:14.073 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:14.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:14.177 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:14.203 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:14.428 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 37512/156999 objects degraded (23.893%), 34 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:14.428 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:14 smithi167 ceph-mon[192880]: pgmap v1136: 139 pgs: 1 active+clean+remapped, 1 remapped+peering, 6 active+recovery_wait+degraded, 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 52 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 35052/155214 objects degraded (22.583%); 8351/155214 objects misplaced (5.380%); 7.2 MiB/s, 256 objects/s recovering 2023-09-30T09:42:14.428 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3452667380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:14.428 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:14 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:14.448 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:42:14.451 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:14.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:14.556 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 37512/156999 objects degraded (23.893%), 34 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:14 smithi019 ceph-mon[224043]: pgmap v1136: 139 pgs: 1 active+clean+remapped, 1 remapped+peering, 6 active+recovery_wait+degraded, 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 52 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 35052/155214 objects degraded (22.583%); 8351/155214 objects misplaced (5.380%); 7.2 MiB/s, 256 objects/s recovering 2023-09-30T09:42:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3452667380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:14 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 37512/156999 objects degraded (23.893%), 34 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:14.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:14 smithi019 ceph-mon[216815]: pgmap v1136: 139 pgs: 1 active+clean+remapped, 1 remapped+peering, 6 active+recovery_wait+degraded, 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 52 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 35052/155214 objects degraded (22.583%); 8351/155214 objects misplaced (5.380%); 7.2 MiB/s, 256 objects/s recovering 2023-09-30T09:42:14.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3452667380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:14.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:14 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:14.586 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:14.636 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:42:14.640 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:14.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:14.748 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:14.781 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:14.840 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:42:14.879 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:14.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:14.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:14.984 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:15.034 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:15.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:15 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:15 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:42:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:15 smithi019 ceph-mon[216815]: osdmap e439: 8 total, 7 up, 6 in 2023-09-30T09:42:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:15 smithi019 ceph-mon[216815]: failed to encode map e439 with expected crc 2023-09-30T09:42:15.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:15 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:15.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:15 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:42:15.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:15 smithi019 ceph-mon[224043]: osdmap e439: 8 total, 7 up, 6 in 2023-09-30T09:42:15.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:15 smithi019 ceph-mon[224043]: failed to encode map e439 with expected crc 2023-09-30T09:42:15.346 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:15 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:15.346 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:15 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:42:15.346 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:15 smithi167 ceph-mon[192880]: osdmap e439: 8 total, 7 up, 6 in 2023-09-30T09:42:15.346 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:15 smithi167 ceph-mon[192880]: failed to encode map e439 with expected crc 2023-09-30T09:42:15.703 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:15.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:15.807 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:15.865 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:16.408 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:42:16.408 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:42:16.438 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:16.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:16 smithi167 ceph-mon[192880]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:16 smithi167 ceph-mon[192880]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:16 smithi167 ceph-mon[192880]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:16 smithi167 ceph-mon[192880]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:16 smithi167 ceph-mon[192880]: pgmap v1138: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 263 op/s; 36140/152622 objects degraded (23.679%); 10479/152622 objects misplaced (6.866%); 8.7 MiB/s, 145 objects/s recovering 2023-09-30T09:42:16.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[216815]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[216815]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[216815]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[216815]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[216815]: pgmap v1138: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 263 op/s; 36140/152622 objects degraded (23.679%); 10479/152622 objects misplaced (6.866%); 8.7 MiB/s, 145 objects/s recovering 2023-09-30T09:42:16.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[224043]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[224043]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[224043]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[224043]: failed to encode map e439 with expected crc 2023-09-30T09:42:16.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:16 smithi019 ceph-mon[224043]: pgmap v1138: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 263 op/s; 36140/152622 objects degraded (23.679%); 10479/152622 objects misplaced (6.866%); 8.7 MiB/s, 145 objects/s recovering 2023-09-30T09:42:16.593 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:16.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:16.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:16.700 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:16.782 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:16.833 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:16.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:16.937 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:17.001 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:17.189 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:42:17.388 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:42:17.398 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:17.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:17.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:17.421 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:42:17.476 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:42:17.512 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:17.518 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:42:17.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2410104975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:17.555 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:42:17.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2410104975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:17.566 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2410104975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:17.578 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:17.585 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:42:17.589 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:42:17.624 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:42:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:42:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:42:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:42:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:42:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:42:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:42:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:42:17.625 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:42:17.658 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:17.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:17.762 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:17.815 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:18 smithi167 ceph-mon[192880]: pgmap v1139: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 35571/151251 objects degraded (23.518%); 10379/151251 objects misplaced (6.862%); 8.6 MiB/s, 143 objects/s recovering 2023-09-30T09:42:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:18.754 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:42:18.789 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:18 smithi019 ceph-mon[224043]: pgmap v1139: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 35571/151251 objects degraded (23.518%); 10379/151251 objects misplaced (6.862%); 8.6 MiB/s, 143 objects/s recovering 2023-09-30T09:42:18.789 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:18.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:18 smithi019 ceph-mon[216815]: pgmap v1139: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 35571/151251 objects degraded (23.518%); 10379/151251 objects misplaced (6.862%); 8.6 MiB/s, 143 objects/s recovering 2023-09-30T09:42:18.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:19.195 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:19.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:19.269 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:42:19.270 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:42:19.297 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:42:19.297 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:42:19.297 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:42:19.297 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:42:19.303 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:19.334 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:19.449 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:19.489 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:19.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:19.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:19.575 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 35571/151251 objects degraded (23.518%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:19.576 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 35571/151251 objects degraded (23.518%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:19.594 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:19.620 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:19.662 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:19.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 35571/151251 objects degraded (23.518%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:19.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:19.766 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:19.817 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:19.920 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:19.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:19.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:20.049 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:20.080 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:20.083 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:42:20.131 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:20.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:20.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:20.238 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:20.276 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:20.398 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:20.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:20.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:20.446 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.0 to 0.804445 (cdf0) 2023-09-30T09:42:20.503 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:20.506 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:42:20.534 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:20 smithi019 ceph-mon[216815]: pgmap v1140: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 35200/148530 objects degraded (23.699%); 9748/148530 objects misplaced (6.563%); 12 MiB/s, 200 objects/s recovering 2023-09-30T09:42:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1530690668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]: dispatch 2023-09-30T09:42:20.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:20 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]: dispatch 2023-09-30T09:42:20.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:20 smithi019 ceph-mon[224043]: pgmap v1140: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 35200/148530 objects degraded (23.699%); 9748/148530 objects misplaced (6.563%); 12 MiB/s, 200 objects/s recovering 2023-09-30T09:42:20.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1530690668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]: dispatch 2023-09-30T09:42:20.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:20 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]: dispatch 2023-09-30T09:42:20.586 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:20.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:20.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:20.688 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:20.714 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:20 smithi167 ceph-mon[192880]: pgmap v1140: 139 pgs: 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 21 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 52 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 35200/148530 objects degraded (23.699%); 9748/148530 objects misplaced (6.563%); 12 MiB/s, 200 objects/s recovering 2023-09-30T09:42:20.715 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1530690668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]: dispatch 2023-09-30T09:42:20.715 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:20 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]: dispatch 2023-09-30T09:42:20.718 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:20.772 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:20.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:20.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:20.876 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:20.904 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:21.003 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:21.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:21.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:21.106 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:21.140 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:21.271 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:42:21.437 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]': finished 2023-09-30T09:42:21.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[216815]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[216815]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[216815]: osdmap e440: 8 total, 7 up, 6 in 2023-09-30T09:42:21.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[216815]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[216815]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.665 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:21.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:21.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:21.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]': finished 2023-09-30T09:42:21.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[224043]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[224043]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[224043]: osdmap e440: 8 total, 7 up, 6 in 2023-09-30T09:42:21.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[224043]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:21 smithi019 ceph-mon[224043]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:21 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8044449851977984}]': finished 2023-09-30T09:42:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:21 smithi167 ceph-mon[192880]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:21 smithi167 ceph-mon[192880]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:21 smithi167 ceph-mon[192880]: osdmap e440: 8 total, 7 up, 6 in 2023-09-30T09:42:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:21 smithi167 ceph-mon[192880]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:21 smithi167 ceph-mon[192880]: failed to encode map e440 with expected crc 2023-09-30T09:42:21.774 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:21.806 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:22.115 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:42:22.117 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:42:22.117 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:42:22.117 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:42:22.118 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:42:22.368 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:22.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:22.473 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:22.509 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:22.699 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[224043]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[224043]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.700 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[224043]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.701 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[224043]: osdmap e441: 8 total, 7 up, 6 in 2023-09-30T09:42:22.701 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[224043]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.701 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[224043]: pgmap v1143: 139 pgs: 1 remapped+peering, 3 peering, 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 20 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 49 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 34844/147477 objects degraded (23.627%); 9566/147477 objects misplaced (6.486%); 9.5 MiB/s, 152 objects/s recovering 2023-09-30T09:42:22.701 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[224043]: osdmap e442: 8 total, 7 up, 6 in 2023-09-30T09:42:22.702 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[216815]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.702 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[216815]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.702 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[216815]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.702 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[216815]: osdmap e441: 8 total, 7 up, 6 in 2023-09-30T09:42:22.702 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[216815]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.702 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[216815]: pgmap v1143: 139 pgs: 1 remapped+peering, 3 peering, 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 20 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 49 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 34844/147477 objects degraded (23.627%); 9566/147477 objects misplaced (6.486%); 9.5 MiB/s, 152 objects/s recovering 2023-09-30T09:42:22.703 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:22 smithi019 ceph-mon[216815]: osdmap e442: 8 total, 7 up, 6 in 2023-09-30T09:42:22.710 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:22.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:22.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:22 smithi167 ceph-mon[192880]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:22 smithi167 ceph-mon[192880]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:22 smithi167 ceph-mon[192880]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:22 smithi167 ceph-mon[192880]: osdmap e441: 8 total, 7 up, 6 in 2023-09-30T09:42:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:22 smithi167 ceph-mon[192880]: failed to encode map e441 with expected crc 2023-09-30T09:42:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:22 smithi167 ceph-mon[192880]: pgmap v1143: 139 pgs: 1 remapped+peering, 3 peering, 1 active+clean+remapped, 7 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 20 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 49 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 34844/147477 objects degraded (23.627%); 9566/147477 objects misplaced (6.486%); 9.5 MiB/s, 152 objects/s recovering 2023-09-30T09:42:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:22 smithi167 ceph-mon[192880]: osdmap e442: 8 total, 7 up, 6 in 2023-09-30T09:42:22.815 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:22.850 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:22.871 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:22.923 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:22.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:23.031 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:23.064 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:23.145 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:42:23.162 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:23.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:23.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:23.270 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:23.349 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:23.439 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:23.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:42:23.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/862805964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:23.446 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:23.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:23.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:23.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:23 smithi167 ceph-mon[192880]: osdmap e443: 8 total, 7 up, 6 in 2023-09-30T09:42:23.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:42:23.448 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/862805964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:23.448 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:23.448 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:23.448 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:23.448 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[216815]: osdmap e443: 8 total, 7 up, 6 in 2023-09-30T09:42:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:23.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:23.467 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:42:23.552 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:23.576 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:42:23.641 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: failed to encode map e442 with expected crc 2023-09-30T09:42:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:42:23.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/862805964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:23.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:23.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:23.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:23.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:23 smithi019 ceph-mon[224043]: osdmap e443: 8 total, 7 up, 6 in 2023-09-30T09:42:24.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:42:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:42:23] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.43.0" 2023-09-30T09:42:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:24 smithi167 ceph-mon[192880]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:24 smithi167 ceph-mon[192880]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:24 smithi167 ceph-mon[192880]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:24 smithi167 ceph-mon[192880]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:24 smithi167 ceph-mon[192880]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 34844/147477 objects degraded (23.627%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:24 smithi167 ceph-mon[192880]: pgmap v1146: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 4 peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 36916/145800 objects degraded (25.320%); 7508/145800 objects misplaced (5.150%); 11 MiB/s, 170 objects/s recovering 2023-09-30T09:42:24.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:24 smithi167 ceph-mon[192880]: osdmap e444: 8 total, 7 up, 6 in 2023-09-30T09:42:24.830 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:24.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[216815]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[216815]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[216815]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[216815]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[216815]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.833 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 34844/147477 objects degraded (23.627%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:24.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[216815]: pgmap v1146: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 4 peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 36916/145800 objects degraded (25.320%); 7508/145800 objects misplaced (5.150%); 11 MiB/s, 170 objects/s recovering 2023-09-30T09:42:24.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[216815]: osdmap e444: 8 total, 7 up, 6 in 2023-09-30T09:42:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:24.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:24.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[224043]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[224043]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[224043]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[224043]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[224043]: failed to encode map e443 with expected crc 2023-09-30T09:42:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 34844/147477 objects degraded (23.627%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[224043]: pgmap v1146: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 4 peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 50 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 36916/145800 objects degraded (25.320%); 7508/145800 objects misplaced (5.150%); 11 MiB/s, 170 objects/s recovering 2023-09-30T09:42:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:24 smithi019 ceph-mon[224043]: osdmap e444: 8 total, 7 up, 6 in 2023-09-30T09:42:24.935 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:24.978 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:25.062 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:25.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:25.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:25.170 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:25.206 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:25.268 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:25.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:25.378 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:25.407 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:25.458 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:25.460 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:25 smithi167 ceph-mon[192880]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.460 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:25 smithi167 ceph-mon[192880]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.460 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:25 smithi167 ceph-mon[192880]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.460 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:25 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:42:25.460 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:25 smithi167 ceph-mon[192880]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:25.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:25.562 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:25.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[216815]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[216815]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[216815]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:42:25.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[216815]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[224043]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[224043]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[224043]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:42:25.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:25 smithi019 ceph-mon[224043]: failed to encode map e444 with expected crc 2023-09-30T09:42:25.620 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:25.622 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:42:25.657 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:42:25.693 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:42:25.713 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:25.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:25.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:25.726 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:42:25.752 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:42:25.776 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:42:25.803 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:42:25.803 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:42:25.803 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:42:25.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:42:25.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:42:25.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:42:25.804 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:42:25.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:42:25.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:42:25.815 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:42:25.815 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:42:25.822 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:25.854 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:26.019 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:26.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:26.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:26.123 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:26.152 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:26.203 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:26.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:26.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:26.311 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:26.337 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:26.379 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:26.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:26.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:26.457 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/940915237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:26.457 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:26 smithi167 ceph-mon[192880]: pgmap v1148: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 1 peering, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 51 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 35353/143946 objects degraded (24.560%); 7872/143946 objects misplaced (5.469%); 15 MiB/s, 270 objects/s recovering 2023-09-30T09:42:26.482 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:26.518 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/940915237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:26 smithi019 ceph-mon[216815]: pgmap v1148: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 1 peering, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 51 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 35353/143946 objects degraded (24.560%); 7872/143946 objects misplaced (5.469%); 15 MiB/s, 270 objects/s recovering 2023-09-30T09:42:26.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/940915237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:26.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:26 smithi019 ceph-mon[224043]: pgmap v1148: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 1 peering, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 51 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 35353/143946 objects degraded (24.560%); 7872/143946 objects misplaced (5.469%); 15 MiB/s, 270 objects/s recovering 2023-09-30T09:42:26.852 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:26.870 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:42:26.870 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:42:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:26.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:26.972 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:27.001 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:27.041 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:42:27.045 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:42:27.057 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:27.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:27.165 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:27.229 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:27.286 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:27.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:27.393 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:27.420 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:27.468 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:27.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:27.574 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:27.578 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:27.614 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:27.618 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0] out_osds: [3, 6] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:42:27.618 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:42:27.618 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-30T09:42:27.618 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:42:27.619 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:42:27.619 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-30T09:42:27.620 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:27.914 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:27.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:28.028 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:28.097 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:28.873 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:42:29.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:28 smithi167 ceph-mon[192880]: pgmap v1149: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 1 peering, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 51 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 35353/143946 objects degraded (24.560%); 7872/143946 objects misplaced (5.469%); 11 MiB/s, 192 objects/s recovering 2023-09-30T09:42:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:28 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 35353/143946 objects degraded (24.560%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:28 smithi019 ceph-mon[216815]: pgmap v1149: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 1 peering, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 51 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 35353/143946 objects degraded (24.560%); 7872/143946 objects misplaced (5.469%); 11 MiB/s, 192 objects/s recovering 2023-09-30T09:42:29.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:28 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 35353/143946 objects degraded (24.560%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:28 smithi019 ceph-mon[224043]: pgmap v1149: 139 pgs: 2 remapped+peering, 3 active+clean+remapped, 1 peering, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 51 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 35353/143946 objects degraded (24.560%); 7872/143946 objects misplaced (5.469%); 11 MiB/s, 192 objects/s recovering 2023-09-30T09:42:29.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:28 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 35353/143946 objects degraded (24.560%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:29.287 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:29.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:29.392 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:29.432 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:29.856 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:29.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:29.964 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:29.992 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:30.035 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:30.040 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:30.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:30.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:30.095 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:30.144 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:30.171 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:30.217 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:30.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:30.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:30.325 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:30.369 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:30.761 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:30 smithi019 ceph-mon[224043]: pgmap v1150: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 411 op/s; 34248/138498 objects degraded (24.728%); 7664/138498 objects misplaced (5.534%); 13 MiB/s, 217 objects/s recovering 2023-09-30T09:42:30.803 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:42:30.803 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:42:30.909 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:30.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:30.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:31.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:31.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:30 smithi167 ceph-mon[192880]: pgmap v1150: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 411 op/s; 34248/138498 objects degraded (24.728%); 7664/138498 objects misplaced (5.534%); 13 MiB/s, 217 objects/s recovering 2023-09-30T09:42:31.048 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:31.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:30 smithi019 ceph-mon[216815]: pgmap v1150: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 411 op/s; 34248/138498 objects degraded (24.728%); 7664/138498 objects misplaced (5.534%); 13 MiB/s, 217 objects/s recovering 2023-09-30T09:42:31.095 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:31.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:31.200 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:31.229 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:31.277 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:31.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:31.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:31.385 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:31.415 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:31.795 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:42:32.015 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:31 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3511644272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:42:32.015 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:42:32.016 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:31 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3511644272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:42:32.016 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:42:32.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:31 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3511644272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:42:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:42:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[224043]: pgmap v1151: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 369 op/s; 33941/137358 objects degraded (24.710%); 7468/137358 objects misplaced (5.437%); 9.4 MiB/s, 243 objects/s recovering 2023-09-30T09:42:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:42:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[224043]: osdmap e445: 8 total, 7 up, 6 in 2023-09-30T09:42:33.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[224043]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[224043]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[224043]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[216815]: pgmap v1151: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 369 op/s; 33941/137358 objects degraded (24.710%); 7468/137358 objects misplaced (5.437%); 9.4 MiB/s, 243 objects/s recovering 2023-09-30T09:42:33.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:33.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:42:33.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[216815]: osdmap e445: 8 total, 7 up, 6 in 2023-09-30T09:42:33.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[216815]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[216815]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:32 smithi019 ceph-mon[216815]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:32 smithi167 ceph-mon[192880]: pgmap v1151: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 369 op/s; 33941/137358 objects degraded (24.710%); 7468/137358 objects misplaced (5.437%); 9.4 MiB/s, 243 objects/s recovering 2023-09-30T09:42:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:32 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:32 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:42:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:32 smithi167 ceph-mon[192880]: osdmap e445: 8 total, 7 up, 6 in 2023-09-30T09:42:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:32 smithi167 ceph-mon[192880]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:32 smithi167 ceph-mon[192880]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:32 smithi167 ceph-mon[192880]: failed to encode map e445 with expected crc 2023-09-30T09:42:33.980 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:33.984 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:33.984 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 33941/137358 objects degraded (24.710%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:33.984 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:42:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:42:33] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-09-30T09:42:33.984 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:33.984 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 33941/137358 objects degraded (24.710%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:33.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:33.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:34.087 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:34.114 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:34.114 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 33941/137358 objects degraded (24.710%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:34.117 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:34.161 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:42:34.166 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:34.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:34.274 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:34.347 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:34.473 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:34.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:34.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:34.582 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:34.619 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:34.673 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:34.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:34.781 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:34.848 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:34.965 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:42:34.965 INFO:teuthology.orchestra.run.smithi019.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:42:34.966 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:42:34.974 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:34.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:34.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:34.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:34 smithi019 ceph-mon[224043]: pgmap v1153: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 381 op/s; 33263/134088 objects degraded (24.807%); 7323/134088 objects misplaced (5.461%); 6.8 MiB/s, 191 objects/s recovering 2023-09-30T09:42:34.986 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:34 smithi019 ceph-mon[216815]: pgmap v1153: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 381 op/s; 33263/134088 objects degraded (24.807%); 7323/134088 objects misplaced (5.461%); 6.8 MiB/s, 191 objects/s recovering 2023-09-30T09:42:35.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:34 smithi167 ceph-mon[192880]: pgmap v1153: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 381 op/s; 33263/134088 objects degraded (24.807%); 7323/134088 objects misplaced (5.461%); 6.8 MiB/s, 191 objects/s recovering 2023-09-30T09:42:35.086 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:35.116 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:42:35.152 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:35.154 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:42:35.188 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:42:35.228 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:42:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:35.246 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:35.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:35.262 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:42:35.306 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:42:35.335 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:42:35.335 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:42:35.336 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:42:35.336 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:42:35.336 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:42:35.336 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:42:35.336 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:42:35.336 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:42:35.336 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:42:35.347 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:35.375 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:35.419 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:35.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:35.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:35.523 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:35.549 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:35.591 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:35.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:35.697 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:35.747 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:36.032 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:42:36.073 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3199672876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:36.074 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3199672876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:36.094 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:36.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:36.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:36.200 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:36.228 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:36.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3199672876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:36.685 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:36.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:36.773 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:36.792 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:36.819 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:42:36.823 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:42:36.865 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:36.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:36.962 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:36.970 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:36.999 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:37.032 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:36 smithi019 ceph-mon[224043]: pgmap v1154: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 32474/131040 objects degraded (24.782%); 6335/131040 objects misplaced (4.834%); 10 MiB/s, 230 objects/s recovering 2023-09-30T09:42:37.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:36 smithi019 ceph-mon[216815]: pgmap v1154: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 32474/131040 objects degraded (24.782%); 6335/131040 objects misplaced (4.834%); 10 MiB/s, 230 objects/s recovering 2023-09-30T09:42:37.045 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:37.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:37.153 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:37.178 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:36 smithi167 ceph-mon[192880]: pgmap v1154: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 32474/131040 objects degraded (24.782%); 6335/131040 objects misplaced (4.834%); 10 MiB/s, 230 objects/s recovering 2023-09-30T09:42:37.181 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:37.228 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:37.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:37.334 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:37.406 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:37.641 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:37.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:37.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:37.748 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:37.806 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:38.180 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:38.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:38.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:38.284 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:38.324 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:38.393 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:38.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:38.500 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:38.528 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:38.819 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:38 smithi019 ceph-mon[224043]: pgmap v1155: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 32474/131040 objects degraded (24.782%); 6335/131040 objects misplaced (4.834%); 10 MiB/s, 230 objects/s recovering 2023-09-30T09:42:38.819 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:38 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 32474/131040 objects degraded (24.782%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:38.825 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:42:39.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:38 smithi019 ceph-mon[216815]: pgmap v1155: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 32474/131040 objects degraded (24.782%); 6335/131040 objects misplaced (4.834%); 10 MiB/s, 230 objects/s recovering 2023-09-30T09:42:39.078 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:38 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 32474/131040 objects degraded (24.782%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:39.113 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:39.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:39.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:39.224 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:39.248 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:38 smithi167 ceph-mon[192880]: pgmap v1155: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 52 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 32474/131040 objects degraded (24.782%); 6335/131040 objects misplaced (4.834%); 10 MiB/s, 230 objects/s recovering 2023-09-30T09:42:39.248 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:38 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 32474/131040 objects degraded (24.782%), 42 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:39.251 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:39.300 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:39.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:39.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:39.408 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:39.441 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:39.488 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:39.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:39.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:39.594 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:39.663 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:39.845 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:39.883 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:42:39.883 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:42:39.883 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:42:39.883 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:42:39.884 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:42:39.904 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:39.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:39.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:40.009 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:40.062 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:40.161 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:40.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:40.265 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:40.295 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:40.344 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:40.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:40.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:40.451 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:40.499 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:40.580 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:40.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:40.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:40.686 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:40.712 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:40.762 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:40.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:40.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:40.825 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:40 smithi019 ceph-mon[224043]: pgmap v1156: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 409 op/s; 31370/126213 objects degraded (24.855%); 6164/126213 objects misplaced (4.884%); 10 MiB/s, 237 objects/s recovering 2023-09-30T09:42:40.826 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1135141573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:40.826 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:40 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:40.826 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:40 smithi019 ceph-mon[216815]: pgmap v1156: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 409 op/s; 31370/126213 objects degraded (24.855%); 6164/126213 objects misplaced (4.884%); 10 MiB/s, 237 objects/s recovering 2023-09-30T09:42:40.826 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1135141573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:40.826 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:40 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:40.847 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:42:40.868 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:40.912 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:40 smithi167 ceph-mon[192880]: pgmap v1156: 139 pgs: 5 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 409 op/s; 31370/126213 objects degraded (24.855%); 6164/126213 objects misplaced (4.884%); 10 MiB/s, 237 objects/s recovering 2023-09-30T09:42:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1135141573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:41.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:40 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:42:41.189 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:41.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:41.221 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:42:41.293 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:41.323 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:41.367 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:41.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:41.456 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0] out_osds: [3, 6] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:42:41.456 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:42:41.456 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-30T09:42:41.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-30T09:42:41.475 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:41.501 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:41.542 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:41.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:41.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:41.647 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:41.682 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:42.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:42:42.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:42.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[224043]: osdmap e446: 8 total, 7 up, 6 in 2023-09-30T09:42:42.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[224043]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[224043]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[224043]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:42:42.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:42.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[216815]: osdmap e446: 8 total, 7 up, 6 in 2023-09-30T09:42:42.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[216815]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[216815]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:41 smithi019 ceph-mon[216815]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:41 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:42:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:41 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:42:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:41 smithi167 ceph-mon[192880]: osdmap e446: 8 total, 7 up, 6 in 2023-09-30T09:42:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:41 smithi167 ceph-mon[192880]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:41 smithi167 ceph-mon[192880]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:41 smithi167 ceph-mon[192880]: failed to encode map e446 with expected crc 2023-09-30T09:42:42.519 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:42.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:42.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:42.630 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:42.657 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:42.725 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:42.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:42.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:42.830 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:42.858 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:42.907 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:42.912 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:42.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:42.913 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:42 smithi167 ceph-mon[192880]: pgmap v1158: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 414 op/s; 29765/125046 objects degraded (23.803%); 6789/125046 objects misplaced (5.429%); 13 MiB/s, 248 objects/s recovering 2023-09-30T09:42:42.981 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:43.014 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:43.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:42 smithi019 ceph-mon[216815]: pgmap v1158: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 414 op/s; 29765/125046 objects degraded (23.803%); 6789/125046 objects misplaced (5.429%); 13 MiB/s, 248 objects/s recovering 2023-09-30T09:42:43.060 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:42 smithi019 ceph-mon[224043]: pgmap v1158: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 414 op/s; 29765/125046 objects degraded (23.803%); 6789/125046 objects misplaced (5.429%); 13 MiB/s, 248 objects/s recovering 2023-09-30T09:42:43.065 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:43.220 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:43.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:43.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:43.326 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:43.372 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:43.525 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:42:43.557 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:42:43.588 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:42:43.621 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:42:43.657 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:42:43.699 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:42:43.730 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:42:43.731 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:42:43.731 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:42:43.731 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:42:43.731 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:42:43.731 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:42:43.731 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:42:43.731 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:42:43.731 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:42:43.821 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:43.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:43.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:43.941 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:43.971 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:44.017 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:44.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:44.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:44.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/383454761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:44.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 29765/125046 objects degraded (23.803%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/383454761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:44.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 29765/125046 objects degraded (23.803%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:44.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:42:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:42:43] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-09-30T09:42:44.122 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:44.147 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/383454761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:44.147 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 29765/125046 objects degraded (23.803%), 41 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:44.150 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:44.197 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:44.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:44.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:44.304 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:44.330 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:44.372 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:44.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:44.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:44.476 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:44.506 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:44.552 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:42:45.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:44 smithi167 ceph-mon[192880]: pgmap v1159: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 400 op/s; 29053/122076 objects degraded (23.799%); 6730/122076 objects misplaced (5.513%); 13 MiB/s, 246 objects/s recovering 2023-09-30T09:42:45.311 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:44 smithi019 ceph-mon[224043]: pgmap v1159: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 400 op/s; 29053/122076 objects degraded (23.799%); 6730/122076 objects misplaced (5.513%); 13 MiB/s, 246 objects/s recovering 2023-09-30T09:42:45.312 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:44 smithi019 ceph-mon[216815]: pgmap v1159: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 400 op/s; 29053/122076 objects degraded (23.799%); 6730/122076 objects misplaced (5.513%); 13 MiB/s, 246 objects/s recovering 2023-09-30T09:42:45.511 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:45.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:45.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:45.622 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:45.651 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:45.701 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:45.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:45.806 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:45.832 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:45.873 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:45.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:45.979 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:46.018 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:46.065 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:46.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:46.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:46.171 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:46.199 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:46.251 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:46.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:42:46.328 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:42:46.357 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:46.393 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:46.522 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:46.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:46.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:46.618 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:46.625 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:46.653 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:46.698 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:46.699 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:42:46.699 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:42:46.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:46.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:46.807 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:46.859 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:47.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:46 smithi167 ceph-mon[192880]: pgmap v1160: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 28325/118944 objects degraded (23.814%); 6433/118944 objects misplaced (5.408%); 12 MiB/s, 227 objects/s recovering 2023-09-30T09:42:47.099 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:47.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:46 smithi019 ceph-mon[216815]: pgmap v1160: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 28325/118944 objects degraded (23.814%); 6433/118944 objects misplaced (5.408%); 12 MiB/s, 227 objects/s recovering 2023-09-30T09:42:47.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:46 smithi019 ceph-mon[224043]: pgmap v1160: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 28325/118944 objects degraded (23.814%); 6433/118944 objects misplaced (5.408%); 12 MiB/s, 227 objects/s recovering 2023-09-30T09:42:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:47.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:47.206 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:47.276 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:47.362 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:47 smithi019 conmon[216791]: 2023-09-30T09:42:47.358+0000 7faf52971700 -1 mon.a@0(leader).osd e446 definitely_dead 0 2023-09-30T09:42:47.897 INFO:teuthology.orchestra.run.smithi019.stderr:marked in osd.3. 2023-09-30T09:42:47.929 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1943858273' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-30T09:42:47.930 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1943858273' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-30T09:42:48.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1943858273' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-30T09:42:48.329 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:42:48.802 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:42:48.949 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:48.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:49.053 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:49.079 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: pgmap v1161: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 28325/118944 objects degraded (23.814%); 6433/118944 objects misplaced (5.408%); 12 MiB/s, 227 objects/s recovering 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1943858273' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: osdmap e447: 8 total, 7 up, 7 in 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:49.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28325/118944 objects degraded (23.814%), 41 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[216815]: osdmap e448: 8 total, 7 up, 7 in 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: pgmap v1161: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 28325/118944 objects degraded (23.814%); 6433/118944 objects misplaced (5.408%); 12 MiB/s, 227 objects/s recovering 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1943858273' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: osdmap e447: 8 total, 7 up, 7 in 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:49.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:49.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28325/118944 objects degraded (23.814%), 41 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:49.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:48 smithi019 ceph-mon[224043]: osdmap e448: 8 total, 7 up, 7 in 2023-09-30T09:42:49.082 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:49.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: pgmap v1161: 139 pgs: 6 active+clean+remapped, 9 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 20 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 52 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 28325/118944 objects degraded (23.814%); 6433/118944 objects misplaced (5.408%); 12 MiB/s, 227 objects/s recovering 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1943858273' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: osdmap e447: 8 total, 7 up, 7 in 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: failed to encode map e447 with expected crc 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28325/118944 objects degraded (23.814%), 41 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:48 smithi167 ceph-mon[192880]: osdmap e448: 8 total, 7 up, 7 in 2023-09-30T09:42:49.562 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-30T09:42:49.742 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:49.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:49.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:49.851 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:42:49.877 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:49.921 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:42:49.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:49.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:42:50.014 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:50.027 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:50.041 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:50.053 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:50.097 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:50.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:50.202 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:50.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.8 starting backfill to osd.3 from (0'0,370'3444] MIN to 446'7085 2023-09-30T09:42:50.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.c starting backfill to osd.3 from (0'0,370'3493] MAX to 446'8948 2023-09-30T09:42:50.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.1c starting backfill to osd.3 from (0'0,370'3493] MAX to 446'8977 2023-09-30T09:42:50.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.b starting backfill to osd.3 from (0'0,370'3313] MAX to 446'8568 2023-09-30T09:42:50.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: osdmap e449: 8 total, 7 up, 7 in 2023-09-30T09:42:50.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.20 starting backfill to osd.3 from (0'0,370'3470] MIN to 446'6975 2023-09-30T09:42:50.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.10 starting backfill to osd.3 from (0'0,0'0] MAX to 446'7169 2023-09-30T09:42:50.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.f starting backfill to osd.3 from (0'0,370'3289] MAX to 446'8550 2023-09-30T09:42:50.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.1b starting backfill to osd.3 from (0'0,370'3313] MAX to 446'8541 2023-09-30T09:42:50.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.1f starting backfill to osd.3 from (0'0,370'3289] MAX to 446'8584 2023-09-30T09:42:50.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[224043]: 8.0 starting backfill to osd.3 from (0'0,370'3470] MIN to 446'6952 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.8 starting backfill to osd.3 from (0'0,370'3444] MIN to 446'7085 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.c starting backfill to osd.3 from (0'0,370'3493] MAX to 446'8948 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.1c starting backfill to osd.3 from (0'0,370'3493] MAX to 446'8977 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.b starting backfill to osd.3 from (0'0,370'3313] MAX to 446'8568 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: osdmap e449: 8 total, 7 up, 7 in 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.20 starting backfill to osd.3 from (0'0,370'3470] MIN to 446'6975 2023-09-30T09:42:50.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.10 starting backfill to osd.3 from (0'0,0'0] MAX to 446'7169 2023-09-30T09:42:50.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.f starting backfill to osd.3 from (0'0,370'3289] MAX to 446'8550 2023-09-30T09:42:50.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.1b starting backfill to osd.3 from (0'0,370'3313] MAX to 446'8541 2023-09-30T09:42:50.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.1f starting backfill to osd.3 from (0'0,370'3289] MAX to 446'8584 2023-09-30T09:42:50.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:49 smithi019 ceph-mon[216815]: 8.0 starting backfill to osd.3 from (0'0,370'3470] MIN to 446'6952 2023-09-30T09:42:50.249 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.8 starting backfill to osd.3 from (0'0,370'3444] MIN to 446'7085 2023-09-30T09:42:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.c starting backfill to osd.3 from (0'0,370'3493] MAX to 446'8948 2023-09-30T09:42:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.1c starting backfill to osd.3 from (0'0,370'3493] MAX to 446'8977 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: failed to encode map e448 with expected crc 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.b starting backfill to osd.3 from (0'0,370'3313] MAX to 446'8568 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: osdmap e449: 8 total, 7 up, 7 in 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.20 starting backfill to osd.3 from (0'0,370'3470] MIN to 446'6975 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.10 starting backfill to osd.3 from (0'0,0'0] MAX to 446'7169 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.f starting backfill to osd.3 from (0'0,370'3289] MAX to 446'8550 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.1b starting backfill to osd.3 from (0'0,370'3313] MAX to 446'8541 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.1f starting backfill to osd.3 from (0'0,370'3289] MAX to 446'8584 2023-09-30T09:42:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:49 smithi167 ceph-mon[192880]: 8.0 starting backfill to osd.3 from (0'0,370'3470] MIN to 446'6952 2023-09-30T09:42:50.371 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:50.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:50.476 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:50.514 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:50.582 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:50.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:50.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:50.686 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:50.730 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:51.049 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:51.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 446'6903 2023-09-30T09:42:51.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: 8.19 starting backfill to osd.3 from (0'0,0'0] MAX to 446'7018 2023-09-30T09:42:51.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: pgmap v1165: 139 pgs: 11 remapped+peering, 11 peering, 5 active+clean+remapped, 8 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 20 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 46 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 383 op/s; 20519/115173 objects degraded (17.816%); 5136/115173 objects misplaced (4.459%); 15 MiB/s, 296 objects/s recovering 2023-09-30T09:42:51.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:42:51.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: osdmap e450: 8 total, 7 up, 7 in 2023-09-30T09:42:51.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3556425831' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:51.052 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.052 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.052 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.052 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 446'6903 2023-09-30T09:42:51.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: 8.19 starting backfill to osd.3 from (0'0,0'0] MAX to 446'7018 2023-09-30T09:42:51.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: pgmap v1165: 139 pgs: 11 remapped+peering, 11 peering, 5 active+clean+remapped, 8 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 20 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 46 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 383 op/s; 20519/115173 objects degraded (17.816%); 5136/115173 objects misplaced (4.459%); 15 MiB/s, 296 objects/s recovering 2023-09-30T09:42:51.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:42:51.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: osdmap e450: 8 total, 7 up, 7 in 2023-09-30T09:42:51.053 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3556425831' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:51.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:51.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:51.155 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:42:51.189 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 446'6903 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: 8.19 starting backfill to osd.3 from (0'0,0'0] MAX to 446'7018 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: pgmap v1165: 139 pgs: 11 remapped+peering, 11 peering, 5 active+clean+remapped, 8 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 20 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 46 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 383 op/s; 20519/115173 objects degraded (17.816%); 5136/115173 objects misplaced (4.459%); 15 MiB/s, 296 objects/s recovering 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: failed to encode map e449 with expected crc 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: osdmap e450: 8 total, 7 up, 7 in 2023-09-30T09:42:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3556425831' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:42:51.641 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:42:52.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:52.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3556425831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[216815]: osdmap e451: 8 total, 7 up, 7 in 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3556425831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:42:52.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:51 smithi019 ceph-mon[224043]: osdmap e451: 8 total, 7 up, 7 in 2023-09-30T09:42:52.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: failed to encode map e450 with expected crc 2023-09-30T09:42:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:42:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3556425831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:42:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:51 smithi167 ceph-mon[192880]: osdmap e451: 8 total, 7 up, 7 in 2023-09-30T09:42:52.326 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:42:52.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:52.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:42:52.433 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:52.459 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:52.551 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:52.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:52.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:52.656 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:52.695 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:52 smithi167 ceph-mon[192880]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:52 smithi167 ceph-mon[192880]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:52 smithi167 ceph-mon[192880]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:52 smithi167 ceph-mon[192880]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:52 smithi167 ceph-mon[192880]: pgmap v1168: 139 pgs: 1 activating+undersized+remapped, 11 remapped+peering, 11 peering, 5 active+clean+remapped, 6 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+remapped+backfill_wait, 16 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+degraded+remapped, 44 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 21239/113553 objects degraded (18.704%); 11425/113553 objects misplaced (10.061%); 15 MiB/s, 313 objects/s recovering 2023-09-30T09:42:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:52 smithi167 ceph-mon[192880]: osdmap e452: 8 total, 7 up, 7 in 2023-09-30T09:42:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[224043]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[224043]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[224043]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[224043]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[224043]: pgmap v1168: 139 pgs: 1 activating+undersized+remapped, 11 remapped+peering, 11 peering, 5 active+clean+remapped, 6 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+remapped+backfill_wait, 16 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+degraded+remapped, 44 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 21239/113553 objects degraded (18.704%); 11425/113553 objects misplaced (10.061%); 15 MiB/s, 313 objects/s recovering 2023-09-30T09:42:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[224043]: osdmap e452: 8 total, 7 up, 7 in 2023-09-30T09:42:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[216815]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[216815]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[216815]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[216815]: failed to encode map e451 with expected crc 2023-09-30T09:42:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[216815]: pgmap v1168: 139 pgs: 1 activating+undersized+remapped, 11 remapped+peering, 11 peering, 5 active+clean+remapped, 6 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+remapped+backfill_wait, 16 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+degraded+remapped, 44 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 21239/113553 objects degraded (18.704%); 11425/113553 objects misplaced (10.061%); 15 MiB/s, 313 objects/s recovering 2023-09-30T09:42:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:52 smithi019 ceph-mon[216815]: osdmap e452: 8 total, 7 up, 7 in 2023-09-30T09:42:53.505 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:53.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:53.611 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:53.638 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:53.686 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:53.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:53.792 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:53.870 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:53.977 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[224043]: failed to encode map e452 with expected crc 2023-09-30T09:42:53.977 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[224043]: failed to encode map e452 with expected crc 2023-09-30T09:42:53.977 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[224043]: failed to encode map e452 with expected crc 2023-09-30T09:42:53.977 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[224043]: failed to encode map e452 with expected crc 2023-09-30T09:42:53.978 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 21239/113553 objects degraded (18.704%), 38 pgs degraded, 40 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:53.978 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[224043]: osdmap e453: 8 total, 7 up, 7 in 2023-09-30T09:42:53.978 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:42:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:42:53] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2023-09-30T09:42:53.978 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[216815]: failed to encode map e452 with expected crc 2023-09-30T09:42:53.978 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[216815]: failed to encode map e452 with expected crc 2023-09-30T09:42:53.979 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[216815]: failed to encode map e452 with expected crc 2023-09-30T09:42:53.979 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[216815]: failed to encode map e452 with expected crc 2023-09-30T09:42:53.979 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 21239/113553 objects degraded (18.704%), 38 pgs degraded, 40 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:53.979 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:53 smithi019 ceph-mon[216815]: osdmap e453: 8 total, 7 up, 7 in 2023-09-30T09:42:54.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:53 smithi167 ceph-mon[192880]: failed to encode map e452 with expected crc 2023-09-30T09:42:54.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:53 smithi167 ceph-mon[192880]: failed to encode map e452 with expected crc 2023-09-30T09:42:54.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:53 smithi167 ceph-mon[192880]: failed to encode map e452 with expected crc 2023-09-30T09:42:54.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:53 smithi167 ceph-mon[192880]: failed to encode map e452 with expected crc 2023-09-30T09:42:54.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:53 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 21239/113553 objects degraded (18.704%), 38 pgs degraded, 40 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:54.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:53 smithi167 ceph-mon[192880]: osdmap e453: 8 total, 7 up, 7 in 2023-09-30T09:42:54.052 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:42:54.052 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:42:54.348 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:42:54.386 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:54.388 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:42:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:54.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:54.429 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:42:54.467 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:42:54.508 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:42:54.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:54.539 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:54.551 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:42:54.563 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3] out_osds: [6] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:42:54.563 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:42:54.564 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-30T09:42:54.564 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-30T09:42:54.586 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:54.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:54.691 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:42:54.720 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:54.767 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:42:54.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:54.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:42:54.777 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:42:54.777 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:42:54.778 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:42:54.778 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:42:54.778 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:42:54.778 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:42:54.778 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:42:54.778 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:42:54.778 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:42:54.871 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:54.894 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[216815]: failed to encode map e453 with expected crc 2023-09-30T09:42:54.894 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[216815]: failed to encode map e453 with expected crc 2023-09-30T09:42:54.894 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[216815]: failed to encode map e453 with expected crc 2023-09-30T09:42:54.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[216815]: pgmap v1171: 139 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 17 active+clean+remapped, 6 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 57 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 30067/112584 objects degraded (26.706%); 16483/112584 objects misplaced (14.641%); 4.4 MiB/s, 70 objects/s recovering 2023-09-30T09:42:54.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[216815]: failed to encode map e453 with expected crc 2023-09-30T09:42:54.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2926982290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:54.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-30T09:42:54.942 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:54 smithi167 ceph-mon[192880]: failed to encode map e453 with expected crc 2023-09-30T09:42:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:54 smithi167 ceph-mon[192880]: failed to encode map e453 with expected crc 2023-09-30T09:42:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:54 smithi167 ceph-mon[192880]: failed to encode map e453 with expected crc 2023-09-30T09:42:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:54 smithi167 ceph-mon[192880]: pgmap v1171: 139 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 17 active+clean+remapped, 6 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 57 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 30067/112584 objects degraded (26.706%); 16483/112584 objects misplaced (14.641%); 4.4 MiB/s, 70 objects/s recovering 2023-09-30T09:42:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:54 smithi167 ceph-mon[192880]: failed to encode map e453 with expected crc 2023-09-30T09:42:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:54 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2926982290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:54 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-30T09:42:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[224043]: failed to encode map e453 with expected crc 2023-09-30T09:42:55.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[224043]: failed to encode map e453 with expected crc 2023-09-30T09:42:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[224043]: failed to encode map e453 with expected crc 2023-09-30T09:42:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[224043]: pgmap v1171: 139 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 17 active+clean+remapped, 6 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 57 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 30067/112584 objects degraded (26.706%); 16483/112584 objects misplaced (14.641%); 4.4 MiB/s, 70 objects/s recovering 2023-09-30T09:42:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[224043]: failed to encode map e453 with expected crc 2023-09-30T09:42:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2926982290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:42:55.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:54 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-30T09:42:55.453 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:42:55.557 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:42:55.579 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:55.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:55.685 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:42:55.714 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:55.758 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:42:55.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:55.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:42:55.863 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:55.890 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:55.934 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:55.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:55.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:56.041 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:56.125 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:42:56.207 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:42:56.241 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:56.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:56.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:56.351 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:56.378 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:56.426 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:56.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:56.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:56.531 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:42:56.544 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:56.552 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:56.626 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:42:56.635 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:56.905 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:56 smithi019 ceph-mon[216815]: pgmap v1172: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 28532/110991 objects degraded (25.707%); 17311/110991 objects misplaced (15.597%); 3.4 MiB/s, 212 objects/s recovering 2023-09-30T09:42:56.907 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:42:56.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:56.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:42:57.012 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:57.085 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:57.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:56 smithi019 ceph-mon[224043]: pgmap v1172: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 28532/110991 objects degraded (25.707%); 17311/110991 objects misplaced (15.597%); 3.4 MiB/s, 212 objects/s recovering 2023-09-30T09:42:57.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:56 smithi167 ceph-mon[192880]: pgmap v1172: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 28532/110991 objects degraded (25.707%); 17311/110991 objects misplaced (15.597%); 3.4 MiB/s, 212 objects/s recovering 2023-09-30T09:42:57.627 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:57.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:57.737 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:57.819 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:58.208 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:42:59.009 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:59.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:59.122 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:42:59.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:58 smithi019 ceph-mon[224043]: pgmap v1173: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 28336/109449 objects degraded (25.890%); 16812/109449 objects misplaced (15.361%); 1022 KiB/s, 150 objects/s recovering 2023-09-30T09:42:59.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:42:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 28336/109449 objects degraded (25.890%), 38 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:59.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:58 smithi019 ceph-mon[216815]: pgmap v1173: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 28336/109449 objects degraded (25.890%); 16812/109449 objects misplaced (15.361%); 1022 KiB/s, 150 objects/s recovering 2023-09-30T09:42:59.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:42:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 28336/109449 objects degraded (25.890%), 38 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:59.188 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:58 smithi167 ceph-mon[192880]: pgmap v1173: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 28336/109449 objects degraded (25.890%); 16812/109449 objects misplaced (15.361%); 1022 KiB/s, 150 objects/s recovering 2023-09-30T09:42:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:42:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 28336/109449 objects degraded (25.890%), 38 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-30T09:42:59.683 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:42:59.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:59.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:42:59.805 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:42:59.832 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:42:59.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:42:59.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:42:59.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:42:59.984 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:00.013 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:00.059 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:00.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:00.164 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:00.220 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:00.744 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:00.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:00.910 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:00.939 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:00.989 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:00.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:00.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:01.015 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:00 smithi019 ceph-mon[216815]: pgmap v1174: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 27487/102726 objects degraded (26.758%); 16362/102726 objects misplaced (15.928%); 1.2 MiB/s, 120 objects/s recovering 2023-09-30T09:43:01.015 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:00 smithi019 ceph-mon[224043]: pgmap v1174: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 27487/102726 objects degraded (26.758%); 16362/102726 objects misplaced (15.928%); 1.2 MiB/s, 120 objects/s recovering 2023-09-30T09:43:01.095 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:01.265 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:01.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:00 smithi167 ceph-mon[192880]: pgmap v1174: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 27487/102726 objects degraded (26.758%); 16362/102726 objects misplaced (15.928%); 1.2 MiB/s, 120 objects/s recovering 2023-09-30T09:43:01.456 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:43:01.457 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:43:01.457 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:43:01.457 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:43:01.457 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:43:01.491 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:01.728 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:01.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:01.829 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:01.884 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:01 smithi019 conmon[216791]: 2023-09-30T09:43:01.695+0000 7faf52971700 -1 mon.a@0(leader).osd e453 definitely_dead 0 2023-09-30T09:43:01.913 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:01.982 INFO:teuthology.orchestra.run.smithi019.stderr:marked in osd.6. 2023-09-30T09:43:01.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:01.991 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:01.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:02.092 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:02.149 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:02.190 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:02.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:02.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3446055450' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-30T09:43:02.297 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:02.330 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:02.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3446055450' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-30T09:43:02.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3446055450' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-30T09:43:02.374 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:02.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:02.480 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:02.511 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:02.554 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:02.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:02.682 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:02.716 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:02.818 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:02.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:02.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:02.969 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:02.985 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:43:03.022 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:03.075 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:03.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:03.180 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:03.210 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:03.257 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:03.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: pgmap v1175: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 26841/99117 objects degraded (27.080%); 15254/99117 objects misplaced (15.390%); 1.0 MiB/s, 106 objects/s recovering 2023-09-30T09:43:03.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3446055450' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-30T09:43:03.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: osdmap e454: 8 total, 7 up, 8 in 2023-09-30T09:43:03.288 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1421257592' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:03.289 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:02 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:03.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: pgmap v1175: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 26841/99117 objects degraded (27.080%); 15254/99117 objects misplaced (15.390%); 1.0 MiB/s, 106 objects/s recovering 2023-09-30T09:43:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3446055450' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-30T09:43:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: osdmap e454: 8 total, 7 up, 8 in 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1421257592' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: pgmap v1175: 139 pgs: 20 active+clean+remapped, 3 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 18 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 26841/99117 objects degraded (27.080%); 15254/99117 objects misplaced (15.390%); 1.0 MiB/s, 106 objects/s recovering 2023-09-30T09:43:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3446055450' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-30T09:43:03.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: osdmap e454: 8 total, 7 up, 8 in 2023-09-30T09:43:03.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: failed to encode map e454 with expected crc 2023-09-30T09:43:03.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1421257592' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:03.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:02 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:03.391 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:03.419 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:03.468 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:03.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:03.573 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:03.605 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:04.005 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:43:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:43:03] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-09-30T09:43:04.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:04.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:04.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: osdmap e455: 8 total, 7 up, 8 in 2023-09-30T09:43:04.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:04.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:04.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 26841/99117 objects degraded (27.080%), 38 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:04.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[216815]: osdmap e456: 8 total, 7 up, 8 in 2023-09-30T09:43:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: osdmap e455: 8 total, 7 up, 8 in 2023-09-30T09:43:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 26841/99117 objects degraded (27.080%), 38 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:04 smithi167 ceph-mon[192880]: osdmap e456: 8 total, 7 up, 8 in 2023-09-30T09:43:04.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:04.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:04.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: osdmap e455: 8 total, 7 up, 8 in 2023-09-30T09:43:04.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: failed to encode map e455 with expected crc 2023-09-30T09:43:04.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:04.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:04.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 26841/99117 objects degraded (27.080%), 38 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:04.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:04 smithi019 ceph-mon[224043]: osdmap e456: 8 total, 7 up, 8 in 2023-09-30T09:43:04.833 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-30T09:43:04.837 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:43:05.034 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:05.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:05.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: pgmap v1178: 139 pgs: 4 peering, 14 remapped+peering, 13 active+clean+remapped, 2 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 57 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 24935/96720 objects degraded (25.781%); 13184/96720 objects misplaced (13.631%); 408 KiB/s, 13 objects/s recovering 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: 8.5 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'7205 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: 8.19 starting backfill to osd.6 from (0'0,388'5541] MIN to 453'7205 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: 8.13 starting backfill to osd.6 from (0'0,388'5503] MIN to 453'7163 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: 8.3 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'6805 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: 8.23 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'6828 2023-09-30T09:43:05.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[224043]: osdmap e457: 8 total, 7 up, 8 in 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: pgmap v1178: 139 pgs: 4 peering, 14 remapped+peering, 13 active+clean+remapped, 2 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 57 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 24935/96720 objects degraded (25.781%); 13184/96720 objects misplaced (13.631%); 408 KiB/s, 13 objects/s recovering 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: 8.5 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'7205 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: 8.19 starting backfill to osd.6 from (0'0,388'5541] MIN to 453'7205 2023-09-30T09:43:05.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: 8.13 starting backfill to osd.6 from (0'0,388'5503] MIN to 453'7163 2023-09-30T09:43:05.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: 8.3 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'6805 2023-09-30T09:43:05.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: 8.23 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'6828 2023-09-30T09:43:05.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:05 smithi019 ceph-mon[216815]: osdmap e457: 8 total, 7 up, 8 in 2023-09-30T09:43:05.139 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:05.213 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: pgmap v1178: 139 pgs: 4 peering, 14 remapped+peering, 13 active+clean+remapped, 2 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 57 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 24935/96720 objects degraded (25.781%); 13184/96720 objects misplaced (13.631%); 408 KiB/s, 13 objects/s recovering 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: 8.5 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'7205 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: failed to encode map e456 with expected crc 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: 8.19 starting backfill to osd.6 from (0'0,388'5541] MIN to 453'7205 2023-09-30T09:43:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: 8.13 starting backfill to osd.6 from (0'0,388'5503] MIN to 453'7163 2023-09-30T09:43:05.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: 8.3 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'6805 2023-09-30T09:43:05.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: 8.23 starting backfill to osd.6 from (0'0,388'5531] MIN to 453'6828 2023-09-30T09:43:05.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:05 smithi167 ceph-mon[192880]: osdmap e457: 8 total, 7 up, 8 in 2023-09-30T09:43:05.385 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:05.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:05.509 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:05.552 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:05.608 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:05.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:05.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:05.713 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:05.739 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:05.792 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:05.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:05.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:05.898 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:05.927 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:05.990 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:05.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:06.093 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:06.122 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:06.173 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:06.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.175 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[216815]: osdmap e458: 8 total, 7 up, 8 in 2023-09-30T09:43:06.176 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:06 smithi019 ceph-mon[224043]: osdmap e458: 8 total, 7 up, 8 in 2023-09-30T09:43:06.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:06.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: failed to encode map e457 with expected crc 2023-09-30T09:43:06.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:06 smithi167 ceph-mon[192880]: osdmap e458: 8 total, 7 up, 8 in 2023-09-30T09:43:06.279 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:06.307 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:06.356 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:06.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:06.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:06.462 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:06.490 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:06.541 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:06.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:06.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:06.646 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:06.678 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:06.955 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:06.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:07.058 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:07.128 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:07.144 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:07.196 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:43:07.202 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:07.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:07.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:07.233 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[216815]: pgmap v1181: 139 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 peering, 15 remapped+peering, 18 active+clean+remapped, 2 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 58 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 25524/95088 objects degraded (26.843%); 12364/95088 objects misplaced (13.003%); 6.2 MiB/s, 186 objects/s recovering 2023-09-30T09:43:07.233 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[216815]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.233 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[216815]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.233 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[216815]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.233 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[216815]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[216815]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[216815]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[216815]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[224043]: pgmap v1181: 139 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 peering, 15 remapped+peering, 18 active+clean+remapped, 2 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 58 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 25524/95088 objects degraded (26.843%); 12364/95088 objects misplaced (13.003%); 6.2 MiB/s, 186 objects/s recovering 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[224043]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[224043]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[224043]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[224043]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[224043]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[224043]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.235 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:07 smithi019 ceph-mon[224043]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:07 smithi167 ceph-mon[192880]: pgmap v1181: 139 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 peering, 15 remapped+peering, 18 active+clean+remapped, 2 active+recovery_wait+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 58 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 25524/95088 objects degraded (26.843%); 12364/95088 objects misplaced (13.003%); 6.2 MiB/s, 186 objects/s recovering 2023-09-30T09:43:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:07 smithi167 ceph-mon[192880]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:07 smithi167 ceph-mon[192880]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:07 smithi167 ceph-mon[192880]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:07 smithi167 ceph-mon[192880]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:07 smithi167 ceph-mon[192880]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:07 smithi167 ceph-mon[192880]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:07 smithi167 ceph-mon[192880]: failed to encode map e458 with expected crc 2023-09-30T09:43:07.307 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:07.346 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:07.425 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:07.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:07.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:07.534 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:07.561 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:07.607 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:07.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:07.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:07.713 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:07.737 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:43:07.755 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:07.755 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:43:07.777 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:43:07.786 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:07.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:07.891 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:07.918 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:07.964 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:07.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:07.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:08.033 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[216815]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.033 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[216815]: osdmap e459: 8 total, 7 up, 8 in 2023-09-30T09:43:08.033 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[216815]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.033 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[216815]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.033 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3347282833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:08.070 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:08.078 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:43:08.100 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:08.108 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:43:08.133 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:43:08.147 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:08.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:08.163 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:43:08.163 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:43:08.163 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:43:08.163 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:43:08.163 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:43:08.163 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:43:08.163 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:43:08.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:43:08.164 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:43:08.254 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:08.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:08 smithi167 ceph-mon[192880]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:08 smithi167 ceph-mon[192880]: osdmap e459: 8 total, 7 up, 8 in 2023-09-30T09:43:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:08 smithi167 ceph-mon[192880]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:08 smithi167 ceph-mon[192880]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3347282833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:08.281 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:08.325 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:08.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:08.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:08.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[224043]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[224043]: osdmap e459: 8 total, 7 up, 8 in 2023-09-30T09:43:08.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[224043]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[224043]: failed to encode map e459 with expected crc 2023-09-30T09:43:08.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3347282833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:08.433 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:08.485 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:09.268 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:09.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:09.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:09.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[224043]: pgmap v1184: 139 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 peering, 15 remapped+peering, 18 active+clean+remapped, 2 active+recovery_wait+degraded, 8 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 2 active+remapped+backfill_wait, 9 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 199 op/s; 24392/94323 objects degraded (25.860%); 15171/94323 objects misplaced (16.084%); 6.2 MiB/s, 407 objects/s recovering 2023-09-30T09:43:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[224043]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[224043]: osdmap e460: 8 total, 7 up, 8 in 2023-09-30T09:43:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[224043]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[224043]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 24392/94323 objects degraded (25.860%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[224043]: osdmap e461: 8 total, 7 up, 8 in 2023-09-30T09:43:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[216815]: pgmap v1184: 139 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 peering, 15 remapped+peering, 18 active+clean+remapped, 2 active+recovery_wait+degraded, 8 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 2 active+remapped+backfill_wait, 9 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 199 op/s; 24392/94323 objects degraded (25.860%); 15171/94323 objects misplaced (16.084%); 6.2 MiB/s, 407 objects/s recovering 2023-09-30T09:43:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[216815]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[216815]: osdmap e460: 8 total, 7 up, 8 in 2023-09-30T09:43:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[216815]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[216815]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 24392/94323 objects degraded (25.860%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:09.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:09 smithi019 ceph-mon[216815]: osdmap e461: 8 total, 7 up, 8 in 2023-09-30T09:43:09.376 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:09.423 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:09 smithi167 ceph-mon[192880]: pgmap v1184: 139 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 peering, 15 remapped+peering, 18 active+clean+remapped, 2 active+recovery_wait+degraded, 8 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 2 active+remapped+backfill_wait, 9 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 199 op/s; 24392/94323 objects degraded (25.860%); 15171/94323 objects misplaced (16.084%); 6.2 MiB/s, 407 objects/s recovering 2023-09-30T09:43:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:09 smithi167 ceph-mon[192880]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:09 smithi167 ceph-mon[192880]: osdmap e460: 8 total, 7 up, 8 in 2023-09-30T09:43:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:09 smithi167 ceph-mon[192880]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:09 smithi167 ceph-mon[192880]: failed to encode map e460 with expected crc 2023-09-30T09:43:09.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 24392/94323 objects degraded (25.860%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:09.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:09 smithi167 ceph-mon[192880]: osdmap e461: 8 total, 7 up, 8 in 2023-09-30T09:43:09.605 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:09.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:09.712 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:09.756 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:09.832 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:09.834 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:43:09.834 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:43:09.834 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:43:09.834 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-09-30T09:43:09.835 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-30T09:43:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:09.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:09.938 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:10.005 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:10.093 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:10.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:10.197 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:10.232 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:10.290 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:10.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:10 smithi167 ceph-mon[192880]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:10 smithi167 ceph-mon[192880]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:10 smithi167 ceph-mon[192880]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:10 smithi167 ceph-mon[192880]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.295 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:10 smithi167 ceph-mon[192880]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.296 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:10 smithi167 ceph-mon[192880]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.296 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:10 smithi167 ceph-mon[192880]: osdmap e462: 8 total, 7 up, 8 in 2023-09-30T09:43:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:10.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:10.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[224043]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[224043]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[224043]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[224043]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[224043]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[224043]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[224043]: osdmap e462: 8 total, 7 up, 8 in 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[216815]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[216815]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[216815]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[216815]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[216815]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[216815]: failed to encode map e461 with expected crc 2023-09-30T09:43:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:10 smithi019 ceph-mon[216815]: osdmap e462: 8 total, 7 up, 8 in 2023-09-30T09:43:10.397 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:10.433 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:11.286 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:11.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:11.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[216815]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[216815]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[216815]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[216815]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[216815]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[216815]: pgmap v1188: 139 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 remapped+peering, 26 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 67 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 31365/93636 objects degraded (33.497%); 18048/93636 objects misplaced (19.275%); 0 B/s, 524 objects/s recovering 2023-09-30T09:43:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[224043]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[224043]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[224043]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[224043]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[224043]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:11 smithi019 ceph-mon[224043]: pgmap v1188: 139 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 remapped+peering, 26 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 67 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 31365/93636 objects degraded (33.497%); 18048/93636 objects misplaced (19.275%); 0 B/s, 524 objects/s recovering 2023-09-30T09:43:11.393 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:11.416 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:11 smithi167 ceph-mon[192880]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.417 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:11 smithi167 ceph-mon[192880]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.417 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:11 smithi167 ceph-mon[192880]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.417 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:11 smithi167 ceph-mon[192880]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.417 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:11 smithi167 ceph-mon[192880]: failed to encode map e462 with expected crc 2023-09-30T09:43:11.417 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:11 smithi167 ceph-mon[192880]: pgmap v1188: 139 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 remapped+peering, 26 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 67 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 31365/93636 objects degraded (33.497%); 18048/93636 objects misplaced (19.275%); 0 B/s, 524 objects/s recovering 2023-09-30T09:43:11.419 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:11.476 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:11.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:11.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:11.582 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:11.603 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:11.614 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:11.675 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:11.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:43:11.764 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:43:11.782 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:11.810 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:11.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:11.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:11.988 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:12.051 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:12.059 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:12.454 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:12.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:12.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:12.501 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:43:12.501 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:43:12.558 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:12.598 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: pgmap v1189: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 28 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 222 op/s; 29543/91167 objects degraded (32.405%); 18394/91167 objects misplaced (20.176%); 5.4 MiB/s, 516 objects/s recovering 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: osdmap e463: 8 total, 7 up, 8 in 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[224043]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: pgmap v1189: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 28 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 222 op/s; 29543/91167 objects degraded (32.405%); 18394/91167 objects misplaced (20.176%); 5.4 MiB/s, 516 objects/s recovering 2023-09-30T09:43:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: osdmap e463: 8 total, 7 up, 8 in 2023-09-30T09:43:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:13 smithi019 ceph-mon[216815]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: pgmap v1189: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 28 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 222 op/s; 29543/91167 objects degraded (32.405%); 18394/91167 objects misplaced (20.176%); 5.4 MiB/s, 516 objects/s recovering 2023-09-30T09:43:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: osdmap e463: 8 total, 7 up, 8 in 2023-09-30T09:43:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:13 smithi167 ceph-mon[192880]: failed to encode map e463 with expected crc 2023-09-30T09:43:13.765 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:43:14.077 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:43:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:43:13] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-09-30T09:43:14.311 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:43:14.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[224043]: osdmap e464: 8 total, 7 up, 8 in 2023-09-30T09:43:14.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[224043]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[224043]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[224043]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[224043]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:43:14.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 29543/91167 objects degraded (32.405%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:14.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[216815]: osdmap e464: 8 total, 7 up, 8 in 2023-09-30T09:43:14.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[216815]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[216815]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[216815]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[216815]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:43:14.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 29543/91167 objects degraded (32.405%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:14.428 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:14.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:14.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:14 smithi167 ceph-mon[192880]: osdmap e464: 8 total, 7 up, 8 in 2023-09-30T09:43:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:14 smithi167 ceph-mon[192880]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:14 smithi167 ceph-mon[192880]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:14 smithi167 ceph-mon[192880]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:14 smithi167 ceph-mon[192880]: failed to encode map e464 with expected crc 2023-09-30T09:43:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:14 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:43:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 29543/91167 objects degraded (32.405%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:14.543 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:14.610 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:14.799 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:14.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:14.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:14.903 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:14.945 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[224043]: pgmap v1192: 139 pgs: 1 remapped+peering, 2 peering, 28 active+clean+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 71 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 28999/88680 objects degraded (32.701%); 17286/88680 objects misplaced (19.493%); 9.6 MiB/s, 481 objects/s recovering 2023-09-30T09:43:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[224043]: osdmap e465: 8 total, 7 up, 8 in 2023-09-30T09:43:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[224043]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[224043]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[224043]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[224043]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[224043]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[224043]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[216815]: pgmap v1192: 139 pgs: 1 remapped+peering, 2 peering, 28 active+clean+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 71 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 28999/88680 objects degraded (32.701%); 17286/88680 objects misplaced (19.493%); 9.6 MiB/s, 481 objects/s recovering 2023-09-30T09:43:15.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[216815]: osdmap e465: 8 total, 7 up, 8 in 2023-09-30T09:43:15.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[216815]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[216815]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[216815]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[216815]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[216815]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:15 smithi019 ceph-mon[216815]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:15 smithi167 ceph-mon[192880]: pgmap v1192: 139 pgs: 1 remapped+peering, 2 peering, 28 active+clean+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 71 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 28999/88680 objects degraded (32.701%); 17286/88680 objects misplaced (19.493%); 9.6 MiB/s, 481 objects/s recovering 2023-09-30T09:43:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:15 smithi167 ceph-mon[192880]: osdmap e465: 8 total, 7 up, 8 in 2023-09-30T09:43:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:15 smithi167 ceph-mon[192880]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:15 smithi167 ceph-mon[192880]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:15 smithi167 ceph-mon[192880]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:15 smithi167 ceph-mon[192880]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:15 smithi167 ceph-mon[192880]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:15 smithi167 ceph-mon[192880]: failed to encode map e465 with expected crc 2023-09-30T09:43:15.691 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:15.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:15.799 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:15.819 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:15.846 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:16.008 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:16.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:16.117 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:16.187 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:16.366 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:16.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:16.419 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:16.478 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:16.570 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:16.881 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:16.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:16.987 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:17.037 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:17.087 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:17.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:17.118 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-30T09:43:17.118 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:43:17.191 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:17.252 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:17.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:17 smithi167 ceph-mon[192880]: pgmap v1194: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 remapped+peering, 1 peering, 28 active+clean+remapped, 1 active+recovering+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 26747/87141 objects degraded (30.694%); 17076/87141 objects misplaced (19.596%); 14 MiB/s, 681 objects/s recovering 2023-09-30T09:43:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2557770446' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:43:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:43:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/781442394' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:43:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:17 smithi019 ceph-mon[216815]: pgmap v1194: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 remapped+peering, 1 peering, 28 active+clean+remapped, 1 active+recovering+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 26747/87141 objects degraded (30.694%); 17076/87141 objects misplaced (19.596%); 14 MiB/s, 681 objects/s recovering 2023-09-30T09:43:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2557770446' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:43:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:43:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/781442394' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:43:17.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:17 smithi019 ceph-mon[224043]: pgmap v1194: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 remapped+peering, 1 peering, 28 active+clean+remapped, 1 active+recovering+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 26747/87141 objects degraded (30.694%); 17076/87141 objects misplaced (19.596%); 14 MiB/s, 681 objects/s recovering 2023-09-30T09:43:17.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2557770446' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:43:17.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:43:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/781442394' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:43:18.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:18.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:43:18.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/781442394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:43:18.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: failed to encode map e466 with expected crc 2023-09-30T09:43:18.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: osdmap e466: 8 total, 7 up, 8 in 2023-09-30T09:43:18.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: failed to encode map e466 with expected crc 2023-09-30T09:43:18.382 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:18.382 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:18.382 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:18.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:18.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:43:18.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/781442394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:43:18.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: failed to encode map e466 with expected crc 2023-09-30T09:43:18.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: osdmap e466: 8 total, 7 up, 8 in 2023-09-30T09:43:18.383 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: failed to encode map e466 with expected crc 2023-09-30T09:43:18.383 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:18.383 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:18.383 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:43:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/781442394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:43:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: failed to encode map e466 with expected crc 2023-09-30T09:43:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: osdmap e466: 8 total, 7 up, 8 in 2023-09-30T09:43:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: failed to encode map e466 with expected crc 2023-09-30T09:43:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:18.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:19.217 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:19.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:19.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:19.325 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:19.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: pgmap v1196: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 remapped+peering, 1 peering, 28 active+clean+remapped, 1 active+recovering+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 26540/85914 objects degraded (30.891%); 16667/85914 objects misplaced (19.400%); 10 MiB/s, 479 objects/s recovering 2023-09-30T09:43:19.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: osdmap e467: 8 total, 7 up, 8 in 2023-09-30T09:43:19.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 26540/85914 objects degraded (30.891%), 22 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:19.385 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: pgmap v1196: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 remapped+peering, 1 peering, 28 active+clean+remapped, 1 active+recovering+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 26540/85914 objects degraded (30.891%); 16667/85914 objects misplaced (19.400%); 10 MiB/s, 479 objects/s recovering 2023-09-30T09:43:19.385 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: osdmap e467: 8 total, 7 up, 8 in 2023-09-30T09:43:19.385 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.385 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.386 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.386 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.386 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.386 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.386 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 26540/85914 objects degraded (30.891%), 22 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:19.393 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:19.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: pgmap v1196: 139 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 remapped+peering, 1 peering, 28 active+clean+remapped, 1 active+recovering+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 26540/85914 objects degraded (30.891%); 16667/85914 objects misplaced (19.400%); 10 MiB/s, 479 objects/s recovering 2023-09-30T09:43:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: osdmap e467: 8 total, 7 up, 8 in 2023-09-30T09:43:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: failed to encode map e467 with expected crc 2023-09-30T09:43:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 26540/85914 objects degraded (30.891%), 22 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:20.094 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:20.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:20.199 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:20.265 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:21.267 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:21.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:21.341 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:43:21.381 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:21.402 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:21 smithi019 ceph-mon[216815]: pgmap v1198: 139 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 320 op/s; 24888/82911 objects degraded (30.018%); 16371/82911 objects misplaced (19.745%); 12 MiB/s, 429 objects/s recovering 2023-09-30T09:43:21.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:21 smithi019 ceph-mon[224043]: pgmap v1198: 139 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 320 op/s; 24888/82911 objects degraded (30.018%); 16371/82911 objects misplaced (19.745%); 12 MiB/s, 429 objects/s recovering 2023-09-30T09:43:21.454 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:21.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:21 smithi167 ceph-mon[192880]: pgmap v1198: 139 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 320 op/s; 24888/82911 objects degraded (30.018%); 16371/82911 objects misplaced (19.745%); 12 MiB/s, 429 objects/s recovering 2023-09-30T09:43:21.584 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:21.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:21.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:21.625 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:43:21.678 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:43:21.696 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:21.724 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:43:21.765 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:43:21.807 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:21.816 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:43:21.864 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:43:21.913 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:43:21.913 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:43:21.913 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:43:21.913 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:43:21.914 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:43:21.914 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:43:21.914 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:43:21.914 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:43:21.914 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:43:22.097 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:43:22.097 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:43:22.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.098 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:159743 8.19 8:9bb42475:::benchmark_data_smithi019_478679_object67295:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-30T09:43:22.098 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:22.089395+0000", 2023-09-30T09:43:22.098 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0067142970000000001, 2023-09-30T09:43:22.098 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0067433930000000003, 2023-09-30T09:43:22.098 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:22.098 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:43:22.098 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:22.098 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:22.099 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:22.099 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 159743 2023-09-30T09:43:22.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.099 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:22.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:22.100 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089395+0000", 2023-09-30T09:43:22.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:22.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:22.100 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089395+0000", 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089396+0000", 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.852e-06 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089403+0000", 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4110000000000001e-06 2023-09-30T09:43:22.101 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089423+0000", 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9692999999999999e-05 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089432+0000", 2023-09-30T09:43:22.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9849999999999995e-06 2023-09-30T09:43:22.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:22.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089494+0000", 2023-09-30T09:43:22.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.211e-05 2023-09-30T09:43:22.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089639+0000", 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000145375 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,6", 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.090086+0000", 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00044671000000000001 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092391+0000", 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023052340000000002 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:159734 8.1 e467/435 8:839b55fd:::benchmark_data_smithi019_478679_object67286:head v 467'6865, mlcod=467'6863)", 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:22.089615+0000", 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0064936100000000004, 2023-09-30T09:43:22.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0066295269999999996, 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089615+0000", 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:22.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089615+0000", 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089616+0000", 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9999999999999998e-07 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089620+0000", 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.775e-06 2023-09-30T09:43:22.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:22.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089628+0000", 2023-09-30T09:43:22.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7160000000000003e-06 2023-09-30T09:43:22.108 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:22.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.089635+0000", 2023-09-30T09:43:22.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5579999999999997e-06 2023-09-30T09:43:22.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.109 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:22.109 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.091781+0000", 2023-09-30T09:43:22.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002146561 2023-09-30T09:43:22.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.091799+0000", 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8159e-05 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.110 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:159744 8.1a 8:5b7f3615:::benchmark_data_smithi019_478679_object67296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:22.092370+0000", 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.003738398, 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0039114500000000003, 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 159744 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.111 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092370+0000", 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092370+0000", 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092372+0000", 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.497e-06 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092377+0000", 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.276e-06 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:22.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092390+0000", 2023-09-30T09:43:22.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2917e-05 2023-09-30T09:43:22.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:22.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092399+0000", 2023-09-30T09:43:22.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0440000000000001e-06 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092460+0000", 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0825999999999997e-05 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092614+0000", 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015359099999999999 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1", 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.092968+0000", 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035478499999999999 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:22.094841+0000", 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00187295 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:22.116 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:22.117 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2023-09-30T09:43:22.117 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 3 2023-09-30T09:43:22.117 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:43:22.117 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:22.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:22.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:22.229 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:22.263 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:22.313 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:22.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:22.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:22.425 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:22.490 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1846725483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:22.491 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1846725483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:22.496 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:22.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1846725483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:23.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:23 smithi167 ceph-mon[192880]: pgmap v1199: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 22140/79128 objects degraded (27.980%); 15007/79128 objects misplaced (18.965%); 12 MiB/s, 377 objects/s recovering 2023-09-30T09:43:23.580 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:23 smithi019 ceph-mon[224043]: pgmap v1199: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 22140/79128 objects degraded (27.980%); 15007/79128 objects misplaced (18.965%); 12 MiB/s, 377 objects/s recovering 2023-09-30T09:43:23.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:23 smithi019 ceph-mon[216815]: pgmap v1199: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 22140/79128 objects degraded (27.980%); 15007/79128 objects misplaced (18.965%); 12 MiB/s, 377 objects/s recovering 2023-09-30T09:43:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:43:23.618 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:43:23.749 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:23.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:23.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:23.834 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:43:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:43:23] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-09-30T09:43:23.853 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:23.867 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:43:23.926 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:23.938 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:24.058 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:24.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:24.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:24.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22140/79128 objects degraded (27.980%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:24.169 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22140/79128 objects degraded (27.980%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:24.169 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:24.200 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:24.254 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:24.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:24.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:24.360 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:24.388 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:24.439 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:24.445 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22140/79128 objects degraded (27.980%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:24.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:24.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:24.547 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:24.606 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:24.900 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:24.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:24.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:25.008 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:25.037 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:25.081 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:25.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:25.186 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:25.212 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:25.253 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:25.254 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:43:25.254 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:43:25.254 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-30T09:43:25.255 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:43:25.255 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:43:25.255 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-30T09:43:25.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:25.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:25.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:25 smithi019 ceph-mon[216815]: pgmap v1200: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 21706/75513 objects degraded (28.745%); 14611/75513 objects misplaced (19.349%); 8.4 MiB/s, 183 objects/s recovering 2023-09-30T09:43:25.281 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:25 smithi019 ceph-mon[224043]: pgmap v1200: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 21706/75513 objects degraded (28.745%); 14611/75513 objects misplaced (19.349%); 8.4 MiB/s, 183 objects/s recovering 2023-09-30T09:43:25.358 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:25.459 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:25.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:25 smithi167 ceph-mon[192880]: pgmap v1200: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 21706/75513 objects degraded (28.745%); 14611/75513 objects misplaced (19.349%); 8.4 MiB/s, 183 objects/s recovering 2023-09-30T09:43:25.618 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:43:25.800 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:25.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:25.913 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:25.920 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:25.969 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:26.085 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:26.221 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:26.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:26.228 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:26 smithi019 ceph-mon[216815]: pgmap v1201: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 21318/73299 objects degraded (29.084%); 14225/73299 objects misplaced (19.407%); 9.9 MiB/s, 204 objects/s recovering 2023-09-30T09:43:26.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:26 smithi019 ceph-mon[224043]: pgmap v1201: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 21318/73299 objects degraded (29.084%); 14225/73299 objects misplaced (19.407%); 9.9 MiB/s, 204 objects/s recovering 2023-09-30T09:43:26.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:26.329 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:26.424 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:26.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:26 smithi167 ceph-mon[192880]: pgmap v1201: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 21318/73299 objects degraded (29.084%); 14225/73299 objects misplaced (19.407%); 9.9 MiB/s, 204 objects/s recovering 2023-09-30T09:43:26.831 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:26.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:26.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:26.940 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:27.024 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:27.232 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:43:27.364 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3133524225' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:27.364 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:27 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:27.364 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3133524225' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:27.365 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:27 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:27.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3133524225' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:27.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:27 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:28 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:43:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:28 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:28 smithi167 ceph-mon[192880]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:28 smithi167 ceph-mon[192880]: osdmap e468: 8 total, 7 up, 8 in 2023-09-30T09:43:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:28 smithi167 ceph-mon[192880]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:28 smithi167 ceph-mon[192880]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:28 smithi167 ceph-mon[192880]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:28 smithi167 ceph-mon[192880]: pgmap v1203: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 21051/71493 objects degraded (29.445%); 13623/71493 objects misplaced (19.055%); 4.9 MiB/s, 119 objects/s recovering 2023-09-30T09:43:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:43:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[216815]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[216815]: osdmap e468: 8 total, 7 up, 8 in 2023-09-30T09:43:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[216815]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[216815]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[216815]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[216815]: pgmap v1203: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 21051/71493 objects degraded (29.445%); 13623/71493 objects misplaced (19.055%); 4.9 MiB/s, 119 objects/s recovering 2023-09-30T09:43:28.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:43:28.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:28.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[224043]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[224043]: osdmap e468: 8 total, 7 up, 8 in 2023-09-30T09:43:28.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[224043]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[224043]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[224043]: failed to encode map e468 with expected crc 2023-09-30T09:43:28.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:28 smithi019 ceph-mon[224043]: pgmap v1203: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 21051/71493 objects degraded (29.445%); 13623/71493 objects misplaced (19.055%); 4.9 MiB/s, 119 objects/s recovering 2023-09-30T09:43:28.613 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:28.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:28.720 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:28.794 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:29.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:29 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 21051/71493 objects degraded (29.445%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:29 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 21051/71493 objects degraded (29.445%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:29.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:29 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 21051/71493 objects degraded (29.445%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:29.625 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:29.634 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:29.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:29.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:29.739 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:29.797 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:30.640 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:30.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:30.745 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:30.776 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:30.834 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:30.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:30.939 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:31.073 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:31.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:30 smithi019 ceph-mon[216815]: pgmap v1204: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 20577/67677 objects degraded (30.405%); 13350/67677 objects misplaced (19.726%); 5.7 MiB/s, 109 objects/s recovering 2023-09-30T09:43:31.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:30 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:31.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:30 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:31.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:30 smithi019 ceph-mon[224043]: pgmap v1204: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 20577/67677 objects degraded (30.405%); 13350/67677 objects misplaced (19.726%); 5.7 MiB/s, 109 objects/s recovering 2023-09-30T09:43:31.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:30 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:31.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:30 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:31.221 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:43:31.221 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:43:31.221 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:43:31.221 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:43:31.222 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:43:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:30 smithi167 ceph-mon[192880]: pgmap v1204: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 20577/67677 objects degraded (30.405%); 13350/67677 objects misplaced (19.726%); 5.7 MiB/s, 109 objects/s recovering 2023-09-30T09:43:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:30 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:30 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:31.459 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:31.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:31.567 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:31.603 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:31.655 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:31.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:31.689 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:43:31.689 INFO:teuthology.orchestra.run.smithi019.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:43:31.689 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:43:31.763 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:31.851 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:32.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:43:32.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:43:32.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:32.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:43:32.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:43:32.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:32.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:43:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:43:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:32.863 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:32.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:32.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:33.014 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:33.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:32 smithi019 ceph-mon[216815]: pgmap v1205: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 19865/63651 objects degraded (31.209%); 12318/63651 objects misplaced (19.352%); 6.5 MiB/s, 134 objects/s recovering 2023-09-30T09:43:33.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:32 smithi019 ceph-mon[224043]: pgmap v1205: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 19865/63651 objects degraded (31.209%); 12318/63651 objects misplaced (19.352%); 6.5 MiB/s, 134 objects/s recovering 2023-09-30T09:43:33.059 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:33.218 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:43:33.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:32 smithi167 ceph-mon[192880]: pgmap v1205: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 19865/63651 objects degraded (31.209%); 12318/63651 objects misplaced (19.352%); 6.5 MiB/s, 134 objects/s recovering 2023-09-30T09:43:33.359 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:43:33.810 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:43:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:43:33] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-09-30T09:43:33.914 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:33.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:34.036 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:34.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 19865/63651 objects degraded (31.209%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:34.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:34.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:34.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 19865/63651 objects degraded (31.209%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:34.126 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:34.219 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:34.241 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:34.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:34.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 19865/63651 objects degraded (31.209%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:34.349 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:34.430 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:34.583 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:34.685 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:34.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:34.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:34.728 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:43:34.773 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:43:34.801 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:34.813 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:43:34.857 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:43:34.869 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:34.904 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:43:34.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:43:35.054 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:43:35.055 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:43:35.055 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:43:35.055 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:43:35.055 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:43:35.055 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:43:35.055 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:43:35.055 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:43:35.055 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:43:35.210 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:35.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:35.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:34 smithi167 ceph-mon[192880]: pgmap v1206: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 19337/59970 objects degraded (32.244%); 12027/59970 objects misplaced (20.055%); 6.6 MiB/s, 136 objects/s recovering 2023-09-30T09:43:35.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2480432258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:35.282 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:34 smithi019 ceph-mon[224043]: pgmap v1206: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 19337/59970 objects degraded (32.244%); 12027/59970 objects misplaced (20.055%); 6.6 MiB/s, 136 objects/s recovering 2023-09-30T09:43:35.282 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2480432258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:35.283 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:34 smithi019 ceph-mon[216815]: pgmap v1206: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 19337/59970 objects degraded (32.244%); 12027/59970 objects misplaced (20.055%); 6.6 MiB/s, 136 objects/s recovering 2023-09-30T09:43:35.283 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2480432258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:35.316 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:35.369 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:36.962 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:36.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:36.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:37.077 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:37.108 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:37.161 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:37.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:37.267 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:37.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:37 smithi167 ceph-mon[192880]: pgmap v1207: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 18872/57330 objects degraded (32.918%); 11590/57330 objects misplaced (20.216%); 4.8 MiB/s, 159 objects/s recovering 2023-09-30T09:43:37.292 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:37.334 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:37.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:37 smithi019 ceph-mon[216815]: pgmap v1207: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 18872/57330 objects degraded (32.918%); 11590/57330 objects misplaced (20.216%); 4.8 MiB/s, 159 objects/s recovering 2023-09-30T09:43:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:37 smithi019 ceph-mon[224043]: pgmap v1207: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 18872/57330 objects degraded (32.918%); 11590/57330 objects misplaced (20.216%); 4.8 MiB/s, 159 objects/s recovering 2023-09-30T09:43:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:37.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:37.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:43:37.673 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:37.674 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:43:37.992 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:38.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:38.111 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:38.140 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:38.192 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:38.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:38.297 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:38.298 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:43:38.299 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:43:38.327 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:38.374 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:38.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:38.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:38.480 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:38.508 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:38.561 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:38.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:38.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:38.671 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:38.725 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:38.907 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:43:38.908 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:43:38.908 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:43:38.908 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.270675 2023-09-30T09:43:38.908 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0.2706747426527387 2023-09-30T09:43:39.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:39 smithi019 ceph-mon[216815]: pgmap v1208: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 18569/55458 objects degraded (33.483%); 10966/55458 objects misplaced (19.774%); 4.6 MiB/s, 151 objects/s recovering 2023-09-30T09:43:39.023 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:39 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18569/55458 objects degraded (33.483%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:39 smithi019 ceph-mon[224043]: pgmap v1208: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 18569/55458 objects degraded (33.483%); 10966/55458 objects misplaced (19.774%); 4.6 MiB/s, 151 objects/s recovering 2023-09-30T09:43:39.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:39 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18569/55458 objects degraded (33.483%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:39.384 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:39.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:39.498 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:39.506 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:43:39.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:39 smithi167 ceph-mon[192880]: pgmap v1208: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 18569/55458 objects degraded (33.483%); 10966/55458 objects misplaced (19.774%); 4.6 MiB/s, 151 objects/s recovering 2023-09-30T09:43:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:39 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18569/55458 objects degraded (33.483%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:39.572 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:39.696 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:39.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:39.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:39.706 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:43:39.780 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:39.804 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:39.971 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:40.057 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:43:40.224 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/251729302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:43:40.224 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/251729302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:43:40.266 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:40.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:40.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:40.378 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:40.409 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/251729302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:43:40.413 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:40.469 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:40.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:40.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:40.575 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:40.609 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:40.660 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:40.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:40.767 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:40.829 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:41.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[216815]: pgmap v1209: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 18080/51660 objects degraded (34.998%); 10702/51660 objects misplaced (20.716%); 4.9 MiB/s, 156 objects/s recovering 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/251729302' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[216815]: failed to encode map e469 with expected crc 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[216815]: osdmap e469: 8 total, 7 up, 8 in 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[216815]: failed to encode map e469 with expected crc 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[224043]: pgmap v1209: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 18080/51660 objects degraded (34.998%); 10702/51660 objects misplaced (20.716%); 4.9 MiB/s, 156 objects/s recovering 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/251729302' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:43:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[224043]: failed to encode map e469 with expected crc 2023-09-30T09:43:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[224043]: osdmap e469: 8 total, 7 up, 8 in 2023-09-30T09:43:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:41 smithi019 ceph-mon[224043]: failed to encode map e469 with expected crc 2023-09-30T09:43:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:41 smithi167 ceph-mon[192880]: pgmap v1209: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 18080/51660 objects degraded (34.998%); 10702/51660 objects misplaced (20.716%); 4.9 MiB/s, 156 objects/s recovering 2023-09-30T09:43:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:41 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/251729302' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:43:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:41 smithi167 ceph-mon[192880]: failed to encode map e469 with expected crc 2023-09-30T09:43:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:41 smithi167 ceph-mon[192880]: osdmap e469: 8 total, 7 up, 8 in 2023-09-30T09:43:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:41 smithi167 ceph-mon[192880]: failed to encode map e469 with expected crc 2023-09-30T09:43:42.059 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:42 smithi019 ceph-mon[216815]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.059 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:42 smithi019 ceph-mon[216815]: osdmap e470: 8 total, 7 up, 8 in 2023-09-30T09:43:42.059 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:42 smithi019 ceph-mon[216815]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.059 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:42 smithi019 ceph-mon[216815]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.288 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:42.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:42.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:42 smithi019 ceph-mon[224043]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:42 smithi019 ceph-mon[224043]: osdmap e470: 8 total, 7 up, 8 in 2023-09-30T09:43:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:42 smithi019 ceph-mon[224043]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:42 smithi019 ceph-mon[224043]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.402 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:42.454 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:42.497 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:42.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:42 smithi167 ceph-mon[192880]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:42 smithi167 ceph-mon[192880]: osdmap e470: 8 total, 7 up, 8 in 2023-09-30T09:43:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:42 smithi167 ceph-mon[192880]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:42 smithi167 ceph-mon[192880]: failed to encode map e470 with expected crc 2023-09-30T09:43:42.554 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:42.702 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:42.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:42.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:42.812 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:42.868 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:43.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[224043]: pgmap v1212: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 990 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 17313/47376 objects degraded (36.544%); 9630/47376 objects misplaced (20.327%); 1.3 MiB/s, 223 objects/s recovering 2023-09-30T09:43:43.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[224043]: failed to encode map e471 with expected crc 2023-09-30T09:43:43.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[224043]: failed to encode map e471 with expected crc 2023-09-30T09:43:43.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[224043]: osdmap e471: 8 total, 7 up, 8 in 2023-09-30T09:43:43.133 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[224043]: failed to encode map e471 with expected crc 2023-09-30T09:43:43.133 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[216815]: pgmap v1212: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 990 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 17313/47376 objects degraded (36.544%); 9630/47376 objects misplaced (20.327%); 1.3 MiB/s, 223 objects/s recovering 2023-09-30T09:43:43.133 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[216815]: failed to encode map e471 with expected crc 2023-09-30T09:43:43.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[216815]: failed to encode map e471 with expected crc 2023-09-30T09:43:43.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[216815]: osdmap e471: 8 total, 7 up, 8 in 2023-09-30T09:43:43.134 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:43 smithi019 ceph-mon[216815]: failed to encode map e471 with expected crc 2023-09-30T09:43:43.190 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:43.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:43.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:43.303 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:43.435 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:43.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:43 smithi167 ceph-mon[192880]: pgmap v1212: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 990 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 17313/47376 objects degraded (36.544%); 9630/47376 objects misplaced (20.327%); 1.3 MiB/s, 223 objects/s recovering 2023-09-30T09:43:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:43 smithi167 ceph-mon[192880]: failed to encode map e471 with expected crc 2023-09-30T09:43:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:43 smithi167 ceph-mon[192880]: failed to encode map e471 with expected crc 2023-09-30T09:43:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:43 smithi167 ceph-mon[192880]: osdmap e471: 8 total, 7 up, 8 in 2023-09-30T09:43:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:43 smithi167 ceph-mon[192880]: failed to encode map e471 with expected crc 2023-09-30T09:43:44.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:43:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:43:43] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-09-30T09:43:44.130 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.3 primary-affinity to 0.270675 (8177382) 2023-09-30T09:43:44.287 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:43:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/292281200' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]: dispatch 2023-09-30T09:43:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:44 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]: dispatch 2023-09-30T09:43:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:44 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 17313/47376 objects degraded (36.544%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:44.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/292281200' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]: dispatch 2023-09-30T09:43:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:44 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]: dispatch 2023-09-30T09:43:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:44 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 17313/47376 objects degraded (36.544%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:44.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/292281200' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]: dispatch 2023-09-30T09:43:44.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:44 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]: dispatch 2023-09-30T09:43:44.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:44 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 17313/47376 objects degraded (36.544%), 21 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:45.047 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:45.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:45.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:45.151 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:45.191 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:45.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: pgmap v1214: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 75 active+clean; 915 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 16817/43818 objects degraded (38.379%); 8147/43818 objects misplaced (18.593%); 3.6 MiB/s, 298 objects/s recovering 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]': finished 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: osdmap e472: 8 total, 7 up, 8 in 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[216815]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: pgmap v1214: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 75 active+clean; 915 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 16817/43818 objects degraded (38.379%); 8147/43818 objects misplaced (18.593%); 3.6 MiB/s, 298 objects/s recovering 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]': finished 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: osdmap e472: 8 total, 7 up, 8 in 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:45 smithi019 ceph-mon[224043]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: pgmap v1214: 139 pgs: 1 active+recovery_wait+remapped, 29 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 75 active+clean; 915 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 16817/43818 objects degraded (38.379%); 8147/43818 objects misplaced (18.593%); 3.6 MiB/s, 298 objects/s recovering 2023-09-30T09:43:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2706747426527387}]': finished 2023-09-30T09:43:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: osdmap e472: 8 total, 7 up, 8 in 2023-09-30T09:43:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: failed to encode map e472 with expected crc 2023-09-30T09:43:45.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:45 smithi167 ceph-mon[192880]: failed to encode map e472 with expected crc 2023-09-30T09:43:46.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: osdmap e473: 8 total, 7 up, 8 in 2023-09-30T09:43:46.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[216815]: 8.8 continuing backfill to osd.3 from (363'1053,471'7833] MIN to 471'7833 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: osdmap e473: 8 total, 7 up, 8 in 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:46 smithi019 ceph-mon[224043]: 8.8 continuing backfill to osd.3 from (363'1053,471'7833] MIN to 471'7833 2023-09-30T09:43:46.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: osdmap e473: 8 total, 7 up, 8 in 2023-09-30T09:43:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: failed to encode map e473 with expected crc 2023-09-30T09:43:46.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:46 smithi167 ceph-mon[192880]: 8.8 continuing backfill to osd.3 from (363'1053,471'7833] MIN to 471'7833 2023-09-30T09:43:46.703 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:46.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:46.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:46.807 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:46.845 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:46.903 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:47.013 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:47.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:47.117 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:47.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: pgmap v1217: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 3 peering, 30 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 72 active+clean; 866 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 15464/41448 objects degraded (37.309%); 5666/41448 objects misplaced (13.670%); 5.0 MiB/s, 203 objects/s recovering 2023-09-30T09:43:47.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: osdmap e474: 8 total, 7 up, 8 in 2023-09-30T09:43:47.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: 8.1f continuing backfill to osd.3 from (376'3968,471'9384] MIN to 471'9384 2023-09-30T09:43:47.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: 8.f continuing backfill to osd.3 from (376'3968,471'9304] MIN to 471'9304 2023-09-30T09:43:47.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: 8.1b continuing backfill to osd.3 from (377'4338,471'9319] MIN to 471'9319 2023-09-30T09:43:47.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[216815]: 8.b continuing backfill to osd.3 from (377'4338,471'9337] MIN to 471'9337 2023-09-30T09:43:47.161 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:47.406 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:47.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:47.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:47.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: pgmap v1217: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 3 peering, 30 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 72 active+clean; 866 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 15464/41448 objects degraded (37.309%); 5666/41448 objects misplaced (13.670%); 5.0 MiB/s, 203 objects/s recovering 2023-09-30T09:43:47.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: osdmap e474: 8 total, 7 up, 8 in 2023-09-30T09:43:47.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.417 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.417 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: 8.1f continuing backfill to osd.3 from (376'3968,471'9384] MIN to 471'9384 2023-09-30T09:43:47.417 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: 8.f continuing backfill to osd.3 from (376'3968,471'9304] MIN to 471'9304 2023-09-30T09:43:47.417 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: 8.1b continuing backfill to osd.3 from (377'4338,471'9319] MIN to 471'9319 2023-09-30T09:43:47.417 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:47 smithi019 ceph-mon[224043]: 8.b continuing backfill to osd.3 from (377'4338,471'9337] MIN to 471'9337 2023-09-30T09:43:47.492 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:43:47.516 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: pgmap v1217: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 3 peering, 30 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 72 active+clean; 866 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 15464/41448 objects degraded (37.309%); 5666/41448 objects misplaced (13.670%); 5.0 MiB/s, 203 objects/s recovering 2023-09-30T09:43:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: osdmap e474: 8 total, 7 up, 8 in 2023-09-30T09:43:47.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: failed to encode map e474 with expected crc 2023-09-30T09:43:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: 8.1f continuing backfill to osd.3 from (376'3968,471'9384] MIN to 471'9384 2023-09-30T09:43:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: 8.f continuing backfill to osd.3 from (376'3968,471'9304] MIN to 471'9304 2023-09-30T09:43:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: 8.1b continuing backfill to osd.3 from (377'4338,471'9319] MIN to 471'9319 2023-09-30T09:43:47.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:47 smithi167 ceph-mon[192880]: 8.b continuing backfill to osd.3 from (377'4338,471'9337] MIN to 471'9337 2023-09-30T09:43:47.540 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:43:47.544 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:47.578 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:43:47.600 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:47.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:47.626 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:43:47.678 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:43:47.705 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:47.733 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:47.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:43:47.784 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:47.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:47.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:47.803 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:43:47.803 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:43:47.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:43:47.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:43:47.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:43:47.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:43:47.804 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:43:47.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:43:47.804 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:43:47.821 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:172400 8.f e475/473 8:f513ad46:::benchmark_data_smithi019_478679_object79952:head v 475'9334, mlcod=475'9331)", 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.776739+0000", 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.027619219, 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027662586999999999, 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.822 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.776739+0000", 2023-09-30T09:43:47.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.776739+0000", 2023-09-30T09:43:47.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.776740+0000", 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3e-07 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.776747+0000", 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0790000000000003e-06 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.776759+0000", 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.184e-05 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.776766+0000", 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8110000000000003e-06 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.785591+0000", 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.008824274 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.785609+0000", 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8473000000000001e-05 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.826 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:172370 8.17 8:ec3a465a:::benchmark_data_smithi019_478679_object79922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.731967+0000", 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.072390905000000005, 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.072511041999999998, 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for rw locks", 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172370 2023-09-30T09:43:47.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.731967+0000", 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.731967+0000", 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.731968+0000", 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1909999999999999e-06 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.731972+0000", 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1360000000000001e-06 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.731981+0000", 2023-09-30T09:43:47.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2150000000000007e-06 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.731987+0000", 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0549999999999996e-06 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.740887+0000", 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088997969999999992 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.742126+0000", 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012390630000000001 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.761419+0000", 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.019293471999999999 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.762522+0000", 2023-09-30T09:43:47.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011024470000000001 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:172410 8.7 8.ca2e2e07 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.790627+0000", 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.013730743, 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013936143, 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "queued for pg", 2023-09-30T09:43:47.832 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172410 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.790627+0000", 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.790627+0000", 2023-09-30T09:43:47.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.790629+0000", 2023-09-30T09:43:47.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.19e-06 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.790633+0000", 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6450000000000004e-06 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.790644+0000", 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.06e-05 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.790650+0000", 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0390000000000001e-06 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(8.17 475/455 [PushOp(8:efa46155:::benchmark_data_smithi019_478679_object79947:head, version: 382'4974, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:efa46155:::benchmark_data_smithi019_478679_object79947:head@382'4974, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.792615+0000", 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.011743497, 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012036407000000001, 2023-09-30T09:43:47.836 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.792615+0000", 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.792615+0000", 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.792615+0000", 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2600000000000001e-07 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.792678+0000", 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2497999999999997e-05 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.792701+0000", 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2993000000000002e-05 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.792719+0000", 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8321e-05 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.798370+0000", 2023-09-30T09:43:47.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0056512089999999999 2023-09-30T09:43:47.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:43:47.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.798492+0000", 2023-09-30T09:43:47.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000121809 2023-09-30T09:43:47.840 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.840 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.840 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.841 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop_reply(client.35820.0:172398 8.7 e475/455 ondisk, result = 0)", 2023-09-30T09:43:47.841 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.800669+0000", 2023-09-30T09:43:47.841 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0036892439999999999, 2023-09-30T09:43:47.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040463620000000004, 2023-09-30T09:43:47.841 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.842 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:43:47.842 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.800669+0000", 2023-09-30T09:43:47.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.800669+0000", 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.800670+0000", 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1800000000000005e-07 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.800680+0000", 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0054000000000001e-05 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.800691+0000", 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1086e-05 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.800698+0000", 2023-09-30T09:43:47.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.878e-06 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.800958+0000", 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000260831 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.801055+0000", 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6518e-05 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.845 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop_reply(client.35820.0:172398 8.7 e475/455)", 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.804717+0000", 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 4294967295.9996409, 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1693999999999996e-05, 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "queued for pg", 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.804717+0000", 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.804717+0000", 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.804718+0000", 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1429999999999999e-06 2023-09-30T09:43:47.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.804724+0000", 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9800000000000003e-06 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.804748+0000", 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3476999999999999e-05 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.804756+0000", 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8849999999999999e-06 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:172388 8.7 8:e41277ee:::benchmark_data_smithi019_478679_object79940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.751528+0000", 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.052830004999999999, 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.053307618000000001, 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.849 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172388 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.751528+0000", 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.751528+0000", 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.751529+0000", 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6800000000000001e-07 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.751531+0000", 2023-09-30T09:43:47.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6529999999999998e-06 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.751538+0000", 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.657e-06 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.751543+0000", 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0019999999999997e-06 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.779122+0000", 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.027578828999999999 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.780131+0000", 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0010095429999999999 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.781905+0000", 2023-09-30T09:43:47.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001774255 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.788907+0000", 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0070012429999999999 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.789926+0000", 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001019121 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.803866+0000", 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013940409000000001 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:172343 8.17 8:ebff346f:::benchmark_data_smithi019_478679_object79895:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.690231+0000", 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.114127477, 2023-09-30T09:43:47.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.11470632, 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for rw locks", 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172343 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.690231+0000", 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.690231+0000", 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.690232+0000", 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6670000000000001e-06 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.690239+0000", 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2260000000000002e-06 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.690253+0000", 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4301000000000001e-05 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.690262+0000", 2023-09-30T09:43:47.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1570000000000006e-06 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.707768+0000", 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.017506339999999999 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.712460+0000", 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0046915510000000004 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.739087+0000", 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026627123999999999 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.740322+0000", 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012347339999999999 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.860 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:172395 8.17 8:efa46155:::benchmark_data_smithi019_478679_object79947:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.772334+0000", 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.032023797, 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.032688674000000001, 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for missing object", 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172395 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772334+0000", 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772334+0000", 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772336+0000", 2023-09-30T09:43:47.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8360000000000001e-06 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772341+0000", 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4309999999999996e-06 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772357+0000", 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.626e-05 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772367+0000", 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8579999999999995e-06 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.790372+0000", 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018005653999999999 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.791667+0000", 2023-09-30T09:43:47.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001294714 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:172398 8.7 8:e371db8c:::benchmark_data_smithi019_478679_object79950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.772386+0000", 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.031972304, 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.032712894999999999, 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:47.865 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172398 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772386+0000", 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772386+0000", 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772386+0000", 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4499999999999997e-07 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772390+0000", 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2260000000000001e-06 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772400+0000", 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0264e-05 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.772407+0000", 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9110000000000004e-06 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.791842+0000", 2023-09-30T09:43:47.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.019435411 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.793063+0000", 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001220952 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.795001+0000", 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001938211 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.801147+0000", 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061459920000000003 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:172354 8.17 8:ed5f8b45:::benchmark_data_smithi019_478679_object79906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:47.699406+0000", 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.104952534, 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.10578641599999999, 2023-09-30T09:43:47.870 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "waiting for rw locks", 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 172354 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.699406+0000", 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.699406+0000", 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.699408+0000", 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0049999999999999e-06 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.699413+0000", 2023-09-30T09:43:47.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8799999999999996e-06 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.699428+0000", 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4368e-05 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.699438+0000", 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0227000000000001e-05 2023-09-30T09:43:47.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.712931+0000", 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013493277 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.714759+0000", 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001827331 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:43:47.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.737482+0000", 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022723435 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for rw locks", 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:47.738726+0000", 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00124373 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2023-09-30T09:43:47.875 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 11 2023-09-30T09:43:47.876 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:43:47.894 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:47.994 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:48.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:48 smithi167 ceph-mon[192880]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:48 smithi167 ceph-mon[192880]: osdmap e475: 8 total, 7 up, 8 in 2023-09-30T09:43:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:48 smithi167 ceph-mon[192880]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:48 smithi167 ceph-mon[192880]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3771177366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[216815]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[216815]: osdmap e475: 8 total, 7 up, 8 in 2023-09-30T09:43:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[216815]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[216815]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3771177366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:48.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[224043]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[224043]: osdmap e475: 8 total, 7 up, 8 in 2023-09-30T09:43:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[224043]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[224043]: failed to encode map e475 with expected crc 2023-09-30T09:43:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3771177366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:43:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:43:48.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:43:49.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:49 smithi019 ceph-mon[224043]: pgmap v1220: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 3 peering, 30 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 72 active+clean; 855 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 239 op/s; 15222/40938 objects degraded (37.183%); 5496/40938 objects misplaced (13.425%); 4.2 MiB/s, 127 objects/s recovering 2023-09-30T09:43:49.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:49 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 15222/40938 objects degraded (37.183%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:49.416 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:49 smithi019 ceph-mon[216815]: pgmap v1220: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 3 peering, 30 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 72 active+clean; 855 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 239 op/s; 15222/40938 objects degraded (37.183%); 5496/40938 objects misplaced (13.425%); 4.2 MiB/s, 127 objects/s recovering 2023-09-30T09:43:49.416 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:49 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 15222/40938 objects degraded (37.183%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:49.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:49 smithi167 ceph-mon[192880]: pgmap v1220: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 3 peering, 30 active+clean+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 72 active+clean; 855 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 239 op/s; 15222/40938 objects degraded (37.183%); 5496/40938 objects misplaced (13.425%); 4.2 MiB/s, 127 objects/s recovering 2023-09-30T09:43:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:49 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 15222/40938 objects degraded (37.183%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:50.117 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:50.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:50.170 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:43:50.225 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:50.255 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:50.308 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:50.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:50.323 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:43:50.323 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:43:50.413 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:50.498 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:50.620 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:50.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:50.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:50.741 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:50.825 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:50.929 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:50.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:51.039 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:51.045 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:51.074 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:51.129 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:51.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:51.235 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:43:51.268 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:51.326 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:43:51.329 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:51 smithi167 ceph-mon[192880]: pgmap v1221: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 31 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 4 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 787 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 10344/37650 objects degraded (27.474%); 6157/37650 objects misplaced (16.353%); 7.9 MiB/s, 261 objects/s recovering 2023-09-30T09:43:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:51.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:43:51.417 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:51 smithi019 ceph-mon[224043]: pgmap v1221: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 31 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 4 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 787 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 10344/37650 objects degraded (27.474%); 6157/37650 objects misplaced (16.353%); 7.9 MiB/s, 261 objects/s recovering 2023-09-30T09:43:51.419 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:51 smithi019 ceph-mon[216815]: pgmap v1221: 139 pgs: 4 remapped+peering, 1 active+remapped+backfilling, 31 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+undersized+remapped, 4 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 787 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 10344/37650 objects degraded (27.474%); 6157/37650 objects misplaced (16.353%); 7.9 MiB/s, 261 objects/s recovering 2023-09-30T09:43:51.431 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:43:51.465 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:51.532 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:43:51.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:51.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:43:51.637 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:51.721 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:51.776 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:43:51.776 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:43:51.776 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:43:51.776 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2023-09-30T09:43:51.776 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-30T09:43:51.897 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:51.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:51.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:52.002 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:43:52.036 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:52.101 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:43:52.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:52.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:43:52.200 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:43:52.208 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:52.294 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:52.324 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:43:52.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1080715381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:52 smithi167 ceph-mon[192880]: pgmap v1222: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 696 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 9991/33282 objects degraded (30.019%); 6649/33282 objects misplaced (19.978%); 8.1 MiB/s, 253 objects/s recovering 2023-09-30T09:43:52.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1080715381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:52 smithi019 ceph-mon[216815]: pgmap v1222: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 696 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 9991/33282 objects degraded (30.019%); 6649/33282 objects misplaced (19.978%); 8.1 MiB/s, 253 objects/s recovering 2023-09-30T09:43:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1080715381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:43:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:52 smithi019 ceph-mon[224043]: pgmap v1222: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 696 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 9991/33282 objects degraded (30.019%); 6649/33282 objects misplaced (19.978%); 8.1 MiB/s, 253 objects/s recovering 2023-09-30T09:43:52.979 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:52.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:52.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:53.091 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:53.181 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1080715381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[216815]: failed to encode map e476 with expected crc 2023-09-30T09:43:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[216815]: osdmap e476: 8 total, 7 up, 8 in 2023-09-30T09:43:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[216815]: failed to encode map e476 with expected crc 2023-09-30T09:43:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1080715381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[224043]: failed to encode map e476 with expected crc 2023-09-30T09:43:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[224043]: osdmap e476: 8 total, 7 up, 8 in 2023-09-30T09:43:53.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:53 smithi019 ceph-mon[224043]: failed to encode map e476 with expected crc 2023-09-30T09:43:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:53 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:43:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1080715381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:43:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:53 smithi167 ceph-mon[192880]: failed to encode map e476 with expected crc 2023-09-30T09:43:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:53 smithi167 ceph-mon[192880]: osdmap e476: 8 total, 7 up, 8 in 2023-09-30T09:43:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:53 smithi167 ceph-mon[192880]: failed to encode map e476 with expected crc 2023-09-30T09:43:53.828 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:53.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:53.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:53.935 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:54.022 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:43:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:43:53] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-09-30T09:43:54.036 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:54.259 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:54.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:54.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:54.286 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:54 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9991/33282 objects degraded (30.019%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:54.286 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:54 smithi019 ceph-mon[224043]: pgmap v1224: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 635 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 9655/30336 objects degraded (31.827%); 5896/30336 objects misplaced (19.436%); 7.2 MiB/s, 222 objects/s recovering 2023-09-30T09:43:54.290 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:54 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9991/33282 objects degraded (30.019%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:54.290 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:54 smithi019 ceph-mon[216815]: pgmap v1224: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 635 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 9655/30336 objects degraded (31.827%); 5896/30336 objects misplaced (19.436%); 7.2 MiB/s, 222 objects/s recovering 2023-09-30T09:43:54.373 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:54.397 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:54.463 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:54.469 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:54 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9991/33282 objects degraded (30.019%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:54.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:54 smithi167 ceph-mon[192880]: pgmap v1224: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 635 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 9655/30336 objects degraded (31.827%); 5896/30336 objects misplaced (19.436%); 7.2 MiB/s, 222 objects/s recovering 2023-09-30T09:43:54.730 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:54.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:54.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:54.839 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:43:54.851 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:43:54.851 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:43:54.852 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:43:54.852 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:43:54.852 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:43:54.957 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:56.068 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:43:56.814 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:56 smithi019 ceph-mon[216815]: pgmap v1225: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 475 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 9028/22674 objects degraded (39.817%); 4470/22674 objects misplaced (19.714%); 8.1 MiB/s, 226 objects/s recovering 2023-09-30T09:43:57.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:56 smithi019 ceph-mon[224043]: pgmap v1225: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 475 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 9028/22674 objects degraded (39.817%); 4470/22674 objects misplaced (19.714%); 8.1 MiB/s, 226 objects/s recovering 2023-09-30T09:43:57.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:56 smithi167 ceph-mon[192880]: pgmap v1225: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 475 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 9028/22674 objects degraded (39.817%); 4470/22674 objects misplaced (19.714%); 8.1 MiB/s, 226 objects/s recovering 2023-09-30T09:43:57.654 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:43:58.094 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:43:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:58.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:43:58.201 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:43:58.277 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:58.556 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:43:58.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:58.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:43:58.676 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:58.795 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:59.009 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:59.009 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:43:59.010 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:43:59.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:59.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:43:59.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:58 smithi019 ceph-mon[216815]: pgmap v1226: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 451 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 8665/21519 objects degraded (40.267%); 4085/21519 objects misplaced (18.983%); 7.0 MiB/s, 195 objects/s recovering 2023-09-30T09:43:59.091 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:58 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 8665/21519 objects degraded (40.267%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:59.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:58 smithi019 ceph-mon[224043]: pgmap v1226: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 451 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 8665/21519 objects degraded (40.267%); 4085/21519 objects misplaced (18.983%); 7.0 MiB/s, 195 objects/s recovering 2023-09-30T09:43:59.093 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:58 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 8665/21519 objects degraded (40.267%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:59.117 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:43:59.230 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:59.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:58 smithi167 ceph-mon[192880]: pgmap v1226: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 451 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 8665/21519 objects degraded (40.267%); 4085/21519 objects misplaced (18.983%); 7.0 MiB/s, 195 objects/s recovering 2023-09-30T09:43:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:58 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 8665/21519 objects degraded (40.267%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:43:59.495 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:43:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:43:59.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:43:59.597 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:43:59.753 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:43:59.871 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:43:59.929 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:43:59.971 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:44:00.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:43:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/276350940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:00.035 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:43:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/276350940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:00.122 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:44:00.162 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:44:00.210 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:44:00.252 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:44:00.252 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:44:00.253 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:44:00.253 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:44:00.253 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:44:00.253 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:44:00.253 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:44:00.253 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:44:00.253 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:44:00.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:43:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/276350940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:00.866 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-30T09:44:01.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:00 smithi167 ceph-mon[192880]: pgmap v1227: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 319 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 8286/15189 objects degraded (54.553%); 3089/15189 objects misplaced (20.337%); 5.9 MiB/s, 124 objects/s recovering 2023-09-30T09:44:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4094431557' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:44:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:00 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:44:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:00 smithi019 ceph-mon[216815]: pgmap v1227: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 319 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 8286/15189 objects degraded (54.553%); 3089/15189 objects misplaced (20.337%); 5.9 MiB/s, 124 objects/s recovering 2023-09-30T09:44:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4094431557' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:44:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:00 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:44:01.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:00 smithi019 ceph-mon[224043]: pgmap v1227: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 319 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 8286/15189 objects degraded (54.553%); 3089/15189 objects misplaced (20.337%); 5.9 MiB/s, 124 objects/s recovering 2023-09-30T09:44:01.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4094431557' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:44:01.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:00 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:44:01.430 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:01.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:01.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:01.547 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:01.578 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:01.650 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:01.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:01.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:01.758 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:01.829 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:02.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:01 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:44:02.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:01 smithi019 ceph-mon[224043]: osdmap e477: 8 total, 7 up, 8 in 2023-09-30T09:44:02.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:01 smithi019 ceph-mon[224043]: failed to encode map e477 with expected crc 2023-09-30T09:44:02.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:01 smithi019 ceph-mon[224043]: failed to encode map e477 with expected crc 2023-09-30T09:44:02.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:01 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:44:02.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:01 smithi019 ceph-mon[216815]: osdmap e477: 8 total, 7 up, 8 in 2023-09-30T09:44:02.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:01 smithi019 ceph-mon[216815]: failed to encode map e477 with expected crc 2023-09-30T09:44:02.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:01 smithi019 ceph-mon[216815]: failed to encode map e477 with expected crc 2023-09-30T09:44:02.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:01 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:44:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:01 smithi167 ceph-mon[192880]: osdmap e477: 8 total, 7 up, 8 in 2023-09-30T09:44:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:01 smithi167 ceph-mon[192880]: failed to encode map e477 with expected crc 2023-09-30T09:44:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:01 smithi167 ceph-mon[192880]: failed to encode map e477 with expected crc 2023-09-30T09:44:02.884 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:02.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:02.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:03.001 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:03.031 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:02 smithi167 ceph-mon[192880]: pgmap v1229: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 202 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7310/9582 objects degraded (76.289%); 1751/9582 objects misplaced (18.274%); 5.2 MiB/s, 120 objects/s recovering 2023-09-30T09:44:03.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:02 smithi019 ceph-mon[224043]: pgmap v1229: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 202 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7310/9582 objects degraded (76.289%); 1751/9582 objects misplaced (18.274%); 5.2 MiB/s, 120 objects/s recovering 2023-09-30T09:44:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:02 smithi019 ceph-mon[216815]: pgmap v1229: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 9 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 202 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7310/9582 objects degraded (76.289%); 1751/9582 objects misplaced (18.274%); 5.2 MiB/s, 120 objects/s recovering 2023-09-30T09:44:03.549 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Removed 92445 objects 2023-09-30T09:44:03.549 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Clean up completed and total clean up time :267.989 2023-09-30T09:44:03.780 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:03.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:44:03.805 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:44:03.834 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:44:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:44:03] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-09-30T09:44:03.886 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:03.913 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:03.958 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:03.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:04.063 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:04.099 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:04.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:03 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7310/9582 objects degraded (76.289%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:04.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:04.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:03 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7310/9582 objects degraded (76.289%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:04.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:04.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:03 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7310/9582 objects degraded (76.289%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:04.355 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:04.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:04.460 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:04.498 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:04.554 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:04.556 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:44:04.556 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:44:04.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:04.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:04.658 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:04.689 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:04.798 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:04.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:04.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:04.903 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:04.932 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:04.977 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:04.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:04.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:05.081 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:05.107 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:05.149 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:05.153 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:04 smithi167 ceph-mon[192880]: pgmap v1230: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 147 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 7005/6921 objects degraded (101.214%); 1148/6921 objects misplaced (16.587%); 4.5 MiB/s, 129 objects/s recovering 2023-09-30T09:44:05.153 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1382789836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:44:05.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:05.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:05.252 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-30T09:44:05.252 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-30T09:44:05.255 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:05.281 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:04 smithi019 ceph-mon[224043]: pgmap v1230: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 147 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 7005/6921 objects degraded (101.214%); 1148/6921 objects misplaced (16.587%); 4.5 MiB/s, 129 objects/s recovering 2023-09-30T09:44:05.281 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1382789836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:44:05.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:04 smithi019 ceph-mon[216815]: pgmap v1230: 139 pgs: 33 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 147 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 7005/6921 objects degraded (101.214%); 1148/6921 objects misplaced (16.587%); 4.5 MiB/s, 129 objects/s recovering 2023-09-30T09:44:05.282 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1382789836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:44:05.284 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:05.580 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:05.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:05.588 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:44:05.656 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:44:05.681 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:05.688 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:05.728 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:05.806 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:44:05.836 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:05.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:05.942 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:05.969 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:06.002 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:06.019 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:06.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:06.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:06.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:05 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1382789836' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:44:06.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:05 smithi019 ceph-mon[216815]: osdmap e478: 8 total, 7 up, 8 in 2023-09-30T09:44:06.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:05 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:06.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1382789836' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:44:06.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:05 smithi019 ceph-mon[224043]: osdmap e478: 8 total, 7 up, 8 in 2023-09-30T09:44:06.124 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:06.150 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:06.212 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:05 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:06.212 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1382789836' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:44:06.212 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:05 smithi167 ceph-mon[192880]: osdmap e478: 8 total, 7 up, 8 in 2023-09-30T09:44:06.350 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:06.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:06.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:06.454 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:06.489 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:06.639 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:06.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:06.743 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:06.775 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:06.930 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:06.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:06.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:07.034 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:07.059 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:06 smithi019 ceph-mon[216815]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:06 smithi019 ceph-mon[216815]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:06 smithi019 ceph-mon[216815]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:06 smithi019 ceph-mon[216815]: pgmap v1232: 139 pgs: 33 active+clean+remapped, 2 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 7 active+remapped+backfill_wait, 6 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 32 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 6667/1437 objects degraded (463.953%); 370/1437 objects misplaced (25.748%); 3.1 MiB/s, 211 objects/s recovering 2023-09-30T09:44:07.060 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:06 smithi019 ceph-mon[224043]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.060 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:06 smithi019 ceph-mon[224043]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.060 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:06 smithi019 ceph-mon[224043]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.060 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:06 smithi019 ceph-mon[224043]: pgmap v1232: 139 pgs: 33 active+clean+remapped, 2 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 7 active+remapped+backfill_wait, 6 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 32 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 6667/1437 objects degraded (463.953%); 370/1437 objects misplaced (25.748%); 3.1 MiB/s, 211 objects/s recovering 2023-09-30T09:44:07.062 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:07.113 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:07.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:07.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:07.216 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:07.242 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:06 smithi167 ceph-mon[192880]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.242 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:06 smithi167 ceph-mon[192880]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.242 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:06 smithi167 ceph-mon[192880]: failed to encode map e478 with expected crc 2023-09-30T09:44:07.242 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:06 smithi167 ceph-mon[192880]: pgmap v1232: 139 pgs: 33 active+clean+remapped, 2 active+recovering+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 7 active+remapped+backfill_wait, 6 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 32 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 6667/1437 objects degraded (463.953%); 370/1437 objects misplaced (25.748%); 3.1 MiB/s, 211 objects/s recovering 2023-09-30T09:44:07.245 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:07.292 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:07.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:07.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:07.396 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:07.422 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:07.467 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:07.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:07.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:07.549 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:07.572 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:07.609 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:07.692 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:07.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:07.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:07.796 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:07.822 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:07.867 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:07.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:07.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:07.928 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-30T09:44:07.928 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-09-30T09:44:07.928 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-09-30T09:44:07.928 DEBUG:teuthology.orchestra.run.smithi167:osd.5> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-30T09:44:07.931 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl start ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.5 2023-09-30T09:44:07.957 INFO:journalctl@ceph.osd.5.smithi167.stdout:-- Logs begin at Sat 2023-09-30 08:49:08 UTC. -- 2023-09-30T09:44:07.971 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:08.005 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:08.240 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:07 smithi167 systemd[1]: Starting Ceph osd.5 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:44:08.533 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:44:08.669 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:08.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:08.777 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:08.811 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:09.050 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:09.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:09.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:09.153 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:09.157 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:09.195 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: pgmap v1233: 139 pgs: 3 active+undersized+remapped+backfill_wait, 33 active+clean+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 7 active+remapped+backfill_wait, 6 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 27 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 6593/1191 objects degraded (553.568%); 288/1191 objects misplaced (24.181%); 1.1 MiB/s, 178 objects/s recovering 2023-09-30T09:44:09.195 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: osdmap e479: 8 total, 7 up, 8 in 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: 8.12 scrub starts 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6593/1191 objects degraded (553.568%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: osdmap e480: 8 total, 7 up, 8 in 2023-09-30T09:44:09.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2985418442' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: pgmap v1233: 139 pgs: 3 active+undersized+remapped+backfill_wait, 33 active+clean+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 7 active+remapped+backfill_wait, 6 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 27 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 6593/1191 objects degraded (553.568%); 288/1191 objects misplaced (24.181%); 1.1 MiB/s, 178 objects/s recovering 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: osdmap e479: 8 total, 7 up, 8 in 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: 8.12 scrub starts 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6593/1191 objects degraded (553.568%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: osdmap e480: 8 total, 7 up, 8 in 2023-09-30T09:44:09.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2985418442' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:44:09.200 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: pgmap v1233: 139 pgs: 3 active+undersized+remapped+backfill_wait, 33 active+clean+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 7 active+remapped+backfill_wait, 6 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 27 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 6593/1191 objects degraded (553.568%); 288/1191 objects misplaced (24.181%); 1.1 MiB/s, 178 objects/s recovering 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: osdmap e479: 8 total, 7 up, 8 in 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: failed to encode map e479 with expected crc 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: 8.12 scrub starts 2023-09-30T09:44:09.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6593/1191 objects degraded (553.568%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: osdmap e480: 8 total, 7 up, 8 in 2023-09-30T09:44:09.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2985418442' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:44:09.280 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:09.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:09.384 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:09.415 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:09.680 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_3' removed 2023-09-30T09:44:09.984 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: 8.2 scrub starts 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2985418442' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[216815]: osdmap e481: 8 total, 7 up, 8 in 2023-09-30T09:44:09.985 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: failed to encode map e480 with expected crc 2023-09-30T09:44:09.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: 8.2 scrub starts 2023-09-30T09:44:09.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2985418442' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:44:09.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:09 smithi019 ceph-mon[224043]: osdmap e481: 8 total, 7 up, 8 in 2023-09-30T09:44:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: failed to encode map e480 with expected crc 2023-09-30T09:44:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: failed to encode map e480 with expected crc 2023-09-30T09:44:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: failed to encode map e480 with expected crc 2023-09-30T09:44:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: failed to encode map e480 with expected crc 2023-09-30T09:44:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: failed to encode map e480 with expected crc 2023-09-30T09:44:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: failed to encode map e480 with expected crc 2023-09-30T09:44:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: 8.2 scrub starts 2023-09-30T09:44:10.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2985418442' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:44:10.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:09 smithi167 ceph-mon[192880]: osdmap e481: 8 total, 7 up, 8 in 2023-09-30T09:44:10.831 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:44:10.850 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:44:10.870 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:44:10.891 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:44:10.910 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:44:10.929 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:44:10.949 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:44:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:10 smithi167 ceph-mon[192880]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:10 smithi167 ceph-mon[192880]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:10 smithi167 ceph-mon[192880]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:10 smithi167 ceph-mon[192880]: pgmap v1237: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:11.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3561498677' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:11.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[216815]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[216815]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[216815]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[216815]: pgmap v1237: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3561498677' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[224043]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[224043]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[224043]: failed to encode map e481 with expected crc 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[224043]: pgmap v1237: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:11.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3561498677' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:11.599 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-30T09:44:11.600 INFO:tasks.radosbench:Beginning radosbench... 2023-09-30T09:44:11.600 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-30T09:44:11.601 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-30T09:44:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:11 smithi167 ceph-mon[192880]: 1.12 scrub starts 2023-09-30T09:44:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:11 smithi167 ceph-mon[192880]: 1.12 scrub ok 2023-09-30T09:44:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:11 smithi019 ceph-mon[216815]: 1.12 scrub starts 2023-09-30T09:44:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:11 smithi019 ceph-mon[216815]: 1.12 scrub ok 2023-09-30T09:44:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:11 smithi019 ceph-mon[224043]: 1.12 scrub starts 2023-09-30T09:44:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:11 smithi019 ceph-mon[224043]: 1.12 scrub ok 2023-09-30T09:44:12.656 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:44:12.656 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:44:12.727 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:12.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:12 smithi019 ceph-mon[216815]: 1.f scrub starts 2023-09-30T09:44:12.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:12 smithi019 ceph-mon[216815]: 1.f scrub ok 2023-09-30T09:44:12.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:12 smithi019 ceph-mon[216815]: pgmap v1238: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:12.890 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:12 smithi019 ceph-mon[224043]: 1.f scrub starts 2023-09-30T09:44:13.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:12 smithi019 ceph-mon[224043]: 1.f scrub ok 2023-09-30T09:44:13.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:12 smithi019 ceph-mon[224043]: pgmap v1238: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:13.256 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:13.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:12 smithi167 ceph-mon[192880]: 1.f scrub starts 2023-09-30T09:44:13.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:12 smithi167 ceph-mon[192880]: 1.f scrub ok 2023-09-30T09:44:13.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:12 smithi167 ceph-mon[192880]: pgmap v1238: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:13.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:13.361 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:13.393 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:13.446 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:13.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:13.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:13.553 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:13.582 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:13.626 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:13.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:13.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:13.729 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:13.760 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:13.808 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:13.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:13.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:13.908 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:44:13.912 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:13.943 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:14.018 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:44:14.019 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:44:14.019 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:44:14.019 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:44:14.039 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:14.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3068501278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:13 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:14.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:13 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 3/81 objects degraded (3.704%), 3 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3068501278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:13 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:14.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:13 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 3/81 objects degraded (3.704%), 3 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:14.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:44:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:44:13] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-09-30T09:44:14.143 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:14.174 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:14.187 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3068501278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:14.187 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:13 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:14.187 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:13 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 3/81 objects degraded (3.704%), 3 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:14.657 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:44:14.710 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:14.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:14.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:14.728 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:44:14.817 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:14.844 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:14.943 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:14.945 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:14 smithi167 bash[342499]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-30T09:44:14.945 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:14 smithi167 bash[342499]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-30T09:44:14.945 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:14 smithi167 bash[342499]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2023-09-30T09:44:14.945 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:14 smithi167 bash[342499]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2023-09-30T09:44:14.945 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:14 smithi167 bash[342499]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2023-09-30T09:44:14.945 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:14 smithi167 bash[342499]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-30T09:44:14.945 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:14 smithi167 bash[342499]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-30T09:44:14.946 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:14 smithi167 bash[342499]: --> ceph-volume lvm activate successful for osd ID: 5 2023-09-30T09:44:14.946 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: 1.d scrub starts 2023-09-30T09:44:14.946 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: 1.d scrub ok 2023-09-30T09:44:14.946 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: pgmap v1239: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:14.946 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:44:14.946 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:14.946 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: osdmap e482: 8 total, 7 up, 8 in 2023-09-30T09:44:14.947 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: failed to encode map e482 with expected crc 2023-09-30T09:44:14.947 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: failed to encode map e482 with expected crc 2023-09-30T09:44:14.947 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: failed to encode map e482 with expected crc 2023-09-30T09:44:14.947 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: 1.13 scrub starts 2023-09-30T09:44:14.947 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: 1.13 scrub ok 2023-09-30T09:44:14.947 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: 1.1d scrub starts 2023-09-30T09:44:14.947 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:14 smithi167 ceph-mon[192880]: 1.1d scrub ok 2023-09-30T09:44:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:14.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:15.049 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:15.078 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:15.176 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: 1.d scrub starts 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: 1.d scrub ok 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: pgmap v1239: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: osdmap e482: 8 total, 7 up, 8 in 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: failed to encode map e482 with expected crc 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: failed to encode map e482 with expected crc 2023-09-30T09:44:15.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: failed to encode map e482 with expected crc 2023-09-30T09:44:15.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: 1.13 scrub starts 2023-09-30T09:44:15.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: 1.13 scrub ok 2023-09-30T09:44:15.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: 1.1d scrub starts 2023-09-30T09:44:15.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[224043]: 1.1d scrub ok 2023-09-30T09:44:15.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: 1.d scrub starts 2023-09-30T09:44:15.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: 1.d scrub ok 2023-09-30T09:44:15.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: pgmap v1239: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:15.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: osdmap e482: 8 total, 7 up, 8 in 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: failed to encode map e482 with expected crc 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: failed to encode map e482 with expected crc 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: failed to encode map e482 with expected crc 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: 1.13 scrub starts 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: 1.13 scrub ok 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: 1.1d scrub starts 2023-09-30T09:44:15.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:14 smithi019 ceph-mon[216815]: 1.1d scrub ok 2023-09-30T09:44:15.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:15.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:15.220 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:15.282 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:15.310 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:15.463 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:15.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:15.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:15.497 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:15.568 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:15.596 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:15.694 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:15.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:15.798 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:15.826 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:15.974 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:15.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:15.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:15.991 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_4' created 2023-09-30T09:44:16.079 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:16.109 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:16.179 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:15 smithi167 ceph-mon[192880]: 1.1e scrub starts 2023-09-30T09:44:16.179 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:15 smithi167 ceph-mon[192880]: 1.1e scrub ok 2023-09-30T09:44:16.202 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:16.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:16.306 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:16.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:15 smithi019 ceph-mon[216815]: 1.1e scrub starts 2023-09-30T09:44:16.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:15 smithi019 ceph-mon[216815]: 1.1e scrub ok 2023-09-30T09:44:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:15 smithi019 ceph-mon[224043]: 1.1e scrub starts 2023-09-30T09:44:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:15 smithi019 ceph-mon[224043]: 1.1e scrub ok 2023-09-30T09:44:16.338 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:16.487 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:16.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:16.626 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:16.655 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:16.716 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:44:16.804 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:16.804 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:44:16.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:16.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:16.909 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:16.933 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-osd[343732]: -- 172.21.15.167:0/326515085 <== mon.2 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563d561eab60 con 0x563d561b1000 2023-09-30T09:44:16.933 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:16 smithi167 bash[343431]: 3ae29cf3e4041c22113679517653d49d03b88cc9abb72f3b98181d7e8e8d68cb 2023-09-30T09:44:16.934 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:16 smithi167 systemd[1]: Started Ceph osd.5 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:44:16.934 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: 1.c scrub starts 2023-09-30T09:44:16.934 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: 1.c scrub ok 2023-09-30T09:44:16.934 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:44:16.934 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: 1.1c scrub ok 2023-09-30T09:44:16.937 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:17.039 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:17.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:17.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:17.143 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:17.173 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:17.219 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3346326950' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-30T09:44:17.219 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: pgmap v1241: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:17.219 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3346326950' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-30T09:44:17.219 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: failed to encode map e483 with expected crc 2023-09-30T09:44:17.219 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: osdmap e483: 8 total, 7 up, 8 in 2023-09-30T09:44:17.220 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:16 smithi167 ceph-mon[192880]: failed to encode map e483 with expected crc 2023-09-30T09:44:17.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: 1.c scrub starts 2023-09-30T09:44:17.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: 1.c scrub ok 2023-09-30T09:44:17.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:44:17.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: 1.1c scrub ok 2023-09-30T09:44:17.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3346326950' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: pgmap v1241: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3346326950' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: failed to encode map e483 with expected crc 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: osdmap e483: 8 total, 7 up, 8 in 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[224043]: failed to encode map e483 with expected crc 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: 1.c scrub starts 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: 1.c scrub ok 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:44:17.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: 1.1c scrub ok 2023-09-30T09:44:17.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3346326950' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-30T09:44:17.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: pgmap v1241: 103 pgs: 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:17.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3346326950' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-30T09:44:17.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: failed to encode map e483 with expected crc 2023-09-30T09:44:17.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: osdmap e483: 8 total, 7 up, 8 in 2023-09-30T09:44:17.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:16 smithi019 ceph-mon[216815]: failed to encode map e483 with expected crc 2023-09-30T09:44:17.274 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:17.274 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:44:17.274 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:44:17.274 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:44:17.275 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:44:17.275 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:44:17.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:17.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:17.287 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:17.352 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-30T09:44:17.381 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:17.423 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:17.478 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:17.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:17.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:17.502 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:17.521 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:17 smithi167 conmon[343682]: 2023-09-30T09:44:17.216+0000 7f325e68f540 -1 Falling back to public interface 2023-09-30T09:44:17.552 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:44:17.574 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:44:17.583 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:17.595 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:44:17.613 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:17.617 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:44:17.636 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:44:17.658 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:44:17.678 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:44:17.678 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:44:17.679 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:44:17.679 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:44:17.679 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:44:17.679 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:44:17.679 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:44:17.679 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:44:17.679 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:44:17.876 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:17.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:17.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:17.934 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:44:17.980 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:17.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[224043]: failed to encode map e483 with expected crc 2023-09-30T09:44:17.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[224043]: osdmap e484: 8 total, 7 up, 8 in 2023-09-30T09:44:17.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[224043]: failed to encode map e484 with expected crc 2023-09-30T09:44:17.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[224043]: failed to encode map e484 with expected crc 2023-09-30T09:44:17.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[224043]: failed to encode map e484 with expected crc 2023-09-30T09:44:17.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[224043]: failed to encode map e484 with expected crc 2023-09-30T09:44:17.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4007191919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:17.996 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:17.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[216815]: failed to encode map e483 with expected crc 2023-09-30T09:44:17.997 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[216815]: osdmap e484: 8 total, 7 up, 8 in 2023-09-30T09:44:17.997 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[216815]: failed to encode map e484 with expected crc 2023-09-30T09:44:17.997 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[216815]: failed to encode map e484 with expected crc 2023-09-30T09:44:17.997 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[216815]: failed to encode map e484 with expected crc 2023-09-30T09:44:17.997 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[216815]: failed to encode map e484 with expected crc 2023-09-30T09:44:17.997 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4007191919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:17.997 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:17 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:18.016 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:18.156 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:17 smithi167 ceph-mon[192880]: failed to encode map e483 with expected crc 2023-09-30T09:44:18.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:17 smithi167 ceph-mon[192880]: osdmap e484: 8 total, 7 up, 8 in 2023-09-30T09:44:18.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:17 smithi167 ceph-mon[192880]: failed to encode map e484 with expected crc 2023-09-30T09:44:18.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:17 smithi167 ceph-mon[192880]: failed to encode map e484 with expected crc 2023-09-30T09:44:18.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:17 smithi167 ceph-mon[192880]: failed to encode map e484 with expected crc 2023-09-30T09:44:18.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:17 smithi167 ceph-mon[192880]: failed to encode map e484 with expected crc 2023-09-30T09:44:18.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4007191919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:18.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:17 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:18.468 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:18.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:18.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:18.573 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:18.605 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:18.660 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:44:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:18.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:18.664 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:44:18.765 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:18.796 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:18.841 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:18.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:18.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:18.927 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:18.945 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:18.972 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:19.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[216815]: pgmap v1244: 119 pgs: 1 creating+peering, 15 unknown, 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[216815]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[216815]: osdmap e485: 8 total, 7 up, 8 in 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[216815]: failed to encode map e485 with expected crc 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[216815]: failed to encode map e485 with expected crc 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[224043]: pgmap v1244: 119 pgs: 1 creating+peering, 15 unknown, 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[224043]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:44:19.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[224043]: osdmap e485: 8 total, 7 up, 8 in 2023-09-30T09:44:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[224043]: failed to encode map e485 with expected crc 2023-09-30T09:44:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:18 smithi019 ceph-mon[224043]: failed to encode map e485 with expected crc 2023-09-30T09:44:19.270 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:19.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:19.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:18 smithi167 ceph-mon[192880]: pgmap v1244: 119 pgs: 1 creating+peering, 15 unknown, 25 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 68 active+clean; 4.3 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3/81 objects degraded (3.704%); 5/81 objects misplaced (6.173%) 2023-09-30T09:44:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:18 smithi167 ceph-mon[192880]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:44:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:18 smithi167 ceph-mon[192880]: osdmap e485: 8 total, 7 up, 8 in 2023-09-30T09:44:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:18 smithi167 ceph-mon[192880]: failed to encode map e485 with expected crc 2023-09-30T09:44:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:18 smithi167 ceph-mon[192880]: failed to encode map e485 with expected crc 2023-09-30T09:44:19.355 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:19.374 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:19.406 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:19.504 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:19.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:19.546 INFO:teuthology.orchestra.run.smithi167.stderr:no valid command found; 10 closest matches: 2023-09-30T09:44:19.546 INFO:teuthology.orchestra.run.smithi167.stderr:0 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:1 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:2 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:abort 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:assert 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:config diff 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:config diff get 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:config get 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:config help [] 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:config set ... 2023-09-30T09:44:19.547 INFO:teuthology.orchestra.run.smithi167.stderr:admin_socket: invalid command 2023-09-30T09:44:19.609 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:19.639 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:19.788 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:19.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:19.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:19.891 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:19.923 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:20.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:20 smithi019 ceph-mon[216815]: failed to encode map e485 with expected crc 2023-09-30T09:44:20.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:20 smithi019 ceph-mon[224043]: failed to encode map e485 with expected crc 2023-09-30T09:44:20.226 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:20.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:20.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:20 smithi167 ceph-mon[192880]: failed to encode map e485 with expected crc 2023-09-30T09:44:20.331 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:20.359 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:20.447 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:20.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:20.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:20.513 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:44:20.513 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-09-30T09:44:20.551 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:20.581 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:20.629 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:20.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:20.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:20.665 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:44:20.733 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:20.761 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:20.816 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:20.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:20.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:20.923 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:20.954 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:21.015 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:44:21.059 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:44:21.059 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:44:21.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:21 smithi019 ceph-mon[216815]: pgmap v1246: 119 pgs: 1 creating+peering, 7 unknown, 25 active+clean+remapped, 12 active+undersized, 3 active+undersized+degraded, 71 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:21.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:21 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:21.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1758999265' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:44:21.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:21 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:44:21.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:21 smithi019 ceph-mon[224043]: pgmap v1246: 119 pgs: 1 creating+peering, 7 unknown, 25 active+clean+remapped, 12 active+undersized, 3 active+undersized+degraded, 71 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:21.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:21 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:21.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1758999265' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:44:21.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:21 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:44:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:21 smithi167 ceph-mon[192880]: pgmap v1246: 119 pgs: 1 creating+peering, 7 unknown, 25 active+clean+remapped, 12 active+undersized, 3 active+undersized+degraded, 71 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:21 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1758999265' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:44:21.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:21 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:44:21.804 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:44:21.908 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:21.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:22.012 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:22.018 INFO:teuthology.orchestra.run.smithi019.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-30T09:44:22.035 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:22.042 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:22 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:22.042 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:22 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:44:22.042 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:22 smithi167 ceph-mon[192880]: osdmap e486: 8 total, 7 up, 8 in 2023-09-30T09:44:22.042 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:22 smithi167 ceph-mon[192880]: failed to encode map e486 with expected crc 2023-09-30T09:44:22.042 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/854220237' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:44:22.045 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:22.094 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:22.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:22.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:22.181 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:22.201 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:22.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:22.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:44:22.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[216815]: osdmap e486: 8 total, 7 up, 8 in 2023-09-30T09:44:22.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[216815]: failed to encode map e486 with expected crc 2023-09-30T09:44:22.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/854220237' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:44:22.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:22.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:44:22.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[224043]: osdmap e486: 8 total, 7 up, 8 in 2023-09-30T09:44:22.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[224043]: failed to encode map e486 with expected crc 2023-09-30T09:44:22.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/854220237' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:44:22.232 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:22.278 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:22.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:22.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:22.381 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:22.408 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:22.778 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:22.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:22.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:22.882 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:22.910 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:23.207 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:23.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:23.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:23.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:23 smithi167 ceph-mon[192880]: pgmap v1248: 119 pgs: 25 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 75 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/854220237' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:44:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:23 smithi167 ceph-mon[192880]: osdmap e487: 8 total, 7 up, 8 in 2023-09-30T09:44:23.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:23 smithi167 ceph-mon[192880]: failed to encode map e487 with expected crc 2023-09-30T09:44:23.311 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:23.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:23 smithi019 ceph-mon[224043]: pgmap v1248: 119 pgs: 25 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 75 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/854220237' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:44:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:23 smithi019 ceph-mon[224043]: osdmap e487: 8 total, 7 up, 8 in 2023-09-30T09:44:23.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:23 smithi019 ceph-mon[224043]: failed to encode map e487 with expected crc 2023-09-30T09:44:23.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:23 smithi019 ceph-mon[216815]: pgmap v1248: 119 pgs: 25 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 75 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:23.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/854220237' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:44:23.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:23 smithi019 ceph-mon[216815]: osdmap e487: 8 total, 7 up, 8 in 2023-09-30T09:44:23.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:23 smithi019 ceph-mon[216815]: failed to encode map e487 with expected crc 2023-09-30T09:44:23.340 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:23.387 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:23.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:23.490 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:23.517 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:23.686 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:23.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:23.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:23.696 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:23 smithi167 conmon[343682]: 2023-09-30T09:44:23.435+0000 7f325e68f540 -1 osd.5 427 log_to_monitors true 2023-09-30T09:44:23.793 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:23.822 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:23.836 DEBUG:teuthology.orchestra.run.smithi019:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-30T09:44:23.838 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-30T09:44:23.838 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-30T09:44:23.888 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:hints = 1 2023-09-30T09:44:23.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-30T09:44:23.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object prefix: benchmark_data_smithi019_560486 2023-09-30T09:44:23.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:44:23.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 0 0 0 0 0 0 - 0 2023-09-30T09:44:23.893 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:44:23.940 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:44:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:44:23] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.43.0" 2023-09-30T09:44:23.952 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:44:23.952 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:44:23.972 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:23.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:24.089 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:24.164 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:24.245 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:24.252 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:24 smithi019 ceph-mon[224043]: from='osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:44:24.253 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:24 smithi019 ceph-mon[224043]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:44:24.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:24 smithi019 ceph-mon[216815]: from='osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:44:24.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:24 smithi019 ceph-mon[216815]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:44:24.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:24.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:24 smithi167 ceph-mon[192880]: from='osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:44:24.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:24 smithi167 ceph-mon[192880]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-30T09:44:24.319 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:44:24.341 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:44:24.354 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:24.374 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:44:24.415 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:24.421 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:44:24.459 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:44:24.490 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:24.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:24.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:24.493 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:44:24.509 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:24.595 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:24.622 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:24.771 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:24.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:44:24 smithi167 conmon[343682]: 2023-09-30T09:44:24.292+0000 7f324e46c700 -1 osd.5 488 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:44:24.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:24.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:24.812 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:44:24.812 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:44:24.812 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:44:24.812 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:44:24.812 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:44:24.812 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:44:24.813 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-30T09:44:24.813 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:44:24.813 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:44:24.876 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:24.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 1 16 1849 1833 114.556 114.562 0.00432684 0.00867203 2023-09-30T09:44:24.936 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:25.208 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:44:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: failed to encode map e487 with expected crc 2023-09-30T09:44:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: failed to encode map e487 with expected crc 2023-09-30T09:44:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: pgmap v1250: 119 pgs: 25 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 75 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: osdmap e488: 8 total, 7 up, 8 in 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: from='osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1854925706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[224043]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:44:25.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: failed to encode map e487 with expected crc 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: failed to encode map e487 with expected crc 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: pgmap v1250: 119 pgs: 25 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 75 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: osdmap e488: 8 total, 7 up, 8 in 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: from='osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1854925706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:25.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:25 smithi019 ceph-mon[216815]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:44:25.431 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:25.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:25.515 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:44:25.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: failed to encode map e487 with expected crc 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: failed to encode map e487 with expected crc 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: pgmap v1250: 119 pgs: 25 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 75 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 3/84 objects degraded (3.571%); 5/84 objects misplaced (5.952%) 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: osdmap e488: 8 total, 7 up, 8 in 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: from='osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-09-30T09:44:25.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: failed to encode map e488 with expected crc 2023-09-30T09:44:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1854925706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:25.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:25 smithi167 ceph-mon[192880]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:44:25.535 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:25.558 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:25.564 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:25.612 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:25.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:25.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:25.716 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:25.744 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:25.841 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:25.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:25.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:25.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 2 16 2312 2296 71.7412 28.9375 0.00304757 0.00805226 2023-09-30T09:44:25.945 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:25.972 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:26.015 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:26.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:26.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:26.062 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:44:26.062 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215] boot 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: osdmap e489: 8 total, 8 up, 8 in 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.063 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:26 smithi167 ceph-mon[192880]: osdmap e490: 8 total, 8 up, 8 in 2023-09-30T09:44:26.118 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:26.146 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:26.240 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:26.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:26.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215] boot 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: osdmap e489: 8 total, 8 up, 8 in 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[224043]: osdmap e490: 8 total, 8 up, 8 in 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: osd.5 [v2:172.21.15.167:6808/3203515215,v1:172.21.15.167:6809/3203515215] boot 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: osdmap e489: 8 total, 8 up, 8 in 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-30T09:44:26.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:26.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:26 smithi019 ceph-mon[216815]: osdmap e490: 8 total, 8 up, 8 in 2023-09-30T09:44:26.343 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:26.378 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:26.785 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:44:26.786 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:44:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:26.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:26.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 3 16 2312 2296 47.828 0 - 0.00805226 2023-09-30T09:44:26.891 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:26.919 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:27.016 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:27.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:27.059 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:27.120 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:27.181 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e428 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e468 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e489 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: pgmap v1253: 119 pgs: 2 peering, 6 remapped+peering, 19 active+clean+remapped, 13 active+undersized, 4 active+undersized+degraded, 75 active+clean; 60 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 12 MiB/s wr, 188 op/s; 42/2763 objects degraded (1.520%); 3/2763 objects misplaced (0.109%) 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.182 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.183 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.183 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 42/2763 objects degraded (1.520%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:27.183 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:27 smithi167 ceph-mon[192880]: osdmap e491: 8 total, 8 up, 8 in 2023-09-30T09:44:27.188 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e428 with expected crc 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e468 with expected crc 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e489 with expected crc 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: pgmap v1253: 119 pgs: 2 peering, 6 remapped+peering, 19 active+clean+remapped, 13 active+undersized, 4 active+undersized+degraded, 75 active+clean; 60 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 12 MiB/s wr, 188 op/s; 42/2763 objects degraded (1.520%); 3/2763 objects misplaced (0.109%) 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 42/2763 objects degraded (1.520%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:27.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[224043]: osdmap e491: 8 total, 8 up, 8 in 2023-09-30T09:44:27.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e428 with expected crc 2023-09-30T09:44:27.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e468 with expected crc 2023-09-30T09:44:27.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e489 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: pgmap v1253: 119 pgs: 2 peering, 6 remapped+peering, 19 active+clean+remapped, 13 active+undersized, 4 active+undersized+degraded, 75 active+clean; 60 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 12 MiB/s wr, 188 op/s; 42/2763 objects degraded (1.520%); 3/2763 objects misplaced (0.109%) 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: failed to encode map e490 with expected crc 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 42/2763 objects degraded (1.520%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-30T09:44:27.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:27 smithi019 ceph-mon[216815]: osdmap e491: 8 total, 8 up, 8 in 2023-09-30T09:44:27.393 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:27.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:27.505 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:27.558 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:27.716 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:44:27.717 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:44:27.717 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:44:27.717 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:44:27.717 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:44:27.758 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:27.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:27.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:27.863 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:27.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 4 16 4563 4547 71.0393 70.3438 0.00259808 0.0140591 2023-09-30T09:44:27.924 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:28.031 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:28.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:28.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:28.090 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:44:28.137 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:28.176 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/774131244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:28.295 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.319 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.319 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.319 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.319 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.319 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/774131244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.320 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: failed to encode map e491 with expected crc 2023-09-30T09:44:28.321 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/774131244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:28.787 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:44:28.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 5 16 5302 5286 66.0679 46.1875 0.00196411 0.0138814 2023-09-30T09:44:29.078 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: pgmap v1256: 119 pgs: 2 peering, 7 remapped+peering, 19 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 75 active+clean; 69 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 16 MiB/s wr, 258 op/s; 42/3192 objects degraded (1.316%); 3/3192 objects misplaced (0.094%) 2023-09-30T09:44:29.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:29.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/774131244' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:29.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: pgmap v1256: 119 pgs: 2 peering, 7 remapped+peering, 19 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 75 active+clean; 69 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 16 MiB/s wr, 258 op/s; 42/3192 objects degraded (1.316%); 3/3192 objects misplaced (0.094%) 2023-09-30T09:44:29.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:29.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/774131244' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:29.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: osdmap e492: 8 total, 8 up, 8 in 2023-09-30T09:44:29.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[216815]: osdmap e493: 8 total, 8 up, 8 in 2023-09-30T09:44:29.083 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:29.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:29.189 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:29.218 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: osdmap e492: 8 total, 8 up, 8 in 2023-09-30T09:44:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:29 smithi019 ceph-mon[224043]: osdmap e493: 8 total, 8 up, 8 in 2023-09-30T09:44:29.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: pgmap v1256: 119 pgs: 2 peering, 7 remapped+peering, 19 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 75 active+clean; 69 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 16 MiB/s wr, 258 op/s; 42/3192 objects degraded (1.316%); 3/3192 objects misplaced (0.094%) 2023-09-30T09:44:29.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:29.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/774131244' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: osdmap e492: 8 total, 8 up, 8 in 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: failed to encode map e492 with expected crc 2023-09-30T09:44:29.413 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:29 smithi167 ceph-mon[192880]: osdmap e493: 8 total, 8 up, 8 in 2023-09-30T09:44:29.458 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:29.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:29.562 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:29.585 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:29.588 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:29.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 6 16 5654 5638 58.7226 22 0.00138976 0.016537 2023-09-30T09:44:30.011 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:44:30.045 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:30.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:30.124 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[224043]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.124 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[224043]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.124 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[224043]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.124 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[224043]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.124 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[224043]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.125 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[224043]: osdmap e494: 8 total, 8 up, 8 in 2023-09-30T09:44:30.125 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[216815]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.125 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[216815]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.125 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[216815]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.125 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[216815]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.125 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[216815]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.125 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:30 smithi019 ceph-mon[216815]: osdmap e494: 8 total, 8 up, 8 in 2023-09-30T09:44:30.149 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:30.178 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:30.230 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:30.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:30 smithi167 ceph-mon[192880]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:30 smithi167 ceph-mon[192880]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:30 smithi167 ceph-mon[192880]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:30 smithi167 ceph-mon[192880]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:30 smithi167 ceph-mon[192880]: failed to encode map e493 with expected crc 2023-09-30T09:44:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:30 smithi167 ceph-mon[192880]: osdmap e494: 8 total, 8 up, 8 in 2023-09-30T09:44:30.335 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:30.365 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:30.410 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:30.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:30.514 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:30.542 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:30.582 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:44:30.583 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:44:30.583 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.583 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5545 9.f 9.f655645f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.583 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:44:30.583 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.87523796899999995, 2023-09-30T09:44:30.583 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.875263071, 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5545 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:44:30.584 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705597+0000", 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705693+0000", 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5401000000000004e-05 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705706+0000", 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2665e-05 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.586 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705713+0000", 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9940000000000003e-06 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705830+0000", 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011760900000000001 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705836+0000", 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7610000000000004e-06 2023-09-30T09:44:30.587 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5500 9.f 9.714deedf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.88967562200000005, 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.88975946900000003, 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.588 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5500 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691160+0000", 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691215+0000", 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5435e-05 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.590 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691223+0000", 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9340000000000006e-06 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691227+0000", 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7459999999999998e-06 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691295+0000", 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8145999999999994e-05 2023-09-30T09:44:30.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691307+0000", 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1516e-05 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5654 9.f 9.1043fd2f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.767698+0000", 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.81313829100000001, 2023-09-30T09:44:30.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81326656399999997, 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5654 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.767698+0000", 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.767698+0000", 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.767697+0000", 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.767750+0000", 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3436999999999997e-05 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.767760+0000", 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8539999999999992e-06 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.767766+0000", 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8799999999999996e-06 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.767807+0000", 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1205999999999998e-05 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.596 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.767813+0000", 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3419999999999998e-06 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5551 9.f 9.2b2e759f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.87432290899999998, 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.87448806199999995, 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.597 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5551 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.598 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706513+0000", 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706568+0000", 2023-09-30T09:44:30.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5432000000000001e-05 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706580+0000", 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1494999999999999e-05 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706586+0000", 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7530000000000004e-06 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.600 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706645+0000", 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8591000000000002e-05 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706653+0000", 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5250000000000005e-06 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.601 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5614 9.f 9.2237a53f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.84842217900000005, 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.84862548599999998, 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5614 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.602 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732413+0000", 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732525+0000", 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112094 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732544+0000", 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9120999999999999e-05 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732551+0000", 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9e-06 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732616+0000", 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4272999999999999e-05 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732625+0000", 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.166e-06 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5581 9.f 9.f574ca5f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.865098747, 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.86533873299999997, 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.606 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5581 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.607 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715737+0000", 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.608 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715765+0000", 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8323e-05 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715780+0000", 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5475999999999999e-05 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715787+0000", 2023-09-30T09:44:30.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9149999999999999e-06 2023-09-30T09:44:30.610 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.610 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715853+0000", 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5413000000000005e-05 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715862+0000", 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.594e-06 2023-09-30T09:44:30.611 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5618 9.f 9.401fc23f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.84762668200000002, 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.84790649799999995, 2023-09-30T09:44:30.612 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.613 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.613 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.613 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.613 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.613 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5618 2023-09-30T09:44:30.613 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.613 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.613 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.614 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.614 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:44:30.614 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.614 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.614 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.614 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.614 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:44:30.614 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733209+0000", 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733267+0000", 2023-09-30T09:44:30.615 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7673999999999997e-05 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733281+0000", 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4701999999999999e-05 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.616 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733287+0000", 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9839999999999998e-06 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733333+0000", 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5609000000000003e-05 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733339+0000", 2023-09-30T09:44:30.617 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7960000000000001e-06 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5641 9.f 9.4e9c8a7f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.749994+0000", 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.83084253699999999, 2023-09-30T09:44:30.618 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.831160707, 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5641 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.619 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.749994+0000", 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.749994+0000", 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.620 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.621 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.749993+0000", 2023-09-30T09:44:30.621 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.621 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.621 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.621 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.621 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.750015+0000", 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2028999999999999e-05 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.750024+0000", 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2839999999999995e-06 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.622 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.750030+0000", 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9909999999999999e-06 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.750086+0000", 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6001000000000001e-05 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.623 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.750095+0000", 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.003e-06 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.624 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5624 9.f 9.23853bcf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.736348+0000", 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.84448854500000003, 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.84484245700000005, 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.625 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5624 2023-09-30T09:44:30.626 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.626 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.626 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.626 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.626 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.736348+0000", 2023-09-30T09:44:30.626 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.626 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.626 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.736348+0000", 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.736347+0000", 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:30.627 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.736395+0000", 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7852999999999997e-05 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.736403+0000", 2023-09-30T09:44:30.628 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0239999999999997e-06 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.736409+0000", 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3619999999999997e-06 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.629 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.736442+0000", 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3677000000000003e-05 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.736450+0000", 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3679999999999996e-06 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.630 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5591 9.f 9.bf2428df (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.86186669999999999, 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.86225600400000002, 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.631 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5591 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.632 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718969+0000", 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.633 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719014+0000", 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4433e-05 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719031+0000", 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7078000000000001e-05 2023-09-30T09:44:30.634 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719036+0000", 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2240000000000003e-06 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719098+0000", 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2292999999999995e-05 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.635 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719107+0000", 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3319999999999992e-06 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.636 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5593 9.f 9.c431f2ef (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.86149820499999996, 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.86192235399999995, 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5593 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.637 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:44:30.638 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.639 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.639 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.639 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.639 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:44:30.639 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:30.639 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.639 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.639 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.641 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719402+0000", 2023-09-30T09:44:30.641 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4404000000000002e-05 2023-09-30T09:44:30.641 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.641 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.641 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719414+0000", 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1895999999999999e-05 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719420+0000", 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.365e-06 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.642 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719641+0000", 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000221122 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719653+0000", 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1884e-05 2023-09-30T09:44:30.643 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5531 9.f 9.7917373f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.88018481299999995, 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.88064809099999997, 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.644 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5531 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:44:30.645 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700650+0000", 2023-09-30T09:44:30.646 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700746+0000", 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5857000000000001e-05 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700758+0000", 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1983e-05 2023-09-30T09:44:30.647 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700765+0000", 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8279999999999997e-06 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700839+0000", 2023-09-30T09:44:30.648 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4226000000000004e-05 2023-09-30T09:44:30.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.649 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.649 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700857+0000", 2023-09-30T09:44:30.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7193e-05 2023-09-30T09:44:30.649 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.649 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.649 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5488 9.f 9.b17383af (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.89288571000000005, 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.89338461300000005, 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.650 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.651 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.651 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5488 2023-09-30T09:44:30.651 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.651 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.651 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.651 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.651 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:44:30.651 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.652 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687950+0000", 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687988+0000", 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8231e-05 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.653 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688001+0000", 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2577e-05 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688007+0000", 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3740000000000003e-06 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.654 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688066+0000", 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8805999999999999e-05 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688075+0000", 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7499999999999998e-06 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.655 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5456 9.f 9.864382ef (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.90285252199999999, 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90338833600000001, 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.656 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5456 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.657 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.658 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677983+0000", 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678041+0000", 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7439999999999999e-05 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678050+0000", 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2380000000000003e-06 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678055+0000", 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2480000000000004e-06 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678212+0000", 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015715 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678221+0000", 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2140000000000002e-06 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5604 9.f 9.2864884f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.85660547399999998, 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.857186632, 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.662 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5604 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.663 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724230+0000", 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.664 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724380+0000", 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015008099999999999 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724392+0000", 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1518999999999999e-05 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724398+0000", 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5540000000000003e-06 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724454+0000", 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5358000000000001e-05 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.666 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724465+0000", 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1226000000000001e-05 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5571 9.f 9.bc08320f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:44:30.667 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.86816773700000005, 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.86880013599999995, 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "delayed", 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5571 2023-09-30T09:44:30.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:44:30.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712668+0000", 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:30.670 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713054+0000", 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00038651499999999998 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713063+0000", 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6440000000000006e-06 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:30.671 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713068+0000", 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2120000000000002e-06 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713200+0000", 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000131977 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:30.672 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713210+0000", 2023-09-30T09:44:30.673 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4250000000000004e-06 2023-09-30T09:44:30.673 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.673 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:30.673 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.673 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:30.673 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:44:30.673 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 16 2023-09-30T09:44:30.673 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:44:30.676 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:30.676 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:30.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:30.777 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:30.807 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:30.851 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:30.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:30.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 7 16 5696 5680 50.7089 2.625 0.00143549 0.0191413 2023-09-30T09:44:30.954 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:30.981 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:31.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:31 smithi167 ceph-mon[192880]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:31 smithi167 ceph-mon[192880]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:31 smithi167 ceph-mon[192880]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:31 smithi167 ceph-mon[192880]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:31 smithi167 ceph-mon[192880]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:31 smithi167 ceph-mon[192880]: pgmap v1260: 119 pgs: 1 creating+peering, 3 peering, 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 270 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 493/12801 objects degraded (3.851%); 7.1 MiB/s, 113 objects/s recovering 2023-09-30T09:44:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:31 smithi167 ceph-mon[192880]: osdmap e495: 8 total, 8 up, 8 in 2023-09-30T09:44:31.331 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[216815]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.331 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[216815]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.331 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[216815]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.331 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[216815]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.331 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[216815]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[216815]: pgmap v1260: 119 pgs: 1 creating+peering, 3 peering, 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 270 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 493/12801 objects degraded (3.851%); 7.1 MiB/s, 113 objects/s recovering 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[216815]: osdmap e495: 8 total, 8 up, 8 in 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[224043]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[224043]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[224043]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[224043]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[224043]: failed to encode map e494 with expected crc 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[224043]: pgmap v1260: 119 pgs: 1 creating+peering, 3 peering, 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 270 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 493/12801 objects degraded (3.851%); 7.1 MiB/s, 113 objects/s recovering 2023-09-30T09:44:31.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:31 smithi019 ceph-mon[224043]: osdmap e495: 8 total, 8 up, 8 in 2023-09-30T09:44:31.418 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:31.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:31.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:31.521 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:31.530 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:44:31.548 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:31.578 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-30T09:44:31.643 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:44:31.647 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:31.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:31.676 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:44:31.722 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:44:31.751 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:31.762 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:44:31.805 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:44:31.857 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:31.870 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:44:31.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 8 16 6070 6054 47.2921 23.375 0.00905031 0.0211193 2023-09-30T09:44:31.910 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:44:31.951 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:44:31.969 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:31.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:44:31.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:44:31.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:44:31.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:44:31.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:44:31.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:44:31.970 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:44:31.971 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:44:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:31.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:32.076 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:44:32.129 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:32.177 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:44:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:32.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:44:32.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:32 smithi167 ceph-mon[192880]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:32 smithi167 ceph-mon[192880]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:32 smithi167 ceph-mon[192880]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:32 smithi167 ceph-mon[192880]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:32 smithi167 ceph-mon[192880]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:32 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:44:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2148801357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:32.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:32 smithi167 ceph-mon[192880]: osdmap e496: 8 total, 8 up, 8 in 2023-09-30T09:44:32.281 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:32.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[224043]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[224043]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[224043]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[224043]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[224043]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:44:32.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2148801357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:32.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[224043]: osdmap e496: 8 total, 8 up, 8 in 2023-09-30T09:44:32.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[216815]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[216815]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[216815]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[216815]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[216815]: failed to encode map e495 with expected crc 2023-09-30T09:44:32.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:44:32.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2148801357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:32.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:32 smithi019 ceph-mon[216815]: osdmap e496: 8 total, 8 up, 8 in 2023-09-30T09:44:32.312 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:32.464 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:32.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:32.569 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:32.599 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:32.816 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:32.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:32.889 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 9 16 7335 7319 50.8207 79.0625 0.00391815 0.0196661 2023-09-30T09:44:32.923 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:32.970 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:33.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: pgmap v1263: 119 pgs: 1 creating+peering, 1 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 332 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 493/15810 objects degraded (3.118%); 9.9 MiB/s, 157 objects/s recovering 2023-09-30T09:44:33.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 493/15810 objects degraded (3.118%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:44:33.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:33.215 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:33.275 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:33.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.297 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: pgmap v1263: 119 pgs: 1 creating+peering, 1 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 332 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 493/15810 objects degraded (3.118%); 9.9 MiB/s, 157 objects/s recovering 2023-09-30T09:44:33.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 493/15810 objects degraded (3.118%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:44:33.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:33.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:33.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.301 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: failed to encode map e496 with expected crc 2023-09-30T09:44:33.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: pgmap v1263: 119 pgs: 1 creating+peering, 1 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 332 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 493/15810 objects degraded (3.118%); 9.9 MiB/s, 157 objects/s recovering 2023-09-30T09:44:33.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 493/15810 objects degraded (3.118%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:44:33.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:33.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:33.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:33.358 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:33.379 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:33.404 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:33.469 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:33.546 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:44:33.546 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:44:33.578 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:33.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:33.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:33.684 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:33.749 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:33.834 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:44:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:44:33] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-09-30T09:44:33.890 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 10 16 9864 9848 61.5412 158.062 0.00846818 0.0162405 2023-09-30T09:44:34.890 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 11 16 12706 12690 72.0923 177.625 0.00763323 0.0138633 2023-09-30T09:44:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:34 smithi167 ceph-mon[192880]: pgmap v1264: 119 pgs: 1 creating+peering, 1 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 359 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 347/17112 objects degraded (2.028%); 7.0 MiB/s, 111 objects/s recovering 2023-09-30T09:44:35.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:34 smithi019 ceph-mon[216815]: pgmap v1264: 119 pgs: 1 creating+peering, 1 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 359 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 347/17112 objects degraded (2.028%); 7.0 MiB/s, 111 objects/s recovering 2023-09-30T09:44:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:34 smithi019 ceph-mon[224043]: pgmap v1264: 119 pgs: 1 creating+peering, 1 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 359 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 347/17112 objects degraded (2.028%); 7.0 MiB/s, 111 objects/s recovering 2023-09-30T09:44:35.304 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:44:35.890 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 12 16 14910 14894 77.562 137.75 0.0019212 0.0127531 2023-09-30T09:44:35.922 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:35.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:35.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:36.027 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:36.050 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:44:36.070 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:35 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 347/17112 objects degraded (2.028%), 3 pgs degraded) 2023-09-30T09:44:36.070 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:35 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 347/17112 objects degraded (2.028%), 3 pgs degraded) 2023-09-30T09:44:36.076 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:36.198 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:36.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:36.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:36.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:35 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 347/17112 objects degraded (2.028%), 3 pgs degraded) 2023-09-30T09:44:36.303 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:36.331 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:36.726 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:36.891 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 13 16 16407 16391 78.7872 93.5625 0.00935138 0.012683 2023-09-30T09:44:36.944 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:36.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:36.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:37.054 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:37.114 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:37.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:36 smithi019 ceph-mon[216815]: pgmap v1265: 119 pgs: 119 active+clean; 714 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8.7 MiB/s, 138 objects/s recovering 2023-09-30T09:44:37.117 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:36 smithi019 ceph-mon[224043]: pgmap v1265: 119 pgs: 119 active+clean; 714 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8.7 MiB/s, 138 objects/s recovering 2023-09-30T09:44:37.118 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:36 smithi167 ceph-mon[192880]: pgmap v1265: 119 pgs: 119 active+clean; 714 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8.7 MiB/s, 138 objects/s recovering 2023-09-30T09:44:37.133 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:37.304 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:37.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:37.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:37.409 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:37.467 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:37.732 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:37.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:37.836 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:37.893 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 14 16 18054 18038 80.4975 102.938 0.00893166 0.012415 2023-09-30T09:44:37.926 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:38.116 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:38.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:38.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:38.224 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:44:38.259 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:38.434 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:44:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:38.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:44:38.540 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:38.570 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:38.888 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:44:38.889 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.1 to 1 (10000) 2023-09-30T09:44:38.894 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 15 16 19109 19093 79.5266 65.9375 0.0023046 0.0124462 2023-09-30T09:44:39.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:38 smithi167 ceph-mon[192880]: pgmap v1266: 119 pgs: 119 active+clean; 714 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 6.6 MiB/s, 105 objects/s recovering 2023-09-30T09:44:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1327131426' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:44:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:38 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:44:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3372143274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:44:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:38 smithi019 ceph-mon[216815]: pgmap v1266: 119 pgs: 119 active+clean; 714 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 6.6 MiB/s, 105 objects/s recovering 2023-09-30T09:44:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1327131426' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:44:39.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:38 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:44:39.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3372143274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:44:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:38 smithi019 ceph-mon[224043]: pgmap v1266: 119 pgs: 119 active+clean; 714 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 6.6 MiB/s, 105 objects/s recovering 2023-09-30T09:44:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1327131426' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:44:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:38 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:44:39.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3372143274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:44:39.895 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 16 16 20898 20882 81.5361 111.812 0.00784699 0.0122575 2023-09-30T09:44:40.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:39 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:40.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:44:40.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3372143274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:44:40.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:39 smithi167 ceph-mon[192880]: failed to encode map e497 with expected crc 2023-09-30T09:44:40.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:39 smithi167 ceph-mon[192880]: osdmap e497: 8 total, 8 up, 8 in 2023-09-30T09:44:40.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:39 smithi167 ceph-mon[192880]: failed to encode map e497 with expected crc 2023-09-30T09:44:40.145 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:40.146 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:44:40.146 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3372143274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:44:40.146 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[224043]: failed to encode map e497 with expected crc 2023-09-30T09:44:40.146 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[224043]: osdmap e497: 8 total, 8 up, 8 in 2023-09-30T09:44:40.146 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[224043]: failed to encode map e497 with expected crc 2023-09-30T09:44:40.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:44:40.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:44:40.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3372143274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:44:40.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[216815]: failed to encode map e497 with expected crc 2023-09-30T09:44:40.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[216815]: osdmap e497: 8 total, 8 up, 8 in 2023-09-30T09:44:40.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:39 smithi019 ceph-mon[216815]: failed to encode map e497 with expected crc 2023-09-30T09:44:40.153 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:44:40.153 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:44:40.159 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:40.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:40.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:40.269 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:40.299 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:40.362 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:40.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:40.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:40.466 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:40.522 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:40.896 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 17 16 22776 22760 83.6415 117.375 0.00919046 0.0119495 2023-09-30T09:44:41.012 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-30T09:44:41.064 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:41.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:41.171 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:41.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:40 smithi167 ceph-mon[192880]: 1.12 scrub starts 2023-09-30T09:44:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:40 smithi167 ceph-mon[192880]: 1.12 scrub ok 2023-09-30T09:44:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:40 smithi167 ceph-mon[192880]: pgmap v1268: 119 pgs: 119 active+clean; 1.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 5.3 MiB/s, 85 objects/s recovering 2023-09-30T09:44:41.274 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:40 smithi019 ceph-mon[224043]: 1.12 scrub starts 2023-09-30T09:44:41.274 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:40 smithi019 ceph-mon[224043]: 1.12 scrub ok 2023-09-30T09:44:41.274 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:40 smithi019 ceph-mon[224043]: pgmap v1268: 119 pgs: 119 active+clean; 1.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 5.3 MiB/s, 85 objects/s recovering 2023-09-30T09:44:41.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:40 smithi019 ceph-mon[216815]: 1.12 scrub starts 2023-09-30T09:44:41.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:40 smithi019 ceph-mon[216815]: 1.12 scrub ok 2023-09-30T09:44:41.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:40 smithi019 ceph-mon[216815]: pgmap v1268: 119 pgs: 119 active+clean; 1.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 5.3 MiB/s, 85 objects/s recovering 2023-09-30T09:44:41.288 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:41.407 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:41.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:41.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:41.512 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:41.577 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:41.678 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:41.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:41.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:41.763 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:44:41.774 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:41.788 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:41.808 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:44:41.853 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:44:41.857 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:41.894 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:44:41.897 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 18 16 24307 24291 84.31 95.6875 0.00689261 0.0118541 2023-09-30T09:44:41.932 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:44:41.975 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:44:41.979 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:44:42.020 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:44:42.056 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:44:42.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:44:42.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:44:42.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:44:42.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:44:42.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:44:42.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:44:42.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:44:42.062 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:44:42.195 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/184435573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:42.196 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/184435573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:42.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/184435573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:42.376 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:42.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:42.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:42.483 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:42.547 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:42.810 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:42.896 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 19 16 26288 26272 86.3879 123.812 0.00359849 0.0115672 2023-09-30T09:44:43.087 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:44:43.139 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:42 smithi019 ceph-mon[224043]: pgmap v1269: 119 pgs: 119 active+clean; 1.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s; 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:44:43.140 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:42 smithi019 ceph-mon[216815]: pgmap v1269: 119 pgs: 119 active+clean; 1.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s; 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:44:43.179 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:42 smithi167 ceph-mon[192880]: pgmap v1269: 119 pgs: 119 active+clean; 1.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s; 4.3 MiB/s, 68 objects/s recovering 2023-09-30T09:44:43.897 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:44:43.896464+0000 min lat: 0.00110418 max lat: 2.06823 avg lat: 0.0117275 2023-09-30T09:44:43.897 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:44:43.897 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 20 16 27297 27281 85.2217 63.0625 0.0119257 0.0117275 2023-09-30T09:44:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[216815]: 1.15 scrub starts 2023-09-30T09:44:44.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[216815]: 1.15 scrub ok 2023-09-30T09:44:44.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:44:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:44:43] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-09-30T09:44:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[224043]: 1.15 scrub starts 2023-09-30T09:44:44.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[224043]: 1.15 scrub ok 2023-09-30T09:44:44.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:43 smithi167 ceph-mon[192880]: 1.15 scrub starts 2023-09-30T09:44:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:43 smithi167 ceph-mon[192880]: 1.15 scrub ok 2023-09-30T09:44:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:44.357 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:44.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:44.462 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:44.491 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:44.799 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:44.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:44.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:44.896 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 21 16 28970 28954 86.142 104.562 0.0062332 0.0115992 2023-09-30T09:44:44.907 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:44:44.956 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:45.055 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:44:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:45.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:44:45.063 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:44 smithi019 ceph-mon[216815]: pgmap v1270: 119 pgs: 119 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 3.4 MiB/s, 54 objects/s recovering 2023-09-30T09:44:45.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:44 smithi019 ceph-mon[224043]: pgmap v1270: 119 pgs: 119 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 3.4 MiB/s, 54 objects/s recovering 2023-09-30T09:44:45.164 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:44:45.218 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:44 smithi167 ceph-mon[192880]: pgmap v1270: 119 pgs: 119 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 3.4 MiB/s, 54 objects/s recovering 2023-09-30T09:44:45.224 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:45.306 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:44:45.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:45.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:44:45.412 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:45.468 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:45.613 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:45.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:45.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:45.713 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:45.724 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:45.791 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:44:45.791 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:44:45.791 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:44:45.791 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:44:45.792 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:44:45.817 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:45.896 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 22 16 30719 30703 87.1945 109.312 0.00396327 0.0114642 2023-09-30T09:44:46.051 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:46.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:46.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:46.319 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:46.363 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:46.435 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:46.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:46.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:46.540 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:46.579 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:46.661 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:44:46.896 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 23 16 31959 31943 86.7728 77.5 0.00623397 0.0115183 2023-09-30T09:44:46.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:46 smithi019 ceph-mon[216815]: pgmap v1271: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:44:46.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2723848195' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:44:46.985 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:46 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:44:46.991 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.4 to 1 (10000) 2023-09-30T09:44:47.245 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:46 smithi167 ceph-mon[192880]: pgmap v1271: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:44:47.245 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2723848195' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:44:47.245 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:46 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:44:47.275 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:44:47.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:46 smithi019 ceph-mon[224043]: pgmap v1271: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-30T09:44:47.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2723848195' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:44:47.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:46 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:44:47.692 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:44:47.896 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 24 16 33178 33162 86.3315 76.1875 0.00435187 0.0115774 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: osdmap e498: 8 total, 8 up, 8 in 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:47 smithi167 ceph-mon[192880]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: osdmap e498: 8 total, 8 up, 8 in 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[216815]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: osdmap e498: 8 total, 8 up, 8 in 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:47 smithi019 ceph-mon[224043]: failed to encode map e498 with expected crc 2023-09-30T09:44:48.896 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 25 16 34412 34396 85.963 77.125 0.00211253 0.0115573 2023-09-30T09:44:48.976 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-30T09:44:49.059 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:49.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:49.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:49.128 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:49.168 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:44:49.208 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:49.260 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[224043]: pgmap v1273: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[216815]: pgmap v1273: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:49.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:49 smithi167 ceph-mon[192880]: pgmap v1273: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-30T09:44:49.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:44:49.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:49.673 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:44:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:49.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:44:49.785 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:49.848 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:49.897 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 26 16 35648 35632 85.6277 77.25 0.00447517 0.0116732 2023-09-30T09:44:50.178 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:50.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:44:50.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:44:50.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:50.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:44:50.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:44:50.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:50.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:50.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:44:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:44:50.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:44:50.284 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:50.336 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:50.408 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:50.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101707 8.c 8:32eb269e:::benchmark_data_smithi019_478679_object9259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 287.11383613499999, 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.50943429299999998, 2023-09-30T09:44:50.432 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101707 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:44:50.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314889+0000", 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:50.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314894+0000", 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3600000000000004e-06 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314906+0000", 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1963999999999999e-05 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314912+0000", 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2939999999999996e-06 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.774633+0000", 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.45972106400000001 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.436 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.778979+0000", 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043463390000000003 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.785183+0000", 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062043089999999999 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.797548+0000", 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012364841 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.798913+0000", 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013653890000000001 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.811924+0000", 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013010661999999999 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812345+0000", 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00042060699999999998 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.814649+0000", 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023046479999999999 2023-09-30T09:44:50.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.824324+0000", 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0096749469999999997 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101711 8.c 8:342f96d5:::benchmark_data_smithi019_478679_object9263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:44:50.440 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 287.04905780399997, 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.50859565699999998, 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101711 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379667+0000", 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379676+0000", 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1120000000000002e-06 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379696+0000", 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9777999999999999e-05 2023-09-30T09:44:50.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379705+0000", 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0049999999999993e-06 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.824943+0000", 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.44523816599999999 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.830542+0000", 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055991670000000004 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.837804+0000", 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0072618020000000004 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.871495+0000", 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.033691169999999999 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.872040+0000", 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00054504200000000003 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.873778+0000", 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00173759 2023-09-30T09:44:50.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.874777+0000", 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00099985400000000011 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.876989+0000", 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022113549999999999 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.888264+0000", 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011275253000000001 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101717 8.c 8:35bdb3c9:::benchmark_data_smithi019_478679_object9269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 287.02381030499998, 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.49835376599999998, 2023-09-30T09:44:50.448 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101717 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404914+0000", 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404923+0000", 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4540000000000007e-06 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404937+0000", 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4056000000000001e-05 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404944+0000", 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3300000000000001e-06 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.845543+0000", 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.44059901200000001 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.851287+0000", 2023-09-30T09:44:50.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057437360000000002 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.859094+0000", 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0078075109999999996 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.872354+0000", 2023-09-30T09:44:50.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013259979 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.889815+0000", 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.017460304999999999 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.891113+0000", 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001298738 2023-09-30T09:44:50.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.892101+0000", 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00098792200000000002 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.894073+0000", 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019714590000000001 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.903270+0000", 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091969100000000008 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101732 8.c 8:37cad26d:::benchmark_data_smithi019_478679_object9284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 286.92475679, 2023-09-30T09:44:50.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.45659628400000002, 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101732 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503968+0000", 2023-09-30T09:44:50.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503977+0000", 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1269999999999992e-06 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503994+0000", 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.702e-05 2023-09-30T09:44:50.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.504002+0000", 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0490000000000004e-06 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.903740+0000", 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.39973803200000002 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.908656+0000", 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049154869999999996 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.916127+0000", 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007471742 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.946021+0000", 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.029893491000000001 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.947537+0000", 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001516107 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.948747+0000", 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001209768 2023-09-30T09:44:50.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.949116+0000", 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000369735 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.951057+0000", 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001940499 2023-09-30T09:44:50.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.960566+0000", 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0095088229999999996 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101748 8.c 8:3570d742:::benchmark_data_smithi019_478679_object9300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 286.89031890899997, 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.43868606199999999, 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101748 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.465 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538406+0000", 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538413+0000", 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.25e-06 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538425+0000", 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1833e-05 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538431+0000", 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.541e-06 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.923820+0000", 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.38538876100000002 2023-09-30T09:44:50.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.929942+0000", 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0061226579999999996 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.937368+0000", 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007425783 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.961017+0000", 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.023649196000000001 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.962281+0000", 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001264285 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.963794+0000", 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015121139999999999 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.965116+0000", 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001322611 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.967433+0000", 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023167349999999999 2023-09-30T09:44:50.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.977093+0000", 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0096604380000000004 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101750 8.c 8:3710a7dd:::benchmark_data_smithi019_478679_object9302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:44:50.472 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 286.87285730899998, 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.47879937299999997, 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101750 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555867+0000", 2023-09-30T09:44:50.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555876+0000", 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9660000000000002e-06 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555894+0000", 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7373000000000001e-05 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555904+0000", 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0555e-05 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.977641+0000", 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.42173738700000002 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.982678+0000", 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050369020000000002 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.989714+0000", 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007035683 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.477 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.018051+0000", 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.028337256000000002 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.019726+0000", 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00167417 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.020234+0000", 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00050844099999999999 2023-09-30T09:44:50.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.023410+0000", 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003176201 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.025302+0000", 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018919290000000001 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.034668+0000", 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0093661639999999997 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101764 8.c 8:3df3ae0b:::benchmark_data_smithi019_478679_object9316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:44:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 286.80103749599999, 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.421116035, 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101764 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627687+0000", 2023-09-30T09:44:50.482 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627696+0000", 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3260000000000002e-06 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627713+0000", 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7232999999999999e-05 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.483 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627722+0000", 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6969999999999999e-06 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.996707+0000", 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.36898468600000001 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.484 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.001172+0000", 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044654580000000003 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.008935+0000", 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077631189999999997 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.020869+0000", 2023-09-30T09:44:50.485 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011933208000000001 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.022114+0000", 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012456979999999999 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.036065+0000", 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013950393 2023-09-30T09:44:50.486 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.037310+0000", 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001245377 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.039037+0000", 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017264960000000001 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.487 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.048805+0000", 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097681729999999998 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101769 8.c 8:36635324:::benchmark_data_smithi019_478679_object9321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:44:50.488 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 286.69630090499999, 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.374847404, 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101769 2023-09-30T09:44:50.489 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:44:50.490 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732424+0000", 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.491 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732431+0000", 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3520000000000003e-06 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732444+0000", 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3142000000000001e-05 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732450+0000", 2023-09-30T09:44:50.492 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9399999999999999e-06 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.049283+0000", 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.31683297199999999 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054085+0000", 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048020700000000003 2023-09-30T09:44:50.493 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060789+0000", 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006704448 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.090971+0000", 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.030181990999999998 2023-09-30T09:44:50.494 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.092186+0000", 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001215198 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.093180+0000", 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00099359499999999989 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.495 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.094984+0000", 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018042119999999999 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.097274+0000", 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022901610000000002 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.496 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.107273+0000", 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0099984359999999994 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126845 8.c 8:36dee011:::benchmark_data_smithi019_478679_object34397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 184.65686755900001, 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.93607846500000003, 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.497 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126845 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.498 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771858+0000", 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.499 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771863+0000", 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6930000000000004e-06 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771879+0000", 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6004000000000001e-05 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.500 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771886+0000", 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4760000000000003e-06 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771947+0000", 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1393e-05 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771958+0000", 2023-09-30T09:44:50.501 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0495e-05 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.687778+0000", 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.91581990400000002 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.687930+0000", 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015184700000000001 2023-09-30T09:44:50.502 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.688083+0000", 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000153113 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.694570+0000", 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064871449999999997 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.503 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695773+0000", 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012035189999999999 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707761+0000", 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011987507999999999 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.504 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707817+0000", 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6196999999999997e-05 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707937+0000", 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000120059 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.505 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126846 8.c 8:38abd25d:::benchmark_data_smithi019_478679_object34398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 184.64852893299999, 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92797822600000002, 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126846 2023-09-30T09:44:50.506 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.507 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780196+0000", 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780202+0000", 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0950000000000001e-06 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.508 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780217+0000", 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4602e-05 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780223+0000", 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6860000000000003e-06 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780256+0000", 2023-09-30T09:44:50.509 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2988999999999999e-05 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780263+0000", 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1359999999999999e-06 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691354+0000", 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.911091706 2023-09-30T09:44:50.510 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691496+0000", 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000142058 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691644+0000", 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014777599999999999 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.511 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695526+0000", 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003882108 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695780+0000", 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000253804 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708058+0000", 2023-09-30T09:44:50.512 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012278213 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708097+0000", 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8503e-05 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708175+0000", 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8568999999999999e-05 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.513 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:132587 8.d 8:b57b0da1:::benchmark_data_smithi019_478679_object40139:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e435)", 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 162.522657458, 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.74966547400000005, 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.514 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 132587 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.515 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906068+0000", 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.516 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906071+0000", 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1879999999999998e-06 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906078+0000", 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7440000000000001e-06 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.517 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906083+0000", 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5639999999999998e-06 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.914573+0000", 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0084897930000000007 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.914581+0000", 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8370000000000003e-06 2023-09-30T09:44:50.518 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644509+0000", 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.72992777600000003 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644642+0000", 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013305200000000001 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.519 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2", 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644719+0000", 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6363000000000001e-05 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655647+0000", 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010928208 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.520 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655652+0000", 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.967e-06 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655686+0000", 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4452999999999997e-05 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655734+0000", 2023-09-30T09:44:50.521 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7939000000000003e-05 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:176082 8.18 8:1f93b3c9:::benchmark_data_smithi019_478679_object83634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 58.146498211999997, 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.35076756999999997, 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.522 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 176082 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.523 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.524 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282230+0000", 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6960000000000001e-06 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282237+0000", 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7569999999999998e-06 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282241+0000", 2023-09-30T09:44:50.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9249999999999997e-06 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.284838+0000", 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002597125 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.284993+0000", 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015470999999999999 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.285408+0000", 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00041504500000000001 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.287512+0000", 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021044219999999999 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.500557+0000", 2023-09-30T09:44:50.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21304519899999999 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632421+0000", 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.13186376399999999 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632470+0000", 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8624000000000001e-05 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.528 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632996+0000", 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052566699999999995 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:176090 8.8 8:17545422:::benchmark_data_smithi019_478679_object83642:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 58.139490053000003, 2023-09-30T09:44:50.529 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34340390900000001, 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 176090 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.530 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:50.531 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289239+0000", 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.264e-06 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289246+0000", 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8859999999999998e-06 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.532 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289250+0000", 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4379999999999997e-06 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289291+0000", 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1014999999999999e-05 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289420+0000", 2023-09-30T09:44:50.533 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128823 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2,3", 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289844+0000", 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00042338899999999998 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.292151+0000", 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023077509999999998 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.534 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.294495+0000", 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023442430000000002 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.500522+0000", 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20602649000000001 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632458+0000", 2023-09-30T09:44:50.535 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.13193581900000001 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632514+0000", 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5893000000000002e-05 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632640+0000", 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000126327 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.536 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2297 9.d 9:b0cd7046:::benchmark_data_smithi019_560486_object2296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 25.397244700999998, 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0633178289999998, 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2297 2023-09-30T09:44:50.537 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.538 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031516+0000", 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.464e-05 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.539 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031523+0000", 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8779999999999998e-06 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031528+0000", 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6970000000000001e-06 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.540 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032166+0000", 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00063765200000000005 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032280+0000", 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000113811 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032419+0000", 2023-09-30T09:44:50.541 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013967100000000001 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.033675+0000", 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001255363 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.542 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053674+0000", 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.019999847000000001 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053684+0000", 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9990000000000003e-06 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.059238+0000", 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.00555335 2023-09-30T09:44:50.543 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.059246+0000", 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4940000000000003e-06 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.086761+0000", 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.027514383 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.544 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.088028+0000", 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012668429999999999 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094723+0000", 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0066955230000000001 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.545 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094799+0000", 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6225999999999998e-05 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2293 9.a 9:552d93c3:::benchmark_data_smithi019_560486_object2292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 24.383658362999999, 2023-09-30T09:44:50.546 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.051156384, 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2293 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:44:50.547 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045066+0000", 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045217+0000", 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150221 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045234+0000", 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7676000000000002e-05 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045243+0000", 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0429999999999996e-06 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045361+0000", 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011739900000000001 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045370+0000", 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0049999999999993e-06 2023-09-30T09:44:50.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058484+0000", 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013114657 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058494+0000", 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3559999999999999e-06 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.085277+0000", 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.026783555 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.087303+0000", 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020253559999999999 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.552 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096095+0000", 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0087920210000000006 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096224+0000", 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129453 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2299 9.a 9:5ff10925:::benchmark_data_smithi019_560486_object2298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:44:50.553 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 24.383469216999998, 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0514241090000001, 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2299 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.554 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045256+0000", 2023-09-30T09:44:50.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045294+0000", 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7457999999999999e-05 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045304+0000", 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0271999999999999e-05 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.556 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045310+0000", 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7760000000000003e-06 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045439+0000", 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128909 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045445+0000", 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0229999999999998e-06 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.557 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058514+0000", 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013069717999999999 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058520+0000", 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2920000000000003e-06 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.087342+0000", 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.028822465 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.088285+0000", 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00094316300000000001 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.559 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096601+0000", 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0083155179999999992 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096681+0000", 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0162999999999998e-05 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.560 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5657 9.d 9:b0e10006:::benchmark_data_smithi019_560486_object5656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.752034546000001, 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.006709437, 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5657 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.561 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.562 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676689+0000", 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676814+0000", 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012537599999999999 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676838+0000", 2023-09-30T09:44:50.563 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3623000000000001e-05 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676847+0000", 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2590000000000006e-06 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676916+0000", 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8670000000000005e-05 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.564 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676927+0000", 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0849000000000001e-05 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680187+0000", 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0032605560000001 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680317+0000", 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129453 2023-09-30T09:44:50.565 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680486+0000", 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016957800000000001 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682922+0000", 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024357670000000001 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.566 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683200+0000", 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027740300000000001 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683261+0000", 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1591e-05 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.567 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683346+0000", 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4541000000000003e-05 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683401+0000", 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5316999999999999e-05 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.568 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5664 9.d 9:b51be770:::benchmark_data_smithi019_560486_object5663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.750878389, 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.005731259, 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5664 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.569 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677847+0000", 2023-09-30T09:44:50.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677904+0000", 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7519999999999998e-05 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677923+0000", 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8369e-05 2023-09-30T09:44:50.571 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677930+0000", 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4680000000000003e-06 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677977+0000", 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7170999999999999e-05 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.572 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677988+0000", 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0855e-05 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680921+0000", 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0029331560000001 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681034+0000", 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011288199999999999 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.573 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681184+0000", 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150094 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682930+0000", 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017458339999999999 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.574 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683222+0000", 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00029221800000000001 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683451+0000", 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00022835100000000001 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683528+0000", 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6852000000000002e-05 2023-09-30T09:44:50.575 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683579+0000", 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1418999999999997e-05 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5673 9.d 9:b96c3c13:::benchmark_data_smithi019_560486_object5672:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.748940511000001, 2023-09-30T09:44:50.576 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0071881890000001, 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5673 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.577 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.578 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679785+0000", 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679870+0000", 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4948000000000001e-05 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679884+0000", 2023-09-30T09:44:50.579 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4348000000000001e-05 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679891+0000", 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9589999999999998e-06 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679948+0000", 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7120999999999997e-05 2023-09-30T09:44:50.580 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679955+0000", 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7870000000000004e-06 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681621+0000", 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.001666497 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.581 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681730+0000", 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010867 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681870+0000", 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013995699999999999 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.582 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682935+0000", 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010647829999999999 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.685992+0000", 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030569799999999999 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.583 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686739+0000", 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00074698799999999997 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686937+0000", 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000197858 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686974+0000", 2023-09-30T09:44:50.584 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7280999999999998e-05 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5686 9.d 9:b2765571:::benchmark_data_smithi019_560486_object5685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 19.743422033000002, 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.002027996, 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:44:50.585 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5686 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.586 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:44:50.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685339+0000", 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5487000000000001e-05 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685347+0000", 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.072e-06 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685353+0000", 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8860000000000001e-06 2023-09-30T09:44:50.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685417+0000", 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4487999999999996e-05 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685433+0000", 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5132e-05 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682219+0000", 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99678628499999999 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682328+0000", 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109515 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682483+0000", 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015417100000000001 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686027+0000", 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035446449999999999 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.591 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687236+0000", 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012086670000000001 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687247+0000", 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.136e-05 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687295+0000", 2023-09-30T09:44:50.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8118999999999998e-05 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687332+0000", 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6742999999999997e-05 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:44:50.593 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:44:50.594 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:50.641 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:50.746 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:50.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:50.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:50.854 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:50.897 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 27 16 37026 37010 85.645 86.125 0.00358768 0.0116707 2023-09-30T09:44:50.915 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:51.022 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:44:51.078 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:51.083 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:51.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:51.190 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:51.246 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:51.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:51 smithi019 ceph-mon[216815]: pgmap v1274: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-30T09:44:51.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4147908408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:51.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:51 smithi019 ceph-mon[224043]: pgmap v1274: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-30T09:44:51.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4147908408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:51 smithi167 ceph-mon[192880]: pgmap v1274: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-30T09:44:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4147908408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:44:51.725 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:51.734 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:44:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:51.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:51.837 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:51.897 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 28 16 38233 38217 85.2803 75.4375 0.0127816 0.0117209 2023-09-30T09:44:51.931 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:52.018 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:52 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:44:52.019 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:52 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:44:52.019 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4147908408' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:52.019 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:52 smithi167 ceph-mon[192880]: osdmap e499: 8 total, 8 up, 8 in 2023-09-30T09:44:52.019 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:52 smithi167 ceph-mon[192880]: failed to encode map e499 with expected crc 2023-09-30T09:44:52.019 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:52 smithi167 ceph-mon[192880]: failed to encode map e499 with expected crc 2023-09-30T09:44:52.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:44:52.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:44:52.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4147908408' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:52.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[224043]: osdmap e499: 8 total, 8 up, 8 in 2023-09-30T09:44:52.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[224043]: failed to encode map e499 with expected crc 2023-09-30T09:44:52.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[224043]: failed to encode map e499 with expected crc 2023-09-30T09:44:52.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:44:52.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:44:52.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4147908408' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:44:52.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[216815]: osdmap e499: 8 total, 8 up, 8 in 2023-09-30T09:44:52.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[216815]: failed to encode map e499 with expected crc 2023-09-30T09:44:52.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:52 smithi019 ceph-mon[216815]: failed to encode map e499 with expected crc 2023-09-30T09:44:52.313 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:52.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:52.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:52.421 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:44:52.487 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:52.587 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:44:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:52.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:44:52.695 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:52.773 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:52.868 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:52.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:52.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:52.898 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 29 16 39825 39809 85.7683 99.5 0.00905214 0.0116523 2023-09-30T09:44:52.982 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:53.009 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:53.055 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:53.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:53.159 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:44:53.185 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:53.230 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:44:53.244 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:44:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:53.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:44:53.268 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:44:53.308 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:44:53.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:53 smithi019 ceph-mon[224043]: pgmap v1276: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-30T09:44:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2343102692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:53 smithi019 ceph-mon[216815]: pgmap v1276: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-30T09:44:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2343102692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:53.349 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:44:53.353 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:53.386 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:44:53.420 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:53.427 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:44:53.460 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:44:53.499 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:44:53.507 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:44:53.507 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:44:53.508 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:44:53.508 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:44:53.508 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:44:53.508 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:44:53.508 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:44:53.508 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:44:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:53 smithi167 ceph-mon[192880]: pgmap v1276: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-30T09:44:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2343102692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:44:53.898 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 30 16 40964 40948 85.2822 71.1875 0.0051413 0.0117125 2023-09-30T09:44:54.087 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:44:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:44:53] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2023-09-30T09:44:54.095 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:54.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:44:54.203 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:44:54.216 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:44:54.262 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:54.342 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:44:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:54.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:44:54.452 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:44:54.539 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:54.578 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:54.645 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:44:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:54.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:44:54.759 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:44:54.799 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:54 smithi019 ceph-mon[216815]: pgmap v1277: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-30T09:44:54.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:54 smithi019 ceph-mon[224043]: pgmap v1277: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-30T09:44:54.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:44:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:54.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:44:54.898 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 31 16 42106 42090 84.8335 71.375 0.00243422 0.011775 2023-09-30T09:44:54.984 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:44:55.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:54 smithi167 ceph-mon[192880]: pgmap v1277: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-30T09:44:55.033 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:55.118 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:44:55.126 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:44:55.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:55.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:44:55.234 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:55.284 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:55.357 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:55.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:55.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:55.463 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:55.627 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:55.898 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 32 16 43523 43507 84.9497 88.5625 0.00885093 0.0117667 2023-09-30T09:44:56.414 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:56.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:56.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:56.522 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:44:56.542 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:44:56.567 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:56.635 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:44:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:56.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:44:56.744 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:56.815 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:56.898 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 33 16 44841 44825 84.8715 82.375 0.0101169 0.0117762 2023-09-30T09:44:56.944 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:56 smithi019 ceph-mon[216815]: pgmap v1278: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-30T09:44:56.944 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:56 smithi019 ceph-mon[224043]: pgmap v1278: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-30T09:44:57.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:56 smithi167 ceph-mon[192880]: pgmap v1278: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-30T09:44:57.344 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:57.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:57.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:57.451 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:44:57.520 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:57.858 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.5 to 1 (10000) 2023-09-30T09:44:57.898 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 34 15 46459 46444 85.3511 101.188 0.00840969 0.011711 2023-09-30T09:44:58.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:57 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2504620793' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:44:58.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:57 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:44:58.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:57 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2504620793' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:44:58.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:57 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:44:58.231 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:44:58.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:58.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:44:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:57 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2504620793' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:44:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:57 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:44:58.338 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:44:58.409 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:58.898 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 35 16 47632 47616 85.0052 73.25 0.0108371 0.0117582 2023-09-30T09:44:59.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[216815]: pgmap v1279: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-30T09:44:59.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:44:59.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[216815]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[216815]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[216815]: osdmap e500: 8 total, 8 up, 8 in 2023-09-30T09:44:59.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[216815]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[216815]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[224043]: pgmap v1279: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-30T09:44:59.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:44:59.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[224043]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[224043]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[224043]: osdmap e500: 8 total, 8 up, 8 in 2023-09-30T09:44:59.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[224043]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:44:58 smithi019 ceph-mon[224043]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.088 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:44:59.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:59.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:44:59.198 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:44:59.271 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:44:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:58 smithi167 ceph-mon[192880]: pgmap v1279: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-30T09:44:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:58 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:44:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:58 smithi167 ceph-mon[192880]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:58 smithi167 ceph-mon[192880]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:58 smithi167 ceph-mon[192880]: osdmap e500: 8 total, 8 up, 8 in 2023-09-30T09:44:59.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:58 smithi167 ceph-mon[192880]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:44:58 smithi167 ceph-mon[192880]: failed to encode map e500 with expected crc 2023-09-30T09:44:59.369 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-30T09:44:59.424 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:44:59.425 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:44:59.898 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 36 16 49026 49010 85.0638 87.125 0.00966003 0.0117509 2023-09-30T09:44:59.935 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:44:59.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:44:59.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:00.048 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:00.077 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:00.594 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:00.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:00.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:00.661 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:00.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:45:00.705 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:00.748 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:00.817 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:00.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:00.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:00.868 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:00.887 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:45:00.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 37 16 50251 50235 84.8309 76.5625 0.00551845 0.0117826 2023-09-30T09:45:00.919 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:00 smithi019 ceph-mon[216815]: pgmap v1281: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-30T09:45:00.920 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4029577346' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:00.920 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:00 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:00.921 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:00 smithi019 ceph-mon[224043]: pgmap v1281: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-30T09:45:00.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4029577346' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:00.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:00 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:00.923 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:00 smithi167 ceph-mon[192880]: pgmap v1281: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-30T09:45:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4029577346' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:01.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:00 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:01.025 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:01.903 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 38 16 51406 51390 84.4956 72.1875 0.00779849 0.0118295 2023-09-30T09:45:02.161 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:02.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:02.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:02.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:02.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:45:02.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[216815]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[216815]: osdmap e501: 8 total, 8 up, 8 in 2023-09-30T09:45:02.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[216815]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[216815]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.165 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[216815]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:02.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:45:02.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[224043]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[224043]: osdmap e501: 8 total, 8 up, 8 in 2023-09-30T09:45:02.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[224043]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[224043]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:01 smithi019 ceph-mon[224043]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.266 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:02.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:01 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:02.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:01 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:45:02.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:01 smithi167 ceph-mon[192880]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:01 smithi167 ceph-mon[192880]: osdmap e501: 8 total, 8 up, 8 in 2023-09-30T09:45:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:01 smithi167 ceph-mon[192880]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:01 smithi167 ceph-mon[192880]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:01 smithi167 ceph-mon[192880]: failed to encode map e501 with expected crc 2023-09-30T09:45:02.328 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:02.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 39 16 52811 52795 84.5804 87.8125 0.00986779 0.0118162 2023-09-30T09:45:03.054 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:03.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:03.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:03.163 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:03.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:02 smithi019 ceph-mon[224043]: pgmap v1283: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-30T09:45:03.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:02 smithi019 ceph-mon[216815]: pgmap v1283: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-30T09:45:03.230 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:03.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:02 smithi167 ceph-mon[192880]: pgmap v1283: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-30T09:45:03.375 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:03.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:03.480 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:03.507 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:03.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:45:03.901616+0000 min lat: 0.00110418 max lat: 2.06823 avg lat: 0.0118825 2023-09-30T09:45:03.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:45:03.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 40 16 53872 53856 84.1237 66.3125 0.00632206 0.0118825 2023-09-30T09:45:04.003 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:04.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:04.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:04.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:45:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:45:03] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-09-30T09:45:04.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:04.109 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:04.153 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:04.154 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:04.159 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:04.234 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:04.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:04.344 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:04.392 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:04.487 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:04.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:04.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:04.596 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:04.656 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:04.731 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:04.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:04.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:04.742 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:45:04.792 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:45:04.829 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:45:04.843 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:04.874 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:45:04.892 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:04.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 41 16 55214 55198 84.117 83.875 0.0169508 0.0118828 2023-09-30T09:45:04.909 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:45:04.934 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:45:04.964 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:45:04.967 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:04.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:04.981 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:04 smithi019 ceph-mon[216815]: pgmap v1284: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-30T09:45:04.981 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3628396032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:04.981 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:04 smithi019 ceph-mon[224043]: pgmap v1284: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-30T09:45:04.981 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3628396032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:05.006 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:45:05.043 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:45:05.045 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:45:05.046 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:45:05.046 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:45:05.046 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:45:05.046 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:45:05.046 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:45:05.046 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:45:05.046 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:45:05.079 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:05.210 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:05.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:04 smithi167 ceph-mon[192880]: pgmap v1284: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-30T09:45:05.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3628396032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:05.801 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:05.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:05.881 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.2 to 1 (10000) 2023-09-30T09:45:05.901 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 42 16 56129 56113 83.4758 57.1875 0.00328142 0.0119493 2023-09-30T09:45:05.911 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:05.937 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:05 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/41072330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:45:06.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:05 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:45:06.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:05 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/41072330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:45:06.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:05 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:45:06.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:05 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/41072330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:45:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:05 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:45:06.682 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:06.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:06.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:06.789 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:06.833 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:06.902 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 43 16 57071 57055 82.9036 58.875 0.0102158 0.0120345 2023-09-30T09:45:07.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[216815]: pgmap v1285: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-09-30T09:45:07.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:45:07.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[216815]: failed to encode map e502 with expected crc 2023-09-30T09:45:07.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[216815]: osdmap e502: 8 total, 8 up, 8 in 2023-09-30T09:45:07.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[216815]: failed to encode map e502 with expected crc 2023-09-30T09:45:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[224043]: pgmap v1285: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-09-30T09:45:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:45:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[224043]: failed to encode map e502 with expected crc 2023-09-30T09:45:07.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[224043]: osdmap e502: 8 total, 8 up, 8 in 2023-09-30T09:45:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:06 smithi019 ceph-mon[224043]: failed to encode map e502 with expected crc 2023-09-30T09:45:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:06 smithi167 ceph-mon[192880]: pgmap v1285: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-09-30T09:45:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:06 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:45:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:06 smithi167 ceph-mon[192880]: failed to encode map e502 with expected crc 2023-09-30T09:45:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:06 smithi167 ceph-mon[192880]: osdmap e502: 8 total, 8 up, 8 in 2023-09-30T09:45:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:06 smithi167 ceph-mon[192880]: failed to encode map e502 with expected crc 2023-09-30T09:45:07.308 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:07.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:07.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:07.413 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:07.462 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:07.530 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-30T09:45:07.537 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:07.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:07.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:07.647 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:07.692 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:07.738 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:07.902 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 44 16 58365 58349 82.8573 80.875 0.0251819 0.0120643 2023-09-30T09:45:07.911 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:07.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:08.025 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:08.074 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:08.144 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:08.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:08.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:08.250 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:08.255 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:45:08.255 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:45:08.255 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:45:08.255 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:45:08.255 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:45:08.335 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:08.452 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:45:08 smithi167 conmon[172457]: logger=cleanup t=2023-09-30T09:45:08.449582949Z level=info msg="Completed cleanup jobs" duration=141.911124ms 2023-09-30T09:45:08.509 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:08.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:08.616 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:08.658 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:08.773 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:08.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:08.881 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:08.902 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 45 16 59606 59590 82.7395 77.5625 0.0135935 0.0120805 2023-09-30T09:45:08.936 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:09.038 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:09.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:09.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:09.043 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:08 smithi167 ceph-mon[192880]: pgmap v1287: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-30T09:45:09.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:08 smithi019 ceph-mon[224043]: pgmap v1287: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-30T09:45:09.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:08 smithi019 ceph-mon[216815]: pgmap v1287: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-30T09:45:09.144 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:09.206 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:09.563 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:45:09.903 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 46 16 60637 60621 82.3392 64.4375 0.00802145 0.01214 2023-09-30T09:45:10.210 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:10.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:10.324 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:10.335 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:10.363 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:45:10.415 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:10.661 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:10.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:10.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:10.784 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:10.838 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:10.903 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 47 16 61813 61797 82.151 73.5 0.0214346 0.0121676 2023-09-30T09:45:10.911 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:10.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:11.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:11.071 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:11.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:10 smithi019 ceph-mon[224043]: pgmap v1288: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-30T09:45:11.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:10 smithi019 ceph-mon[216815]: pgmap v1288: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-30T09:45:11.158 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:11.161 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:11.167 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:10 smithi167 ceph-mon[192880]: pgmap v1288: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-30T09:45:11.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:11.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:11.269 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:11.315 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:11.392 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:11.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:11.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:11.500 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:11.556 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:11.598 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:11.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:11.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:11.705 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:11.731 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:11.773 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:11.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:11.879 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:11.903 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 48 16 62706 62690 81.6021 55.8125 0.0101296 0.0122489 2023-09-30T09:45:11.936 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:45:11.967 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4002969157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:11.967 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:11 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:11.969 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4002969157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:11.969 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:11 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:12.098 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:12.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4002969157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:12.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:11 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:12.666 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:12.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:12.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:12.772 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:12.797 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:12.888 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:12.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:12.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:12.905 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 49 16 63916 63900 81.4768 75.625 0.00659031 0.0122691 2023-09-30T09:45:12.998 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:13.065 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:13.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:12 smithi167 ceph-mon[192880]: pgmap v1289: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-30T09:45:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:12 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:12 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:45:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:12 smithi167 ceph-mon[192880]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:12 smithi167 ceph-mon[192880]: osdmap e503: 8 total, 8 up, 8 in 2023-09-30T09:45:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:12 smithi167 ceph-mon[192880]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:12 smithi167 ceph-mon[192880]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[224043]: pgmap v1289: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-30T09:45:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:45:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[224043]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[224043]: osdmap e503: 8 total, 8 up, 8 in 2023-09-30T09:45:13.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[224043]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[224043]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[216815]: pgmap v1289: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-30T09:45:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:45:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[216815]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[216815]: osdmap e503: 8 total, 8 up, 8 in 2023-09-30T09:45:13.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[216815]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:12 smithi019 ceph-mon[216815]: failed to encode map e503 with expected crc 2023-09-30T09:45:13.660 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:13.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:13.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:13.770 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:13.833 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:13.906 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 50 16 65065 65049 81.2834 71.8125 0.0111755 0.0122974 2023-09-30T09:45:13.914 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:13.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:14.025 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:14.082 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:45:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:45:13] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-09-30T09:45:14.168 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:14.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:14.278 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:14.304 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:14.345 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:14.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:14.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:14.450 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:14.519 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:14.798 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:14 smithi019 ceph-mon[216815]: pgmap v1291: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-30T09:45:14.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:14 smithi019 ceph-mon[224043]: pgmap v1291: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-30T09:45:14.842 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:14.886 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:14.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:14.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:14.908 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 51 16 66109 66093 80.9658 65.25 0.0137399 0.0123455 2023-09-30T09:45:14.995 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:15.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:14 smithi167 ceph-mon[192880]: pgmap v1291: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-30T09:45:15.042 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:15.127 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:15.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:15.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:15.236 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:15.324 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:15.618 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:45:15.703 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.7 to 1 (10000) 2023-09-30T09:45:15.716 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/90643560' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:45:15.716 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:15 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:45:15.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/90643560' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:45:15.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:15 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:45:15.911 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 52 16 67074 67058 80.5674 60.3125 0.0105607 0.0124074 2023-09-30T09:45:16.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/90643560' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:45:16.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:15 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:45:16.029 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:16.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:16.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:16.111 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:45:16.137 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:16.166 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:45:16.196 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:16.205 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:45:16.239 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:45:16.280 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:16.283 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:45:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:16.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:16.317 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:45:16.388 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:45:16.388 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:45:16.388 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:45:16.388 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:45:16.388 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:45:16.388 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:45:16.388 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:45:16.388 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:45:16.389 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:45:16.389 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:45:16.389 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:16.442 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:16.530 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:16.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:16.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:16.638 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:16.876 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:16.909 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 53 16 68041 68025 80.1876 60.4375 0.00593924 0.0124662 2023-09-30T09:45:17.024 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:45:17.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[216815]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[216815]: osdmap e504: 8 total, 8 up, 8 in 2023-09-30T09:45:17.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[216815]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[216815]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[216815]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[216815]: pgmap v1293: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-30T09:45:17.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2699967098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:17.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:16 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:45:17.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:16 smithi167 ceph-mon[192880]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:16 smithi167 ceph-mon[192880]: osdmap e504: 8 total, 8 up, 8 in 2023-09-30T09:45:17.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:16 smithi167 ceph-mon[192880]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:16 smithi167 ceph-mon[192880]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:16 smithi167 ceph-mon[192880]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:16 smithi167 ceph-mon[192880]: pgmap v1293: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-30T09:45:17.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2699967098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:17.028 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:45:17.028 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[224043]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.028 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[224043]: osdmap e504: 8 total, 8 up, 8 in 2023-09-30T09:45:17.028 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[224043]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.028 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[224043]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.028 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[224043]: failed to encode map e504 with expected crc 2023-09-30T09:45:17.028 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[224043]: pgmap v1293: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-30T09:45:17.029 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2699967098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:17.716 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-30T09:45:17.878 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:17.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:17.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:17.912 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 54 16 69262 69246 80.1111 76.3125 0.0112141 0.012478 2023-09-30T09:45:17.985 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:18.057 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:18.366 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:18.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:18.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:18.471 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:18.516 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:18.593 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:18.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:18.648 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:18.675 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:18.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:18.776 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:18.915 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 55 16 70373 70357 79.9132 69.4375 0.0160445 0.0125087 2023-09-30T09:45:19.067 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:19.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:18 smithi019 ceph-mon[224043]: pgmap v1294: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-30T09:45:19.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:19.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:19.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:19.073 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:18 smithi019 ceph-mon[216815]: pgmap v1294: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-30T09:45:19.074 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:19.174 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:19.234 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:19.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:18 smithi167 ceph-mon[192880]: pgmap v1294: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-30T09:45:19.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:19.455 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:19.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:19.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:19.561 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:19.612 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:19.625 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:45:19.625 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:45:19.703 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:19.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:19.813 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:19.875 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:19.916 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 56 16 71500 71484 79.7422 70.4375 0.0130377 0.0125347 2023-09-30T09:45:19.974 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:19.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:20.081 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:20.141 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:20.228 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:45:20.243 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:20.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:20.354 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:20.398 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:20.481 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:20.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:20.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:20.588 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:20.613 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:20.655 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:20.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:20.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:20.760 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:20.821 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:20.916 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 57 16 72461 72445 79.3969 60.0625 0.00330125 0.0125833 2023-09-30T09:45:21.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:20 smithi167 ceph-mon[192880]: pgmap v1295: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-30T09:45:21.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:20 smithi019 ceph-mon[216815]: pgmap v1295: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-30T09:45:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:20 smithi019 ceph-mon[224043]: pgmap v1295: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-30T09:45:21.638 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:21.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:21.746 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:21.799 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:21.861 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:21.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:21.916 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 58 16 73427 73411 79.0689 60.375 0.00547374 0.0126347 2023-09-30T09:45:21.972 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:22.010 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:22.094 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:22.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:22.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:22.196 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:45:22.205 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:22.279 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:22.446 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:22.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:22.552 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:22.610 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:22.784 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:45:22.826 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:22.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:22.916 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 59 16 74418 74402 78.7786 61.9375 0.0138428 0.0126885 2023-09-30T09:45:22.934 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:22.998 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:23.013 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:23.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:22 smithi019 ceph-mon[216815]: pgmap v1296: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-30T09:45:23.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:22 smithi019 ceph-mon[224043]: pgmap v1296: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-30T09:45:23.095 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:23.100 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:22 smithi167 ceph-mon[192880]: pgmap v1296: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-30T09:45:23.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:23.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:23.202 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:23.256 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:23.399 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:23.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:23.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:23.503 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:23.549 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:23.676 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:23.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:23.780 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:23.820 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:23.896 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:23.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:23.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:23.917 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:45:23.916970+0000 min lat: 0.00110418 max lat: 2.06823 avg lat: 0.0127316 2023-09-30T09:45:23.917 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:45:23.917 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 60 16 75426 75410 78.5156 63 0.0135699 0.0127316 2023-09-30T09:45:24.000 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:24.035 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:24.076 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:24.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:24.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:24.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:45:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:45:23] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-30T09:45:24.180 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:24.204 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:24.246 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:24.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:24.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:24.350 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:24.387 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:24.594 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:24.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:24.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2203088239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:45:24.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:24 smithi019 ceph-mon[216815]: pgmap v1297: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-30T09:45:24.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/716172387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:45:24.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:24 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:45:24.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2203088239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:45:24.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:24 smithi019 ceph-mon[224043]: pgmap v1297: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-30T09:45:24.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/716172387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:45:24.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:24 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:45:24.703 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:24.704 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.0 to 1 (10000) 2023-09-30T09:45:24.708 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:45:24.777 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:24.917 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 61 16 76118 76102 77.9376 43.25 0.00410037 0.0128035 2023-09-30T09:45:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2203088239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:45:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:24 smithi167 ceph-mon[192880]: pgmap v1297: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-30T09:45:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/716172387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:45:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:24 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:45:25.518 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:45:25.519 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:45:25.917 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 62 16 76118 76102 76.6809 0 - 0.0128035 2023-09-30T09:45:25.995 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:25.997 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:25.997 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2203088239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:45:25.997 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:45:25.997 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[224043]: failed to encode map e505 with expected crc 2023-09-30T09:45:25.997 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[224043]: osdmap e505: 8 total, 8 up, 8 in 2023-09-30T09:45:25.997 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[224043]: failed to encode map e505 with expected crc 2023-09-30T09:45:25.997 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[224043]: failed to encode map e505 with expected crc 2023-09-30T09:45:25.998 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[224043]: failed to encode map e505 with expected crc 2023-09-30T09:45:25.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:25.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2203088239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:45:25.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:45:25.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[216815]: failed to encode map e505 with expected crc 2023-09-30T09:45:25.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[216815]: osdmap e505: 8 total, 8 up, 8 in 2023-09-30T09:45:25.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[216815]: failed to encode map e505 with expected crc 2023-09-30T09:45:25.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[216815]: failed to encode map e505 with expected crc 2023-09-30T09:45:25.999 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:25 smithi019 ceph-mon[216815]: failed to encode map e505 with expected crc 2023-09-30T09:45:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:25.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:25 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:25 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2203088239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:45:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:25 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:45:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:25 smithi167 ceph-mon[192880]: failed to encode map e505 with expected crc 2023-09-30T09:45:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:25 smithi167 ceph-mon[192880]: osdmap e505: 8 total, 8 up, 8 in 2023-09-30T09:45:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:25 smithi167 ceph-mon[192880]: failed to encode map e505 with expected crc 2023-09-30T09:45:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:25 smithi167 ceph-mon[192880]: failed to encode map e505 with expected crc 2023-09-30T09:45:26.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:25 smithi167 ceph-mon[192880]: failed to encode map e505 with expected crc 2023-09-30T09:45:26.100 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:26.131 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:26.183 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-30T09:45:26.352 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:26.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:26.423 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:45:26.456 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:26.491 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:26.537 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:26.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:26.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:26.643 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:26.668 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:26.670 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:26.675 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:45:26.703 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:45:26.734 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:45:26.758 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:45:26.777 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:45:26.781 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:45:26.797 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:45:26.816 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:45:26.820 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:26.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:26.837 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:45:26.839 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:45:26.839 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:45:26.839 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:45:26.839 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:45:26.839 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:45:26.839 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:45:26.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:45:26.840 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:45:26.917 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 63 16 76143 76127 75.489 0.78125 0.00797848 0.012942 2023-09-30T09:45:26.926 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:26.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: osdmap e506: 8 total, 8 up, 8 in 2023-09-30T09:45:26.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.963 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: pgmap v1300: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/629084694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: osdmap e506: 8 total, 8 up, 8 in 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: failed to encode map e506 with expected crc 2023-09-30T09:45:26.964 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: pgmap v1300: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-30T09:45:26.965 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/629084694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:27.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: failed to encode map e506 with expected crc 2023-09-30T09:45:27.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: failed to encode map e506 with expected crc 2023-09-30T09:45:27.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: failed to encode map e506 with expected crc 2023-09-30T09:45:27.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: osdmap e506: 8 total, 8 up, 8 in 2023-09-30T09:45:27.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: failed to encode map e506 with expected crc 2023-09-30T09:45:27.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: failed to encode map e506 with expected crc 2023-09-30T09:45:27.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: failed to encode map e506 with expected crc 2023-09-30T09:45:27.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: pgmap v1300: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-30T09:45:27.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/629084694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:27.025 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:27.704 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:27 smithi167 ceph-mon[192880]: failed to encode map e507 with expected crc 2023-09-30T09:45:27.704 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:27 smithi167 ceph-mon[192880]: failed to encode map e507 with expected crc 2023-09-30T09:45:27.704 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:27 smithi167 ceph-mon[192880]: osdmap e507: 8 total, 8 up, 8 in 2023-09-30T09:45:27.705 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:27 smithi167 ceph-mon[192880]: failed to encode map e507 with expected crc 2023-09-30T09:45:27.917 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 64 16 76828 76812 74.9786 42.8125 0.0109859 0.0133314 2023-09-30T09:45:28.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:27 smithi019 ceph-mon[216815]: failed to encode map e507 with expected crc 2023-09-30T09:45:28.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:27 smithi019 ceph-mon[216815]: failed to encode map e507 with expected crc 2023-09-30T09:45:28.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:27 smithi019 ceph-mon[216815]: osdmap e507: 8 total, 8 up, 8 in 2023-09-30T09:45:28.082 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:27 smithi019 ceph-mon[216815]: failed to encode map e507 with expected crc 2023-09-30T09:45:28.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:27 smithi019 ceph-mon[224043]: failed to encode map e507 with expected crc 2023-09-30T09:45:28.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:27 smithi019 ceph-mon[224043]: failed to encode map e507 with expected crc 2023-09-30T09:45:28.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:27 smithi019 ceph-mon[224043]: osdmap e507: 8 total, 8 up, 8 in 2023-09-30T09:45:28.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:27 smithi019 ceph-mon[224043]: failed to encode map e507 with expected crc 2023-09-30T09:45:28.293 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:45:28.417 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:28.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:28.535 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:28.586 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:28.645 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:45:28.645 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:45:28.645 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:45:28.645 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:45:28.645 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:45:28.686 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:28.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:28.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:28.734 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[216815]: osdmap e508: 8 total, 8 up, 8 in 2023-09-30T09:45:28.734 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[216815]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.734 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[216815]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.734 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[216815]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.734 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[216815]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.735 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[216815]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.735 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[216815]: pgmap v1303: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s 2023-09-30T09:45:28.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[224043]: osdmap e508: 8 total, 8 up, 8 in 2023-09-30T09:45:28.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[224043]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[224043]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[224043]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[224043]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[224043]: failed to encode map e508 with expected crc 2023-09-30T09:45:28.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:28 smithi019 ceph-mon[224043]: pgmap v1303: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s 2023-09-30T09:45:28.790 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:28.845 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:28.909 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:28.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:28.917 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 65 16 77688 77672 74.652 53.75 0.0308367 0.0133913 2023-09-30T09:45:29.012 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:28 smithi167 ceph-mon[192880]: osdmap e508: 8 total, 8 up, 8 in 2023-09-30T09:45:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:28 smithi167 ceph-mon[192880]: failed to encode map e508 with expected crc 2023-09-30T09:45:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:28 smithi167 ceph-mon[192880]: failed to encode map e508 with expected crc 2023-09-30T09:45:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:28 smithi167 ceph-mon[192880]: failed to encode map e508 with expected crc 2023-09-30T09:45:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:28 smithi167 ceph-mon[192880]: failed to encode map e508 with expected crc 2023-09-30T09:45:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:28 smithi167 ceph-mon[192880]: failed to encode map e508 with expected crc 2023-09-30T09:45:29.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:28 smithi167 ceph-mon[192880]: pgmap v1303: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s 2023-09-30T09:45:29.100 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:29.139 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:29.214 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:29.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:29.318 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:29.401 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:29.636 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:45:29.660 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:29.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:29.770 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:29.826 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:29.911 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:29.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:29.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:29.917 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:29 smithi167 ceph-mon[192880]: failed to encode map e508 with expected crc 2023-09-30T09:45:29.917 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:29 smithi167 ceph-mon[192880]: failed to encode map e509 with expected crc 2023-09-30T09:45:29.917 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:29 smithi167 ceph-mon[192880]: osdmap e509: 8 total, 8 up, 8 in 2023-09-30T09:45:29.918 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:29 smithi167 ceph-mon[192880]: failed to encode map e509 with expected crc 2023-09-30T09:45:29.918 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:29 smithi167 ceph-mon[192880]: failed to encode map e509 with expected crc 2023-09-30T09:45:29.918 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:29 smithi167 ceph-mon[192880]: failed to encode map e509 with expected crc 2023-09-30T09:45:29.918 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:29 smithi167 ceph-mon[192880]: failed to encode map e509 with expected crc 2023-09-30T09:45:29.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 66 16 78430 78414 74.2236 46.375 0.0179778 0.0134673 2023-09-30T09:45:30.018 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:30.065 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:30.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[216815]: failed to encode map e508 with expected crc 2023-09-30T09:45:30.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[216815]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[216815]: osdmap e509: 8 total, 8 up, 8 in 2023-09-30T09:45:30.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[216815]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[216815]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[216815]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[216815]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[224043]: failed to encode map e508 with expected crc 2023-09-30T09:45:30.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[224043]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[224043]: osdmap e509: 8 total, 8 up, 8 in 2023-09-30T09:45:30.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[224043]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[224043]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[224043]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:29 smithi019 ceph-mon[224043]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.183 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:45:30.222 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:30.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:30.333 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:30.435 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:30.551 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:30.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:30.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:30.603 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:45:30.657 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:30.685 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:30.731 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:30.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:30.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:30.743 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.3 to 1 (10000) 2023-09-30T09:45:30.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:30 smithi167 ceph-mon[192880]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:30 smithi167 ceph-mon[192880]: pgmap v1305: 119 pgs: 1 active+recovering+remapped, 1 peering, 117 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s; 4812/231681 objects misplaced (2.077%); 486 KiB/s, 9 objects/s recovering 2023-09-30T09:45:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2250295290' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:45:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:45:30.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:30 smithi019 ceph-mon[224043]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:30 smithi019 ceph-mon[224043]: pgmap v1305: 119 pgs: 1 active+recovering+remapped, 1 peering, 117 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s; 4812/231681 objects misplaced (2.077%); 486 KiB/s, 9 objects/s recovering 2023-09-30T09:45:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2250295290' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:45:30.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:45:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:30 smithi019 ceph-mon[216815]: failed to encode map e509 with expected crc 2023-09-30T09:45:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:30 smithi019 ceph-mon[216815]: pgmap v1305: 119 pgs: 1 active+recovering+remapped, 1 peering, 117 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s; 4812/231681 objects misplaced (2.077%); 486 KiB/s, 9 objects/s recovering 2023-09-30T09:45:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2250295290' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:45:30.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:45:30.836 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:30.864 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:30.907 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:30.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:30.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:30.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 67 16 79066 79050 73.708 39.75 0.378637 0.0135306 2023-09-30T09:45:31.011 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:31.050 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:31.112 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:31.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:31.217 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:31.274 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:31.921 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 68 16 80043 80027 73.5221 61.0625 0.0152435 0.0135959 2023-09-30T09:45:32.013 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:45:32.014 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[224043]: osdmap e510: 8 total, 8 up, 8 in 2023-09-30T09:45:32.014 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[224043]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.014 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[224043]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.014 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[224043]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.014 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[224043]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.015 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:45:32.015 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[216815]: osdmap e510: 8 total, 8 up, 8 in 2023-09-30T09:45:32.015 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[216815]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.015 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[216815]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.015 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[216815]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.016 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:31 smithi019 ceph-mon[216815]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:45:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:31 smithi167 ceph-mon[192880]: osdmap e510: 8 total, 8 up, 8 in 2023-09-30T09:45:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:31 smithi167 ceph-mon[192880]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:31 smithi167 ceph-mon[192880]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:31 smithi167 ceph-mon[192880]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:31 smithi167 ceph-mon[192880]: failed to encode map e510 with expected crc 2023-09-30T09:45:32.225 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:32.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:32.338 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:32.340 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-30T09:45:32.394 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:32.397 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:45:32.397 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:45:32.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:45:32.397 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:45:32.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.397 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2291 9.f 9.c3faf36f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:45:32.397 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 67.341007477999995, 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99786489, 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2291 2023-09-30T09:45:32.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051293+0000", 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051362+0000", 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9276999999999994e-05 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052566+0000", 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012036099999999999 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052575+0000", 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3910000000000004e-06 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061665+0000", 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0090901720000000005 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061671+0000", 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3539999999999996e-06 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049150+0000", 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98747858099999997 2023-09-30T09:45:32.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049159+0000", 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9099999999999994e-06 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2293 9.a 9.c3c9b4aa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 67.339707359000002, 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98499129799999996, 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2293 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052593+0000", 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052645+0000", 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1387000000000001e-05 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052655+0000", 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8919999999999995e-06 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052660+0000", 2023-09-30T09:45:32.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4809999999999997e-06 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.059982+0000", 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0073216380000000001 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.059988+0000", 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7399999999999998e-06 2023-09-30T09:45:32.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037576+0000", 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97758752599999998 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037585+0000", 2023-09-30T09:45:32.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4930000000000004e-06 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2294 9.f 9.a98a737f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 67.339598637999998, 2023-09-30T09:45:32.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99648747699999995, 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2294 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.412 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052702+0000", 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052798+0000", 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5100999999999996e-05 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052809+0000", 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1833e-05 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052815+0000", 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.356e-06 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061690+0000", 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088752989999999997 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061694+0000", 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0940000000000001e-06 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049184+0000", 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98748982500000004 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049190+0000", 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4779999999999996e-06 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.417 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2296 9.f 9.2d8a7a1f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 67.339477991999999, 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99639533999999996, 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.418 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2296 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052823+0000", 2023-09-30T09:45:32.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052859+0000", 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6331999999999997e-05 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052867+0000", 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8970000000000008e-06 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052873+0000", 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3149999999999999e-06 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061712+0000", 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.008839431 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061716+0000", 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7979999999999999e-06 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049212+0000", 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.987496598 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049219+0000", 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4300000000000003e-06 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.425 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2298 9.7 9.88338c87 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:45:32.425 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 67.339423814, 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99525176199999998, 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2298 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.426 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052877+0000", 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052906+0000", 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8574000000000001e-05 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052912+0000", 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5150000000000003e-06 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052917+0000", 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1230000000000002e-06 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053439+0000", 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052202900000000003 2023-09-30T09:45:32.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053447+0000", 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1620000000000008e-06 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.048119+0000", 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99467239399999996 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.048130+0000", 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0394000000000001e-05 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2299 9.a 9.a4908ffa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 67.339380468000002, 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.984693029, 2023-09-30T09:45:32.432 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2299 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052947+0000", 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6486999999999999e-05 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052954+0000", 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6640000000000001e-06 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052958+0000", 2023-09-30T09:45:32.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.617e-06 2023-09-30T09:45:32.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.436 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.436 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.060007+0000", 2023-09-30T09:45:32.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0070489289999999998 2023-09-30T09:45:32.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.060012+0000", 2023-09-30T09:45:32.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7790000000000003e-06 2023-09-30T09:45:32.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037607+0000", 2023-09-30T09:45:32.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.977594764 2023-09-30T09:45:32.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037614+0000", 2023-09-30T09:45:32.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2309999999999999e-06 2023-09-30T09:45:32.439 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.439 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.439 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.440 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2303 9.f 9.7718716f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:45:32.440 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:45:32.440 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 67.328847046999996, 2023-09-30T09:45:32.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98609501700000002, 2023-09-30T09:45:32.440 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.441 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:45:32.441 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.441 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.441 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.441 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2303 2023-09-30T09:45:32.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.442 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:45:32.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063453+0000", 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063499+0000", 2023-09-30T09:45:32.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5393999999999999e-05 2023-09-30T09:45:32.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063510+0000", 2023-09-30T09:45:32.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1914e-05 2023-09-30T09:45:32.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063518+0000", 2023-09-30T09:45:32.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9719999999999992e-06 2023-09-30T09:45:32.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064775+0000", 2023-09-30T09:45:32.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001256987 2023-09-30T09:45:32.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064783+0000", 2023-09-30T09:45:32.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.267e-06 2023-09-30T09:45:32.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049541+0000", 2023-09-30T09:45:32.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98475831199999997 2023-09-30T09:45:32.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049550+0000", 2023-09-30T09:45:32.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5299999999999996e-06 2023-09-30T09:45:32.449 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.449 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.449 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.449 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5456 9.f 9:f741c261:::benchmark_data_smithi019_560486_object5455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.450 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:45:32.450 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.714317618000003, 2023-09-30T09:45:32.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99827741800000003, 2023-09-30T09:45:32.450 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.450 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.450 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.451 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.451 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.451 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5456 2023-09-30T09:45:32.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.451 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677983+0000", 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678041+0000", 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7439999999999999e-05 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678050+0000", 2023-09-30T09:45:32.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2380000000000003e-06 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678055+0000", 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2480000000000004e-06 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678212+0000", 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015715 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678221+0000", 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2140000000000002e-06 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666463+0000", 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98824183200000004 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666571+0000", 2023-09-30T09:45:32.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000107722 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666679+0000", 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108446 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669881+0000", 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032016509999999998 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676162+0000", 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062807169999999999 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676201+0000", 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9162000000000002e-05 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676235+0000", 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4459000000000003e-05 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676261+0000", 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5897999999999998e-05 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.458 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5488 9.f 9:f5c1ce8d:::benchmark_data_smithi019_560486_object5487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.704350806000001, 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98845970699999997, 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5488 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687950+0000", 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687988+0000", 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8231e-05 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688001+0000", 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2577e-05 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688007+0000", 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3740000000000003e-06 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688066+0000", 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8805999999999999e-05 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688075+0000", 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7499999999999998e-06 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667057+0000", 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97898119299999997 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667125+0000", 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7909000000000003e-05 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667219+0000", 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4759000000000005e-05 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669890+0000", 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026711130000000001 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676296+0000", 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064055370000000002 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676351+0000", 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5433999999999998e-05 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676386+0000", 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4400000000000003e-05 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676410+0000", 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4664e-05 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5500 9.f 9:fb77b28e:::benchmark_data_smithi019_560486_object5499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.701140717999998, 2023-09-30T09:45:32.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98537109199999995, 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5500 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691160+0000", 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691215+0000", 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5435e-05 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691223+0000", 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9340000000000006e-06 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691227+0000", 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7459999999999998e-06 2023-09-30T09:45:32.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691295+0000", 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8145999999999994e-05 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691307+0000", 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1516e-05 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667479+0000", 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97617223600000003 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667613+0000", 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013427400000000001 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667811+0000", 2023-09-30T09:45:32.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019798 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669899+0000", 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002087977 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676442+0000", 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0065432470000000003 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676474+0000", 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1275000000000001e-05 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676512+0000", 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7997000000000002e-05 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676532+0000", 2023-09-30T09:45:32.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0285999999999999e-05 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5531 9.f 9:fcece89e:::benchmark_data_smithi019_560486_object5530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.691649908999999, 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97602003800000003, 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.474 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5531 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700650+0000", 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700746+0000", 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5857000000000001e-05 2023-09-30T09:45:32.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700758+0000", 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1983e-05 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700765+0000", 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8279999999999997e-06 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.477 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700839+0000", 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4226000000000004e-05 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700857+0000", 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7193e-05 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668290+0000", 2023-09-30T09:45:32.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96743334299999995 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668421+0000", 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013117099999999999 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668566+0000", 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014448900000000001 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671187+0000", 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002621529 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676578+0000", 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005391051 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676610+0000", 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2039999999999998e-05 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676647+0000", 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7292000000000001e-05 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676672+0000", 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4232000000000001e-05 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.481 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5545 9.f 9:fa26aa6f:::benchmark_data_smithi019_560486_object5544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.686703065000003, 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97120855500000003, 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.482 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5545 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:45:32.483 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705597+0000", 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705693+0000", 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5401000000000004e-05 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.484 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705706+0000", 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2665e-05 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705713+0000", 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9940000000000003e-06 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.485 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705830+0000", 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011760900000000001 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705836+0000", 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7610000000000004e-06 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668931+0000", 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96309502499999999 2023-09-30T09:45:32.486 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669015+0000", 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3947000000000001e-05 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669120+0000", 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010467900000000001 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.487 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671197+0000", 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020777399999999998 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676699+0000", 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055018530000000001 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676739+0000", 2023-09-30T09:45:32.488 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9949e-05 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676786+0000", 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6755999999999998e-05 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676807+0000", 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1189e-05 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.489 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5551 9.f 9:f9ae74d4:::benchmark_data_smithi019_560486_object5550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.685788004999999, 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97040703399999995, 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.490 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5551 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:45:32.491 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706513+0000", 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706568+0000", 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5432000000000001e-05 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.492 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706580+0000", 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1494999999999999e-05 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706586+0000", 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7530000000000004e-06 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.493 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706645+0000", 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8591000000000002e-05 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706653+0000", 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5250000000000005e-06 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669422+0000", 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96276874499999998 2023-09-30T09:45:32.494 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669547+0000", 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012500799999999999 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669633+0000", 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5400000000000002e-05 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.495 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672244+0000", 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026115729999999998 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676836+0000", 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045915579999999999 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676856+0000", 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.02e-05 2023-09-30T09:45:32.496 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676898+0000", 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1789000000000003e-05 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676921+0000", 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2900000000000001e-05 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.497 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5571 9.f 9:f04c103d:::benchmark_data_smithi019_560486_object5570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.679632832999999, 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96437061999999996, 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.498 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5571 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.499 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712668+0000", 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713054+0000", 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00038651499999999998 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.500 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713063+0000", 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6440000000000006e-06 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713068+0000", 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2120000000000002e-06 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713200+0000", 2023-09-30T09:45:32.501 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000131977 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713210+0000", 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4250000000000004e-06 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669945+0000", 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95673509599999995 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.502 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670055+0000", 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000110145 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670148+0000", 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3212000000000001e-05 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672252+0000", 2023-09-30T09:45:32.503 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021043170000000001 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676960+0000", 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047072759999999998 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676992+0000", 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2808000000000002e-05 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.504 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677019+0000", 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6267000000000001e-05 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677039+0000", 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0647000000000001e-05 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.505 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5581 9.f 9:fa532eaf:::benchmark_data_smithi019_560486_object5580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.676563842999997, 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96139814300000004, 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5581 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.506 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.507 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715737+0000", 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715765+0000", 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8323e-05 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715780+0000", 2023-09-30T09:45:32.508 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5475999999999999e-05 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715787+0000", 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9149999999999999e-06 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715853+0000", 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5413000000000005e-05 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.509 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715862+0000", 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.594e-06 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670527+0000", 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95466473799999996 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.510 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670657+0000", 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012968699999999999 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670839+0000", 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000182461 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673403+0000", 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002564036 2023-09-30T09:45:32.511 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677071+0000", 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036674810000000002 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677096+0000", 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5429999999999999e-05 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.512 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677119+0000", 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3090000000000001e-05 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677136+0000", 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6702e-05 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.513 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5591 9.f 9:fb1424fd:::benchmark_data_smithi019_560486_object5590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.673331795999999, 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95826697900000002, 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5591 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.514 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.515 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718969+0000", 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719014+0000", 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4433e-05 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719031+0000", 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7078000000000001e-05 2023-09-30T09:45:32.516 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719036+0000", 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2240000000000003e-06 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719098+0000", 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2292999999999995e-05 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.517 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719107+0000", 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3319999999999992e-06 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671243+0000", 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95213663199999998 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671330+0000", 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6706999999999995e-05 2023-09-30T09:45:32.518 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671432+0000", 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010215 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673409+0000", 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019769060000000001 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.519 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677162+0000", 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037529379999999999 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677190+0000", 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7702e-05 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677216+0000", 2023-09-30T09:45:32.520 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6894e-05 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677237+0000", 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0253000000000001e-05 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5593 9.f 9:f74f8c23:::benchmark_data_smithi019_560486_object5592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.521 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.672963301000003, 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95807467700000004, 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5593 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.522 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:45:32.523 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719402+0000", 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4404000000000002e-05 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719414+0000", 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1895999999999999e-05 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.524 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719420+0000", 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.365e-06 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719641+0000", 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000221122 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719653+0000", 2023-09-30T09:45:32.525 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1884e-05 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671683+0000", 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95203014699999999 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671776+0000", 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2331999999999996e-05 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.526 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671902+0000", 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000126666 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.674505+0000", 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026023959999999999 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.527 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677294+0000", 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027896100000000001 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677329+0000", 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4668000000000001e-05 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677380+0000", 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0841999999999999e-05 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.528 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677413+0000", 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3028999999999998e-05 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5604 9.f 9:f2112614:::benchmark_data_smithi019_560486_object5603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.668070569999998, 2023-09-30T09:45:32.529 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95332888500000001, 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5604 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:45:32.530 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724230+0000", 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.531 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724380+0000", 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015008099999999999 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724392+0000", 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1518999999999999e-05 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.532 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724398+0000", 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5540000000000003e-06 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724454+0000", 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5358000000000001e-05 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724465+0000", 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1226000000000001e-05 2023-09-30T09:45:32.533 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672293+0000", 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94782834900000001 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672431+0000", 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013751799999999999 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.534 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672607+0000", 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000176517 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.674510+0000", 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019026710000000001 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677453+0000", 2023-09-30T09:45:32.535 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029434510000000001 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677481+0000", 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7932e-05 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677526+0000", 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4234999999999999e-05 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.536 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677560+0000", 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4345999999999999e-05 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5614 9.f 9:fca5ec44:::benchmark_data_smithi019_560486_object5613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.659887275000003, 2023-09-30T09:45:32.537 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.945425556, 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5614 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.538 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732413+0000", 2023-09-30T09:45:32.539 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732525+0000", 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112094 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732544+0000", 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9120999999999999e-05 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.540 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732551+0000", 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9e-06 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732616+0000", 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4272999999999999e-05 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732625+0000", 2023-09-30T09:45:32.541 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.166e-06 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672981+0000", 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94035580399999996 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673059+0000", 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8621999999999994e-05 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.542 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673154+0000", 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4519999999999996e-05 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.675823+0000", 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002669205 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.543 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677652+0000", 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001828795 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677707+0000", 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5056000000000003e-05 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677772+0000", 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4801999999999994e-05 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.544 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677840+0000", 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8238999999999995e-05 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5618 9.f 9:fc43f802:::benchmark_data_smithi019_560486_object5617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 62.659091777999997, 2023-09-30T09:45:32.545 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94477232899999997, 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5618 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:45:32.546 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733209+0000", 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733267+0000", 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7673999999999997e-05 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733281+0000", 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4701999999999999e-05 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733287+0000", 2023-09-30T09:45:32.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9839999999999998e-06 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733333+0000", 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5609000000000003e-05 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733339+0000", 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7960000000000001e-06 2023-09-30T09:45:32.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673445+0000", 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94010585599999996 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673574+0000", 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129469 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673744+0000", 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017018 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.675832+0000", 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002087696 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677879+0000", 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020473380000000001 2023-09-30T09:45:32.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677911+0000", 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1739000000000002e-05 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677950+0000", 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8690999999999999e-05 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:45:32.552 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:45:32.553 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677982+0000", 2023-09-30T09:45:32.553 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2471999999999997e-05 2023-09-30T09:45:32.553 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.553 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.553 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.553 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:45:32.553 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:45:32.553 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:45:32.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:32.554 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:32.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:32.655 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:32.704 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:32.768 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:32.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:32.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:32 smithi019 ceph-mon[216815]: pgmap v1307: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 475 op/s; 4812/234294 objects misplaced (2.054%); 391 KiB/s, 7 objects/s recovering 2023-09-30T09:45:32.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:32 smithi019 ceph-mon[224043]: pgmap v1307: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 475 op/s; 4812/234294 objects misplaced (2.054%); 391 KiB/s, 7 objects/s recovering 2023-09-30T09:45:32.845 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:32.873 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:32.902 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:32 smithi167 ceph-mon[192880]: pgmap v1307: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 475 op/s; 4812/234294 objects misplaced (2.054%); 391 KiB/s, 7 objects/s recovering 2023-09-30T09:45:32.906 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:32.918 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 69 16 80890 80874 73.2238 52.9375 0.0108364 0.0136504 2023-09-30T09:45:32.947 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:32.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:32.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:33.050 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:33.092 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:33.112 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:33.432 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:33.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:33.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:33.502 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:45:33.537 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:33.591 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:33.646 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:33.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:33.752 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:33.758 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:33.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:33.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:33 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2774998897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:33.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:33 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:33.815 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:33.815 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:33 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2774998897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:33.815 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:33 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:33.815 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:45:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:45:33] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-09-30T09:45:33.826 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:45:33.849 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:33.919 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 70 16 81560 81544 72.7759 41.875 0.0125391 0.0137352 2023-09-30T09:45:34.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:33 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2774998897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:34.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:33 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:45:34.919 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 71 16 82443 82427 72.5274 55.1875 0.0144598 0.0137743 2023-09-30T09:45:35.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:45:35.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:45:35.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: pgmap v1308: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 4675/236517 objects misplaced (1.977%); 1.7 MiB/s, 28 objects/s recovering 2023-09-30T09:45:35.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:45:35.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: osdmap e511: 8 total, 8 up, 8 in 2023-09-30T09:45:35.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.027 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:34 smithi167 ceph-mon[192880]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:45:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:45:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: pgmap v1308: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 4675/236517 objects misplaced (1.977%); 1.7 MiB/s, 28 objects/s recovering 2023-09-30T09:45:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:45:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: osdmap e511: 8 total, 8 up, 8 in 2023-09-30T09:45:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[224043]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:45:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:45:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: pgmap v1308: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 4675/236517 objects misplaced (1.977%); 1.7 MiB/s, 28 objects/s recovering 2023-09-30T09:45:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:45:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: osdmap e511: 8 total, 8 up, 8 in 2023-09-30T09:45:35.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:34 smithi019 ceph-mon[216815]: failed to encode map e511 with expected crc 2023-09-30T09:45:35.920 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 72 16 83306 83290 72.2692 53.9375 0.015123 0.0138323 2023-09-30T09:45:36.029 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:36.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:36.138 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:36.242 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:36.920 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 73 16 84177 84161 72.025 54.4375 0.0136195 0.013879 2023-09-30T09:45:37.052 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:37.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:37.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:37.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:36 smithi019 ceph-mon[216815]: pgmap v1310: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 4675/246285 objects misplaced (1.898%); 1.2 MiB/s, 20 objects/s recovering 2023-09-30T09:45:37.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:36 smithi019 ceph-mon[224043]: pgmap v1310: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 4675/246285 objects misplaced (1.898%); 1.2 MiB/s, 20 objects/s recovering 2023-09-30T09:45:37.159 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:37.204 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:36 smithi167 ceph-mon[192880]: pgmap v1310: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 4675/246285 objects misplaced (1.898%); 1.2 MiB/s, 20 objects/s recovering 2023-09-30T09:45:37.207 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:37.281 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:37.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:37.387 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:37.427 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:37.495 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:37.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:37.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:37.545 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:37.603 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:37.721 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:37.753 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:45:37.798 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:45:37.831 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:45:37.870 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:45:37.913 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:45:37.919 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 74 16 84961 84945 71.7139 49 0.00905192 0.0139366 2023-09-30T09:45:37.924 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:37.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:37.940 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3717620185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:37.940 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3717620185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:37.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:37.958 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:45:38.019 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:45:38.041 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:38.069 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:45:38.088 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:45:38.088 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:45:38.088 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:45:38.089 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:45:38.089 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:45:38.089 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:45:38.089 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:45:38.089 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:45:38.164 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:38.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3717620185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:38.895 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.6 to 1 (10000) 2023-09-30T09:45:38.920 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 75 16 85806 85790 71.4619 52.8125 0.0132552 0.0139869 2023-09-30T09:45:39.230 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:45:39.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:38 smithi167 ceph-mon[192880]: pgmap v1311: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 607 op/s; 4675/246285 objects misplaced (1.898%); 1.1 MiB/s, 17 objects/s recovering 2023-09-30T09:45:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1062611468' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:45:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:38 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:45:39.278 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:38 smithi019 ceph-mon[216815]: pgmap v1311: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 607 op/s; 4675/246285 objects misplaced (1.898%); 1.1 MiB/s, 17 objects/s recovering 2023-09-30T09:45:39.278 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1062611468' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:45:39.278 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:38 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:45:39.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:38 smithi019 ceph-mon[224043]: pgmap v1311: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 607 op/s; 4675/246285 objects misplaced (1.898%); 1.1 MiB/s, 17 objects/s recovering 2023-09-30T09:45:39.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1062611468' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:45:39.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:38 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:45:39.920 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 76 16 86573 86557 71.1524 47.9375 0.0138264 0.0140494 2023-09-30T09:45:40.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:45:40.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:39 smithi019 ceph-mon[224043]: failed to encode map e512 with expected crc 2023-09-30T09:45:40.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:39 smithi019 ceph-mon[224043]: failed to encode map e512 with expected crc 2023-09-30T09:45:40.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:39 smithi019 ceph-mon[224043]: osdmap e512: 8 total, 8 up, 8 in 2023-09-30T09:45:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:45:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:39 smithi019 ceph-mon[216815]: failed to encode map e512 with expected crc 2023-09-30T09:45:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:39 smithi019 ceph-mon[216815]: failed to encode map e512 with expected crc 2023-09-30T09:45:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:39 smithi019 ceph-mon[216815]: osdmap e512: 8 total, 8 up, 8 in 2023-09-30T09:45:40.241 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:40.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:40.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:45:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:39 smithi167 ceph-mon[192880]: failed to encode map e512 with expected crc 2023-09-30T09:45:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:39 smithi167 ceph-mon[192880]: failed to encode map e512 with expected crc 2023-09-30T09:45:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:39 smithi167 ceph-mon[192880]: osdmap e512: 8 total, 8 up, 8 in 2023-09-30T09:45:40.354 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:40.402 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:40.480 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:40.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:40.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:40.591 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:40.666 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:40.833 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:40.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:40.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:40.905 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:40 smithi019 ceph-mon[224043]: pgmap v1313: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 4562/254856 objects misplaced (1.790%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:45:40.906 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:40 smithi019 ceph-mon[216815]: pgmap v1313: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 4562/254856 objects misplaced (1.790%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:45:40.920 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 77 16 87410 87394 70.9077 52.3125 0.0196236 0.0140967 2023-09-30T09:45:40.939 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:41.011 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:41.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:40 smithi167 ceph-mon[192880]: pgmap v1313: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 4562/254856 objects misplaced (1.790%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:45:41.474 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:41.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:41.585 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:41.627 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:41.680 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-30T09:45:41.680 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:45:41.694 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:41.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:41.798 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:41.878 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:41.925 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 78 16 88185 88169 70.6164 48.4375 0.0241709 0.0141543 2023-09-30T09:45:42.076 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:42.104 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:42.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:42.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:42.210 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:42.241 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:45:42.241 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:45:42.298 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:42.513 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:42.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:42.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:42.628 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:42.670 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:45:42.680 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:42.765 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:42.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:42.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:42.873 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:42.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 79 16 88975 88959 70.3459 49.375 0.0454874 0.0142105 2023-09-30T09:45:42.930 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:43.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:42 smithi167 ceph-mon[192880]: pgmap v1314: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 4562/258897 objects misplaced (1.762%); 903 KiB/s, 14 objects/s recovering 2023-09-30T09:45:43.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:42 smithi019 ceph-mon[216815]: pgmap v1314: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 4562/258897 objects misplaced (1.762%); 903 KiB/s, 14 objects/s recovering 2023-09-30T09:45:43.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:42 smithi019 ceph-mon[224043]: pgmap v1314: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 4562/258897 objects misplaced (1.762%); 903 KiB/s, 14 objects/s recovering 2023-09-30T09:45:43.621 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:43.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:43.731 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:43.764 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:43.774 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:43.816 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:45:43.838 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:43.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:43.880 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:45:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:45:43] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-09-30T09:45:43.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:45:43.926297+0000 min lat: 0.00110418 max lat: 2.17303 avg lat: 0.0142709 2023-09-30T09:45:43.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:45:43.926 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 80 15 89708 89693 70.0401 45.875 0.0120325 0.0142709 2023-09-30T09:45:43.951 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:44.065 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:44.559 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:44.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:44.676 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:44.716 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:44.757 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:44.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:44.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:44.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:44 smithi019 ceph-mon[216815]: pgmap v1315: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 4416/261054 objects misplaced (1.692%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:44.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:44 smithi019 ceph-mon[224043]: pgmap v1315: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 4416/261054 objects misplaced (1.692%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:44.864 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:44.927 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:44.930 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 81 16 90453 90437 69.7471 46.5 0.0226491 0.0143317 2023-09-30T09:45:45.005 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:45.009 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:44 smithi167 ceph-mon[192880]: pgmap v1315: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 4416/261054 objects misplaced (1.692%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:45.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:45.112 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:45.191 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:45.366 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:45.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:45.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:45.472 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:45.511 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:45.573 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:45.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:45.674 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:45.685 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:45.725 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:45.794 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:45.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:45.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:45.907 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:45.928 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 82 16 91272 91256 69.5208 51.1875 0.0087343 0.0143799 2023-09-30T09:45:45.976 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:46.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 83 16 91947 91931 69.1914 42.1875 0.0260385 0.0144476 2023-09-30T09:45:47.068 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:47.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:47.184 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:47.217 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:46 smithi019 ceph-mon[216815]: pgmap v1316: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 4416/270489 objects misplaced (1.633%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:47.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:46 smithi019 ceph-mon[224043]: pgmap v1316: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 4416/270489 objects misplaced (1.633%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:47.228 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:46 smithi167 ceph-mon[192880]: pgmap v1316: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 4416/270489 objects misplaced (1.633%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:47.233 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:47.294 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:47.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:47.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:47.318 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:47.402 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:47.548 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:47.929 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 84 16 92662 92646 68.8998 44.6875 0.0217595 0.0145074 2023-09-30T09:45:47.958 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:47.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:47.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:48.077 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:48.167 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:48.465 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:48.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:48.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:48.570 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:45:48.675 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:48.723 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:45:48.734 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:45:48.899 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:45:48.932 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 85 16 93323 93307 68.5733 41.3125 0.0128409 0.0145771 2023-09-30T09:45:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[224043]: pgmap v1317: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 4416/270489 objects misplaced (1.633%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1062760807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:49.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:49.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[216815]: pgmap v1317: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 4416/270489 objects misplaced (1.633%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:49.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1062760807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:49.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:49.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:49.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:49.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:48 smithi167 ceph-mon[192880]: pgmap v1317: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 4416/270489 objects misplaced (1.633%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1062760807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:45:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:45:49.932 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 86 16 94071 94055 68.3196 46.75 0.0263419 0.0146317 2023-09-30T09:45:50.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:49 smithi167 ceph-mon[192880]: from='client.46325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:45:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:49 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:45:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:49 smithi167 ceph-mon[192880]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:49 smithi167 ceph-mon[192880]: osdmap e513: 8 total, 8 up, 8 in 2023-09-30T09:45:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:49 smithi167 ceph-mon[192880]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:49 smithi167 ceph-mon[192880]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:45:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[216815]: from='client.46325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:45:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:45:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[216815]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[216815]: osdmap e513: 8 total, 8 up, 8 in 2023-09-30T09:45:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[216815]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[216815]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[224043]: from='client.46325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[224043]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[224043]: osdmap e513: 8 total, 8 up, 8 in 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[224043]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[224043]: failed to encode map e513 with expected crc 2023-09-30T09:45:50.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:45:50.933 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 87 16 94792 94776 68.0517 45.0625 0.0194796 0.0146897 2023-09-30T09:45:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:50 smithi167 ceph-mon[192880]: pgmap v1319: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 4306/277920 objects misplaced (1.549%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:51.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:50 smithi019 ceph-mon[216815]: pgmap v1319: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 4306/277920 objects misplaced (1.549%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:51.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:50 smithi019 ceph-mon[224043]: pgmap v1319: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 4306/277920 objects misplaced (1.549%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:51.933 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 88 16 95449 95433 67.7447 41.0625 0.0143497 0.0147566 2023-09-30T09:45:52.114 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1317,"stamp":"2023-09-30T09:45:47.781883+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5911877543,"num_objects":90163,"num_object_clones":0,"num_object_copies":270489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4416,"num_objects_unfound":0,"num_objects_dirty":90163,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":91042,"num_write_kb":5781365,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1977,"num_bytes_recovered":123819634,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96157,"ondisk_log_size":96157,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21862664,"kb_used_data":17420168,"kb_used_omap":30,"kb_used_meta":4442401,"kb_avail":728098552,"statfs":{"total":767960285184,"available":745572917248,"internally_reserved":0,"allocated":17838252032,"data_stored":17819556588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4549018978},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":528744448,"num_objects":8068,"num_object_clones":0,"num_object_copies":24204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-259,"num_objects_unfound":0,"num_objects_dirty":8068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8068,"num_write_kb":516352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":259,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003689"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":935,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779335+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:45:33.779335+0000","last_peered":"2023-09-30T09:45:33.779335+0000","last_clean":"2023-09-30T09:45:33.779335+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:45:33.779335+0000","last_undegraded":"2023-09-30T09:45:33.779335+0000","last_fullsized":"2023-09-30T09:45:33.779335+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":530,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.785868+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:45:33.785868+0000","last_peered":"2023-09-30T09:45:33.785868+0000","last_clean":"2023-09-30T09:45:33.785868+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:45:33.785868+0000","last_undegraded":"2023-09-30T09:45:33.785868+0000","last_fullsized":"2023-09-30T09:45:33.785868+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":427,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786923+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:45:33.786923+0000","last_peered":"2023-09-30T09:45:33.786923+0000","last_clean":"2023-09-30T09:45:33.786923+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:45:33.786923+0000","last_undegraded":"2023-09-30T09:45:33.786923+0000","last_fullsized":"2023-09-30T09:45:33.786923+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":937,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779477+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:45:33.779477+0000","last_peered":"2023-09-30T09:45:33.779477+0000","last_clean":"2023-09-30T09:45:33.779477+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:45:33.779477+0000","last_undegraded":"2023-09-30T09:45:33.779477+0000","last_fullsized":"2023-09-30T09:45:33.779477+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":76,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.794549+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:45:33.794549+0000","last_peered":"2023-09-30T09:45:33.794549+0000","last_clean":"2023-09-30T09:45:33.794549+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:45:33.794549+0000","last_undegraded":"2023-09-30T09:45:33.794549+0000","last_fullsized":"2023-09-30T09:45:33.794549+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":117,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.791867+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:45:33.791867+0000","last_peered":"2023-09-30T09:45:33.791867+0000","last_clean":"2023-09-30T09:45:33.791867+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:45:33.791867+0000","last_undegraded":"2023-09-30T09:45:33.791867+0000","last_fullsized":"2023-09-30T09:45:33.791867+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":745,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.785952+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:45:33.785952+0000","last_peered":"2023-09-30T09:45:33.785952+0000","last_clean":"2023-09-30T09:45:33.785952+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:45:33.785952+0000","last_undegraded":"2023-09-30T09:45:33.785952+0000","last_fullsized":"2023-09-30T09:45:33.785952+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":17,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790959+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:45:33.790959+0000","last_peered":"2023-09-30T09:45:33.790959+0000","last_clean":"2023-09-30T09:45:33.790959+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:45:33.790959+0000","last_undegraded":"2023-09-30T09:45:33.790959+0000","last_fullsized":"2023-09-30T09:45:33.790959+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787740+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:45:33.787740+0000","last_peered":"2023-09-30T09:45:33.787740+0000","last_clean":"2023-09-30T09:45:33.787740+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:45:33.787740+0000","last_undegraded":"2023-09-30T09:45:33.787740+0000","last_fullsized":"2023-09-30T09:45:33.787740+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":913,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783452+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:45:33.783452+0000","last_peered":"2023-09-30T09:45:33.783452+0000","last_clean":"2023-09-30T09:45:33.783452+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:45:33.783452+0000","last_undegraded":"2023-09-30T09:45:33.783452+0000","last_fullsized":"2023-09-30T09:45:33.783452+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":132,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.792075+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:45:33.792075+0000","last_peered":"2023-09-30T09:45:33.792075+0000","last_clean":"2023-09-30T09:45:33.792075+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:45:33.792075+0000","last_undegraded":"2023-09-30T09:45:33.792075+0000","last_fullsized":"2023-09-30T09:45:33.792075+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.793381+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:45:33.793381+0000","last_peered":"2023-09-30T09:45:33.793381+0000","last_clean":"2023-09-30T09:45:33.793381+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:45:33.793381+0000","last_undegraded":"2023-09-30T09:45:33.793381+0000","last_fullsized":"2023-09-30T09:45:33.793381+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":701,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787808+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:45:33.787808+0000","last_peered":"2023-09-30T09:45:33.787808+0000","last_clean":"2023-09-30T09:45:33.787808+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:45:33.787808+0000","last_undegraded":"2023-09-30T09:45:33.787808+0000","last_fullsized":"2023-09-30T09:45:33.787808+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":332,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779820+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:45:33.779820+0000","last_peered":"2023-09-30T09:45:33.779820+0000","last_clean":"2023-09-30T09:45:33.779820+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:45:33.779820+0000","last_undegraded":"2023-09-30T09:45:33.779820+0000","last_fullsized":"2023-09-30T09:45:33.779820+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":468,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.782981+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:45:33.782981+0000","last_peered":"2023-09-30T09:45:33.782981+0000","last_clean":"2023-09-30T09:45:33.782981+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:45:33.782981+0000","last_undegraded":"2023-09-30T09:45:33.782981+0000","last_fullsized":"2023-09-30T09:45:33.782981+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":641,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779938+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:45:33.779938+0000","last_peered":"2023-09-30T09:45:33.779938+0000","last_clean":"2023-09-30T09:45:33.779938+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:45:33.779938+0000","last_undegraded":"2023-09-30T09:45:33.779938+0000","last_fullsized":"2023-09-30T09:45:33.779938+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":226,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783728+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:45:33.783728+0000","last_peered":"2023-09-30T09:45:33.783728+0000","last_clean":"2023-09-30T09:45:33.783728+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:45:33.783728+0000","last_undegraded":"2023-09-30T09:45:33.783728+0000","last_fullsized":"2023-09-30T09:45:33.783728+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":67,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786721+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:45:33.786721+0000","last_peered":"2023-09-30T09:45:33.786721+0000","last_clean":"2023-09-30T09:45:33.786721+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:45:33.786721+0000","last_undegraded":"2023-09-30T09:45:33.786721+0000","last_fullsized":"2023-09-30T09:45:33.786721+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":567,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.791951+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:45:33.791951+0000","last_peered":"2023-09-30T09:45:33.791951+0000","last_clean":"2023-09-30T09:45:33.791951+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:45:33.791951+0000","last_undegraded":"2023-09-30T09:45:33.791951+0000","last_fullsized":"2023-09-30T09:45:33.791951+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":556,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.788428+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:45:33.788428+0000","last_peered":"2023-09-30T09:45:33.788428+0000","last_clean":"2023-09-30T09:45:33.788428+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:45:33.788428+0000","last_undegraded":"2023-09-30T09:45:33.788428+0000","last_fullsized":"2023-09-30T09:45:33.788428+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":534,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.788268+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:45:33.788268+0000","last_peered":"2023-09-30T09:45:33.788268+0000","last_clean":"2023-09-30T09:45:33.788268+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:45:33.788268+0000","last_undegraded":"2023-09-30T09:45:33.788268+0000","last_fullsized":"2023-09-30T09:45:33.788268+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":765,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790278+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:45:33.790278+0000","last_peered":"2023-09-30T09:45:33.790278+0000","last_clean":"2023-09-30T09:45:33.790278+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:45:33.790278+0000","last_undegraded":"2023-09-30T09:45:33.790278+0000","last_fullsized":"2023-09-30T09:45:33.790278+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":162,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783361+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:45:33.783361+0000","last_peered":"2023-09-30T09:45:33.783361+0000","last_clean":"2023-09-30T09:45:33.783361+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:45:33.783361+0000","last_undegraded":"2023-09-30T09:45:33.783361+0000","last_fullsized":"2023-09-30T09:45:33.783361+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":489,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783526+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:45:33.783526+0000","last_peered":"2023-09-30T09:45:33.783526+0000","last_clean":"2023-09-30T09:45:33.783526+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:45:33.783526+0000","last_undegraded":"2023-09-30T09:45:33.783526+0000","last_fullsized":"2023-09-30T09:45:33.783526+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":813,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779252+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:45:33.779252+0000","last_peered":"2023-09-30T09:45:33.779252+0000","last_clean":"2023-09-30T09:45:33.779252+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:45:33.779252+0000","last_undegraded":"2023-09-30T09:45:33.779252+0000","last_fullsized":"2023-09-30T09:45:33.779252+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":91,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790671+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:45:33.790671+0000","last_peered":"2023-09-30T09:45:33.790671+0000","last_clean":"2023-09-30T09:45:33.790671+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:45:33.790671+0000","last_undegraded":"2023-09-30T09:45:33.790671+0000","last_fullsized":"2023-09-30T09:45:33.790671+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":187,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779670+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:45:33.779670+0000","last_peered":"2023-09-30T09:45:33.779670+0000","last_clean":"2023-09-30T09:45:33.779670+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:45:33.779670+0000","last_undegraded":"2023-09-30T09:45:33.779670+0000","last_fullsized":"2023-09-30T09:45:33.779670+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":913,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.792758+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:45:33.792758+0000","last_peered":"2023-09-30T09:45:33.792758+0000","last_clean":"2023-09-30T09:45:33.792758+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:45:33.792758+0000","last_undegraded":"2023-09-30T09:45:33.792758+0000","last_fullsized":"2023-09-30T09:45:33.792758+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":196,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783107+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:45:33.783107+0000","last_peered":"2023-09-30T09:45:33.783107+0000","last_clean":"2023-09-30T09:45:33.783107+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:45:33.783107+0000","last_undegraded":"2023-09-30T09:45:33.783107+0000","last_fullsized":"2023-09-30T09:45:33.783107+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":91,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.791631+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:45:33.791631+0000","last_peered":"2023-09-30T09:45:33.791631+0000","last_clean":"2023-09-30T09:45:33.791631+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:45:33.791631+0000","last_undegraded":"2023-09-30T09:45:33.791631+0000","last_fullsized":"2023-09-30T09:45:33.791631+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":922,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779526+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:45:33.779526+0000","last_peered":"2023-09-30T09:45:33.779526+0000","last_clean":"2023-09-30T09:45:33.779526+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:45:33.779526+0000","last_undegraded":"2023-09-30T09:45:33.779526+0000","last_fullsized":"2023-09-30T09:45:33.779526+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":108,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786262+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:45:33.786262+0000","last_peered":"2023-09-30T09:45:33.786262+0000","last_clean":"2023-09-30T09:45:33.786262+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:45:33.786262+0000","last_undegraded":"2023-09-30T09:45:33.786262+0000","last_fullsized":"2023-09-30T09:45:33.786262+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":76,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.796653+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:45:33.796653+0000","last_peered":"2023-09-30T09:45:33.796653+0000","last_clean":"2023-09-30T09:45:33.796653+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:45:33.796653+0000","last_undegraded":"2023-09-30T09:45:33.796653+0000","last_fullsized":"2023-09-30T09:45:33.796653+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"512'5559","reported_seq":5860,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:44.087082+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:45:44.087082+0000","last_peered":"2023-09-30T09:45:44.087082+0000","last_clean":"2023-09-30T09:45:44.087082+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:45:44.087082+0000","last_undegraded":"2023-09-30T09:45:44.087082+0000","last_fullsized":"2023-09-30T09:45:44.087082+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364314624,"num_objects":5559,"num_object_clones":0,"num_object_copies":16677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5559,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":888,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783630+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:45:33.783630+0000","last_peered":"2023-09-30T09:45:33.783630+0000","last_clean":"2023-09-30T09:45:33.783630+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:45:33.783630+0000","last_undegraded":"2023-09-30T09:45:33.783630+0000","last_fullsized":"2023-09-30T09:45:33.783630+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":188,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.784029+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:45:33.784029+0000","last_peered":"2023-09-30T09:45:33.784029+0000","last_clean":"2023-09-30T09:45:33.784029+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:45:33.784029+0000","last_undegraded":"2023-09-30T09:45:33.784029+0000","last_fullsized":"2023-09-30T09:45:33.784029+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":133,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779158+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:45:33.779158+0000","last_peered":"2023-09-30T09:45:33.779158+0000","last_clean":"2023-09-30T09:45:33.779158+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:45:33.779158+0000","last_undegraded":"2023-09-30T09:45:33.779158+0000","last_fullsized":"2023-09-30T09:45:33.779158+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"512'5590","reported_seq":5627,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:45.207902+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:45:45.207902+0000","last_peered":"2023-09-30T09:45:45.207902+0000","last_clean":"2023-09-30T09:45:45.207902+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:45:45.207902+0000","last_undegraded":"2023-09-30T09:45:45.207902+0000","last_fullsized":"2023-09-30T09:45:45.207902+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5590,"log_dups_size":0,"ondisk_log_size":5590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366346240,"num_objects":5590,"num_object_clones":0,"num_object_copies":16770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5590,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":905,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.799129+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:45:33.799129+0000","last_peered":"2023-09-30T09:45:33.799129+0000","last_clean":"2023-09-30T09:45:33.799129+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:45:33.799129+0000","last_undegraded":"2023-09-30T09:45:33.799129+0000","last_fullsized":"2023-09-30T09:45:33.799129+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":158,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783424+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:45:33.783424+0000","last_peered":"2023-09-30T09:45:33.783424+0000","last_clean":"2023-09-30T09:45:33.783424+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:45:33.783424+0000","last_undegraded":"2023-09-30T09:45:33.783424+0000","last_fullsized":"2023-09-30T09:45:33.783424+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":563,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.785886+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:45:33.785886+0000","last_peered":"2023-09-30T09:45:33.785886+0000","last_clean":"2023-09-30T09:45:33.785886+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:45:33.785886+0000","last_undegraded":"2023-09-30T09:45:33.785886+0000","last_fullsized":"2023-09-30T09:45:33.785886+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"512'5631","reported_seq":5893,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:44.011822+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:45:44.011822+0000","last_peered":"2023-09-30T09:45:44.011822+0000","last_clean":"2023-09-30T09:45:44.011822+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:45:44.011822+0000","last_undegraded":"2023-09-30T09:45:44.011822+0000","last_fullsized":"2023-09-30T09:45:44.011822+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369033216,"num_objects":5631,"num_object_clones":0,"num_object_copies":16893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5631,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":682,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787819+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:45:33.787819+0000","last_peered":"2023-09-30T09:45:33.787819+0000","last_clean":"2023-09-30T09:45:33.787819+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:45:33.787819+0000","last_undegraded":"2023-09-30T09:45:33.787819+0000","last_fullsized":"2023-09-30T09:45:33.787819+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":225,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783319+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:45:33.783319+0000","last_peered":"2023-09-30T09:45:33.783319+0000","last_clean":"2023-09-30T09:45:33.783319+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:45:33.783319+0000","last_undegraded":"2023-09-30T09:45:33.783319+0000","last_fullsized":"2023-09-30T09:45:33.783319+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":81,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786793+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:45:33.786793+0000","last_peered":"2023-09-30T09:45:33.786793+0000","last_clean":"2023-09-30T09:45:33.786793+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:45:33.786793+0000","last_undegraded":"2023-09-30T09:45:33.786793+0000","last_fullsized":"2023-09-30T09:45:33.786793+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"512'5525","reported_seq":5877,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:43.685776+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:45:43.685776+0000","last_peered":"2023-09-30T09:45:43.685776+0000","last_clean":"2023-09-30T09:45:43.685776+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:45:43.685776+0000","last_undegraded":"2023-09-30T09:45:43.685776+0000","last_fullsized":"2023-09-30T09:45:43.685776+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5525,"log_dups_size":0,"ondisk_log_size":5525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362086400,"num_objects":5525,"num_object_clones":0,"num_object_copies":16575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5525,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":747,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787319+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:45:33.787319+0000","last_peered":"2023-09-30T09:45:33.787319+0000","last_clean":"2023-09-30T09:45:33.787319+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:45:33.787319+0000","last_undegraded":"2023-09-30T09:45:33.787319+0000","last_fullsized":"2023-09-30T09:45:33.787319+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":447,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779694+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:45:33.779694+0000","last_peered":"2023-09-30T09:45:33.779694+0000","last_clean":"2023-09-30T09:45:33.779694+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:45:33.779694+0000","last_undegraded":"2023-09-30T09:45:33.779694+0000","last_fullsized":"2023-09-30T09:45:33.779694+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":111,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790676+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:45:33.790676+0000","last_peered":"2023-09-30T09:45:33.790676+0000","last_clean":"2023-09-30T09:45:33.790676+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:45:33.790676+0000","last_undegraded":"2023-09-30T09:45:33.790676+0000","last_fullsized":"2023-09-30T09:45:33.790676+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"512'5658","reported_seq":5695,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:44.218762+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:45:44.218762+0000","last_peered":"2023-09-30T09:45:44.218762+0000","last_clean":"2023-09-30T09:45:44.218762+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:45:44.218762+0000","last_undegraded":"2023-09-30T09:45:44.218762+0000","last_fullsized":"2023-09-30T09:45:44.218762+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5658,"log_dups_size":0,"ondisk_log_size":5658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370802688,"num_objects":5658,"num_object_clones":0,"num_object_copies":16974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5658,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":994,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783075+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:45:33.783075+0000","last_peered":"2023-09-30T09:45:33.783075+0000","last_clean":"2023-09-30T09:45:33.783075+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:45:33.783075+0000","last_undegraded":"2023-09-30T09:45:33.783075+0000","last_fullsized":"2023-09-30T09:45:33.783075+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790455+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:45:33.790455+0000","last_peered":"2023-09-30T09:45:33.790455+0000","last_clean":"2023-09-30T09:45:33.790455+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:45:33.790455+0000","last_undegraded":"2023-09-30T09:45:33.790455+0000","last_fullsized":"2023-09-30T09:45:33.790455+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":510,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.788423+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:45:33.788423+0000","last_peered":"2023-09-30T09:45:33.788423+0000","last_clean":"2023-09-30T09:45:33.788423+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:45:33.788423+0000","last_undegraded":"2023-09-30T09:45:33.788423+0000","last_fullsized":"2023-09-30T09:45:33.788423+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"512'5529","reported_seq":5566,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:44.268215+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:45:44.268215+0000","last_peered":"2023-09-30T09:45:44.268215+0000","last_clean":"2023-09-30T09:45:44.268215+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:45:44.268215+0000","last_undegraded":"2023-09-30T09:45:44.268215+0000","last_fullsized":"2023-09-30T09:45:44.268215+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362348544,"num_objects":5529,"num_object_clones":0,"num_object_copies":16587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5529,"num_write_kb":353856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":992,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787465+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:45:33.787465+0000","last_peered":"2023-09-30T09:45:33.787465+0000","last_clean":"2023-09-30T09:45:33.787465+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:45:33.787465+0000","last_undegraded":"2023-09-30T09:45:33.787465+0000","last_fullsized":"2023-09-30T09:45:33.787465+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":67,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786643+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:45:33.786643+0000","last_peered":"2023-09-30T09:45:33.786643+0000","last_clean":"2023-09-30T09:45:33.786643+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:45:33.786643+0000","last_undegraded":"2023-09-30T09:45:33.786643+0000","last_fullsized":"2023-09-30T09:45:33.786643+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":206,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786455+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:45:33.786455+0000","last_peered":"2023-09-30T09:45:33.786455+0000","last_clean":"2023-09-30T09:45:33.786455+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:45:33.786455+0000","last_undegraded":"2023-09-30T09:45:33.786455+0000","last_fullsized":"2023-09-30T09:45:33.786455+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"512'5772","reported_seq":5809,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:45.592340+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:45:45.592340+0000","last_peered":"2023-09-30T09:45:45.592340+0000","last_clean":"2023-09-30T09:45:45.592340+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:45:45.592340+0000","last_undegraded":"2023-09-30T09:45:45.592340+0000","last_fullsized":"2023-09-30T09:45:45.592340+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5772,"log_dups_size":0,"ondisk_log_size":5772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378273792,"num_objects":5772,"num_object_clones":0,"num_object_copies":17316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5772,"num_write_kb":369408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":850,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790760+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:45:33.790760+0000","last_peered":"2023-09-30T09:45:33.790760+0000","last_clean":"2023-09-30T09:45:33.790760+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:45:33.790760+0000","last_undegraded":"2023-09-30T09:45:33.790760+0000","last_fullsized":"2023-09-30T09:45:33.790760+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":517,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.785743+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:45:33.785743+0000","last_peered":"2023-09-30T09:45:33.785743+0000","last_clean":"2023-09-30T09:45:33.785743+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:45:33.785743+0000","last_undegraded":"2023-09-30T09:45:33.785743+0000","last_fullsized":"2023-09-30T09:45:33.785743+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":524,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787431+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:45:33.787431+0000","last_peered":"2023-09-30T09:45:33.787431+0000","last_clean":"2023-09-30T09:45:33.787431+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:45:33.787431+0000","last_undegraded":"2023-09-30T09:45:33.787431+0000","last_fullsized":"2023-09-30T09:45:33.787431+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"512'5531","reported_seq":5571,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:43.684982+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:45:43.684982+0000","last_peered":"2023-09-30T09:45:43.684982+0000","last_clean":"2023-09-30T09:45:43.684982+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:45:43.684982+0000","last_undegraded":"2023-09-30T09:45:43.684982+0000","last_fullsized":"2023-09-30T09:45:43.684982+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5531,"log_dups_size":0,"ondisk_log_size":5531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362479616,"num_objects":5531,"num_object_clones":0,"num_object_copies":16593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5531,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":714,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787767+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:45:33.787767+0000","last_peered":"2023-09-30T09:45:33.787767+0000","last_clean":"2023-09-30T09:45:33.787767+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:45:33.787767+0000","last_undegraded":"2023-09-30T09:45:33.787767+0000","last_fullsized":"2023-09-30T09:45:33.787767+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.793300+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:45:33.793300+0000","last_peered":"2023-09-30T09:45:33.793300+0000","last_clean":"2023-09-30T09:45:33.793300+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:45:33.793300+0000","last_undegraded":"2023-09-30T09:45:33.793300+0000","last_fullsized":"2023-09-30T09:45:33.793300+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":197,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787109+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:45:33.787109+0000","last_peered":"2023-09-30T09:45:33.787109+0000","last_clean":"2023-09-30T09:45:33.787109+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:45:33.787109+0000","last_undegraded":"2023-09-30T09:45:33.787109+0000","last_fullsized":"2023-09-30T09:45:33.787109+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"512'5649","reported_seq":5945,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:44.047111+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:45:44.047111+0000","last_peered":"2023-09-30T09:45:44.047111+0000","last_clean":"2023-09-30T09:45:44.047111+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:45:44.047111+0000","last_undegraded":"2023-09-30T09:45:44.047111+0000","last_fullsized":"2023-09-30T09:45:44.047111+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5649,"log_dups_size":0,"ondisk_log_size":5649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370212864,"num_objects":5649,"num_object_clones":0,"num_object_copies":16947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5649,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":754,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787632+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:45:33.787632+0000","last_peered":"2023-09-30T09:45:33.787632+0000","last_clean":"2023-09-30T09:45:33.787632+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:45:33.787632+0000","last_undegraded":"2023-09-30T09:45:33.787632+0000","last_fullsized":"2023-09-30T09:45:33.787632+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":742,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.791452+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:45:33.791452+0000","last_peered":"2023-09-30T09:45:33.791452+0000","last_clean":"2023-09-30T09:45:33.791452+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:45:33.791452+0000","last_undegraded":"2023-09-30T09:45:33.791452+0000","last_fullsized":"2023-09-30T09:45:33.791452+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":440,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779803+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:45:33.779803+0000","last_peered":"2023-09-30T09:45:33.779803+0000","last_clean":"2023-09-30T09:45:33.779803+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:45:33.779803+0000","last_undegraded":"2023-09-30T09:45:33.779803+0000","last_fullsized":"2023-09-30T09:45:33.779803+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":129,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779277+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:45:33.779277+0000","last_peered":"2023-09-30T09:45:33.779277+0000","last_clean":"2023-09-30T09:45:33.779277+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:45:33.779277+0000","last_undegraded":"2023-09-30T09:45:33.779277+0000","last_fullsized":"2023-09-30T09:45:33.779277+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"512'5625","reported_seq":5966,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:44.087372+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:45:44.087372+0000","last_peered":"2023-09-30T09:45:44.087372+0000","last_clean":"2023-09-30T09:45:44.087372+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:45:44.087372+0000","last_undegraded":"2023-09-30T09:45:44.087372+0000","last_fullsized":"2023-09-30T09:45:44.087372+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368640000,"num_objects":5625,"num_object_clones":0,"num_object_copies":16875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5625,"num_write_kb":360000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":953,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779982+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:45:33.779982+0000","last_peered":"2023-09-30T09:45:33.779982+0000","last_clean":"2023-09-30T09:45:33.779982+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:45:33.779982+0000","last_undegraded":"2023-09-30T09:45:33.779982+0000","last_fullsized":"2023-09-30T09:45:33.779982+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":459,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786871+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:45:33.786871+0000","last_peered":"2023-09-30T09:45:33.786871+0000","last_clean":"2023-09-30T09:45:33.786871+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:45:33.786871+0000","last_undegraded":"2023-09-30T09:45:33.786871+0000","last_fullsized":"2023-09-30T09:45:33.786871+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":531,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.788357+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:45:33.788357+0000","last_peered":"2023-09-30T09:45:33.788357+0000","last_clean":"2023-09-30T09:45:33.788357+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:45:33.788357+0000","last_undegraded":"2023-09-30T09:45:33.788357+0000","last_fullsized":"2023-09-30T09:45:33.788357+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":271,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880739+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:45:38.880739+0000","last_peered":"2023-09-30T09:45:38.880739+0000","last_clean":"2023-09-30T09:45:38.880739+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:45:38.880739+0000","last_undegraded":"2023-09-30T09:45:38.880739+0000","last_fullsized":"2023-09-30T09:45:38.880739+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"512'5666","reported_seq":6008,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:44.101246+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:45:44.101246+0000","last_peered":"2023-09-30T09:45:44.101246+0000","last_clean":"2023-09-30T09:45:44.101246+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:45:44.101246+0000","last_undegraded":"2023-09-30T09:45:44.101246+0000","last_fullsized":"2023-09-30T09:45:44.101246+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371326976,"num_objects":5666,"num_object_clones":0,"num_object_copies":16998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5666,"num_write_kb":362624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":931,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783795+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:45:33.783795+0000","last_peered":"2023-09-30T09:45:33.783795+0000","last_clean":"2023-09-30T09:45:33.783795+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:45:33.783795+0000","last_undegraded":"2023-09-30T09:45:33.783795+0000","last_fullsized":"2023-09-30T09:45:33.783795+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":780,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.788508+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:45:33.788508+0000","last_peered":"2023-09-30T09:45:33.788508+0000","last_clean":"2023-09-30T09:45:33.788508+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:45:33.788508+0000","last_undegraded":"2023-09-30T09:45:33.788508+0000","last_fullsized":"2023-09-30T09:45:33.788508+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":274,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787322+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:45:33.787322+0000","last_peered":"2023-09-30T09:45:33.787322+0000","last_clean":"2023-09-30T09:45:33.787322+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:45:33.787322+0000","last_undegraded":"2023-09-30T09:45:33.787322+0000","last_fullsized":"2023-09-30T09:45:33.787322+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":361,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.780073+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:45:33.780073+0000","last_peered":"2023-09-30T09:45:33.780073+0000","last_clean":"2023-09-30T09:45:33.780073+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:45:33.780073+0000","last_undegraded":"2023-09-30T09:45:33.780073+0000","last_fullsized":"2023-09-30T09:45:33.780073+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"512'5617","reported_seq":5918,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:44.107685+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:45:44.107685+0000","last_peered":"2023-09-30T09:45:44.107685+0000","last_clean":"2023-09-30T09:45:44.107685+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:45:44.107685+0000","last_undegraded":"2023-09-30T09:45:44.107685+0000","last_fullsized":"2023-09-30T09:45:44.107685+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5617,"log_dups_size":0,"ondisk_log_size":5617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368115712,"num_objects":5617,"num_object_clones":0,"num_object_copies":16851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5617,"num_write_kb":359488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1073,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779555+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:45:33.779555+0000","last_peered":"2023-09-30T09:45:33.779555+0000","last_clean":"2023-09-30T09:45:33.779555+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:45:33.779555+0000","last_undegraded":"2023-09-30T09:45:33.779555+0000","last_fullsized":"2023-09-30T09:45:33.779555+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":874,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783100+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:45:33.783100+0000","last_peered":"2023-09-30T09:45:33.783100+0000","last_clean":"2023-09-30T09:45:33.783100+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:45:33.783100+0000","last_undegraded":"2023-09-30T09:45:33.783100+0000","last_fullsized":"2023-09-30T09:45:33.783100+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":88,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779394+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:45:33.779394+0000","last_peered":"2023-09-30T09:45:33.779394+0000","last_clean":"2023-09-30T09:45:33.779394+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:45:33.779394+0000","last_undegraded":"2023-09-30T09:45:33.779394+0000","last_fullsized":"2023-09-30T09:45:33.779394+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":67,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787077+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:45:33.787077+0000","last_peered":"2023-09-30T09:45:33.787077+0000","last_clean":"2023-09-30T09:45:33.787077+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:45:33.787077+0000","last_undegraded":"2023-09-30T09:45:33.787077+0000","last_fullsized":"2023-09-30T09:45:33.787077+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"512'5709","reported_seq":6878,"reported_epoch":512,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:45:43.684084+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:45:43.684084+0000","last_peered":"2023-09-30T09:45:43.684084+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:45:43.684084+0000","last_undegraded":"2023-09-30T09:45:43.684084+0000","last_fullsized":"2023-09-30T09:45:43.684084+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374145024,"num_objects":5709,"num_object_clones":0,"num_object_copies":17127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4416,"num_objects_unfound":0,"num_objects_dirty":5709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5709,"num_write_kb":365376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":554,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":1293}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":773,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787939+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:45:33.787939+0000","last_peered":"2023-09-30T09:45:33.787939+0000","last_clean":"2023-09-30T09:45:33.787939+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:45:33.787939+0000","last_undegraded":"2023-09-30T09:45:33.787939+0000","last_fullsized":"2023-09-30T09:45:33.787939+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":920,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783052+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:45:33.783052+0000","last_peered":"2023-09-30T09:45:33.783052+0000","last_clean":"2023-09-30T09:45:33.783052+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:45:33.783052+0000","last_undegraded":"2023-09-30T09:45:33.783052+0000","last_fullsized":"2023-09-30T09:45:33.783052+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":124,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790425+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:45:33.790425+0000","last_peered":"2023-09-30T09:45:33.790425+0000","last_clean":"2023-09-30T09:45:33.790425+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:45:33.790425+0000","last_undegraded":"2023-09-30T09:45:33.790425+0000","last_fullsized":"2023-09-30T09:45:33.790425+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":17,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.791553+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:45:33.791553+0000","last_peered":"2023-09-30T09:45:33.791553+0000","last_clean":"2023-09-30T09:45:33.791553+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:45:33.791553+0000","last_undegraded":"2023-09-30T09:45:33.791553+0000","last_fullsized":"2023-09-30T09:45:33.791553+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"512'5674","reported_seq":5711,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:45.633367+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:45:45.633367+0000","last_peered":"2023-09-30T09:45:45.633367+0000","last_clean":"2023-09-30T09:45:45.633367+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:45:45.633367+0000","last_undegraded":"2023-09-30T09:45:45.633367+0000","last_fullsized":"2023-09-30T09:45:45.633367+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371851264,"num_objects":5674,"num_object_clones":0,"num_object_copies":17022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5674,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":764,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787447+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:45:33.787447+0000","last_peered":"2023-09-30T09:45:33.787447+0000","last_clean":"2023-09-30T09:45:33.787447+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:45:33.787447+0000","last_undegraded":"2023-09-30T09:45:33.787447+0000","last_fullsized":"2023-09-30T09:45:33.787447+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1060,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783813+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:45:33.783813+0000","last_peered":"2023-09-30T09:45:33.783813+0000","last_clean":"2023-09-30T09:45:33.783813+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:45:33.783813+0000","last_undegraded":"2023-09-30T09:45:33.783813+0000","last_fullsized":"2023-09-30T09:45:33.783813+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":255,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880645+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:45:38.880645+0000","last_peered":"2023-09-30T09:45:38.880645+0000","last_clean":"2023-09-30T09:45:38.880645+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:45:38.880645+0000","last_undegraded":"2023-09-30T09:45:38.880645+0000","last_fullsized":"2023-09-30T09:45:38.880645+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":124,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790846+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:45:33.790846+0000","last_peered":"2023-09-30T09:45:33.790846+0000","last_clean":"2023-09-30T09:45:33.790846+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:45:33.790846+0000","last_undegraded":"2023-09-30T09:45:33.790846+0000","last_fullsized":"2023-09-30T09:45:33.790846+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"512'5636","reported_seq":6007,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:45.615492+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:45:45.615492+0000","last_peered":"2023-09-30T09:45:45.615492+0000","last_clean":"2023-09-30T09:45:45.615492+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:45:45.615492+0000","last_undegraded":"2023-09-30T09:45:45.615492+0000","last_fullsized":"2023-09-30T09:45:45.615492+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5636,"log_dups_size":0,"ondisk_log_size":5636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369360896,"num_objects":5636,"num_object_clones":0,"num_object_copies":16908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5636,"num_write_kb":360704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1031,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786656+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:45:33.786656+0000","last_peered":"2023-09-30T09:45:33.786656+0000","last_clean":"2023-09-30T09:45:33.786656+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:45:33.786656+0000","last_undegraded":"2023-09-30T09:45:33.786656+0000","last_fullsized":"2023-09-30T09:45:33.786656+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":543,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783982+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:45:33.783982+0000","last_peered":"2023-09-30T09:45:33.783982+0000","last_clean":"2023-09-30T09:45:33.783982+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:45:33.783982+0000","last_undegraded":"2023-09-30T09:45:33.783982+0000","last_fullsized":"2023-09-30T09:45:33.783982+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1230,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787359+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:45:33.787359+0000","last_peered":"2023-09-30T09:45:33.787359+0000","last_clean":"2023-09-30T09:45:33.787359+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:45:33.787359+0000","last_undegraded":"2023-09-30T09:45:33.787359+0000","last_fullsized":"2023-09-30T09:45:33.787359+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":547,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783018+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:45:33.783018+0000","last_peered":"2023-09-30T09:45:33.783018+0000","last_clean":"2023-09-30T09:45:33.783018+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:45:33.783018+0000","last_undegraded":"2023-09-30T09:45:33.783018+0000","last_fullsized":"2023-09-30T09:45:33.783018+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"512'5764","reported_seq":5801,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:45.309613+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:45:45.309613+0000","last_peered":"2023-09-30T09:45:45.309613+0000","last_clean":"2023-09-30T09:45:45.309613+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:45:45.309613+0000","last_undegraded":"2023-09-30T09:45:45.309613+0000","last_fullsized":"2023-09-30T09:45:45.309613+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5764,"log_dups_size":0,"ondisk_log_size":5764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377749504,"num_objects":5764,"num_object_clones":0,"num_object_copies":17292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5764,"num_write_kb":368896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":894,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.798995+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:45:33.798995+0000","last_peered":"2023-09-30T09:45:33.798995+0000","last_clean":"2023-09-30T09:45:33.798995+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:45:33.798995+0000","last_undegraded":"2023-09-30T09:45:33.798995+0000","last_fullsized":"2023-09-30T09:45:33.798995+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":82,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787775+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:45:33.787775+0000","last_peered":"2023-09-30T09:45:33.787775+0000","last_clean":"2023-09-30T09:45:33.787775+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:45:33.787775+0000","last_undegraded":"2023-09-30T09:45:33.787775+0000","last_fullsized":"2023-09-30T09:45:33.787775+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":87,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.791777+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:45:33.791777+0000","last_peered":"2023-09-30T09:45:33.791777+0000","last_clean":"2023-09-30T09:45:33.791777+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:45:33.791777+0000","last_undegraded":"2023-09-30T09:45:33.791777+0000","last_fullsized":"2023-09-30T09:45:33.791777+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":513,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.785817+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:45:33.785817+0000","last_peered":"2023-09-30T09:45:33.785817+0000","last_clean":"2023-09-30T09:45:33.785817+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:45:33.785817+0000","last_undegraded":"2023-09-30T09:45:33.785817+0000","last_fullsized":"2023-09-30T09:45:33.785817+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":831,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790171+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:45:33.790171+0000","last_peered":"2023-09-30T09:45:33.790171+0000","last_clean":"2023-09-30T09:45:33.790171+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:45:33.790171+0000","last_undegraded":"2023-09-30T09:45:33.790171+0000","last_fullsized":"2023-09-30T09:45:33.790171+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":17,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.790774+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:45:33.790774+0000","last_peered":"2023-09-30T09:45:33.790774+0000","last_clean":"2023-09-30T09:45:33.790774+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:45:33.790774+0000","last_undegraded":"2023-09-30T09:45:33.790774+0000","last_fullsized":"2023-09-30T09:45:33.790774+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":217,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787613+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:45:33.787613+0000","last_peered":"2023-09-30T09:45:33.787613+0000","last_clean":"2023-09-30T09:45:33.787613+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:45:33.787613+0000","last_undegraded":"2023-09-30T09:45:33.787613+0000","last_fullsized":"2023-09-30T09:45:33.787613+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":626,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787529+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:45:33.787529+0000","last_peered":"2023-09-30T09:45:33.787529+0000","last_clean":"2023-09-30T09:45:33.787529+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:45:33.787529+0000","last_undegraded":"2023-09-30T09:45:33.787529+0000","last_fullsized":"2023-09-30T09:45:33.787529+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":440,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779650+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:45:33.779650+0000","last_peered":"2023-09-30T09:45:33.779650+0000","last_clean":"2023-09-30T09:45:33.779650+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:45:33.779650+0000","last_undegraded":"2023-09-30T09:45:33.779650+0000","last_fullsized":"2023-09-30T09:45:33.779650+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":226,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787184+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:45:33.787184+0000","last_peered":"2023-09-30T09:45:33.787184+0000","last_clean":"2023-09-30T09:45:33.787184+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:45:33.787184+0000","last_undegraded":"2023-09-30T09:45:33.787184+0000","last_fullsized":"2023-09-30T09:45:33.787184+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":382,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.779899+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:45:33.779899+0000","last_peered":"2023-09-30T09:45:33.779899+0000","last_clean":"2023-09-30T09:45:33.779899+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:45:33.779899+0000","last_undegraded":"2023-09-30T09:45:33.779899+0000","last_fullsized":"2023-09-30T09:45:33.779899+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":246,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.785784+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:45:33.785784+0000","last_peered":"2023-09-30T09:45:33.785784+0000","last_clean":"2023-09-30T09:45:33.785784+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:45:33.785784+0000","last_undegraded":"2023-09-30T09:45:33.785784+0000","last_fullsized":"2023-09-30T09:45:33.785784+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":839,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.783451+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:45:33.783451+0000","last_peered":"2023-09-30T09:45:33.783451+0000","last_clean":"2023-09-30T09:45:33.783451+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:45:33.783451+0000","last_undegraded":"2023-09-30T09:45:33.783451+0000","last_fullsized":"2023-09-30T09:45:33.783451+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":69,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.787281+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:45:33.787281+0000","last_peered":"2023-09-30T09:45:33.787281+0000","last_clean":"2023-09-30T09:45:33.787281+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:45:33.787281+0000","last_undegraded":"2023-09-30T09:45:33.787281+0000","last_fullsized":"2023-09-30T09:45:33.787281+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":462,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.788424+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:45:33.788424+0000","last_peered":"2023-09-30T09:45:33.788424+0000","last_clean":"2023-09-30T09:45:33.788424+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:45:33.788424+0000","last_undegraded":"2023-09-30T09:45:33.788424+0000","last_fullsized":"2023-09-30T09:45:33.788424+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":882,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.786533+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:45:33.786533+0000","last_peered":"2023-09-30T09:45:33.786533+0000","last_clean":"2023-09-30T09:45:33.786533+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:45:33.786533+0000","last_undegraded":"2023-09-30T09:45:33.786533+0000","last_fullsized":"2023-09-30T09:45:33.786533+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":23,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.791777+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:45:33.791777+0000","last_peered":"2023-09-30T09:45:33.791777+0000","last_clean":"2023-09-30T09:45:33.791777+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:45:33.791777+0000","last_undegraded":"2023-09-30T09:45:33.791777+0000","last_fullsized":"2023-09-30T09:45:33.791777+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":527,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-30T09:45:33.785651+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:45:33.785651+0000","last_peered":"2023-09-30T09:45:33.785651+0000","last_clean":"2023-09-30T09:45:33.785651+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:45:33.785651+0000","last_undegraded":"2023-09-30T09:45:33.785651+0000","last_fullsized":"2023-09-30T09:45:33.785651+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5907087360,"num_objects":90135,"num_object_clones":0,"num_object_copies":270405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4416,"num_objects_unfound":0,"num_objects_dirty":90135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90135,"num_write_kb":5768640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1697,"num_bytes_recovered":111214592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17855086592,"data_stored":17855086592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90135,"ondisk_log_size":90135,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234408,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3796088,"kb_used_data":3283632,"kb_used_omap":0,"kb_used_meta":512448,"kb_avail":89949064,"statfs":{"total":95995035648,"available":92107841536,"internally_reserved":0,"allocated":3362439168,"data_stored":3360732066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":524746752},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":3.8450000000000002,"15min":3.7290000000000001},"min":{"1min":0.23300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6360000000000001,"5min":124.74299999999999,"15min":215.416},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":3.984,"15min":3.6459999999999999},"min":{"1min":0.253,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.1779999999999999,"5min":124.69499999999999,"15min":215.041},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:45:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":3.8460000000000001,"15min":3.2160000000000002},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":12.702,"5min":124.577,"15min":124.577},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.218,"5min":3.714,"15min":3.4500000000000002},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.307},"max":{"1min":12.714,"5min":124.623,"15min":124.623},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":4.3570000000000002,"15min":3.5009999999999999},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.24299999999999999},"max":{"1min":5.3789999999999996,"5min":117.14,"15min":117.14},"last":0.39400000000000002},{"interface":"front","average":{"1min":2.3450000000000002,"5min":4.585,"15min":3.528},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":18.603000000000002,"5min":117.044,"15min":117.044},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":8.798,"5min":4.8070000000000004,"15min":8.2530000000000001},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.28899999999999998},"max":{"1min":141.28999999999999,"5min":141.28999999999999,"15min":277.57799999999997},"last":3.2440000000000002},{"interface":"front","average":{"1min":9.1679999999999993,"5min":4.8659999999999997,"15min":7.7830000000000004},"min":{"1min":0.39900000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":142.458,"5min":142.458,"15min":278.69299999999998},"last":3.8780000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":0.96699999999999997,"15min":1.6000000000000001},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":25.192,"5min":25.192,"15min":175.72},"last":0.76900000000000002},{"interface":"front","average":{"1min":2.657,"5min":0.93100000000000005,"15min":1.526},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":24.521000000000001,"5min":24.521000000000001,"15min":179.048},"last":2.7029999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":11.993,"5min":11.993,"15min":11.993},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":197.75700000000001,"5min":197.75700000000001,"15min":197.75700000000001},"last":3.641},{"interface":"front","average":{"1min":11.827,"5min":11.827,"15min":11.827},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":197.63300000000001,"5min":197.63300000000001,"15min":197.63300000000001},"last":1.6739999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":0.61899999999999999,"15min":2.1659999999999999},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.367,"5min":4.367,"15min":206.244},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.6679999999999999,"5min":0.76300000000000001,"15min":2.2000000000000002},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":202.78800000000001},"last":0.34699999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496046,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1599140,"kb_used_data":1087260,"kb_used_omap":0,"kb_used_meta":511872,"kb_avail":92146012,"statfs":{"total":95995035648,"available":94357516288,"internally_reserved":0,"allocated":1113354240,"data_stored":1111666359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":524156928},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":3.1240000000000001,"15min":4.8929999999999998},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":5.0129999999999999,"5min":95.043999999999997,"15min":187.19800000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.1659999999999999,"5min":3.331,"15min":5.0620000000000003},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":6.7960000000000003,"5min":94.855999999999995,"15min":187.46700000000001},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:44:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.94699999999999995,"15min":1.974},"min":{"1min":0.32500000000000001,"5min":0.25,"15min":0.25},"max":{"1min":2.9569999999999999,"5min":32.814,"15min":69.569999999999993},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.998,"15min":2.181},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.6219999999999999,"5min":32.856000000000002,"15min":70.471999999999994},"last":0.42199999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:44:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":4.133,"15min":3.8149999999999999},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.0920000000000001,"5min":207.02500000000001,"15min":207.02500000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.84599999999999997,"5min":4.3140000000000001,"15min":3.9500000000000002},"min":{"1min":0.33500000000000002,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":3.1219999999999999,"5min":207.114,"15min":207.114},"last":0.38500000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":3.5979999999999999,"5min":3.0019999999999998,"15min":4.3170000000000002},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.27600000000000002},"max":{"1min":49.234000000000002,"5min":107.11799999999999,"15min":212.44800000000001},"last":1.1259999999999999},{"interface":"front","average":{"1min":3.6800000000000002,"5min":2.9750000000000001,"15min":4.6109999999999998},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":48.363,"5min":106.982,"15min":210.84299999999999},"last":1.075}]},{"osd":4,"last update":"Sat Sep 30 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":4.2149999999999999,"5min":1.581,"15min":1.377},"min":{"1min":0.29999999999999999,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":63.798000000000002,"5min":63.798000000000002,"15min":63.798000000000002},"last":0.40100000000000002},{"interface":"front","average":{"1min":3.887,"5min":1.4390000000000001,"15min":1.234},"min":{"1min":0.30599999999999999,"5min":0.27800000000000002,"15min":0.23000000000000001},"max":{"1min":63.133000000000003,"5min":63.133000000000003,"15min":63.133000000000003},"last":1.4339999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:45:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.974,"5min":14.974,"15min":14.974},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.6280000000000001,"15min":2.6280000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":16.443000000000001,"5min":16.443000000000001,"15min":16.443000000000001},"last":0.60899999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:44:48 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.70499999999999996,"15min":2.2829999999999999},"min":{"1min":0.28599999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.5950000000000002,"5min":7.9699999999999998,"15min":138.191},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.629,"15min":2.129},"min":{"1min":0.29599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.0699999999999998,"5min":3.5910000000000002,"15min":140.33500000000001},"last":0.63900000000000001}]}]},{"osd":1,"up_from":301,"seq":1292785156244,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3903616,"kb_used_data":3302648,"kb_used_omap":9,"kb_used_meta":600950,"kb_avail":89841536,"statfs":{"total":95995035648,"available":91997732864,"internally_reserved":0,"allocated":3381911552,"data_stored":3378998283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":615372818},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:44:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.9019999999999999,"15min":3.173},"min":{"1min":0.22,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":4.8310000000000004,"5min":50.341000000000001,"15min":138.43799999999999},"last":6.6740000000000004},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.7629999999999999,"15min":3.1480000000000001},"min":{"1min":0.26500000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.52,"5min":50.468000000000004,"15min":138.482},"last":6.9039999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:44:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.802,"15min":2.3679999999999999},"min":{"1min":0.22500000000000001,"5min":0.184,"15min":0.184},"max":{"1min":3.903,"5min":34.939999999999998,"15min":34.939999999999998},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.8049999999999999,"15min":2.5670000000000002},"min":{"1min":0.314,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.5750000000000002,"5min":35.005000000000003,"15min":59.078000000000003},"last":2.9060000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:44:53 2023","interfaces":[{"interface":"back","average":{"1min":13.273,"5min":4.9379999999999997,"15min":6.8170000000000002},"min":{"1min":0.33500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":199.41,"5min":199.41,"15min":199.41},"last":2.581},{"interface":"front","average":{"1min":13.362,"5min":5,"15min":6.9489999999999998},"min":{"1min":0.28999999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":199.58099999999999,"5min":199.58099999999999,"15min":199.58099999999999},"last":2.9420000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:44:53 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.81799999999999995,"15min":2.1840000000000002},"min":{"1min":0.35999999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.929,"5min":4.4660000000000002,"15min":142.77099999999999},"last":2.9780000000000002},{"interface":"front","average":{"1min":1.611,"5min":1.0740000000000001,"15min":2.3220000000000001},"min":{"1min":0.441,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.7779999999999996,"5min":8.0830000000000002,"15min":144.898},"last":2.2509999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.9339999999999999,"15min":1.9339999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.157,"5min":7.157,"15min":7.157},"last":2.2040000000000002},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":5.3810000000000002,"5min":5.3810000000000002,"15min":5.3810000000000002},"last":2.8319999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:44:53 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":0.90000000000000002,"15min":7.508},"min":{"1min":0.40300000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.2629999999999999,"5min":4.1360000000000001,"15min":209.221},"last":3.024},{"interface":"front","average":{"1min":0.998,"5min":0.92000000000000004,"15min":7.4409999999999998},"min":{"1min":0.49099999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.8719999999999999,"5min":4.1639999999999997,"15min":208.43600000000001},"last":3.0750000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:44:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":1.137,"15min":5.3440000000000003},"min":{"1min":0.38500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.964,"5min":9.298,"15min":165.59399999999999},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.181,"15min":5.4359999999999999},"min":{"1min":0.42899999999999999,"5min":0.316,"15min":0.316},"max":{"1min":3.0219999999999998,"5min":9.3399999999999999,"15min":165.833},"last":1.038}]}]},{"osd":0,"up_from":94,"seq":403726926142,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1776008,"kb_used_data":1167296,"kb_used_omap":5,"kb_used_meta":608698,"kb_avail":91969144,"statfs":{"total":95995035648,"available":94176403456,"internally_reserved":0,"allocated":1195311104,"data_stored":1192160407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":623307005},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:44:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.1360000000000001,"15min":2.3780000000000001},"min":{"1min":0.22600000000000001,"5min":0.191,"15min":0.17799999999999999},"max":{"1min":4.258,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.877},{"interface":"front","average":{"1min":1.0700000000000001,"5min":2.3149999999999999,"15min":4.9589999999999996},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.214},"max":{"1min":5.9779999999999998,"5min":57.140999999999998,"15min":138.92400000000001},"last":1.4299999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":4.9539999999999997,"15min":3.1120000000000001},"min":{"1min":0.246,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.3650000000000002,"5min":127.73399999999999,"15min":127.73399999999999},"last":1.282},{"interface":"front","average":{"1min":1.9470000000000001,"5min":5.125,"15min":3.2919999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":8.407,"5min":127.69,"15min":127.69},"last":5.0010000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:45:08 2023","interfaces":[{"interface":"back","average":{"1min":4.9370000000000003,"5min":4.5549999999999997,"15min":5.0339999999999998},"min":{"1min":0.255,"5min":0.23699999999999999,"15min":0.221},"max":{"1min":80.126999999999995,"5min":125.562,"15min":218.06999999999999},"last":4.6219999999999999},{"interface":"front","average":{"1min":5.5439999999999996,"5min":4.3659999999999997,"15min":4.1669999999999998},"min":{"1min":0.34599999999999997,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":80.838999999999999,"5min":117.295,"15min":218.63200000000001},"last":0.63300000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:45:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":2.552,"15min":1.883},"min":{"1min":0.33900000000000002,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":2.391,"5min":72.403999999999996,"15min":72.403999999999996},"last":1.3340000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":2.3780000000000001,"15min":2.6150000000000002},"min":{"1min":0.35299999999999998,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":3.7410000000000001,"5min":72.162999999999997,"15min":136.262},"last":4.5119999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.411,"5min":3.411,"15min":3.411},"last":4.5780000000000003},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.6099999999999999,"5min":3.6099999999999999,"15min":3.6099999999999999},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.696,"15min":3.4540000000000002},"min":{"1min":0.38300000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.54,"5min":57.384,"15min":214.785},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.9850000000000001,"15min":3.6560000000000001},"min":{"1min":0.48499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.2539999999999996,"5min":57.109000000000002,"15min":214.93899999999999},"last":1.361}]},{"osd":7,"last update":"Sat Sep 30 09:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.6379999999999999,"15min":2.8900000000000001},"min":{"1min":0.42699999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.7349999999999999,"5min":57.776000000000003,"15min":150.96899999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.97,"15min":3.0299999999999998},"min":{"1min":0.42199999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.3070000000000004,"5min":57.179000000000002,"15min":150.328},"last":2.6240000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599083,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2208788,"kb_used_data":1421132,"kb_used_omap":5,"kb_used_meta":787642,"kb_avail":91536364,"statfs":{"total":95995035648,"available":93733236736,"internally_reserved":0,"allocated":1455239168,"data_stored":1452364861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":806546061},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:44:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6580000000000004,"5min":2.8570000000000002,"15min":2.468},"min":{"1min":0.192,"5min":0.192,"15min":0.187},"max":{"1min":76.594999999999999,"5min":76.647000000000006,"15min":77.379000000000005},"last":3.6880000000000002},{"interface":"front","average":{"1min":4.3769999999999998,"5min":2.7050000000000001,"15min":2.4129999999999998},"min":{"1min":0.224,"5min":0.217,"15min":0.20499999999999999},"max":{"1min":77.388999999999996,"5min":77.444000000000003,"15min":77.444000000000003},"last":3.8820000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:44:49 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.7569999999999999,"15min":3.125},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.6779999999999999,"5min":28.225000000000001,"15min":64.412999999999997},"last":3.952},{"interface":"front","average":{"1min":1.5309999999999999,"5min":1.821,"15min":2.8239999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.1809999999999992,"5min":27.975999999999999,"15min":63.247999999999998},"last":3.8149999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:45:07 2023","interfaces":[{"interface":"back","average":{"1min":4.1849999999999996,"5min":3.827,"15min":4.5220000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":49.994999999999997,"5min":80.052999999999997,"15min":134.078},"last":7.3040000000000003},{"interface":"front","average":{"1min":4.4870000000000001,"5min":3.754,"15min":4.4470000000000001},"min":{"1min":0.29099999999999998,"5min":0.224,"15min":0.224},"max":{"1min":50.576999999999998,"5min":80.111000000000004,"15min":135.73400000000001},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:44:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.88700000000000001,"15min":1.5700000000000001},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":83.012},"last":3.7810000000000001},{"interface":"front","average":{"1min":1.115,"5min":0.84699999999999998,"15min":1.5720000000000001},"min":{"1min":0.34000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.6289999999999996,"5min":7.6289999999999996,"15min":81.335999999999999},"last":3.6469999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.706,"5min":2.706,"15min":2.706},"last":3.843},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.4449999999999998,"5min":3.4449999999999998,"15min":3.4449999999999998},"last":4.3789999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:44:43 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.96899999999999997,"15min":3.169},"min":{"1min":0.35799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6710000000000003,"5min":7.6710000000000003,"15min":154.417},"last":3.7170000000000001},{"interface":"front","average":{"1min":1.272,"5min":1.2709999999999999,"15min":3.4820000000000002},"min":{"1min":0.372,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.8409999999999993,"5min":9.8409999999999993,"15min":154.04400000000001},"last":3.923}]},{"osd":7,"last update":"Sat Sep 30 09:44:43 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1,"15min":2.895},"min":{"1min":0.38700000000000001,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":7.6989999999999998,"5min":7.6989999999999998,"15min":213.404},"last":3.742},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.3839999999999999,"15min":3.2440000000000002},"min":{"1min":0.41299999999999998,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":7.9039999999999999,"5min":11.074999999999999,"15min":213.27199999999999},"last":4.4370000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435556,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3674244,"kb_used_data":2851900,"kb_used_omap":2,"kb_used_meta":822333,"kb_avail":90070908,"statfs":{"total":95995035648,"available":92232609792,"internally_reserved":0,"allocated":2920345600,"data_stored":2917366514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":842069351},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.6189999999999998,"15min":4.4550000000000001},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.183},"max":{"1min":38.412999999999997,"5min":144.72,"15min":205.73099999999999},"last":0.31},{"interface":"front","average":{"1min":2.673,"5min":5.0030000000000001,"15min":4.4820000000000002},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":34.918999999999997,"5min":142.36600000000001,"15min":205.495},"last":5.1139999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:44:47 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":2.7130000000000001,"15min":9.8239999999999998},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":10.395,"5min":142.434,"15min":273.99599999999998},"last":5.3979999999999997},{"interface":"front","average":{"1min":1.0529999999999999,"5min":2.6299999999999999,"15min":9.1669999999999998},"min":{"1min":0.35399999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.2220000000000004,"5min":142.44900000000001,"15min":274.21600000000001},"last":0.83799999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":5.9569999999999999,"15min":5.5629999999999997},"min":{"1min":0.30499999999999999,"5min":0.249,"15min":0.249},"max":{"1min":10.301,"5min":146.92599999999999,"15min":206.69},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.534,"5min":5.827,"15min":5.3799999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23200000000000001},"max":{"1min":8.4060000000000006,"5min":145.75299999999999,"15min":209.22800000000001},"last":0.71399999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":4.4550000000000001,"5min":1.8899999999999999,"15min":2.0649999999999999},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.27200000000000002},"max":{"1min":81.677000000000007,"5min":81.677000000000007,"15min":81.677000000000007},"last":0.373},{"interface":"front","average":{"1min":4.2699999999999996,"5min":1.9470000000000001,"15min":1.841},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":80.623000000000005,"5min":80.623000000000005,"15min":80.623000000000005},"last":5.1660000000000004}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.1989999999999998,"5min":9.1989999999999998,"15min":9.1989999999999998},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.2719999999999998,"5min":2.2719999999999998,"15min":2.2719999999999998},"last":5.1950000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.0309999999999999,"15min":1.9319999999999999},"min":{"1min":0.41899999999999998,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":5.7690000000000001,"5min":5.915,"15min":60.698},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.093,"15min":1.8320000000000001},"min":{"1min":0.437,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":2.9249999999999998,"5min":10.356999999999999,"15min":12.121},"last":5.2270000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":9.952,"5min":2.7530000000000001,"15min":2.5499999999999998},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":213.548,"5min":213.548,"15min":213.548},"last":0.51500000000000001},{"interface":"front","average":{"1min":9.6460000000000008,"5min":2.7010000000000001,"15min":2.4980000000000002},"min":{"1min":0.44,"5min":0.41099999999999998,"15min":0.34799999999999998},"max":{"1min":213.583,"5min":213.583,"15min":213.583},"last":0.63200000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019319,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1557736,"kb_used_data":1088224,"kb_used_omap":0,"kb_used_meta":469504,"kb_avail":92187416,"statfs":{"total":95995035648,"available":94399913984,"internally_reserved":0,"allocated":1114341376,"data_stored":1112649514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":480772096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:44:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":1.756,"15min":2.3700000000000001},"min":{"1min":0.25,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":31.863,"5min":73.718000000000004,"15min":105.179},"last":34.991999999999997},{"interface":"front","average":{"1min":2.286,"5min":1.8560000000000001,"15min":2.5099999999999998},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":31.98,"5min":74.001999999999995,"15min":104.70999999999999},"last":35.274999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:44:55 2023","interfaces":[{"interface":"back","average":{"1min":5.3440000000000003,"5min":1.6830000000000001,"15min":4.7549999999999999},"min":{"1min":0.253,"5min":0.25,"15min":0.25},"max":{"1min":80.712000000000003,"5min":80.712000000000003,"15min":160.69999999999999},"last":30.538},{"interface":"front","average":{"1min":5.3040000000000003,"5min":1.819,"15min":4.7149999999999999},"min":{"1min":0.30199999999999999,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":80.810000000000002,"5min":80.810000000000002,"15min":160.50700000000001},"last":30.722000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:44:46 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":3.3290000000000002,"15min":2.7050000000000001},"min":{"1min":0.35699999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":30.013999999999999,"5min":142.88999999999999,"15min":142.88999999999999},"last":117.858},{"interface":"front","average":{"1min":2.4540000000000002,"5min":3.258,"15min":2.7519999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":29.977,"5min":142.90700000000001,"15min":142.90700000000001},"last":117.871}]},{"osd":3,"last update":"Sat Sep 30 09:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.96999999999999997,"15min":2.1800000000000002},"min":{"1min":0.39900000000000002,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":2.0800000000000001,"5min":4.8979999999999997,"15min":70.241},"last":0.627},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.95399999999999996,"15min":2.1469999999999998},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.093,"5min":4.9080000000000004,"15min":70.433999999999997},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:45:25 2023","interfaces":[{"interface":"back","average":{"1min":10.734,"5min":10.734,"15min":10.734},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":140.11099999999999,"5min":140.11099999999999,"15min":140.11099999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":10.766,"5min":10.766,"15min":10.766},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":140.065,"5min":140.065,"15min":140.065},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.53800000000000003,"15min":2.1850000000000001},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":4.4299999999999997,"5min":4.4299999999999997,"15min":202.72499999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.61099999999999999,"15min":2.169},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.249},"max":{"1min":5.7199999999999998,"5min":5.7199999999999998,"15min":202.42400000000001},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:44:46 2023","interfaces":[{"interface":"back","average":{"1min":8.3230000000000004,"5min":2.3090000000000002,"15min":3.6920000000000002},"min":{"1min":0.317,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":161.72999999999999,"5min":161.72999999999999,"15min":230.90100000000001},"last":0.434},{"interface":"front","average":{"1min":8.2560000000000002,"5min":2.2469999999999999,"15min":3.5059999999999998},"min":{"1min":0.34999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":162.60900000000001,"5min":162.60900000000001,"15min":229.97},"last":0.55400000000000005}]}]},{"osd":5,"up_from":489,"seq":2100239007761,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3347044,"kb_used_data":3218076,"kb_used_omap":6,"kb_used_meta":128953,"kb_avail":90398108,"statfs":{"total":95995035648,"available":92567662592,"internally_reserved":0,"allocated":3295309824,"data_stored":3293618584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":132047967},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":26.309999999999999,"5min":26.309999999999999,"15min":26.309999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":281.10599999999999,"5min":281.10599999999999,"15min":281.10599999999999},"last":2.0790000000000002},{"interface":"front","average":{"1min":26.599,"5min":26.599,"15min":26.599},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":282.07400000000001,"5min":282.07400000000001,"15min":282.07400000000001},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":14.92,"5min":14.92,"15min":14.92},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":273.48200000000003,"5min":273.48200000000003,"15min":273.48200000000003},"last":2.0529999999999999},{"interface":"front","average":{"1min":14.105,"5min":14.105,"15min":14.105},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":273.50200000000001,"5min":273.50200000000001,"15min":273.50200000000001},"last":1.885}]},{"osd":2,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.0469999999999997,"5min":7.0469999999999997,"15min":7.0469999999999997},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":4.5999999999999996,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":1.244}]},{"osd":3,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":14.749000000000001,"5min":14.749000000000001,"15min":14.749000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":274.01799999999997,"5min":274.01799999999997,"15min":274.01799999999997},"last":1.712},{"interface":"front","average":{"1min":1.732,"5min":1.732,"15min":1.732},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.5830000000000002,"5min":6.5830000000000002,"15min":6.5830000000000002},"last":1.73}]},{"osd":4,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.1769999999999996,"5min":4.1769999999999996,"15min":4.1769999999999996},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.9900000000000002,"5min":3.9900000000000002,"15min":3.9900000000000002},"last":0.33100000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.645,"15min":1.645},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":14.371,"5min":14.371,"15min":14.371},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":14.454000000000001,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.294,"15min":1.294},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.4409999999999998,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.0350000000000001,"5min":3.0350000000000001,"15min":3.0350000000000001},"last":0.432}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1187905536,"data_stored":1187905536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3371958272,"data_stored":3371958272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1459290112,"data_stored":1459290112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2929393664,"data_stored":2929393664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1115226112,"data_stored":1115226112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3315531776,"data_stored":3315531776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1109655552,"data_stored":1109655552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3366125568,"data_stored":3366125568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:45:52.121 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:45:52.169 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:45:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:52.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:45:52.278 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:52.318 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:52.935 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 89 16 96119 96103 67.4526 41.875 0.0376632 0.0148188 2023-09-30T09:45:52.957 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:45:53.006 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:45:53.073 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:45:53.117 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:45:53.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:45:53.205 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:45:53.246 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:53.251 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:52 smithi167 ceph-mon[192880]: pgmap v1320: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 751 op/s; 4306/281454 objects misplaced (1.530%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:53.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:53.259 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:45:53.284 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:53.308 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:45:53.310 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:45:53.310 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:45:53.311 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:45:53.311 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:45:53.311 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:45:53.330 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:45:53.330 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:45:53.330 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:45:53.330 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:45:53.330 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:45:53.330 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:45:53.330 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:45:53.330 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:45:53.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:52 smithi019 ceph-mon[224043]: pgmap v1320: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 751 op/s; 4306/281454 objects misplaced (1.530%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:52 smithi019 ceph-mon[216815]: pgmap v1320: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 751 op/s; 4306/281454 objects misplaced (1.530%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:45:53.353 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:45:53.400 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:53.488 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:45:53.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:53.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:45:53.597 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:45:53.638 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:53.722 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:45:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:53.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:45:53.828 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:53.900 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:53.935 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total time run: 90.0094 2023-09-30T09:45:53.935 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total writes made: 96742 2023-09-30T09:45:53.935 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Write size: 65536 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object size: 65536 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Bandwidth (MB/sec): 67.1749 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Bandwidth: 30.506 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max bandwidth (MB/sec): 177.625 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min bandwidth (MB/sec): 0 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average IOPS: 1074 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev IOPS: 488.108 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max IOPS: 2842 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min IOPS: 0 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average Latency(s): 0.0148826 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Latency(s): 0.0476019 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max latency(s): 2.17303 2023-09-30T09:45:53.936 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min latency(s): 0.00110418 2023-09-30T09:45:53.940 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Cleaning up (deleting benchmark objects) 2023-09-30T09:45:54.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2402595045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:54.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:45:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:45:53] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.43.0" 2023-09-30T09:45:54.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2402595045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:54.131 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:45:54.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2402595045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:45:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:54 smithi167 ceph-mon[192880]: pgmap v1321: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 4188/283332 objects misplaced (1.478%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:54 smithi019 ceph-mon[216815]: pgmap v1321: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 4188/283332 objects misplaced (1.478%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:55.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:54 smithi019 ceph-mon[224043]: pgmap v1321: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 4188/283332 objects misplaced (1.478%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:55.776 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:55.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:55.886 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:45:55.944 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:56.150 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:45:56.541 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:45:56.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:56.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:45:56.653 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:45:56.773 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:56.883 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:45:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:56.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:45:56.988 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:57.065 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:57.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:56 smithi019 ceph-mon[224043]: pgmap v1322: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 716 op/s; 4188/288528 objects misplaced (1.452%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:57.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:57.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:57.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:56 smithi019 ceph-mon[216815]: pgmap v1322: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 716 op/s; 4188/288528 objects misplaced (1.452%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:57.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:57.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:57.110 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:57.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:57.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:57.217 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:57.257 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:56 smithi167 ceph-mon[192880]: pgmap v1322: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 716 op/s; 4188/288528 objects misplaced (1.452%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:57.257 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:57.257 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:45:57.265 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:57.344 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:57.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:57.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:57.453 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:57.497 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:57.574 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:57.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:57.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:57.628 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:57.682 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:57.719 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:57.782 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:45:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:57.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:45:57.810 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:45:57.890 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:57.970 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:58.151 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:58.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:58.254 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:45:58.296 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:58.359 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:45:58.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:58.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:45:58.446 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:45:58.466 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:45:58.646 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:45:59.084 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:45:59.100 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:45:59.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:58 smithi167 ceph-mon[192880]: pgmap v1323: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 716 op/s; 4188/288528 objects misplaced (1.452%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:59.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:58 smithi019 ceph-mon[216815]: pgmap v1323: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 716 op/s; 4188/288528 objects misplaced (1.452%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:59.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:58 smithi019 ceph-mon[224043]: pgmap v1323: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 716 op/s; 4188/288528 objects misplaced (1.452%); 1.4 MiB/s, 22 objects/s recovering 2023-09-30T09:45:59.821 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:45:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:45:59.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:45:59.929 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:45:59.978 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:00.061 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:00.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:00.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:45:59 smithi019 ceph-mon[216815]: from='client.46337 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:00.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:45:59 smithi019 ceph-mon[224043]: from='client.46337 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:00.174 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:00.263 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:45:59 smithi167 ceph-mon[192880]: from='client.46337 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:00.716 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:00.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:00.745 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1323,"stamp":"2023-09-30T09:45:57.790819+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6305879999,"num_objects":96176,"num_object_clones":0,"num_object_copies":288528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4188,"num_objects_unfound":0,"num_objects_dirty":96176,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":98211,"num_write_kb":6203126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2205,"num_bytes_recovered":138761842,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102326,"ondisk_log_size":102326,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23078196,"kb_used_data":18577140,"kb_used_omap":30,"kb_used_meta":4500449,"kb_avail":726883020,"statfs":{"total":767960285184,"available":744328212480,"internally_reserved":0,"allocated":19022991360,"data_stored":19004250940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4608460130},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":394002456,"num_objects":6013,"num_object_clones":0,"num_object_copies":18039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-228,"num_objects_unfound":0,"num_objects_dirty":6013,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7169,"num_write_kb":421761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":228,"num_bytes_recovered":14942208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008935"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":936,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.872917+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:45:38.872917+0000","last_peered":"2023-09-30T09:45:38.872917+0000","last_clean":"2023-09-30T09:45:38.872917+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:45:38.872917+0000","last_undegraded":"2023-09-30T09:45:38.872917+0000","last_fullsized":"2023-09-30T09:45:38.872917+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":531,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.883995+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:45:38.883995+0000","last_peered":"2023-09-30T09:45:38.883995+0000","last_clean":"2023-09-30T09:45:38.883995+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:45:38.883995+0000","last_undegraded":"2023-09-30T09:45:38.883995+0000","last_fullsized":"2023-09-30T09:45:38.883995+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":428,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.885988+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:45:38.885988+0000","last_peered":"2023-09-30T09:45:38.885988+0000","last_clean":"2023-09-30T09:45:38.885988+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:45:38.885988+0000","last_undegraded":"2023-09-30T09:45:38.885988+0000","last_fullsized":"2023-09-30T09:45:38.885988+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":938,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.873043+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:45:38.873043+0000","last_peered":"2023-09-30T09:45:38.873043+0000","last_clean":"2023-09-30T09:45:38.873043+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:45:38.873043+0000","last_undegraded":"2023-09-30T09:45:38.873043+0000","last_fullsized":"2023-09-30T09:45:38.873043+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":77,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878496+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:45:38.878496+0000","last_peered":"2023-09-30T09:45:38.878496+0000","last_clean":"2023-09-30T09:45:38.878496+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:45:38.878496+0000","last_undegraded":"2023-09-30T09:45:38.878496+0000","last_fullsized":"2023-09-30T09:45:38.878496+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":118,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879216+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:45:38.879216+0000","last_peered":"2023-09-30T09:45:38.879216+0000","last_clean":"2023-09-30T09:45:38.879216+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:45:38.879216+0000","last_undegraded":"2023-09-30T09:45:38.879216+0000","last_fullsized":"2023-09-30T09:45:38.879216+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":746,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.884023+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:45:38.884023+0000","last_peered":"2023-09-30T09:45:38.884023+0000","last_clean":"2023-09-30T09:45:38.884023+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:45:38.884023+0000","last_undegraded":"2023-09-30T09:45:38.884023+0000","last_fullsized":"2023-09-30T09:45:38.884023+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":18,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878811+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:45:38.878811+0000","last_peered":"2023-09-30T09:45:38.878811+0000","last_clean":"2023-09-30T09:45:38.878811+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:45:38.878811+0000","last_undegraded":"2023-09-30T09:45:38.878811+0000","last_fullsized":"2023-09-30T09:45:38.878811+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":82,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887878+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:45:38.887878+0000","last_peered":"2023-09-30T09:45:38.887878+0000","last_clean":"2023-09-30T09:45:38.887878+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:45:38.887878+0000","last_undegraded":"2023-09-30T09:45:38.887878+0000","last_fullsized":"2023-09-30T09:45:38.887878+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":914,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877906+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:45:38.877906+0000","last_peered":"2023-09-30T09:45:38.877906+0000","last_clean":"2023-09-30T09:45:38.877906+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:45:38.877906+0000","last_undegraded":"2023-09-30T09:45:38.877906+0000","last_fullsized":"2023-09-30T09:45:38.877906+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":133,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879691+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:45:38.879691+0000","last_peered":"2023-09-30T09:45:38.879691+0000","last_clean":"2023-09-30T09:45:38.879691+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:45:38.879691+0000","last_undegraded":"2023-09-30T09:45:38.879691+0000","last_fullsized":"2023-09-30T09:45:38.879691+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":93,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879299+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:45:38.879299+0000","last_peered":"2023-09-30T09:45:38.879299+0000","last_clean":"2023-09-30T09:45:38.879299+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:45:38.879299+0000","last_undegraded":"2023-09-30T09:45:38.879299+0000","last_fullsized":"2023-09-30T09:45:38.879299+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":702,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880648+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:45:38.880648+0000","last_peered":"2023-09-30T09:45:38.880648+0000","last_clean":"2023-09-30T09:45:38.880648+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:45:38.880648+0000","last_undegraded":"2023-09-30T09:45:38.880648+0000","last_fullsized":"2023-09-30T09:45:38.880648+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":333,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.873058+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:45:38.873058+0000","last_peered":"2023-09-30T09:45:38.873058+0000","last_clean":"2023-09-30T09:45:38.873058+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:45:38.873058+0000","last_undegraded":"2023-09-30T09:45:38.873058+0000","last_fullsized":"2023-09-30T09:45:38.873058+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":469,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.876843+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:45:38.876843+0000","last_peered":"2023-09-30T09:45:38.876843+0000","last_clean":"2023-09-30T09:45:38.876843+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:45:38.876843+0000","last_undegraded":"2023-09-30T09:45:38.876843+0000","last_fullsized":"2023-09-30T09:45:38.876843+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":642,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.874126+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:45:38.874126+0000","last_peered":"2023-09-30T09:45:38.874126+0000","last_clean":"2023-09-30T09:45:38.874126+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:45:38.874126+0000","last_undegraded":"2023-09-30T09:45:38.874126+0000","last_fullsized":"2023-09-30T09:45:38.874126+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":227,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878062+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:45:38.878062+0000","last_peered":"2023-09-30T09:45:38.878062+0000","last_clean":"2023-09-30T09:45:38.878062+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:45:38.878062+0000","last_undegraded":"2023-09-30T09:45:38.878062+0000","last_fullsized":"2023-09-30T09:45:38.878062+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.888018+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:45:38.888018+0000","last_peered":"2023-09-30T09:45:38.888018+0000","last_clean":"2023-09-30T09:45:38.888018+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:45:38.888018+0000","last_undegraded":"2023-09-30T09:45:38.888018+0000","last_fullsized":"2023-09-30T09:45:38.888018+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":568,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879790+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:45:38.879790+0000","last_peered":"2023-09-30T09:45:38.879790+0000","last_clean":"2023-09-30T09:45:38.879790+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:45:38.879790+0000","last_undegraded":"2023-09-30T09:45:38.879790+0000","last_fullsized":"2023-09-30T09:45:38.879790+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":557,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887832+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:45:38.887832+0000","last_peered":"2023-09-30T09:45:38.887832+0000","last_clean":"2023-09-30T09:45:38.887832+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:45:38.887832+0000","last_undegraded":"2023-09-30T09:45:38.887832+0000","last_fullsized":"2023-09-30T09:45:38.887832+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":535,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.884552+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:45:38.884552+0000","last_peered":"2023-09-30T09:45:38.884552+0000","last_clean":"2023-09-30T09:45:38.884552+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:45:38.884552+0000","last_undegraded":"2023-09-30T09:45:38.884552+0000","last_fullsized":"2023-09-30T09:45:38.884552+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":766,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880098+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:45:38.880098+0000","last_peered":"2023-09-30T09:45:38.880098+0000","last_clean":"2023-09-30T09:45:38.880098+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:45:38.880098+0000","last_undegraded":"2023-09-30T09:45:38.880098+0000","last_fullsized":"2023-09-30T09:45:38.880098+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":163,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877313+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:45:38.877313+0000","last_peered":"2023-09-30T09:45:38.877313+0000","last_clean":"2023-09-30T09:45:38.877313+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:45:38.877313+0000","last_undegraded":"2023-09-30T09:45:38.877313+0000","last_fullsized":"2023-09-30T09:45:38.877313+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":490,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877694+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:45:38.877694+0000","last_peered":"2023-09-30T09:45:38.877694+0000","last_clean":"2023-09-30T09:45:38.877694+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:45:38.877694+0000","last_undegraded":"2023-09-30T09:45:38.877694+0000","last_fullsized":"2023-09-30T09:45:38.877694+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":814,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.872817+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:45:38.872817+0000","last_peered":"2023-09-30T09:45:38.872817+0000","last_clean":"2023-09-30T09:45:38.872817+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:45:38.872817+0000","last_undegraded":"2023-09-30T09:45:38.872817+0000","last_fullsized":"2023-09-30T09:45:38.872817+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":92,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878759+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:45:38.878759+0000","last_peered":"2023-09-30T09:45:38.878759+0000","last_clean":"2023-09-30T09:45:38.878759+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:45:38.878759+0000","last_undegraded":"2023-09-30T09:45:38.878759+0000","last_fullsized":"2023-09-30T09:45:38.878759+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":188,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.873405+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:45:38.873405+0000","last_peered":"2023-09-30T09:45:38.873405+0000","last_clean":"2023-09-30T09:45:38.873405+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:45:38.873405+0000","last_undegraded":"2023-09-30T09:45:38.873405+0000","last_fullsized":"2023-09-30T09:45:38.873405+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":914,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878904+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:45:38.878904+0000","last_peered":"2023-09-30T09:45:38.878904+0000","last_clean":"2023-09-30T09:45:38.878904+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:45:38.878904+0000","last_undegraded":"2023-09-30T09:45:38.878904+0000","last_fullsized":"2023-09-30T09:45:38.878904+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":197,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.876782+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:45:38.876782+0000","last_peered":"2023-09-30T09:45:38.876782+0000","last_clean":"2023-09-30T09:45:38.876782+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:45:38.876782+0000","last_undegraded":"2023-09-30T09:45:38.876782+0000","last_fullsized":"2023-09-30T09:45:38.876782+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878955+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:45:38.878955+0000","last_peered":"2023-09-30T09:45:38.878955+0000","last_clean":"2023-09-30T09:45:38.878955+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:45:38.878955+0000","last_undegraded":"2023-09-30T09:45:38.878955+0000","last_fullsized":"2023-09-30T09:45:38.878955+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":923,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877101+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:45:38.877101+0000","last_peered":"2023-09-30T09:45:38.877101+0000","last_clean":"2023-09-30T09:45:38.877101+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:45:38.877101+0000","last_undegraded":"2023-09-30T09:45:38.877101+0000","last_fullsized":"2023-09-30T09:45:38.877101+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":109,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887170+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:45:38.887170+0000","last_peered":"2023-09-30T09:45:38.887170+0000","last_clean":"2023-09-30T09:45:38.887170+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:45:38.887170+0000","last_undegraded":"2023-09-30T09:45:38.887170+0000","last_fullsized":"2023-09-30T09:45:38.887170+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":77,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879325+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:45:38.879325+0000","last_peered":"2023-09-30T09:45:38.879325+0000","last_clean":"2023-09-30T09:45:38.879325+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:45:38.879325+0000","last_undegraded":"2023-09-30T09:45:38.879325+0000","last_fullsized":"2023-09-30T09:45:38.879325+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"513'6003","reported_seq":6305,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:54.118263+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:45:54.118263+0000","last_peered":"2023-09-30T09:45:54.118263+0000","last_clean":"2023-09-30T09:45:54.118263+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:45:54.118263+0000","last_undegraded":"2023-09-30T09:45:54.118263+0000","last_fullsized":"2023-09-30T09:45:54.118263+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6003,"log_dups_size":0,"ondisk_log_size":6003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391315456,"num_objects":5971,"num_object_clones":0,"num_object_copies":17913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6003,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":889,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877992+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:45:38.877992+0000","last_peered":"2023-09-30T09:45:38.877992+0000","last_clean":"2023-09-30T09:45:38.877992+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:45:38.877992+0000","last_undegraded":"2023-09-30T09:45:38.877992+0000","last_fullsized":"2023-09-30T09:45:38.877992+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":189,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877859+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:45:38.877859+0000","last_peered":"2023-09-30T09:45:38.877859+0000","last_clean":"2023-09-30T09:45:38.877859+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:45:38.877859+0000","last_undegraded":"2023-09-30T09:45:38.877859+0000","last_fullsized":"2023-09-30T09:45:38.877859+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":134,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.872746+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:45:38.872746+0000","last_peered":"2023-09-30T09:45:38.872746+0000","last_clean":"2023-09-30T09:45:38.872746+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:45:38.872746+0000","last_undegraded":"2023-09-30T09:45:38.872746+0000","last_fullsized":"2023-09-30T09:45:38.872746+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"513'6044","reported_seq":6082,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:55.265811+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:45:55.265811+0000","last_peered":"2023-09-30T09:45:55.265811+0000","last_clean":"2023-09-30T09:45:55.265811+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:45:55.265811+0000","last_undegraded":"2023-09-30T09:45:55.265811+0000","last_fullsized":"2023-09-30T09:45:55.265811+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6044,"log_dups_size":0,"ondisk_log_size":6044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386662400,"num_objects":5900,"num_object_clones":0,"num_object_copies":17700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6044,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":906,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880770+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:45:38.880770+0000","last_peered":"2023-09-30T09:45:38.880770+0000","last_clean":"2023-09-30T09:45:38.880770+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:45:38.880770+0000","last_undegraded":"2023-09-30T09:45:38.880770+0000","last_fullsized":"2023-09-30T09:45:38.880770+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":159,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878015+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:45:38.878015+0000","last_peered":"2023-09-30T09:45:38.878015+0000","last_clean":"2023-09-30T09:45:38.878015+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:45:38.878015+0000","last_undegraded":"2023-09-30T09:45:38.878015+0000","last_fullsized":"2023-09-30T09:45:38.878015+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":564,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.884094+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:45:38.884094+0000","last_peered":"2023-09-30T09:45:38.884094+0000","last_clean":"2023-09-30T09:45:38.884094+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:45:38.884094+0000","last_undegraded":"2023-09-30T09:45:38.884094+0000","last_fullsized":"2023-09-30T09:45:38.884094+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"513'6105","reported_seq":6368,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:54.085948+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:45:54.085948+0000","last_peered":"2023-09-30T09:45:54.085948+0000","last_clean":"2023-09-30T09:45:54.085948+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:45:54.085948+0000","last_undegraded":"2023-09-30T09:45:54.085948+0000","last_fullsized":"2023-09-30T09:45:54.085948+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6105,"log_dups_size":0,"ondisk_log_size":6105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398262272,"num_objects":6077,"num_object_clones":0,"num_object_copies":18231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6105,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":683,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.888050+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:45:38.888050+0000","last_peered":"2023-09-30T09:45:38.888050+0000","last_clean":"2023-09-30T09:45:38.888050+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:45:38.888050+0000","last_undegraded":"2023-09-30T09:45:38.888050+0000","last_fullsized":"2023-09-30T09:45:38.888050+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":226,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877613+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:45:38.877613+0000","last_peered":"2023-09-30T09:45:38.877613+0000","last_clean":"2023-09-30T09:45:38.877613+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:45:38.877613+0000","last_undegraded":"2023-09-30T09:45:38.877613+0000","last_fullsized":"2023-09-30T09:45:38.877613+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":82,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.888103+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:45:38.888103+0000","last_peered":"2023-09-30T09:45:38.888103+0000","last_clean":"2023-09-30T09:45:38.888103+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:45:38.888103+0000","last_undegraded":"2023-09-30T09:45:38.888103+0000","last_fullsized":"2023-09-30T09:45:38.888103+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"513'5943","reported_seq":6296,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:53.618045+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:45:53.618045+0000","last_peered":"2023-09-30T09:45:53.618045+0000","last_clean":"2023-09-30T09:45:53.618045+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:45:53.618045+0000","last_undegraded":"2023-09-30T09:45:53.618045+0000","last_fullsized":"2023-09-30T09:45:53.618045+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389480448,"num_objects":5943,"num_object_clones":0,"num_object_copies":17829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5943,"num_write_kb":380352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":748,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.886007+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:45:38.886007+0000","last_peered":"2023-09-30T09:45:38.886007+0000","last_clean":"2023-09-30T09:45:38.886007+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:45:38.886007+0000","last_undegraded":"2023-09-30T09:45:38.886007+0000","last_fullsized":"2023-09-30T09:45:38.886007+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":448,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.873132+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:45:38.873132+0000","last_peered":"2023-09-30T09:45:38.873132+0000","last_clean":"2023-09-30T09:45:38.873132+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:45:38.873132+0000","last_undegraded":"2023-09-30T09:45:38.873132+0000","last_fullsized":"2023-09-30T09:45:38.873132+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":112,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879552+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:45:38.879552+0000","last_peered":"2023-09-30T09:45:38.879552+0000","last_clean":"2023-09-30T09:45:38.879552+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:45:38.879552+0000","last_undegraded":"2023-09-30T09:45:38.879552+0000","last_fullsized":"2023-09-30T09:45:38.879552+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"513'6086","reported_seq":6124,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:54.214720+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:45:54.214720+0000","last_peered":"2023-09-30T09:45:54.214720+0000","last_clean":"2023-09-30T09:45:54.214720+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:45:54.214720+0000","last_undegraded":"2023-09-30T09:45:54.214720+0000","last_fullsized":"2023-09-30T09:45:54.214720+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6086,"log_dups_size":0,"ondisk_log_size":6086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396230656,"num_objects":6046,"num_object_clones":0,"num_object_copies":18138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6086,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":995,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.876702+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:45:38.876702+0000","last_peered":"2023-09-30T09:45:38.876702+0000","last_clean":"2023-09-30T09:45:38.876702+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:45:38.876702+0000","last_undegraded":"2023-09-30T09:45:38.876702+0000","last_fullsized":"2023-09-30T09:45:38.876702+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":71,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879340+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:45:38.879340+0000","last_peered":"2023-09-30T09:45:38.879340+0000","last_clean":"2023-09-30T09:45:38.879340+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:45:38.879340+0000","last_undegraded":"2023-09-30T09:45:38.879340+0000","last_fullsized":"2023-09-30T09:45:38.879340+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":511,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887906+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:45:38.887906+0000","last_peered":"2023-09-30T09:45:38.887906+0000","last_clean":"2023-09-30T09:45:38.887906+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:45:38.887906+0000","last_undegraded":"2023-09-30T09:45:38.887906+0000","last_fullsized":"2023-09-30T09:45:38.887906+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"513'5998","reported_seq":6036,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:54.218820+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:45:54.218820+0000","last_peered":"2023-09-30T09:45:54.218820+0000","last_clean":"2023-09-30T09:45:54.218820+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:45:54.218820+0000","last_undegraded":"2023-09-30T09:45:54.218820+0000","last_fullsized":"2023-09-30T09:45:54.218820+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5998,"log_dups_size":0,"ondisk_log_size":5998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390332416,"num_objects":5956,"num_object_clones":0,"num_object_copies":17868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5998,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":993,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880545+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:45:38.880545+0000","last_peered":"2023-09-30T09:45:38.880545+0000","last_clean":"2023-09-30T09:45:38.880545+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:45:38.880545+0000","last_undegraded":"2023-09-30T09:45:38.880545+0000","last_fullsized":"2023-09-30T09:45:38.880545+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":68,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.888143+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:45:38.888143+0000","last_peered":"2023-09-30T09:45:38.888143+0000","last_clean":"2023-09-30T09:45:38.888143+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:45:38.888143+0000","last_undegraded":"2023-09-30T09:45:38.888143+0000","last_fullsized":"2023-09-30T09:45:38.888143+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":207,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.885479+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:45:38.885479+0000","last_peered":"2023-09-30T09:45:38.885479+0000","last_clean":"2023-09-30T09:45:38.885479+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:45:38.885479+0000","last_undegraded":"2023-09-30T09:45:38.885479+0000","last_fullsized":"2023-09-30T09:45:38.885479+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"513'6236","reported_seq":6274,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:55.624518+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:45:55.624518+0000","last_peered":"2023-09-30T09:45:55.624518+0000","last_clean":"2023-09-30T09:45:55.624518+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:45:55.624518+0000","last_undegraded":"2023-09-30T09:45:55.624518+0000","last_fullsized":"2023-09-30T09:45:55.624518+0000","mapping_epoch":483,"log_start":"496'400","ondisk_log_start":"496'400","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5836,"log_dups_size":0,"ondisk_log_size":5836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395444224,"num_objects":6034,"num_object_clones":0,"num_object_copies":18102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6236,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":851,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879637+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:45:38.879637+0000","last_peered":"2023-09-30T09:45:38.879637+0000","last_clean":"2023-09-30T09:45:38.879637+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:45:38.879637+0000","last_undegraded":"2023-09-30T09:45:38.879637+0000","last_fullsized":"2023-09-30T09:45:38.879637+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":518,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.883926+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:45:38.883926+0000","last_peered":"2023-09-30T09:45:38.883926+0000","last_clean":"2023-09-30T09:45:38.883926+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:45:38.883926+0000","last_undegraded":"2023-09-30T09:45:38.883926+0000","last_fullsized":"2023-09-30T09:45:38.883926+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":525,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.886130+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:45:38.886130+0000","last_peered":"2023-09-30T09:45:38.886130+0000","last_clean":"2023-09-30T09:45:38.886130+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:45:38.886130+0000","last_undegraded":"2023-09-30T09:45:38.886130+0000","last_fullsized":"2023-09-30T09:45:38.886130+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"513'5961","reported_seq":6002,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:53.721612+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:45:53.721612+0000","last_peered":"2023-09-30T09:45:53.721612+0000","last_clean":"2023-09-30T09:45:53.721612+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:45:53.721612+0000","last_undegraded":"2023-09-30T09:45:53.721612+0000","last_fullsized":"2023-09-30T09:45:53.721612+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5961,"log_dups_size":0,"ondisk_log_size":5961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390660096,"num_objects":5961,"num_object_clones":0,"num_object_copies":17883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5961,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":715,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.883485+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:45:38.883485+0000","last_peered":"2023-09-30T09:45:38.883485+0000","last_clean":"2023-09-30T09:45:38.883485+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:45:38.883485+0000","last_undegraded":"2023-09-30T09:45:38.883485+0000","last_fullsized":"2023-09-30T09:45:38.883485+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":77,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879173+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:45:38.879173+0000","last_peered":"2023-09-30T09:45:38.879173+0000","last_clean":"2023-09-30T09:45:38.879173+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:45:38.879173+0000","last_undegraded":"2023-09-30T09:45:38.879173+0000","last_fullsized":"2023-09-30T09:45:38.879173+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":198,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.884483+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:45:38.884483+0000","last_peered":"2023-09-30T09:45:38.884483+0000","last_clean":"2023-09-30T09:45:38.884483+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:45:38.884483+0000","last_undegraded":"2023-09-30T09:45:38.884483+0000","last_fullsized":"2023-09-30T09:45:38.884483+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"513'6061","reported_seq":6358,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:54.094896+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:45:54.094896+0000","last_peered":"2023-09-30T09:45:54.094896+0000","last_clean":"2023-09-30T09:45:54.094896+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:45:54.094896+0000","last_undegraded":"2023-09-30T09:45:54.094896+0000","last_fullsized":"2023-09-30T09:45:54.094896+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396034048,"num_objects":6043,"num_object_clones":0,"num_object_copies":18129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6061,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":755,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887806+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:45:38.887806+0000","last_peered":"2023-09-30T09:45:38.887806+0000","last_clean":"2023-09-30T09:45:38.887806+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:45:38.887806+0000","last_undegraded":"2023-09-30T09:45:38.887806+0000","last_fullsized":"2023-09-30T09:45:38.887806+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":743,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878939+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:45:38.878939+0000","last_peered":"2023-09-30T09:45:38.878939+0000","last_clean":"2023-09-30T09:45:38.878939+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:45:38.878939+0000","last_undegraded":"2023-09-30T09:45:38.878939+0000","last_fullsized":"2023-09-30T09:45:38.878939+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":441,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.873541+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:45:38.873541+0000","last_peered":"2023-09-30T09:45:38.873541+0000","last_clean":"2023-09-30T09:45:38.873541+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:45:38.873541+0000","last_undegraded":"2023-09-30T09:45:38.873541+0000","last_fullsized":"2023-09-30T09:45:38.873541+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":130,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.876797+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:45:38.876797+0000","last_peered":"2023-09-30T09:45:38.876797+0000","last_clean":"2023-09-30T09:45:38.876797+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:45:38.876797+0000","last_undegraded":"2023-09-30T09:45:38.876797+0000","last_fullsized":"2023-09-30T09:45:38.876797+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"513'6078","reported_seq":6420,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:54.104390+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:45:54.104390+0000","last_peered":"2023-09-30T09:45:54.104390+0000","last_clean":"2023-09-30T09:45:54.104390+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:45:54.104390+0000","last_undegraded":"2023-09-30T09:45:54.104390+0000","last_fullsized":"2023-09-30T09:45:54.104390+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6078,"log_dups_size":0,"ondisk_log_size":6078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396492800,"num_objects":6050,"num_object_clones":0,"num_object_copies":18150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6078,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":954,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.872913+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:45:38.872913+0000","last_peered":"2023-09-30T09:45:38.872913+0000","last_clean":"2023-09-30T09:45:38.872913+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:45:38.872913+0000","last_undegraded":"2023-09-30T09:45:38.872913+0000","last_fullsized":"2023-09-30T09:45:38.872913+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":460,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.888212+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:45:38.888212+0000","last_peered":"2023-09-30T09:45:38.888212+0000","last_clean":"2023-09-30T09:45:38.888212+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:45:38.888212+0000","last_undegraded":"2023-09-30T09:45:38.888212+0000","last_fullsized":"2023-09-30T09:45:38.888212+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":532,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.884425+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:45:38.884425+0000","last_peered":"2023-09-30T09:45:38.884425+0000","last_clean":"2023-09-30T09:45:38.884425+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:45:38.884425+0000","last_undegraded":"2023-09-30T09:45:38.884425+0000","last_fullsized":"2023-09-30T09:45:38.884425+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":272,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:43.697697+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:45:43.697697+0000","last_peered":"2023-09-30T09:45:43.697697+0000","last_clean":"2023-09-30T09:45:43.697697+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:45:43.697697+0000","last_undegraded":"2023-09-30T09:45:43.697697+0000","last_fullsized":"2023-09-30T09:45:43.697697+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"513'6093","reported_seq":6436,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:54.107203+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:45:54.107203+0000","last_peered":"2023-09-30T09:45:54.107203+0000","last_clean":"2023-09-30T09:45:54.107203+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:45:54.107203+0000","last_undegraded":"2023-09-30T09:45:54.107203+0000","last_fullsized":"2023-09-30T09:45:54.107203+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6093,"log_dups_size":0,"ondisk_log_size":6093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398393344,"num_objects":6079,"num_object_clones":0,"num_object_copies":18237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6093,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":932,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877032+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:45:38.877032+0000","last_peered":"2023-09-30T09:45:38.877032+0000","last_clean":"2023-09-30T09:45:38.877032+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:45:38.877032+0000","last_undegraded":"2023-09-30T09:45:38.877032+0000","last_fullsized":"2023-09-30T09:45:38.877032+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":781,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.881834+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:45:38.881834+0000","last_peered":"2023-09-30T09:45:38.881834+0000","last_clean":"2023-09-30T09:45:38.881834+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:45:38.881834+0000","last_undegraded":"2023-09-30T09:45:38.881834+0000","last_fullsized":"2023-09-30T09:45:38.881834+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":275,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880388+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:45:38.880388+0000","last_peered":"2023-09-30T09:45:38.880388+0000","last_clean":"2023-09-30T09:45:38.880388+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:45:38.880388+0000","last_undegraded":"2023-09-30T09:45:38.880388+0000","last_fullsized":"2023-09-30T09:45:38.880388+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":362,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.874215+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:45:38.874215+0000","last_peered":"2023-09-30T09:45:38.874215+0000","last_clean":"2023-09-30T09:45:38.874215+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:45:38.874215+0000","last_undegraded":"2023-09-30T09:45:38.874215+0000","last_fullsized":"2023-09-30T09:45:38.874215+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"513'6016","reported_seq":6318,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:54.090904+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:45:54.090904+0000","last_peered":"2023-09-30T09:45:54.090904+0000","last_clean":"2023-09-30T09:45:54.090904+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:45:54.090904+0000","last_undegraded":"2023-09-30T09:45:54.090904+0000","last_fullsized":"2023-09-30T09:45:54.090904+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6016,"log_dups_size":0,"ondisk_log_size":6016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393347072,"num_objects":6002,"num_object_clones":0,"num_object_copies":18006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6016,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1074,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.873190+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:45:38.873190+0000","last_peered":"2023-09-30T09:45:38.873190+0000","last_clean":"2023-09-30T09:45:38.873190+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:45:38.873190+0000","last_undegraded":"2023-09-30T09:45:38.873190+0000","last_fullsized":"2023-09-30T09:45:38.873190+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":875,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.875240+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:45:38.875240+0000","last_peered":"2023-09-30T09:45:38.875240+0000","last_clean":"2023-09-30T09:45:38.875240+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:45:38.875240+0000","last_undegraded":"2023-09-30T09:45:38.875240+0000","last_fullsized":"2023-09-30T09:45:38.875240+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":89,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.876875+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:45:38.876875+0000","last_peered":"2023-09-30T09:45:38.876875+0000","last_clean":"2023-09-30T09:45:38.876875+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:45:38.876875+0000","last_undegraded":"2023-09-30T09:45:38.876875+0000","last_fullsized":"2023-09-30T09:45:38.876875+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887726+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:45:38.887726+0000","last_peered":"2023-09-30T09:45:38.887726+0000","last_clean":"2023-09-30T09:45:38.887726+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:45:38.887726+0000","last_undegraded":"2023-09-30T09:45:38.887726+0000","last_fullsized":"2023-09-30T09:45:38.887726+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"513'6158","reported_seq":7787,"reported_epoch":513,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:45:53.728497+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:45:53.728497+0000","last_peered":"2023-09-30T09:45:53.728497+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:45:53.728497+0000","last_undegraded":"2023-09-30T09:45:53.728497+0000","last_fullsized":"2023-09-30T09:45:53.728497+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6158,"log_dups_size":0,"ondisk_log_size":6158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403570688,"num_objects":6158,"num_object_clones":0,"num_object_copies":18474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4188,"num_objects_unfound":0,"num_objects_dirty":6158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6158,"num_write_kb":394112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":782,"num_bytes_recovered":51249152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":1970}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":774,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887779+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:45:38.887779+0000","last_peered":"2023-09-30T09:45:38.887779+0000","last_clean":"2023-09-30T09:45:38.887779+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:45:38.887779+0000","last_undegraded":"2023-09-30T09:45:38.887779+0000","last_fullsized":"2023-09-30T09:45:38.887779+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":921,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.875181+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:45:38.875181+0000","last_peered":"2023-09-30T09:45:38.875181+0000","last_clean":"2023-09-30T09:45:38.875181+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:45:38.875181+0000","last_undegraded":"2023-09-30T09:45:38.875181+0000","last_fullsized":"2023-09-30T09:45:38.875181+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":125,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.882095+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:45:38.882095+0000","last_peered":"2023-09-30T09:45:38.882095+0000","last_clean":"2023-09-30T09:45:38.882095+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:45:38.882095+0000","last_undegraded":"2023-09-30T09:45:38.882095+0000","last_fullsized":"2023-09-30T09:45:38.882095+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":18,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878876+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:45:38.878876+0000","last_peered":"2023-09-30T09:45:38.878876+0000","last_clean":"2023-09-30T09:45:38.878876+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:45:38.878876+0000","last_undegraded":"2023-09-30T09:45:38.878876+0000","last_fullsized":"2023-09-30T09:45:38.878876+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"513'6160","reported_seq":6197,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:55.638769+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:45:55.638769+0000","last_peered":"2023-09-30T09:45:55.638769+0000","last_clean":"2023-09-30T09:45:55.638769+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:45:55.638769+0000","last_undegraded":"2023-09-30T09:45:55.638769+0000","last_fullsized":"2023-09-30T09:45:55.638769+0000","mapping_epoch":483,"log_start":"491'300","ondisk_log_start":"491'300","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5860,"log_dups_size":0,"ondisk_log_size":5860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390070272,"num_objects":5952,"num_object_clones":0,"num_object_copies":17856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6160,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":765,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887705+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:45:38.887705+0000","last_peered":"2023-09-30T09:45:38.887705+0000","last_clean":"2023-09-30T09:45:38.887705+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:45:38.887705+0000","last_undegraded":"2023-09-30T09:45:38.887705+0000","last_fullsized":"2023-09-30T09:45:38.887705+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1061,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.878070+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:45:38.878070+0000","last_peered":"2023-09-30T09:45:38.878070+0000","last_clean":"2023-09-30T09:45:38.878070+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:45:38.878070+0000","last_undegraded":"2023-09-30T09:45:38.878070+0000","last_fullsized":"2023-09-30T09:45:38.878070+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":256,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:43.697667+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:45:43.697667+0000","last_peered":"2023-09-30T09:45:43.697667+0000","last_clean":"2023-09-30T09:45:43.697667+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:45:43.697667+0000","last_undegraded":"2023-09-30T09:45:43.697667+0000","last_fullsized":"2023-09-30T09:45:43.697667+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":125,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879812+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:45:38.879812+0000","last_peered":"2023-09-30T09:45:38.879812+0000","last_clean":"2023-09-30T09:45:38.879812+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:45:38.879812+0000","last_undegraded":"2023-09-30T09:45:38.879812+0000","last_fullsized":"2023-09-30T09:45:38.879812+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"513'6113","reported_seq":6486,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:55.638810+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:45:55.638810+0000","last_peered":"2023-09-30T09:45:55.638810+0000","last_clean":"2023-09-30T09:45:55.638810+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:45:55.638810+0000","last_undegraded":"2023-09-30T09:45:55.638810+0000","last_fullsized":"2023-09-30T09:45:55.638810+0000","mapping_epoch":495,"log_start":"491'300","ondisk_log_start":"491'300","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5813,"log_dups_size":0,"ondisk_log_size":5813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386400280,"num_objects":5897,"num_object_clones":0,"num_object_copies":17691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5897,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6113,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1032,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.885704+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:45:38.885704+0000","last_peered":"2023-09-30T09:45:38.885704+0000","last_clean":"2023-09-30T09:45:38.885704+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:45:38.885704+0000","last_undegraded":"2023-09-30T09:45:38.885704+0000","last_fullsized":"2023-09-30T09:45:38.885704+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":544,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877731+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:45:38.877731+0000","last_peered":"2023-09-30T09:45:38.877731+0000","last_clean":"2023-09-30T09:45:38.877731+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:45:38.877731+0000","last_undegraded":"2023-09-30T09:45:38.877731+0000","last_fullsized":"2023-09-30T09:45:38.877731+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1231,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887610+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:45:38.887610+0000","last_peered":"2023-09-30T09:45:38.887610+0000","last_clean":"2023-09-30T09:45:38.887610+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:45:38.887610+0000","last_undegraded":"2023-09-30T09:45:38.887610+0000","last_fullsized":"2023-09-30T09:45:38.887610+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":548,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.875175+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:45:38.875175+0000","last_peered":"2023-09-30T09:45:38.875175+0000","last_clean":"2023-09-30T09:45:38.875175+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:45:38.875175+0000","last_undegraded":"2023-09-30T09:45:38.875175+0000","last_fullsized":"2023-09-30T09:45:38.875175+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"513'6249","reported_seq":6287,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-30T09:45:55.309616+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:45:55.309616+0000","last_peered":"2023-09-30T09:45:55.309616+0000","last_clean":"2023-09-30T09:45:55.309616+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:45:55.309616+0000","last_undegraded":"2023-09-30T09:45:55.309616+0000","last_fullsized":"2023-09-30T09:45:55.309616+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6249,"log_dups_size":0,"ondisk_log_size":6249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398393344,"num_objects":6079,"num_object_clones":0,"num_object_copies":18237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6249,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":895,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880621+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:45:38.880621+0000","last_peered":"2023-09-30T09:45:38.880621+0000","last_clean":"2023-09-30T09:45:38.880621+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:45:38.880621+0000","last_undegraded":"2023-09-30T09:45:38.880621+0000","last_fullsized":"2023-09-30T09:45:38.880621+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":83,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887975+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:45:38.887975+0000","last_peered":"2023-09-30T09:45:38.887975+0000","last_clean":"2023-09-30T09:45:38.887975+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:45:38.887975+0000","last_undegraded":"2023-09-30T09:45:38.887975+0000","last_fullsized":"2023-09-30T09:45:38.887975+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":88,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879119+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:45:38.879119+0000","last_peered":"2023-09-30T09:45:38.879119+0000","last_clean":"2023-09-30T09:45:38.879119+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:45:38.879119+0000","last_undegraded":"2023-09-30T09:45:38.879119+0000","last_fullsized":"2023-09-30T09:45:38.879119+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":514,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.884012+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:45:38.884012+0000","last_peered":"2023-09-30T09:45:38.884012+0000","last_clean":"2023-09-30T09:45:38.884012+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:45:38.884012+0000","last_undegraded":"2023-09-30T09:45:38.884012+0000","last_fullsized":"2023-09-30T09:45:38.884012+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":832,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880005+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:45:38.880005+0000","last_peered":"2023-09-30T09:45:38.880005+0000","last_clean":"2023-09-30T09:45:38.880005+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:45:38.880005+0000","last_undegraded":"2023-09-30T09:45:38.880005+0000","last_fullsized":"2023-09-30T09:45:38.880005+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":18,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879733+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:45:38.879733+0000","last_peered":"2023-09-30T09:45:38.879733+0000","last_clean":"2023-09-30T09:45:38.879733+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:45:38.879733+0000","last_undegraded":"2023-09-30T09:45:38.879733+0000","last_fullsized":"2023-09-30T09:45:38.879733+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":218,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.880518+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:45:38.880518+0000","last_peered":"2023-09-30T09:45:38.880518+0000","last_clean":"2023-09-30T09:45:38.880518+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:45:38.880518+0000","last_undegraded":"2023-09-30T09:45:38.880518+0000","last_fullsized":"2023-09-30T09:45:38.880518+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":627,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887792+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:45:38.887792+0000","last_peered":"2023-09-30T09:45:38.887792+0000","last_clean":"2023-09-30T09:45:38.887792+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:45:38.887792+0000","last_undegraded":"2023-09-30T09:45:38.887792+0000","last_fullsized":"2023-09-30T09:45:38.887792+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":441,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.873083+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:45:38.873083+0000","last_peered":"2023-09-30T09:45:38.873083+0000","last_clean":"2023-09-30T09:45:38.873083+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:45:38.873083+0000","last_undegraded":"2023-09-30T09:45:38.873083+0000","last_fullsized":"2023-09-30T09:45:38.873083+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":227,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.884220+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:45:38.884220+0000","last_peered":"2023-09-30T09:45:38.884220+0000","last_clean":"2023-09-30T09:45:38.884220+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:45:38.884220+0000","last_undegraded":"2023-09-30T09:45:38.884220+0000","last_fullsized":"2023-09-30T09:45:38.884220+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":383,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.873743+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:45:38.873743+0000","last_peered":"2023-09-30T09:45:38.873743+0000","last_clean":"2023-09-30T09:45:38.873743+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:45:38.873743+0000","last_undegraded":"2023-09-30T09:45:38.873743+0000","last_fullsized":"2023-09-30T09:45:38.873743+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":247,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.883925+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:45:38.883925+0000","last_peered":"2023-09-30T09:45:38.883925+0000","last_clean":"2023-09-30T09:45:38.883925+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:45:38.883925+0000","last_undegraded":"2023-09-30T09:45:38.883925+0000","last_fullsized":"2023-09-30T09:45:38.883925+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":840,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.877566+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:45:38.877566+0000","last_peered":"2023-09-30T09:45:38.877566+0000","last_clean":"2023-09-30T09:45:38.877566+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:45:38.877566+0000","last_undegraded":"2023-09-30T09:45:38.877566+0000","last_fullsized":"2023-09-30T09:45:38.877566+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":70,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.887529+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:45:38.887529+0000","last_peered":"2023-09-30T09:45:38.887529+0000","last_clean":"2023-09-30T09:45:38.887529+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:45:38.887529+0000","last_undegraded":"2023-09-30T09:45:38.887529+0000","last_fullsized":"2023-09-30T09:45:38.887529+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":463,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.881788+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:45:38.881788+0000","last_peered":"2023-09-30T09:45:38.881788+0000","last_clean":"2023-09-30T09:45:38.881788+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:45:38.881788+0000","last_undegraded":"2023-09-30T09:45:38.881788+0000","last_fullsized":"2023-09-30T09:45:38.881788+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":883,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.885573+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:45:38.885573+0000","last_peered":"2023-09-30T09:45:38.885573+0000","last_clean":"2023-09-30T09:45:38.885573+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:45:38.885573+0000","last_undegraded":"2023-09-30T09:45:38.885573+0000","last_fullsized":"2023-09-30T09:45:38.885573+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":24,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.879379+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:45:38.879379+0000","last_peered":"2023-09-30T09:45:38.879379+0000","last_clean":"2023-09-30T09:45:38.879379+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:45:38.879379+0000","last_undegraded":"2023-09-30T09:45:38.879379+0000","last_fullsized":"2023-09-30T09:45:38.879379+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":528,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-30T09:45:38.883861+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:45:38.883861+0000","last_peered":"2023-09-30T09:45:38.883861+0000","last_clean":"2023-09-30T09:45:38.883861+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:45:38.883861+0000","last_undegraded":"2023-09-30T09:45:38.883861+0000","last_fullsized":"2023-09-30T09:45:38.883861+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6301089816,"num_objects":96148,"num_object_clones":0,"num_object_copies":288444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4188,"num_objects_unfound":0,"num_objects_dirty":96148,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":97304,"num_write_kb":6190401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1925,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18952499200,"data_stored":18952486984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96304,"ondisk_log_size":96304,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234410,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3998600,"kb_used_data":3465472,"kb_used_omap":0,"kb_used_meta":532736,"kb_avail":89746552,"statfs":{"total":95995035648,"available":91900469248,"internally_reserved":0,"allocated":3548643328,"data_stored":3546928059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":545521664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":3.8450000000000002,"15min":3.7290000000000001},"min":{"1min":0.23300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6360000000000001,"5min":124.74299999999999,"15min":215.416},"last":1.8680000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":3.984,"15min":3.6459999999999999},"min":{"1min":0.253,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.1779999999999999,"5min":124.69499999999999,"15min":215.041},"last":1.1890000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:45:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":3.8460000000000001,"15min":3.2160000000000002},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":12.702,"5min":124.577,"15min":124.577},"last":5.0350000000000001},{"interface":"front","average":{"1min":1.218,"5min":3.714,"15min":3.4500000000000002},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.307},"max":{"1min":12.714,"5min":124.623,"15min":124.623},"last":5.0499999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":4.3570000000000002,"15min":3.5009999999999999},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.24299999999999999},"max":{"1min":5.3789999999999996,"5min":117.14,"15min":117.14},"last":1.5189999999999999},{"interface":"front","average":{"1min":2.3450000000000002,"5min":4.585,"15min":3.528},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":18.603000000000002,"5min":117.044,"15min":117.044},"last":3.4870000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":8.798,"5min":4.8070000000000004,"15min":8.2530000000000001},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.28899999999999998},"max":{"1min":141.28999999999999,"5min":141.28999999999999,"15min":277.57799999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":9.1679999999999993,"5min":4.8659999999999997,"15min":7.7830000000000004},"min":{"1min":0.39900000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":142.458,"5min":142.458,"15min":278.69299999999998},"last":3.524}]},{"osd":4,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":0.96699999999999997,"15min":1.6000000000000001},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":25.192,"5min":25.192,"15min":175.72},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.657,"5min":0.93100000000000005,"15min":1.526},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":24.521000000000001,"5min":24.521000000000001,"15min":179.048},"last":0.315}]},{"osd":5,"last update":"Sat Sep 30 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":11.993,"5min":11.993,"15min":11.993},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":197.75700000000001,"5min":197.75700000000001,"15min":197.75700000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":11.827,"5min":11.827,"15min":11.827},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":197.63300000000001,"5min":197.63300000000001,"15min":197.63300000000001},"last":0.35499999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":0.61899999999999999,"15min":2.1659999999999999},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.367,"5min":4.367,"15min":206.244},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.6679999999999999,"5min":0.76300000000000001,"15min":2.2000000000000002},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":202.78800000000001},"last":0.54300000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496048,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1692720,"kb_used_data":1171112,"kb_used_omap":0,"kb_used_meta":521600,"kb_avail":92052432,"statfs":{"total":95995035648,"available":94261690368,"internally_reserved":0,"allocated":1199218688,"data_stored":1197526712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":534118400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":3.1240000000000001,"15min":4.8929999999999998},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":5.0129999999999999,"5min":95.043999999999997,"15min":187.19800000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.1659999999999999,"5min":3.331,"15min":5.0620000000000003},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":6.7960000000000003,"5min":94.855999999999995,"15min":187.46700000000001},"last":4.2400000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.1519999999999999,"15min":1.952},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5050000000000008,"5min":32.814,"15min":69.569999999999993},"last":4.577},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.1850000000000001,"15min":2.1400000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":8.5169999999999995,"5min":32.856000000000002,"15min":70.471999999999994},"last":4.6139999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":3.456,"15min":3.798},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6120000000000001,"5min":207.02500000000001,"15min":207.02500000000001},"last":3.1309999999999998},{"interface":"front","average":{"1min":1.661,"5min":3.8170000000000002,"15min":4.0199999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28199999999999997},"max":{"1min":6.1299999999999999,"5min":207.114,"15min":207.114},"last":4.2789999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":3.5979999999999999,"5min":3.0019999999999998,"15min":4.3170000000000002},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.27600000000000002},"max":{"1min":49.234000000000002,"5min":107.11799999999999,"15min":212.44800000000001},"last":3.54},{"interface":"front","average":{"1min":3.6800000000000002,"5min":2.9750000000000001,"15min":4.6109999999999998},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":48.363,"5min":106.982,"15min":210.84299999999999},"last":3.7010000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":4.2149999999999999,"5min":1.581,"15min":1.377},"min":{"1min":0.29999999999999999,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":63.798000000000002,"5min":63.798000000000002,"15min":63.798000000000002},"last":3.1640000000000001},{"interface":"front","average":{"1min":3.887,"5min":1.4390000000000001,"15min":1.234},"min":{"1min":0.30599999999999999,"5min":0.27800000000000002,"15min":0.23000000000000001},"max":{"1min":63.133000000000003,"5min":63.133000000000003,"15min":63.133000000000003},"last":1.653}]},{"osd":5,"last update":"Sat Sep 30 09:45:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.974,"5min":14.974,"15min":14.974},"last":2.802},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.6280000000000001,"15min":2.6280000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":16.443000000000001,"5min":16.443000000000001,"15min":16.443000000000001},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":0.78900000000000003,"15min":1.6579999999999999},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.276999999999999,"5min":11.276999999999999,"15min":138.191},"last":3.266},{"interface":"front","average":{"1min":1.6419999999999999,"5min":0.74299999999999999,"15min":1.51},"min":{"1min":0.38100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":10.909000000000001,"5min":10.909000000000001,"15min":140.33500000000001},"last":2.5960000000000001}]}]},{"osd":1,"up_from":301,"seq":1292785156246,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4003216,"kb_used_data":3389768,"kb_used_omap":9,"kb_used_meta":613430,"kb_avail":89741936,"statfs":{"total":95995035648,"available":91895742464,"internally_reserved":0,"allocated":3471122432,"data_stored":3468200996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":628152338},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":10.134,"5min":3.8159999999999998,"15min":3.6789999999999998},"min":{"1min":0.23899999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":142.08500000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":9.4260000000000002,"5min":3.5449999999999999,"15min":3.5739999999999998},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":142.137,"5min":142.137,"15min":142.137},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":5.3959999999999999,"5min":2.7629999999999999,"15min":2.5409999999999999},"min":{"1min":0.23999999999999999,"5min":0.184,"15min":0.184},"max":{"1min":62.572000000000003,"5min":62.572000000000003,"15min":62.572000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":5.9349999999999996,"5min":2.8769999999999998,"15min":2.782},"min":{"1min":0.26800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":68.373999999999995,"5min":68.373999999999995,"15min":68.373999999999995},"last":0.40999999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":4.4969999999999999,"5min":5.6699999999999999,"15min":6.4870000000000001},"min":{"1min":0.251,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":59.478000000000002,"5min":199.41,"15min":199.41},"last":0.57999999999999996},{"interface":"front","average":{"1min":5.234,"5min":5.923,"15min":6.6189999999999998},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":59.621000000000002,"5min":199.58099999999999,"15min":199.58099999999999},"last":0.48399999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":0.98199999999999998,"15min":2.1499999999999999},"min":{"1min":0.317,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.6020000000000003,"5min":5.6020000000000003,"15min":142.77099999999999},"last":0.372},{"interface":"front","average":{"1min":1.508,"5min":1.244,"15min":2.3359999999999999},"min":{"1min":0.36699999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.976,"5min":8.0830000000000002,"15min":144.898},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.9339999999999999,"15min":1.9339999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.157,"5min":7.157,"15min":7.157},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":5.3810000000000002,"5min":5.3810000000000002,"15min":5.3810000000000002},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":1.3100000000000001,"15min":6.5579999999999998},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":30.986999999999998,"5min":30.986999999999998,"15min":209.221},"last":1.276},{"interface":"front","average":{"1min":2.698,"5min":1.3009999999999999,"15min":6.4859999999999998},"min":{"1min":0.46300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":28.710999999999999,"5min":28.710999999999999,"15min":208.43600000000001},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.2789999999999999,"15min":4.6100000000000003},"min":{"1min":0.38600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.5149999999999997,"5min":9.298,"15min":165.59399999999999},"last":1.383},{"interface":"front","average":{"1min":1.641,"5min":1.3620000000000001,"15min":4.7000000000000002},"min":{"1min":0.48099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":7.5460000000000003,"5min":9.3399999999999999,"15min":165.833},"last":0.56000000000000005}]}]},{"osd":0,"up_from":94,"seq":403726926144,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1905364,"kb_used_data":1291084,"kb_used_omap":5,"kb_used_meta":614266,"kb_avail":91839788,"statfs":{"total":95995035648,"available":94043942912,"internally_reserved":0,"allocated":1322070016,"data_stored":1318915224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":629008637},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":2.1989999999999998,"15min":2.274},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":5.8079999999999998,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.23000000000000001},{"interface":"front","average":{"1min":1.839,"5min":2.4129999999999998,"15min":4.4530000000000003},"min":{"1min":0.33200000000000002,"5min":0.23300000000000001,"15min":0.214},"max":{"1min":5.7149999999999999,"5min":57.140999999999998,"15min":138.92400000000001},"last":2.1560000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":4.9539999999999997,"15min":3.1120000000000001},"min":{"1min":0.246,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.3650000000000002,"5min":127.73399999999999,"15min":127.73399999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.9470000000000001,"5min":5.125,"15min":3.2919999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":8.407,"5min":127.69,"15min":127.69},"last":7.1239999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:45:08 2023","interfaces":[{"interface":"back","average":{"1min":4.9370000000000003,"5min":4.5549999999999997,"15min":5.0339999999999998},"min":{"1min":0.255,"5min":0.23699999999999999,"15min":0.221},"max":{"1min":80.126999999999995,"5min":125.562,"15min":218.06999999999999},"last":5.79},{"interface":"front","average":{"1min":5.5439999999999996,"5min":4.3659999999999997,"15min":4.1669999999999998},"min":{"1min":0.34599999999999997,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":80.838999999999999,"5min":117.295,"15min":218.63200000000001},"last":6.7560000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:45:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":2.552,"15min":1.883},"min":{"1min":0.33900000000000002,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":2.391,"5min":72.403999999999996,"15min":72.403999999999996},"last":6.4610000000000003},{"interface":"front","average":{"1min":1.1399999999999999,"5min":2.3780000000000001,"15min":2.6150000000000002},"min":{"1min":0.35299999999999998,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":3.7410000000000001,"5min":72.162999999999997,"15min":136.262},"last":2.4649999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.411,"5min":3.411,"15min":3.411},"last":2.5640000000000001},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.6099999999999999,"5min":3.6099999999999999,"15min":3.6099999999999999},"last":6.8200000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.696,"15min":3.4540000000000002},"min":{"1min":0.38300000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.54,"5min":57.384,"15min":214.785},"last":6.7919999999999998},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.9850000000000001,"15min":3.6560000000000001},"min":{"1min":0.48499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.2539999999999996,"5min":57.109000000000002,"15min":214.93899999999999},"last":6.4100000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.6379999999999999,"15min":2.8900000000000001},"min":{"1min":0.42699999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.7349999999999999,"5min":57.776000000000003,"15min":150.96899999999999},"last":6.8470000000000004},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.97,"15min":3.0299999999999998},"min":{"1min":0.42199999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.3070000000000004,"5min":57.179000000000002,"15min":150.328},"last":6.4429999999999996}]}]},{"osd":2,"up_from":104,"seq":446676599085,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2333984,"kb_used_data":1538008,"kb_used_omap":5,"kb_used_meta":795962,"kb_avail":91411168,"statfs":{"total":95995035648,"available":93605036032,"internally_reserved":0,"allocated":1574920192,"data_stored":1572041790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":815065741},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":14.532999999999999,"5min":5.5990000000000002,"15min":3.4089999999999998},"min":{"1min":0.26700000000000002,"5min":0.192,"15min":0.187},"max":{"1min":170.678,"5min":170.678,"15min":170.678},"last":13.880000000000001},{"interface":"front","average":{"1min":14.816000000000001,"5min":5.4219999999999997,"15min":3.371},"min":{"1min":0.307,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":172.44999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":12.256}]},{"osd":1,"last update":"Sat Sep 30 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":13.468,"5min":4.3209999999999997,"15min":3.7850000000000001},"min":{"1min":0.27700000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":162.696,"5min":162.696,"15min":162.696},"last":1.804},{"interface":"front","average":{"1min":13.574999999999999,"5min":4.3300000000000001,"15min":3.5249999999999999},"min":{"1min":0.23400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":162.93899999999999,"5min":162.93899999999999,"15min":162.93899999999999},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:45:07 2023","interfaces":[{"interface":"back","average":{"1min":4.1849999999999996,"5min":3.827,"15min":4.5220000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":49.994999999999997,"5min":80.052999999999997,"15min":134.078},"last":0.56799999999999995},{"interface":"front","average":{"1min":4.4870000000000001,"5min":3.754,"15min":4.4470000000000001},"min":{"1min":0.29099999999999998,"5min":0.224,"15min":0.224},"max":{"1min":50.576999999999998,"5min":80.111000000000004,"15min":135.73400000000001},"last":2.2200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":3.9020000000000001,"5min":1.5049999999999999,"15min":1.782},"min":{"1min":0.32800000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":47.463000000000001,"5min":47.463000000000001,"15min":83.012},"last":0.67700000000000005},{"interface":"front","average":{"1min":4.4509999999999996,"5min":1.569,"15min":1.8220000000000001},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":47.412999999999997,"5min":47.412999999999997,"15min":81.335999999999999},"last":2.258}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.706,"5min":2.706,"15min":2.706},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.4449999999999998,"5min":3.4449999999999998,"15min":3.4449999999999998},"last":5.4130000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.1479999999999999,"15min":3.2400000000000002},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.3309999999999995,"5min":9.3309999999999995,"15min":154.417},"last":5.4829999999999997},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.4019999999999999,"15min":3.516},"min":{"1min":0.433,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.7670000000000003,"5min":9.8409999999999993,"15min":154.04400000000001},"last":5.4619999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":1.7989999999999999,"15min":2.355},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":49.816000000000003,"5min":49.816000000000003,"15min":123.294},"last":4.585},{"interface":"front","average":{"1min":4.1269999999999998,"5min":1.873,"15min":2.6440000000000001},"min":{"1min":0.40799999999999997,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":50.956000000000003,"5min":50.956000000000003,"15min":123.16},"last":1.871}]}]},{"osd":3,"up_from":109,"seq":468151435558,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3896976,"kb_used_data":3078792,"kb_used_omap":2,"kb_used_meta":818173,"kb_avail":89848176,"statfs":{"total":95995035648,"available":92004532224,"internally_reserved":0,"allocated":3152683008,"data_stored":3149699827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":837809511},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.6189999999999998,"15min":4.4550000000000001},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.183},"max":{"1min":38.412999999999997,"5min":144.72,"15min":205.73099999999999},"last":1.1180000000000001},{"interface":"front","average":{"1min":2.673,"5min":5.0030000000000001,"15min":4.4820000000000002},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":34.918999999999997,"5min":142.36600000000001,"15min":205.495},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:45:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":3.141,"15min":8.593},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":19.725000000000001,"5min":142.434,"15min":273.99599999999998},"last":8.3450000000000006},{"interface":"front","average":{"1min":3.3340000000000001,"5min":3.1480000000000001,"15min":8.1219999999999999},"min":{"1min":0.33900000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":19.638999999999999,"5min":142.44900000000001,"15min":274.21600000000001},"last":8.2629999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":5.9569999999999999,"15min":5.5629999999999997},"min":{"1min":0.30499999999999999,"5min":0.249,"15min":0.249},"max":{"1min":10.301,"5min":146.92599999999999,"15min":206.69},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.534,"5min":5.827,"15min":5.3799999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23200000000000001},"max":{"1min":8.4060000000000006,"5min":145.75299999999999,"15min":209.22800000000001},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":4.4550000000000001,"5min":1.8899999999999999,"15min":2.0649999999999999},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.27200000000000002},"max":{"1min":81.677000000000007,"5min":81.677000000000007,"15min":81.677000000000007},"last":1.0700000000000001},{"interface":"front","average":{"1min":4.2699999999999996,"5min":1.9470000000000001,"15min":1.841},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":80.623000000000005,"5min":80.623000000000005,"15min":80.623000000000005},"last":0.372}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.1989999999999998,"5min":9.1989999999999998,"15min":9.1989999999999998},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.2719999999999998,"5min":2.2719999999999998,"15min":2.2719999999999998},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.0309999999999999,"15min":1.9319999999999999},"min":{"1min":0.41899999999999998,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":5.7690000000000001,"5min":5.915,"15min":60.698},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.093,"15min":1.8320000000000001},"min":{"1min":0.437,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":2.9249999999999998,"5min":10.356999999999999,"15min":12.121},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:45:18 2023","interfaces":[{"interface":"back","average":{"1min":9.952,"5min":2.7530000000000001,"15min":2.5499999999999998},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":213.548,"5min":213.548,"15min":213.548},"last":0.77800000000000002},{"interface":"front","average":{"1min":9.6460000000000008,"5min":2.7010000000000001,"15min":2.4980000000000002},"min":{"1min":0.44,"5min":0.41099999999999998,"15min":0.34799999999999998},"max":{"1min":213.583,"5min":213.583,"15min":213.583},"last":0.438}]}]},{"osd":4,"up_from":30,"seq":128849019321,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1635380,"kb_used_data":1160236,"kb_used_omap":0,"kb_used_meta":475008,"kb_avail":92109772,"statfs":{"total":95995035648,"available":94320406528,"internally_reserved":0,"allocated":1188081664,"data_stored":1186385707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":486408192},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:44:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":1.756,"15min":2.3700000000000001},"min":{"1min":0.25,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":31.863,"5min":73.718000000000004,"15min":105.179},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.286,"5min":1.8560000000000001,"15min":2.5099999999999998},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":31.98,"5min":74.001999999999995,"15min":104.70999999999999},"last":3.8690000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:44:55 2023","interfaces":[{"interface":"back","average":{"1min":5.3440000000000003,"5min":1.6830000000000001,"15min":4.7549999999999999},"min":{"1min":0.253,"5min":0.25,"15min":0.25},"max":{"1min":80.712000000000003,"5min":80.712000000000003,"15min":160.69999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":5.3040000000000003,"5min":1.819,"15min":4.7149999999999999},"min":{"1min":0.30199999999999999,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":80.810000000000002,"5min":80.810000000000002,"15min":160.50700000000001},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":11.257,"5min":4.6269999999999998,"15min":3.3559999999999999},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":117.858,"5min":142.88999999999999,"15min":142.88999999999999},"last":2.4950000000000001},{"interface":"front","average":{"1min":11.305999999999999,"5min":4.6950000000000003,"15min":3.403},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":117.871,"5min":142.90700000000001,"15min":142.90700000000001},"last":3.6419999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.96999999999999997,"15min":2.1800000000000002},"min":{"1min":0.39900000000000002,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":2.0800000000000001,"5min":4.8979999999999997,"15min":70.241},"last":3.5979999999999999},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.95399999999999996,"15min":2.1469999999999998},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.093,"5min":4.9080000000000004,"15min":70.433999999999997},"last":1.8140000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:45:25 2023","interfaces":[{"interface":"back","average":{"1min":10.734,"5min":10.734,"15min":10.734},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":140.11099999999999,"5min":140.11099999999999,"15min":140.11099999999999},"last":2.8599999999999999},{"interface":"front","average":{"1min":10.766,"5min":10.766,"15min":10.766},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":140.065,"5min":140.065,"15min":140.065},"last":0.69599999999999995}]},{"osd":6,"last update":"Sat Sep 30 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.53800000000000003,"15min":2.1850000000000001},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":4.4299999999999997,"5min":4.4299999999999997,"15min":202.72499999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.61099999999999999,"15min":2.169},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.249},"max":{"1min":5.7199999999999998,"5min":5.7199999999999998,"15min":202.42400000000001},"last":1.778}]},{"osd":7,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.2490000000000001,"15min":3.1819999999999999},"min":{"1min":0.36799999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.8170000000000002,"5min":161.72999999999999,"15min":230.90100000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.4490000000000001,"5min":2.2589999999999999,"15min":3.032},"min":{"1min":0.36199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.1709999999999994,"5min":162.60900000000001,"15min":229.97},"last":0.47599999999999998}]}]},{"osd":5,"up_from":489,"seq":2100239007763,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3611956,"kb_used_data":3482668,"kb_used_omap":6,"kb_used_meta":129273,"kb_avail":90133196,"statfs":{"total":95995035648,"available":92296392704,"internally_reserved":0,"allocated":3566252032,"data_stored":3564552625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":132375647},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":26.309999999999999,"5min":26.309999999999999,"15min":26.309999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":281.10599999999999,"5min":281.10599999999999,"15min":281.10599999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":26.599,"5min":26.599,"15min":26.599},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":282.07400000000001,"5min":282.07400000000001,"15min":282.07400000000001},"last":0.247}]},{"osd":1,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":14.92,"5min":14.92,"15min":14.92},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":273.48200000000003,"5min":273.48200000000003,"15min":273.48200000000003},"last":1.105},{"interface":"front","average":{"1min":14.105,"5min":14.105,"15min":14.105},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":273.50200000000001,"5min":273.50200000000001,"15min":273.50200000000001},"last":0.97399999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.0469999999999997,"5min":7.0469999999999997,"15min":7.0469999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":4.5999999999999996,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.85299999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":14.749000000000001,"5min":14.749000000000001,"15min":14.749000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":274.01799999999997,"5min":274.01799999999997,"15min":274.01799999999997},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.732,"5min":1.732,"15min":1.732},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.5830000000000002,"5min":6.5830000000000002,"15min":6.5830000000000002},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.1769999999999996,"5min":4.1769999999999996,"15min":4.1769999999999996},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.9900000000000002,"5min":3.9900000000000002,"15min":3.9900000000000002},"last":0.442}]},{"osd":6,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.645,"15min":1.645},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":14.371,"5min":14.371,"15min":14.371},"last":0.371},{"interface":"front","average":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":14.454000000000001,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.379}]},{"osd":7,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.294,"15min":1.294},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.4409999999999998,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.0350000000000001,"5min":3.0350000000000001,"15min":3.0350000000000001},"last":0.48999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1310982144,"data_stored":1310982144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3459518464,"data_stored":3459514392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1575092224,"data_stored":1575092224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3158966272,"data_stored":3158966272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1176109056,"data_stored":1176109056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3558805504,"data_stored":3558801432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1189281792,"data_stored":1189281792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3523743744,"data_stored":3523739672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:46:00.752 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:46:00.825 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:00.871 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:00.931 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:00.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:00.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:01.036 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:01.077 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:01.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:00 smithi019 ceph-mon[224043]: pgmap v1324: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 93 B/s rd, 34 MiB/s wr, 836 op/s; 4054/280722 objects misplaced (1.444%); 1.4 MiB/s, 23 objects/s recovering 2023-09-30T09:46:01.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:00 smithi019 ceph-mon[216815]: pgmap v1324: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 93 B/s rd, 34 MiB/s wr, 836 op/s; 4054/280722 objects misplaced (1.444%); 1.4 MiB/s, 23 objects/s recovering 2023-09-30T09:46:01.148 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:01.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:01.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:01.154 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:00 smithi167 ceph-mon[192880]: pgmap v1324: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 93 B/s rd, 34 MiB/s wr, 836 op/s; 4054/280722 objects misplaced (1.444%); 1.4 MiB/s, 23 objects/s recovering 2023-09-30T09:46:01.255 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:01.304 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:01.389 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:01.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:01.502 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:01.548 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:01.589 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:01.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:01.694 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:01.736 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:01.819 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:01.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:01.926 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:01.973 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:02.053 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:02.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:02.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:02.220 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:02.271 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:02.334 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:02.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:02.439 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:02.487 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:02.547 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:02.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:02.560 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:02.652 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:02.741 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:02.930 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:02.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:02.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:02.982 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:02 smithi167 ceph-mon[192880]: pgmap v1325: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 701 op/s; 4054/277449 objects misplaced (1.461%); 1.3 MiB/s, 20 objects/s recovering 2023-09-30T09:46:02.983 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:02 smithi019 ceph-mon[224043]: pgmap v1325: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 701 op/s; 4054/277449 objects misplaced (1.461%); 1.3 MiB/s, 20 objects/s recovering 2023-09-30T09:46:03.048 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:03.088 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:03.144 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:03.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:03.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:03.244 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:02 smithi019 ceph-mon[216815]: pgmap v1325: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 701 op/s; 4054/277449 objects misplaced (1.461%); 1.3 MiB/s, 20 objects/s recovering 2023-09-30T09:46:03.259 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:03.351 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:03.707 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36538 8.6 8:6d039046:::benchmark_data_smithi019_478679_object36537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.552182+0000", 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 457.14871550100003, 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2871744899999999, 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:46:03.708 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36538 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552182+0000", 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.709 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552182+0000", 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552192+0000", 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6220000000000004e-06 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.710 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552217+0000", 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5508e-05 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552237+0000", 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9551999999999998e-05 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.711 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552247+0000", 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8870000000000004e-06 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552313+0000", 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6358000000000004e-05 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552481+0000", 2023-09-30T09:46:03.712 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000167553 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.552628+0000", 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014709400000000001 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.554856+0000", 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022282040000000001 2023-09-30T09:46:03.713 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.555232+0000", 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00037634499999999999 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.650667+0000", 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.095434503000000004 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.714 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.650685+0000", 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7935999999999999e-05 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.660214+0000", 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.009528822 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.715 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.660228+0000", 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4523e-05 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.839232+0000", 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1790039509999999 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.839357+0000", 2023-09-30T09:46:03.716 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012463200000000001 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36573 8.6 8:6478847f:::benchmark_data_smithi019_478679_object36572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.705432+0000", 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 456.99546550299999, 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1429046679999999, 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.717 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36573 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705432+0000", 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.718 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705432+0000", 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705434+0000", 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5799999999999999e-06 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.719 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705459+0000", 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5428000000000001e-05 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705476+0000", 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7261e-05 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.720 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705487+0000", 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0342e-05 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705691+0000", 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000204157 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.721 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705702+0000", 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0879e-05 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.663427+0000", 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95772474200000002 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.722 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.663440+0000", 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3129000000000001e-05 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841104+0000", 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1776643179999999 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841253+0000", 2023-09-30T09:46:03.723 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000148646 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841392+0000", 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000139507 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.844591+0000", 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031987589999999998 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.724 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.845467+0000", 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00087615200000000005 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846194+0000", 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00072731399999999996 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.725 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846271+0000", 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6915999999999993e-05 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.848337+0000", 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002065538 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.726 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36575 8.c 8:30e8fa6b:::benchmark_data_smithi019_478679_object36574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.705506+0000", 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 456.99539205500002, 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1391458339999998, 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36575 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.727 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705506+0000", 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705506+0000", 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.728 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705506+0000", 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9599999999999999e-07 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705525+0000", 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9218000000000001e-05 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.729 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705532+0000", 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7859999999999999e-06 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705537+0000", 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.152e-06 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.730 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705597+0000", 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9351000000000001e-05 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.705606+0000", 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8360000000000004e-06 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659107+0000", 2023-09-30T09:46:03.731 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95350058299999996 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659121+0000", 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3862e-05 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.835026+0000", 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1759052400000001 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.732 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.835210+0000", 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000184208 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.835398+0000", 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018811199999999999 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.840805+0000", 2023-09-30T09:46:03.733 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0054062279999999999 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.840862+0000", 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6898999999999999e-05 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.842785+0000", 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019238009999999999 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.734 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.842841+0000", 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5458000000000003e-05 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.844651+0000", 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018106039999999999 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.735 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36589 8.c 8:398bde77:::benchmark_data_smithi019_478679_object36588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.714335+0000", 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 456.98656251099999, 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1343528040000002, 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36589 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.736 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714335+0000", 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714335+0000", 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.737 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714337+0000", 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.446e-06 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714359+0000", 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2802999999999998e-05 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714372+0000", 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2373e-05 2023-09-30T09:46:03.738 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714379+0000", 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6559999999999992e-06 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714457+0000", 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7766e-05 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.739 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.714469+0000", 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1314999999999999e-05 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659147+0000", 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94467839300000001 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659155+0000", 2023-09-30T09:46:03.740 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1939999999999998e-06 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.838893+0000", 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1797375400000001 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.839042+0000", 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014981900000000001 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.741 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.839238+0000", 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019502300000000001 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846753+0000", 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0075152120000000003 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.742 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846796+0000", 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3318000000000001e-05 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846846+0000", 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9576000000000001e-05 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846901+0000", 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5272999999999997e-05 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.743 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.848688+0000", 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0017870970000000001 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36592 8.c 8:34c28d90:::benchmark_data_smithi019_478679_object36591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.715952+0000", 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 456.98494545400001, 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1369867139999998, 2023-09-30T09:46:03.744 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36592 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715952+0000", 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.745 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715952+0000", 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715953+0000", 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1960000000000001e-06 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.746 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715975+0000", 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1679999999999999e-05 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715985+0000", 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9299999999999998e-06 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.715993+0000", 2023-09-30T09:46:03.747 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5209999999999997e-06 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.716060+0000", 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7007000000000004e-05 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.716067+0000", 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6899999999999992e-06 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.748 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659181+0000", 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94311415899999995 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659196+0000", 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4797000000000001e-05 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.749 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.840909+0000", 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.181712804 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841068+0000", 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000159166 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.841259+0000", 2023-09-30T09:46:03.750 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000190676 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.846810+0000", 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055510280000000004 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.848734+0000", 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019241200000000001 2023-09-30T09:46:03.751 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.850919+0000", 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002184685 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.850988+0000", 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9750000000000001e-05 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.752 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.852939+0000", 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0019505049999999999 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:36604 8.c 8:3abdb19f:::benchmark_data_smithi019_478679_object36603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:26.724974+0000", 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 456.975923846, 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1295574159999999, 2023-09-30T09:46:03.753 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 36604 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724974+0000", 2023-09-30T09:46:03.754 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724974+0000", 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724975+0000", 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7000000000000004e-07 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.755 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724991+0000", 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6755000000000001e-05 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.724998+0000", 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4500000000000001e-06 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.756 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.725002+0000", 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1520000000000002e-06 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.725057+0000", 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4630999999999999e-05 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:26.725063+0000", 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4670000000000001e-06 2023-09-30T09:46:03.757 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659221+0000", 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.93415796600000001 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:27.659229+0000", 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4719999999999995e-06 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.758 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.844688+0000", 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.185458876 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.844837+0000", 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014893799999999999 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.759 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.845033+0000", 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000195634 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.850862+0000", 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058289700000000002 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.760 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.852962+0000", 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021005300000000002 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.853019+0000", 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6668e-05 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.853086+0000", 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6956000000000001e-05 2023-09-30T09:46:03.761 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:28.854531+0000", 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0014452810000000001 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:54238 8.c 8:366ec1be:::benchmark_data_smithi019_478679_object54237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:46:03.762 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 439.06107415600002, 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0706041239999999, 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 54238 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.763 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:46:03.764 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4300000000000001e-07 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639843+0000", 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8371000000000001e-05 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639856+0000", 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3366999999999999e-05 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.765 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639862+0000", 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.161e-06 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639930+0000", 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7676999999999996e-05 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.766 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639945+0000", 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5138e-05 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:45.597093+0000", 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95714748900000002 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:45.597106+0000", 2023-09-30T09:46:03.767 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3617e-05 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676568+0000", 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0794613529999999 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676752+0000", 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018476000000000001 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.768 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676942+0000", 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018961800000000001 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708135+0000", 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031193262999999999 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.769 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708162+0000", 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6557999999999999e-05 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708218+0000", 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5918000000000002e-05 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708303+0000", 2023-09-30T09:46:03.770 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5107999999999999e-05 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.710428+0000", 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021247829999999999 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76883 8.16 8:6fedded8:::benchmark_data_smithi019_478679_object76882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e379)", 2023-09-30T09:46:03.771 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 409.05980211999997, 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.937896308, 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76883 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:46:03.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9400000000000005e-07 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641122+0000", 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5469e-05 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641129+0000", 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7079999999999997e-06 2023-09-30T09:46:03.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641135+0000", 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8420000000000001e-06 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.735104+0000", 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.093968634999999995 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.735114+0000", 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0591000000000001e-05 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.623444+0000", 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.888329232 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.623456+0000", 2023-09-30T09:46:03.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2207e-05 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.570797+0000", 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94734084100000004 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.570930+0000", 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013319100000000001 2023-09-30T09:46:03.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.571052+0000", 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012170700000000001 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.574539+0000", 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034873180000000001 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.778 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576645+0000", 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021061999999999999 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578718+0000", 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020728700000000001 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578793+0000", 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4670000000000002e-05 2023-09-30T09:46:03.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578992+0000", 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019923299999999999 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76913 8.16 8:6fac3ea3:::benchmark_data_smithi019_478679_object76912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-09-30T09:46:03.780 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 408.06837861000002, 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9468952980000001, 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76913 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632521+0000", 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.545e-06 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632547+0000", 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6449999999999999e-05 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632566+0000", 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8515000000000001e-05 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632576+0000", 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0199e-05 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632655+0000", 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9171999999999993e-05 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632669+0000", 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4055e-05 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624641+0000", 2023-09-30T09:46:03.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99197187499999995 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624650+0000", 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9570000000000008e-06 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572117+0000", 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94746728000000002 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572215+0000", 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8041000000000005e-05 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572312+0000", 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6371000000000003e-05 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576694+0000", 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043828060000000004 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576706+0000", 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1773999999999999e-05 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579137+0000", 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024311419999999999 2023-09-30T09:46:03.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579213+0000", 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5665000000000001e-05 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579414+0000", 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020145100000000001 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.789 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76925 8.16 8:6e562ee8:::benchmark_data_smithi019_478679_object76924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 408.0442511, 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9248296740000002, 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76925 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656648+0000", 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.443e-06 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656673+0000", 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4562e-05 2023-09-30T09:46:03.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656689+0000", 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5948999999999999e-05 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656697+0000", 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9789999999999993e-06 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656753+0000", 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6739e-05 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656760+0000", 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4710000000000004e-06 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624673+0000", 2023-09-30T09:46:03.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96791319200000003 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624681+0000", 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9060000000000002e-06 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572984+0000", 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94830274000000003 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573098+0000", 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000114694 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573214+0000", 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115318 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576712+0000", 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003498143 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578659+0000", 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001947308 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581148+0000", 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002488873 2023-09-30T09:46:03.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581230+0000", 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2106000000000003e-05 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581476+0000", 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024625100000000001 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.798 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:126957 8.11 8:8c2c7c44:::benchmark_data_smithi019_478679_object34509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 255.78624618000001, 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8430575170000001, 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.799 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 126957 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:46:03.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914653+0000", 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2550000000000001e-06 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914659+0000", 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5400000000000001e-06 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914676+0000", 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6543999999999999e-05 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914685+0000", 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5109999999999992e-06 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914770+0000", 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4579000000000005e-05 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914933+0000", 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016312100000000001 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915080+0000", 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000146699 2023-09-30T09:46:03.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.917700+0000", 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026206659999999998 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.919771+0000", 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002070985 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757615+0000", 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.837843213 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757648+0000", 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3256000000000001e-05 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757709+0000", 2023-09-30T09:46:03.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1148000000000005e-05 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136440 8.19 8:9e052cf0:::benchmark_data_smithi019_478679_object43992:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 222.272393034, 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8141369350000001, 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.806 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136440 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6399999999999998e-07 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428507+0000", 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0959999999999999e-06 2023-09-30T09:46:03.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428513+0000", 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1639999999999998e-06 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428519+0000", 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.541e-06 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.809 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429278+0000", 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00075965099999999997 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429285+0000", 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.618e-06 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.195903+0000", 2023-09-30T09:46:03.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7666181060000001 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.196239+0000", 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033550600000000001 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.196875+0000", 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000636519 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.199780+0000", 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002904883 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208312+0000", 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0085319590000000004 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242090+0000", 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.033777612999999998 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242261+0000", 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000171507 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242642+0000", 2023-09-30T09:46:03.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00038040799999999999 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136457 8.19 8:9a6b3ff1:::benchmark_data_smithi019_478679_object44009:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 222.27237215, 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.814734099, 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.814 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136457 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.815 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9700000000000002e-07 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428528+0000", 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.905e-06 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428552+0000", 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.366e-05 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428558+0000", 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1160000000000004e-06 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429324+0000", 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00076669300000000004 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429333+0000", 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1120000000000002e-06 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.818 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.199884+0000", 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7705505289999999 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.200283+0000", 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00039902299999999998 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.200894+0000", 2023-09-30T09:46:03.819 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00061094000000000003 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208335+0000", 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0074407869999999999 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208590+0000", 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025573000000000002 2023-09-30T09:46:03.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242752+0000", 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.034161527999999997 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242903+0000", 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000150569 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243260+0000", 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035711 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136497 8.19 8:9ab7e4b6:::benchmark_data_smithi019_478679_object44049:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:46:03.822 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 222.27233084400001, 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.815323341, 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136497 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3499999999999996e-07 2023-09-30T09:46:03.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428570+0000", 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.548e-06 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428577+0000", 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3159999999999999e-06 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428583+0000", 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.592e-06 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429357+0000", 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00077461499999999998 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429365+0000", 2023-09-30T09:46:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4390000000000003e-06 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.202754+0000", 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7733887419999999 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.203035+0000", 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00028132800000000001 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.203627+0000", 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00059232499999999999 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208358+0000", 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004730845 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.828 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.211841+0000", 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034829769999999999 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243353+0000", 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031512327 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243486+0000", 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013215899999999999 2023-09-30T09:46:03.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243890+0000", 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00040459300000000002 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153596 8.6 8:63ca80c0:::benchmark_data_smithi019_478679_object61148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-09-30T09:46:03.830 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 181.69544708000001, 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2178367450000001, 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153596 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005452+0000", 2023-09-30T09:46:03.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-06 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005458+0000", 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0240000000000003e-06 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005476+0000", 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7638999999999998e-05 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005484+0000", 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6649999999999992e-06 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005563+0000", 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8406999999999997e-05 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005575+0000", 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2533e-05 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961360+0000", 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.955784462 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961374+0000", 2023-09-30T09:46:03.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4756999999999999e-05 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197062+0000", 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2356875270000001 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197198+0000", 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013569999999999999 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.837 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:46:03.837 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197626+0000", 2023-09-30T09:46:03.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00042856900000000001 2023-09-30T09:46:03.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.202382+0000", 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0047554479999999998 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.222890+0000", 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020507803000000002 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.222938+0000", 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8897999999999998e-05 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223015+0000", 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6330999999999998e-05 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223287+0000", 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027260099999999997 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153617 8.6 8:671a25b8:::benchmark_data_smithi019_478679_object61169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 181.62155579700001, 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1478323110000002, 2023-09-30T09:46:03.840 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153617 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079343+0000", 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0410000000000001e-06 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079347+0000", 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8829999999999999e-06 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079359+0000", 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1719000000000001e-05 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079364+0000", 2023-09-30T09:46:03.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8889999999999999e-06 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079445+0000", 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0644999999999998e-05 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079460+0000", 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5247e-05 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961403+0000", 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.88194313400000002 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961413+0000", 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9450000000000005e-06 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.199966+0000", 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2385522090000001 2023-09-30T09:46:03.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.200059+0000", 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3688000000000002e-05 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.200467+0000", 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000408093 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223502+0000", 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.023034289999999999 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223531+0000", 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9584e-05 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.226902+0000", 2023-09-30T09:46:03.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033704429999999999 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.226956+0000", 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4129000000000003e-05 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.227174+0000", 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021837199999999999 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.848 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2292 9.8 9:1a1fa048:::benchmark_data_smithi019_560486_object2291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 98.658916958999995, 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.046112114, 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.849 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2292 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.850 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041982+0000", 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.375e-06 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042038+0000", 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6124e-05 2023-09-30T09:46:03.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042052+0000", 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3366e-05 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042061+0000", 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9039999999999998e-06 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.043012+0000", 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00095105899999999998 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.043037+0000", 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5621000000000001e-05 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046269+0000", 2023-09-30T09:46:03.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.003231505 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046280+0000", 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1532000000000001e-05 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080018+0000", 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0337373249999999 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080180+0000", 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000162268 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080377+0000", 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000197295 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080973+0000", 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000596292 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081788+0000", 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00081490800000000004 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087843+0000", 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0060543369999999999 2023-09-30T09:46:03.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087903+0000", 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9904000000000001e-05 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.088093+0000", 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000190299 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2304 9.5 9:a61d12ad:::benchmark_data_smithi019_560486_object2303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 98.646540955999996, 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0366920450000001, 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2304 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.858 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054358+0000", 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.497e-06 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054413+0000", 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.486e-05 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054425+0000", 2023-09-30T09:46:03.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.221e-05 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054436+0000", 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0822000000000001e-05 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.055599+0000", 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001163182 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.055607+0000", 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2219999999999996e-06 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.039370+0000", 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98376339800000001 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.039387+0000", 2023-09-30T09:46:03.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6816999999999998e-05 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081036+0000", 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0416488429999999 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081162+0000", 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012657 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081327+0000", 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000165043 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081830+0000", 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00050259000000000002 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082960+0000", 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001130147 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090727+0000", 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077666380000000002 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090794+0000", 2023-09-30T09:46:03.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7787999999999995e-05 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.091049+0000", 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025441800000000002 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2309 9.8 9:1c660290:::benchmark_data_smithi019_560486_object2308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:46:03.866 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 98.627372649999998, 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0171991079999998, 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2309 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073526+0000", 2023-09-30T09:46:03.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-06 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073572+0000", 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5779999999999999e-05 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073585+0000", 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2870999999999999e-05 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073592+0000", 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5150000000000003e-06 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073661+0000", 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9664000000000003e-05 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073675+0000", 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-05 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046310+0000", 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.972634836 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046319+0000", 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.329e-06 2023-09-30T09:46:03.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081008+0000", 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.034688308 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081125+0000", 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011733199999999999 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081251+0000", 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012646599999999999 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081666+0000", 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00041515399999999999 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082565+0000", 2023-09-30T09:46:03.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00089839100000000001 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090410+0000", 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.007844976 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090481+0000", 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1354999999999997e-05 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090724+0000", 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024294 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2312 9.9 9:939ceb45:::benchmark_data_smithi019_560486_object2311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 98.621384921000001, 2023-09-30T09:46:03.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0117532649999998, 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2312 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:46:03.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079514+0000", 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1200000000000002e-07 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.877 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079550+0000", 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6514000000000003e-05 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079562+0000", 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2275e-05 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:46:03.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079568+0000", 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0109999999999997e-06 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079619+0000", 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0250999999999997e-05 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079631+0000", 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2756e-05 2023-09-30T09:46:03.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043823+0000", 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96419190499999996 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043838+0000", 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4323e-05 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080176+0000", 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.036337939 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080316+0000", 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014040999999999999 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:46:03.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080506+0000", 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019016399999999999 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081229+0000", 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00072260300000000005 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082898+0000", 2023-09-30T09:46:03.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001669366 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090790+0000", 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0078920009999999992 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090859+0000", 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9210999999999999e-05 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:46:03.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.091266+0000", 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00040672400000000002 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:46:03.884 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:04.011 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:46:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:46:03] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-09-30T09:46:04.011 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:04.012 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:04.012 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3781412037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:04.012 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:04 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:04.034 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:46:04.052 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:04.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:04.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:04.157 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:04.189 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3781412037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:04 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3781412037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:04 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:05.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:05 smithi167 ceph-mon[192880]: pgmap v1326: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 661 op/s; 3927/275331 objects misplaced (1.426%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:46:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:05 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:05 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:05 smithi167 ceph-mon[192880]: osdmap e514: 8 total, 8 up, 8 in 2023-09-30T09:46:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:05 smithi167 ceph-mon[192880]: failed to encode map e514 with expected crc 2023-09-30T09:46:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:05 smithi167 ceph-mon[192880]: failed to encode map e514 with expected crc 2023-09-30T09:46:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:05 smithi167 ceph-mon[192880]: failed to encode map e514 with expected crc 2023-09-30T09:46:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[224043]: pgmap v1326: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 661 op/s; 3927/275331 objects misplaced (1.426%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:46:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[224043]: osdmap e514: 8 total, 8 up, 8 in 2023-09-30T09:46:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[224043]: failed to encode map e514 with expected crc 2023-09-30T09:46:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[224043]: failed to encode map e514 with expected crc 2023-09-30T09:46:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[224043]: failed to encode map e514 with expected crc 2023-09-30T09:46:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[216815]: pgmap v1326: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 661 op/s; 3927/275331 objects misplaced (1.426%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:46:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[216815]: osdmap e514: 8 total, 8 up, 8 in 2023-09-30T09:46:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[216815]: failed to encode map e514 with expected crc 2023-09-30T09:46:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[216815]: failed to encode map e514 with expected crc 2023-09-30T09:46:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:05 smithi019 ceph-mon[216815]: failed to encode map e514 with expected crc 2023-09-30T09:46:07.009 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:46:07.045 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:07.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:07.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:07.155 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:07.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:07 smithi019 ceph-mon[224043]: pgmap v1328: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 109 KiB/s wr, 695 op/s; 3927/267765 objects misplaced (1.467%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:07.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:07 smithi019 ceph-mon[216815]: pgmap v1328: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 109 KiB/s wr, 695 op/s; 3927/267765 objects misplaced (1.467%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:07.288 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:07.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:07 smithi167 ceph-mon[192880]: pgmap v1328: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 109 KiB/s wr, 695 op/s; 3927/267765 objects misplaced (1.467%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:07.589 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:07.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:07.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:07.701 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:07.733 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:07.821 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:07.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:07.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:07.933 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:08.037 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:08.764 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:08.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:08.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:08.882 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:08.926 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:08.999 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:09.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:09.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:09.043 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:46:09.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:09 smithi019 ceph-mon[216815]: pgmap v1329: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 109 KiB/s wr, 695 op/s; 3927/267765 objects misplaced (1.467%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:09.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:09.044 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:09.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:09 smithi019 ceph-mon[224043]: pgmap v1329: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 109 KiB/s wr, 695 op/s; 3927/267765 objects misplaced (1.467%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:09.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:09.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:09.106 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:09.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:09 smithi167 ceph-mon[192880]: pgmap v1329: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 109 KiB/s wr, 695 op/s; 3927/267765 objects misplaced (1.467%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:09.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:09.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:09.287 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:09.401 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:09.488 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:09.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:09.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:09.594 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:09.595 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:46:09.626 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:09.636 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:46:09.681 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:46:09.687 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:09.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:09.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:09.732 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:46:09.780 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:46:09.793 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:09.844 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:46:10.335 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:10.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:46:10.779 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:46:10.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:46:10.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:46:10.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:46:10.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:46:10.780 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:46:10.780 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:46:10.780 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:46:10.780 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:46:10.780 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2493119503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:10.780 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2493119503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:10.781 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:10.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:10.782 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:46:10.782 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1708,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":514,"num_osds":8,"num_up_osds":8,"osd_up_since":1696067065,"num_in_osds":8,"osd_in_since":1696066981,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":118},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":85620,"data_bytes":5614081983,"bytes_used":21749006336,"bytes_avail":746211278848,"bytes_total":767960285184,"misplaced_objects":3797,"misplaced_total":256860,"misplaced_ratio":0.014782371720003115,"recovering_objects_per_sec":25,"recovering_bytes_per_sec":1683753,"recovering_keys_per_sec":0,"num_objects_recovered":257,"num_bytes_recovered":16842752,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":795},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-30T09:45:25.765551+0000","services":{}},"progress_events":{"5b5f721d-3b3c-459d-b09b-c0a83655ed5d":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-30T09:46:10.782 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2493119503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:10.883 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:10.956 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:10.964 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:11.037 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:11.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:11.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:11.141 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:11.223 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:11.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[216815]: pgmap v1330: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3797/256860 objects misplaced (1.478%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:11.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:46:11.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:46:11.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:11.360 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2617282643' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:46:11.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[224043]: pgmap v1330: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3797/256860 objects misplaced (1.478%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:11.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:46:11.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:46:11.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:11.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2617282643' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:46:11.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:11 smithi167 ceph-mon[192880]: pgmap v1330: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3797/256860 objects misplaced (1.478%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:46:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:46:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2617282643' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:46:12.429 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:12.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:12.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:12.535 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:12.617 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:13.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:13 smithi019 ceph-mon[216815]: pgmap v1331: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3797/256860 objects misplaced (1.478%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:13 smithi019 ceph-mon[224043]: pgmap v1331: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3797/256860 objects misplaced (1.478%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:13.434 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:46:13.434 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:46:13.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:13 smithi167 ceph-mon[192880]: pgmap v1331: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3797/256860 objects misplaced (1.478%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:13.690 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:13.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:13.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:13.804 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:13.916 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:14.056 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:46:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:46:13] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-09-30T09:46:14.313 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:14.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:14.423 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:14.489 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:14.539 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:14.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:14.644 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:14.696 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:14.757 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:14.760 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:14 smithi167 ceph-mon[192880]: pgmap v1332: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 3797/256860 objects misplaced (1.478%); 832 KiB/s, 12 objects/s recovering 2023-09-30T09:46:14.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:14.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:14.862 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:14.912 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:14.991 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:15.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:15.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:15.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:14 smithi019 ceph-mon[224043]: pgmap v1332: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 3797/256860 objects misplaced (1.478%); 832 KiB/s, 12 objects/s recovering 2023-09-30T09:46:15.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:14 smithi019 ceph-mon[216815]: pgmap v1332: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 3797/256860 objects misplaced (1.478%); 832 KiB/s, 12 objects/s recovering 2023-09-30T09:46:15.105 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:15.152 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:15.230 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:15.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:15.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:15.337 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:15.422 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:15.432 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:46:15.636 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:15.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:15.741 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:15.778 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:15.815 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:46:15.815 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:46:15.857 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:15.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:15.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:15.966 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:16.005 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:16.088 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:16.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:16.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:16.197 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:16.281 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:17.231 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:17.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:17.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:16 smithi019 ceph-mon[216815]: pgmap v1333: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3670/246345 objects misplaced (1.490%); 1.4 MiB/s, 21 objects/s recovering 2023-09-30T09:46:17.237 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:16 smithi019 ceph-mon[224043]: pgmap v1333: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3670/246345 objects misplaced (1.490%); 1.4 MiB/s, 21 objects/s recovering 2023-09-30T09:46:17.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:16 smithi167 ceph-mon[192880]: pgmap v1333: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3670/246345 objects misplaced (1.490%); 1.4 MiB/s, 21 objects/s recovering 2023-09-30T09:46:17.339 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:17.360 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:17.376 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:17.387 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:46:17.446 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:17.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:17.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:17.552 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:17.589 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:17.646 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:17.657 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:17.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:17.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:17.764 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:46:17.797 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:17.886 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:46:17.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:17.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:46:17.895 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:46:17.994 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:18.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3111517383' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:46:18.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:46:18.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3111517383' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:46:18.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:46:18.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3111517383' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:46:18.031 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:46:18.107 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: pgmap v1334: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3670/246345 objects misplaced (1.490%); 1.3 MiB/s, 21 objects/s recovering 2023-09-30T09:46:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:46:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: osdmap e515: 8 total, 8 up, 8 in 2023-09-30T09:46:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: pgmap v1334: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3670/246345 objects misplaced (1.490%); 1.3 MiB/s, 21 objects/s recovering 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: osdmap e515: 8 total, 8 up, 8 in 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:19.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: pgmap v1334: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3670/246345 objects misplaced (1.490%); 1.3 MiB/s, 21 objects/s recovering 2023-09-30T09:46:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:46:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: osdmap e515: 8 total, 8 up, 8 in 2023-09-30T09:46:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: failed to encode map e515 with expected crc 2023-09-30T09:46:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:19.697 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:19.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:19.808 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:19.843 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:20.604 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:20.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:20.713 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:20.747 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:20.815 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:20.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:20.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:20.928 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:20.977 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:21.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:20 smithi167 ceph-mon[192880]: pgmap v1336: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:21.046 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:21.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:21.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:21.155 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:21.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:20 smithi019 ceph-mon[216815]: pgmap v1336: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:21.178 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:20 smithi019 ceph-mon[224043]: pgmap v1336: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:21.226 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:21.401 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:21.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:21.441 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:21.508 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:21.549 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:21.628 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:21.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:21.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:21.738 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:21.802 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:21.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:21.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:21.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:21.993 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:22.070 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:22.520 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:22.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:22.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:22.630 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:46:22.670 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:22.736 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:46:22.746 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:46:22.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:22.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:46:22.761 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:46:22.763 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:46:22.812 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:46:22.849 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:22.853 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:46:22.891 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:46:22.904 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:46:22.945 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:22.958 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:46:23.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:22 smithi167 ceph-mon[192880]: pgmap v1337: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:23.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2931356507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:23.022 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:46:23.059 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:46:23.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:22 smithi019 ceph-mon[224043]: pgmap v1337: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:23.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2931356507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:23.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:22 smithi019 ceph-mon[216815]: pgmap v1337: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:23.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2931356507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:23.106 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:46:23.110 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:23.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:23.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:23.127 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:46:23.128 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:46:23.128 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:46:23.128 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:46:23.128 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:46:23.128 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:46:23.128 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:46:23.128 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:46:23.225 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:23.300 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:24.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:23 smithi019 ceph-mon[224043]: from='client.56245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:46:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:46:23] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2023-09-30T09:46:24.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:23 smithi019 ceph-mon[216815]: from='client.56245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:24.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:23 smithi167 ceph-mon[192880]: from='client.56245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:25.118 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1337,"stamp":"2023-09-30T09:46:21.801662+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5137897407,"num_objects":78354,"num_object_clones":0,"num_object_copies":235062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3527,"num_objects_unfound":0,"num_objects_dirty":78354,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":116067,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":182081138,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113482,"ondisk_log_size":113482,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19839412,"kb_used_data":15346672,"kb_used_omap":30,"kb_used_meta":4492385,"kb_avail":730121804,"statfs":{"total":767960285184,"available":747644727296,"internally_reserved":0,"allocated":15714992128,"data_stored":15696175843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4600202594},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-476184576,"num_objects":-7266,"num_object_clones":0,"num_object_copies":-21798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-270,"num_objects_unfound":0,"num_objects_dirty":-7266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7266,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":270,"num_bytes_recovered":17694720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006406"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":938,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.037183+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:46:04.037183+0000","last_peered":"2023-09-30T09:46:04.037183+0000","last_clean":"2023-09-30T09:46:04.037183+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:46:04.037183+0000","last_undegraded":"2023-09-30T09:46:04.037183+0000","last_fullsized":"2023-09-30T09:46:04.037183+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":533,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040957+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:46:04.040957+0000","last_peered":"2023-09-30T09:46:04.040957+0000","last_clean":"2023-09-30T09:46:04.040957+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:46:04.040957+0000","last_undegraded":"2023-09-30T09:46:04.040957+0000","last_fullsized":"2023-09-30T09:46:04.040957+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":430,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041777+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:46:04.041777+0000","last_peered":"2023-09-30T09:46:04.041777+0000","last_clean":"2023-09-30T09:46:04.041777+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:46:04.041777+0000","last_undegraded":"2023-09-30T09:46:04.041777+0000","last_fullsized":"2023-09-30T09:46:04.041777+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":940,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.038276+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:46:04.038276+0000","last_peered":"2023-09-30T09:46:04.038276+0000","last_clean":"2023-09-30T09:46:04.038276+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:46:04.038276+0000","last_undegraded":"2023-09-30T09:46:04.038276+0000","last_fullsized":"2023-09-30T09:46:04.038276+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.042281+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:46:04.042281+0000","last_peered":"2023-09-30T09:46:04.042281+0000","last_clean":"2023-09-30T09:46:04.042281+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:46:04.042281+0000","last_undegraded":"2023-09-30T09:46:04.042281+0000","last_fullsized":"2023-09-30T09:46:04.042281+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":120,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033810+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:46:04.033810+0000","last_peered":"2023-09-30T09:46:04.033810+0000","last_clean":"2023-09-30T09:46:04.033810+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:46:04.033810+0000","last_undegraded":"2023-09-30T09:46:04.033810+0000","last_fullsized":"2023-09-30T09:46:04.033810+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":748,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040973+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:46:04.040973+0000","last_peered":"2023-09-30T09:46:04.040973+0000","last_clean":"2023-09-30T09:46:04.040973+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:46:04.040973+0000","last_undegraded":"2023-09-30T09:46:04.040973+0000","last_fullsized":"2023-09-30T09:46:04.040973+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033437+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:46:04.033437+0000","last_peered":"2023-09-30T09:46:04.033437+0000","last_clean":"2023-09-30T09:46:04.033437+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:46:04.033437+0000","last_undegraded":"2023-09-30T09:46:04.033437+0000","last_fullsized":"2023-09-30T09:46:04.033437+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":84,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026666+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:46:04.026666+0000","last_peered":"2023-09-30T09:46:04.026666+0000","last_clean":"2023-09-30T09:46:04.026666+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:46:04.026666+0000","last_undegraded":"2023-09-30T09:46:04.026666+0000","last_fullsized":"2023-09-30T09:46:04.026666+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":916,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034513+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:46:04.034513+0000","last_peered":"2023-09-30T09:46:04.034513+0000","last_clean":"2023-09-30T09:46:04.034513+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:46:04.034513+0000","last_undegraded":"2023-09-30T09:46:04.034513+0000","last_fullsized":"2023-09-30T09:46:04.034513+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":135,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034509+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:46:04.034509+0000","last_peered":"2023-09-30T09:46:04.034509+0000","last_clean":"2023-09-30T09:46:04.034509+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:46:04.034509+0000","last_undegraded":"2023-09-30T09:46:04.034509+0000","last_fullsized":"2023-09-30T09:46:04.034509+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":95,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.042202+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:46:04.042202+0000","last_peered":"2023-09-30T09:46:04.042202+0000","last_clean":"2023-09-30T09:46:04.042202+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:46:04.042202+0000","last_undegraded":"2023-09-30T09:46:04.042202+0000","last_fullsized":"2023-09-30T09:46:04.042202+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":704,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035729+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:46:04.035729+0000","last_peered":"2023-09-30T09:46:04.035729+0000","last_clean":"2023-09-30T09:46:04.035729+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:46:04.035729+0000","last_undegraded":"2023-09-30T09:46:04.035729+0000","last_fullsized":"2023-09-30T09:46:04.035729+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":335,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.038082+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:46:04.038082+0000","last_peered":"2023-09-30T09:46:04.038082+0000","last_clean":"2023-09-30T09:46:04.038082+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:46:04.038082+0000","last_undegraded":"2023-09-30T09:46:04.038082+0000","last_fullsized":"2023-09-30T09:46:04.038082+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":471,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033680+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:46:04.033680+0000","last_peered":"2023-09-30T09:46:04.033680+0000","last_clean":"2023-09-30T09:46:04.033680+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:46:04.033680+0000","last_undegraded":"2023-09-30T09:46:04.033680+0000","last_fullsized":"2023-09-30T09:46:04.033680+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":644,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036336+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:46:04.036336+0000","last_peered":"2023-09-30T09:46:04.036336+0000","last_clean":"2023-09-30T09:46:04.036336+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:46:04.036336+0000","last_undegraded":"2023-09-30T09:46:04.036336+0000","last_fullsized":"2023-09-30T09:46:04.036336+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":229,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034613+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:46:04.034613+0000","last_peered":"2023-09-30T09:46:04.034613+0000","last_clean":"2023-09-30T09:46:04.034613+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:46:04.034613+0000","last_undegraded":"2023-09-30T09:46:04.034613+0000","last_fullsized":"2023-09-30T09:46:04.034613+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":70,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026689+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:46:04.026689+0000","last_peered":"2023-09-30T09:46:04.026689+0000","last_clean":"2023-09-30T09:46:04.026689+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:46:04.026689+0000","last_undegraded":"2023-09-30T09:46:04.026689+0000","last_fullsized":"2023-09-30T09:46:04.026689+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":570,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033861+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:46:04.033861+0000","last_peered":"2023-09-30T09:46:04.033861+0000","last_clean":"2023-09-30T09:46:04.033861+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:46:04.033861+0000","last_undegraded":"2023-09-30T09:46:04.033861+0000","last_fullsized":"2023-09-30T09:46:04.033861+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":559,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041373+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:46:04.041373+0000","last_peered":"2023-09-30T09:46:04.041373+0000","last_clean":"2023-09-30T09:46:04.041373+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:46:04.041373+0000","last_undegraded":"2023-09-30T09:46:04.041373+0000","last_fullsized":"2023-09-30T09:46:04.041373+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":537,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041274+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:46:04.041274+0000","last_peered":"2023-09-30T09:46:04.041274+0000","last_clean":"2023-09-30T09:46:04.041274+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:46:04.041274+0000","last_undegraded":"2023-09-30T09:46:04.041274+0000","last_fullsized":"2023-09-30T09:46:04.041274+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":768,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033291+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:46:04.033291+0000","last_peered":"2023-09-30T09:46:04.033291+0000","last_clean":"2023-09-30T09:46:04.033291+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:46:04.033291+0000","last_undegraded":"2023-09-30T09:46:04.033291+0000","last_fullsized":"2023-09-30T09:46:04.033291+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":165,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033864+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:46:04.033864+0000","last_peered":"2023-09-30T09:46:04.033864+0000","last_clean":"2023-09-30T09:46:04.033864+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:46:04.033864+0000","last_undegraded":"2023-09-30T09:46:04.033864+0000","last_fullsized":"2023-09-30T09:46:04.033864+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":492,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034859+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:46:04.034859+0000","last_peered":"2023-09-30T09:46:04.034859+0000","last_clean":"2023-09-30T09:46:04.034859+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:46:04.034859+0000","last_undegraded":"2023-09-30T09:46:04.034859+0000","last_fullsized":"2023-09-30T09:46:04.034859+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":816,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036968+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:46:04.036968+0000","last_peered":"2023-09-30T09:46:04.036968+0000","last_clean":"2023-09-30T09:46:04.036968+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:46:04.036968+0000","last_undegraded":"2023-09-30T09:46:04.036968+0000","last_fullsized":"2023-09-30T09:46:04.036968+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":94,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033160+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:46:04.033160+0000","last_peered":"2023-09-30T09:46:04.033160+0000","last_clean":"2023-09-30T09:46:04.033160+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:46:04.033160+0000","last_undegraded":"2023-09-30T09:46:04.033160+0000","last_fullsized":"2023-09-30T09:46:04.033160+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":190,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035896+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:46:04.035896+0000","last_peered":"2023-09-30T09:46:04.035896+0000","last_clean":"2023-09-30T09:46:04.035896+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:46:04.035896+0000","last_undegraded":"2023-09-30T09:46:04.035896+0000","last_fullsized":"2023-09-30T09:46:04.035896+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":916,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.043117+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:46:04.043117+0000","last_peered":"2023-09-30T09:46:04.043117+0000","last_clean":"2023-09-30T09:46:04.043117+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:46:04.043117+0000","last_undegraded":"2023-09-30T09:46:04.043117+0000","last_fullsized":"2023-09-30T09:46:04.043117+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":199,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033798+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:46:04.033798+0000","last_peered":"2023-09-30T09:46:04.033798+0000","last_clean":"2023-09-30T09:46:04.033798+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:46:04.033798+0000","last_undegraded":"2023-09-30T09:46:04.033798+0000","last_fullsized":"2023-09-30T09:46:04.033798+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":94,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033663+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:46:04.033663+0000","last_peered":"2023-09-30T09:46:04.033663+0000","last_clean":"2023-09-30T09:46:04.033663+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:46:04.033663+0000","last_undegraded":"2023-09-30T09:46:04.033663+0000","last_fullsized":"2023-09-30T09:46:04.033663+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":925,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036168+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:46:04.036168+0000","last_peered":"2023-09-30T09:46:04.036168+0000","last_clean":"2023-09-30T09:46:04.036168+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:46:04.036168+0000","last_undegraded":"2023-09-30T09:46:04.036168+0000","last_fullsized":"2023-09-30T09:46:04.036168+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026487+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:46:04.026487+0000","last_peered":"2023-09-30T09:46:04.026487+0000","last_clean":"2023-09-30T09:46:04.026487+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:46:04.026487+0000","last_undegraded":"2023-09-30T09:46:04.026487+0000","last_fullsized":"2023-09-30T09:46:04.026487+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":79,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.042440+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:46:04.042440+0000","last_peered":"2023-09-30T09:46:04.042440+0000","last_clean":"2023-09-30T09:46:04.042440+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:46:04.042440+0000","last_undegraded":"2023-09-30T09:46:04.042440+0000","last_fullsized":"2023-09-30T09:46:04.042440+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"515'7101","reported_seq":7405,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:19.070947+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:46:19.070947+0000","last_peered":"2023-09-30T09:46:19.070947+0000","last_clean":"2023-09-30T09:46:19.070947+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:46:19.070947+0000","last_undegraded":"2023-09-30T09:46:19.070947+0000","last_fullsized":"2023-09-30T09:46:19.070947+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7101,"log_dups_size":0,"ondisk_log_size":7101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319356928,"num_objects":4873,"num_object_clones":0,"num_object_copies":14619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7101,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":891,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034602+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:46:04.034602+0000","last_peered":"2023-09-30T09:46:04.034602+0000","last_clean":"2023-09-30T09:46:04.034602+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:46:04.034602+0000","last_undegraded":"2023-09-30T09:46:04.034602+0000","last_fullsized":"2023-09-30T09:46:04.034602+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":191,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034750+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:46:04.034750+0000","last_peered":"2023-09-30T09:46:04.034750+0000","last_clean":"2023-09-30T09:46:04.034750+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:46:04.034750+0000","last_undegraded":"2023-09-30T09:46:04.034750+0000","last_fullsized":"2023-09-30T09:46:04.034750+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036887+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:46:04.036887+0000","last_peered":"2023-09-30T09:46:04.036887+0000","last_clean":"2023-09-30T09:46:04.036887+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:46:04.036887+0000","last_undegraded":"2023-09-30T09:46:04.036887+0000","last_fullsized":"2023-09-30T09:46:04.036887+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"515'7090","reported_seq":7130,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:18.776443+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:46:18.776443+0000","last_peered":"2023-09-30T09:46:18.776443+0000","last_clean":"2023-09-30T09:46:18.776443+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:46:18.776443+0000","last_undegraded":"2023-09-30T09:46:18.776443+0000","last_fullsized":"2023-09-30T09:46:18.776443+0000","mapping_epoch":483,"log_start":"496'700","ondisk_log_start":"496'700","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318111744,"num_objects":4854,"num_object_clones":0,"num_object_copies":14562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7090,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":908,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.042832+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:46:04.042832+0000","last_peered":"2023-09-30T09:46:04.042832+0000","last_clean":"2023-09-30T09:46:04.042832+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:46:04.042832+0000","last_undegraded":"2023-09-30T09:46:04.042832+0000","last_fullsized":"2023-09-30T09:46:04.042832+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":161,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034568+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:46:04.034568+0000","last_peered":"2023-09-30T09:46:04.034568+0000","last_clean":"2023-09-30T09:46:04.034568+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:46:04.034568+0000","last_undegraded":"2023-09-30T09:46:04.034568+0000","last_fullsized":"2023-09-30T09:46:04.034568+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":566,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040614+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:46:04.040614+0000","last_peered":"2023-09-30T09:46:04.040614+0000","last_clean":"2023-09-30T09:46:04.040614+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:46:04.040614+0000","last_undegraded":"2023-09-30T09:46:04.040614+0000","last_fullsized":"2023-09-30T09:46:04.040614+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"515'7305","reported_seq":7570,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:19.110649+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:46:19.110649+0000","last_peered":"2023-09-30T09:46:19.110649+0000","last_clean":"2023-09-30T09:46:19.110649+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:46:19.110649+0000","last_undegraded":"2023-09-30T09:46:19.110649+0000","last_fullsized":"2023-09-30T09:46:19.110649+0000","mapping_epoch":493,"log_start":"496'600","ondisk_log_start":"496'600","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6705,"log_dups_size":0,"ondisk_log_size":6705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319619072,"num_objects":4877,"num_object_clones":0,"num_object_copies":14631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7305,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":685,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026813+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:46:04.026813+0000","last_peered":"2023-09-30T09:46:04.026813+0000","last_clean":"2023-09-30T09:46:04.026813+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:46:04.026813+0000","last_undegraded":"2023-09-30T09:46:04.026813+0000","last_fullsized":"2023-09-30T09:46:04.026813+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":228,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034147+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:46:04.034147+0000","last_peered":"2023-09-30T09:46:04.034147+0000","last_clean":"2023-09-30T09:46:04.034147+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:46:04.034147+0000","last_undegraded":"2023-09-30T09:46:04.034147+0000","last_fullsized":"2023-09-30T09:46:04.034147+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":84,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026761+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:46:04.026761+0000","last_peered":"2023-09-30T09:46:04.026761+0000","last_clean":"2023-09-30T09:46:04.026761+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:46:04.026761+0000","last_undegraded":"2023-09-30T09:46:04.026761+0000","last_fullsized":"2023-09-30T09:46:04.026761+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"515'7079","reported_seq":7434,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:18.806419+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:46:18.806419+0000","last_peered":"2023-09-30T09:46:18.806419+0000","last_clean":"2023-09-30T09:46:18.806419+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:46:18.806419+0000","last_undegraded":"2023-09-30T09:46:18.806419+0000","last_fullsized":"2023-09-30T09:46:18.806419+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7079,"log_dups_size":0,"ondisk_log_size":7079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315686912,"num_objects":4817,"num_object_clones":0,"num_object_copies":14451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7079,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":750,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041652+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:46:04.041652+0000","last_peered":"2023-09-30T09:46:04.041652+0000","last_clean":"2023-09-30T09:46:04.041652+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:46:04.041652+0000","last_undegraded":"2023-09-30T09:46:04.041652+0000","last_fullsized":"2023-09-30T09:46:04.041652+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":450,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036060+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:46:04.036060+0000","last_peered":"2023-09-30T09:46:04.036060+0000","last_clean":"2023-09-30T09:46:04.036060+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:46:04.036060+0000","last_undegraded":"2023-09-30T09:46:04.036060+0000","last_fullsized":"2023-09-30T09:46:04.036060+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":114,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034078+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:46:04.034078+0000","last_peered":"2023-09-30T09:46:04.034078+0000","last_clean":"2023-09-30T09:46:04.034078+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:46:04.034078+0000","last_undegraded":"2023-09-30T09:46:04.034078+0000","last_fullsized":"2023-09-30T09:46:04.034078+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"515'7204","reported_seq":7242,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:19.107493+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:46:19.107493+0000","last_peered":"2023-09-30T09:46:19.107493+0000","last_clean":"2023-09-30T09:46:19.107493+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:46:19.107493+0000","last_undegraded":"2023-09-30T09:46:19.107493+0000","last_fullsized":"2023-09-30T09:46:19.107493+0000","mapping_epoch":483,"log_start":"491'200","ondisk_log_start":"491'200","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322961408,"num_objects":4928,"num_object_clones":0,"num_object_copies":14784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7204,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":997,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033785+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:46:04.033785+0000","last_peered":"2023-09-30T09:46:04.033785+0000","last_clean":"2023-09-30T09:46:04.033785+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:46:04.033785+0000","last_undegraded":"2023-09-30T09:46:04.033785+0000","last_fullsized":"2023-09-30T09:46:04.033785+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":73,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033899+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:46:04.033899+0000","last_peered":"2023-09-30T09:46:04.033899+0000","last_clean":"2023-09-30T09:46:04.033899+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:46:04.033899+0000","last_undegraded":"2023-09-30T09:46:04.033899+0000","last_fullsized":"2023-09-30T09:46:04.033899+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":513,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041454+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:46:04.041454+0000","last_peered":"2023-09-30T09:46:04.041454+0000","last_clean":"2023-09-30T09:46:04.041454+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:46:04.041454+0000","last_undegraded":"2023-09-30T09:46:04.041454+0000","last_fullsized":"2023-09-30T09:46:04.041454+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"515'7166","reported_seq":7206,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:19.179390+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:46:19.179390+0000","last_peered":"2023-09-30T09:46:19.179390+0000","last_clean":"2023-09-30T09:46:19.179390+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:46:19.179390+0000","last_undegraded":"2023-09-30T09:46:19.179390+0000","last_fullsized":"2023-09-30T09:46:19.179390+0000","mapping_epoch":483,"log_start":"488'100","ondisk_log_start":"488'100","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7066,"log_dups_size":0,"ondisk_log_size":7066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313786368,"num_objects":4788,"num_object_clones":0,"num_object_copies":14364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7166,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":996,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:13.823790+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:46:13.823790+0000","last_peered":"2023-09-30T09:46:13.823790+0000","last_clean":"2023-09-30T09:46:13.823790+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:46:13.823790+0000","last_undegraded":"2023-09-30T09:46:13.823790+0000","last_fullsized":"2023-09-30T09:46:13.823790+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026670+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:46:04.026670+0000","last_peered":"2023-09-30T09:46:04.026670+0000","last_clean":"2023-09-30T09:46:04.026670+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:46:04.026670+0000","last_undegraded":"2023-09-30T09:46:04.026670+0000","last_fullsized":"2023-09-30T09:46:04.026670+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":209,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041070+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:46:04.041070+0000","last_peered":"2023-09-30T09:46:04.041070+0000","last_clean":"2023-09-30T09:46:04.041070+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:46:04.041070+0000","last_undegraded":"2023-09-30T09:46:04.041070+0000","last_fullsized":"2023-09-30T09:46:04.041070+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"515'7326","reported_seq":7366,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:18.863266+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:46:18.863266+0000","last_peered":"2023-09-30T09:46:18.863266+0000","last_clean":"2023-09-30T09:46:18.863266+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:46:18.863266+0000","last_undegraded":"2023-09-30T09:46:18.863266+0000","last_fullsized":"2023-09-30T09:46:18.863266+0000","mapping_epoch":483,"log_start":"497'1500","ondisk_log_start":"497'1500","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5826,"log_dups_size":0,"ondisk_log_size":5826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324009984,"num_objects":4944,"num_object_clones":0,"num_object_copies":14832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7326,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":853,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034100+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:46:04.034100+0000","last_peered":"2023-09-30T09:46:04.034100+0000","last_clean":"2023-09-30T09:46:04.034100+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:46:04.034100+0000","last_undegraded":"2023-09-30T09:46:04.034100+0000","last_fullsized":"2023-09-30T09:46:04.034100+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":520,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040540+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:46:04.040540+0000","last_peered":"2023-09-30T09:46:04.040540+0000","last_clean":"2023-09-30T09:46:04.040540+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:46:04.040540+0000","last_undegraded":"2023-09-30T09:46:04.040540+0000","last_fullsized":"2023-09-30T09:46:04.040540+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":527,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041741+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:46:04.041741+0000","last_peered":"2023-09-30T09:46:04.041741+0000","last_clean":"2023-09-30T09:46:04.041741+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:46:04.041741+0000","last_undegraded":"2023-09-30T09:46:04.041741+0000","last_fullsized":"2023-09-30T09:46:04.041741+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"515'7113","reported_seq":7156,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:18.806498+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:46:18.806498+0000","last_peered":"2023-09-30T09:46:18.806498+0000","last_clean":"2023-09-30T09:46:18.806498+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:46:18.806498+0000","last_undegraded":"2023-09-30T09:46:18.806498+0000","last_fullsized":"2023-09-30T09:46:18.806498+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7113,"log_dups_size":0,"ondisk_log_size":7113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316080128,"num_objects":4823,"num_object_clones":0,"num_object_copies":14469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7113,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":718,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:13.823869+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:46:13.823869+0000","last_peered":"2023-09-30T09:46:13.823869+0000","last_clean":"2023-09-30T09:46:13.823869+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:46:13.823869+0000","last_undegraded":"2023-09-30T09:46:13.823869+0000","last_fullsized":"2023-09-30T09:46:13.823869+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.042094+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:46:04.042094+0000","last_peered":"2023-09-30T09:46:04.042094+0000","last_clean":"2023-09-30T09:46:04.042094+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:46:04.042094+0000","last_undegraded":"2023-09-30T09:46:04.042094+0000","last_fullsized":"2023-09-30T09:46:04.042094+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":200,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041844+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:46:04.041844+0000","last_peered":"2023-09-30T09:46:04.041844+0000","last_clean":"2023-09-30T09:46:04.041844+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:46:04.041844+0000","last_undegraded":"2023-09-30T09:46:04.041844+0000","last_fullsized":"2023-09-30T09:46:04.041844+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"515'7226","reported_seq":7525,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:19.077311+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:46:19.077311+0000","last_peered":"2023-09-30T09:46:19.077311+0000","last_clean":"2023-09-30T09:46:19.077311+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:46:19.077311+0000","last_undegraded":"2023-09-30T09:46:19.077311+0000","last_fullsized":"2023-09-30T09:46:19.077311+0000","mapping_epoch":494,"log_start":"496'500","ondisk_log_start":"496'500","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319684608,"num_objects":4878,"num_object_clones":0,"num_object_copies":14634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7226,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":757,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026691+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:46:04.026691+0000","last_peered":"2023-09-30T09:46:04.026691+0000","last_clean":"2023-09-30T09:46:04.026691+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:46:04.026691+0000","last_undegraded":"2023-09-30T09:46:04.026691+0000","last_fullsized":"2023-09-30T09:46:04.026691+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":745,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033323+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:46:04.033323+0000","last_peered":"2023-09-30T09:46:04.033323+0000","last_clean":"2023-09-30T09:46:04.033323+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:46:04.033323+0000","last_undegraded":"2023-09-30T09:46:04.033323+0000","last_fullsized":"2023-09-30T09:46:04.033323+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":443,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036045+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:46:04.036045+0000","last_peered":"2023-09-30T09:46:04.036045+0000","last_clean":"2023-09-30T09:46:04.036045+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:46:04.036045+0000","last_undegraded":"2023-09-30T09:46:04.036045+0000","last_fullsized":"2023-09-30T09:46:04.036045+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035859+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:04.035859+0000","last_peered":"2023-09-30T09:46:04.035859+0000","last_clean":"2023-09-30T09:46:04.035859+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:46:04.035859+0000","last_undegraded":"2023-09-30T09:46:04.035859+0000","last_fullsized":"2023-09-30T09:46:04.035859+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"515'7243","reported_seq":7587,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:19.110530+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:46:19.110530+0000","last_peered":"2023-09-30T09:46:19.110530+0000","last_clean":"2023-09-30T09:46:19.110530+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:46:19.110530+0000","last_undegraded":"2023-09-30T09:46:19.110530+0000","last_fullsized":"2023-09-30T09:46:19.110530+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7243,"log_dups_size":0,"ondisk_log_size":7243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320143360,"num_objects":4885,"num_object_clones":0,"num_object_copies":14655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7243,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":956,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035781+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:46:04.035781+0000","last_peered":"2023-09-30T09:46:04.035781+0000","last_clean":"2023-09-30T09:46:04.035781+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:46:04.035781+0000","last_undegraded":"2023-09-30T09:46:04.035781+0000","last_fullsized":"2023-09-30T09:46:04.035781+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":462,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026780+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:46:04.026780+0000","last_peered":"2023-09-30T09:46:04.026780+0000","last_clean":"2023-09-30T09:46:04.026780+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:46:04.026780+0000","last_undegraded":"2023-09-30T09:46:04.026780+0000","last_fullsized":"2023-09-30T09:46:04.026780+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":534,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041312+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:46:04.041312+0000","last_peered":"2023-09-30T09:46:04.041312+0000","last_clean":"2023-09-30T09:46:04.041312+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:46:04.041312+0000","last_undegraded":"2023-09-30T09:46:04.041312+0000","last_fullsized":"2023-09-30T09:46:04.041312+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035805+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:46:04.035805+0000","last_peered":"2023-09-30T09:46:04.035805+0000","last_clean":"2023-09-30T09:46:04.035805+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:46:04.035805+0000","last_undegraded":"2023-09-30T09:46:04.035805+0000","last_fullsized":"2023-09-30T09:46:04.035805+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"515'7244","reported_seq":7589,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:19.046726+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:46:19.046726+0000","last_peered":"2023-09-30T09:46:19.046726+0000","last_clean":"2023-09-30T09:46:19.046726+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:46:19.046726+0000","last_undegraded":"2023-09-30T09:46:19.046726+0000","last_fullsized":"2023-09-30T09:46:19.046726+0000","mapping_epoch":492,"log_start":"496'500","ondisk_log_start":"496'500","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322961408,"num_objects":4928,"num_object_clones":0,"num_object_copies":14784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7244,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":934,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033791+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:46:04.033791+0000","last_peered":"2023-09-30T09:46:04.033791+0000","last_clean":"2023-09-30T09:46:04.033791+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:46:04.033791+0000","last_undegraded":"2023-09-30T09:46:04.033791+0000","last_fullsized":"2023-09-30T09:46:04.033791+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":783,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035705+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:46:04.035705+0000","last_peered":"2023-09-30T09:46:04.035705+0000","last_clean":"2023-09-30T09:46:04.035705+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:46:04.035705+0000","last_undegraded":"2023-09-30T09:46:04.035705+0000","last_fullsized":"2023-09-30T09:46:04.035705+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":277,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035325+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:46:04.035325+0000","last_peered":"2023-09-30T09:46:04.035325+0000","last_clean":"2023-09-30T09:46:04.035325+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:46:04.035325+0000","last_undegraded":"2023-09-30T09:46:04.035325+0000","last_fullsized":"2023-09-30T09:46:04.035325+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":364,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036536+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:46:04.036536+0000","last_peered":"2023-09-30T09:46:04.036536+0000","last_clean":"2023-09-30T09:46:04.036536+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:46:04.036536+0000","last_undegraded":"2023-09-30T09:46:04.036536+0000","last_fullsized":"2023-09-30T09:46:04.036536+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"515'7129","reported_seq":7433,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:19.054800+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:46:19.054800+0000","last_peered":"2023-09-30T09:46:19.054800+0000","last_clean":"2023-09-30T09:46:19.054800+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:46:19.054800+0000","last_undegraded":"2023-09-30T09:46:19.054800+0000","last_fullsized":"2023-09-30T09:46:19.054800+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7129,"log_dups_size":0,"ondisk_log_size":7129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320405504,"num_objects":4889,"num_object_clones":0,"num_object_copies":14667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7129,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1076,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.038419+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:46:04.038419+0000","last_peered":"2023-09-30T09:46:04.038419+0000","last_clean":"2023-09-30T09:46:04.038419+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:46:04.038419+0000","last_undegraded":"2023-09-30T09:46:04.038419+0000","last_fullsized":"2023-09-30T09:46:04.038419+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":877,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033577+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:46:04.033577+0000","last_peered":"2023-09-30T09:46:04.033577+0000","last_clean":"2023-09-30T09:46:04.033577+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:46:04.033577+0000","last_undegraded":"2023-09-30T09:46:04.033577+0000","last_fullsized":"2023-09-30T09:46:04.033577+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035935+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:46:04.035935+0000","last_peered":"2023-09-30T09:46:04.035935+0000","last_clean":"2023-09-30T09:46:04.035935+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:46:04.035935+0000","last_undegraded":"2023-09-30T09:46:04.035935+0000","last_fullsized":"2023-09-30T09:46:04.035935+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":70,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026777+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:46:04.026777+0000","last_peered":"2023-09-30T09:46:04.026777+0000","last_clean":"2023-09-30T09:46:04.026777+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:46:04.026777+0000","last_undegraded":"2023-09-30T09:46:04.026777+0000","last_fullsized":"2023-09-30T09:46:04.026777+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"515'7310","reported_seq":10263,"reported_epoch":515,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:46:18.816615+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:46:18.816615+0000","last_peered":"2023-09-30T09:46:18.816615+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:46:18.816615+0000","last_undegraded":"2023-09-30T09:46:18.816615+0000","last_fullsized":"2023-09-30T09:46:18.816615+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7310,"log_dups_size":0,"ondisk_log_size":7310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328728576,"num_objects":5016,"num_object_clones":0,"num_object_copies":15048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3527,"num_objects_unfound":0,"num_objects_dirty":5016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7310,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1443,"num_bytes_recovered":94568448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":1489}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":776,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.027447+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:46:04.027447+0000","last_peered":"2023-09-30T09:46:04.027447+0000","last_clean":"2023-09-30T09:46:04.027447+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:46:04.027447+0000","last_undegraded":"2023-09-30T09:46:04.027447+0000","last_fullsized":"2023-09-30T09:46:04.027447+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":923,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033525+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:46:04.033525+0000","last_peered":"2023-09-30T09:46:04.033525+0000","last_clean":"2023-09-30T09:46:04.033525+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:46:04.033525+0000","last_undegraded":"2023-09-30T09:46:04.033525+0000","last_fullsized":"2023-09-30T09:46:04.033525+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033441+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:46:04.033441+0000","last_peered":"2023-09-30T09:46:04.033441+0000","last_clean":"2023-09-30T09:46:04.033441+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:46:04.033441+0000","last_undegraded":"2023-09-30T09:46:04.033441+0000","last_fullsized":"2023-09-30T09:46:04.033441+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":20,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033598+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:46:04.033598+0000","last_peered":"2023-09-30T09:46:04.033598+0000","last_clean":"2023-09-30T09:46:04.033598+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:46:04.033598+0000","last_undegraded":"2023-09-30T09:46:04.033598+0000","last_fullsized":"2023-09-30T09:46:04.033598+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"515'7174","reported_seq":7214,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:18.842613+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:46:18.842613+0000","last_peered":"2023-09-30T09:46:18.842613+0000","last_clean":"2023-09-30T09:46:18.842613+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:46:18.842613+0000","last_undegraded":"2023-09-30T09:46:18.842613+0000","last_fullsized":"2023-09-30T09:46:18.842613+0000","mapping_epoch":483,"log_start":"497'1400","ondisk_log_start":"497'1400","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323616768,"num_objects":4938,"num_object_clones":0,"num_object_copies":14814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7174,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":767,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.027093+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:46:04.027093+0000","last_peered":"2023-09-30T09:46:04.027093+0000","last_clean":"2023-09-30T09:46:04.027093+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:46:04.027093+0000","last_undegraded":"2023-09-30T09:46:04.027093+0000","last_fullsized":"2023-09-30T09:46:04.027093+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1063,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034683+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:46:04.034683+0000","last_peered":"2023-09-30T09:46:04.034683+0000","last_clean":"2023-09-30T09:46:04.034683+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:46:04.034683+0000","last_undegraded":"2023-09-30T09:46:04.034683+0000","last_fullsized":"2023-09-30T09:46:04.034683+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":258,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035622+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:46:04.035622+0000","last_peered":"2023-09-30T09:46:04.035622+0000","last_clean":"2023-09-30T09:46:04.035622+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:46:04.035622+0000","last_undegraded":"2023-09-30T09:46:04.035622+0000","last_fullsized":"2023-09-30T09:46:04.035622+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":127,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034349+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:46:04.034349+0000","last_peered":"2023-09-30T09:46:04.034349+0000","last_clean":"2023-09-30T09:46:04.034349+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:46:04.034349+0000","last_undegraded":"2023-09-30T09:46:04.034349+0000","last_fullsized":"2023-09-30T09:46:04.034349+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"515'7078","reported_seq":7453,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:18.866507+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:46:18.866507+0000","last_peered":"2023-09-30T09:46:18.866507+0000","last_clean":"2023-09-30T09:46:18.866507+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:46:18.866507+0000","last_undegraded":"2023-09-30T09:46:18.866507+0000","last_fullsized":"2023-09-30T09:46:18.866507+0000","mapping_epoch":495,"log_start":"497'1300","ondisk_log_start":"497'1300","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5778,"log_dups_size":0,"ondisk_log_size":5778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323158040,"num_objects":4932,"num_object_clones":0,"num_object_copies":14796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4932,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7078,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1034,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041630+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:46:04.041630+0000","last_peered":"2023-09-30T09:46:04.041630+0000","last_clean":"2023-09-30T09:46:04.041630+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:46:04.041630+0000","last_undegraded":"2023-09-30T09:46:04.041630+0000","last_fullsized":"2023-09-30T09:46:04.041630+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":546,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034679+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:46:04.034679+0000","last_peered":"2023-09-30T09:46:04.034679+0000","last_clean":"2023-09-30T09:46:04.034679+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:46:04.034679+0000","last_undegraded":"2023-09-30T09:46:04.034679+0000","last_fullsized":"2023-09-30T09:46:04.034679+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1233,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.027010+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:46:04.027010+0000","last_peered":"2023-09-30T09:46:04.027010+0000","last_clean":"2023-09-30T09:46:04.027010+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:46:04.027010+0000","last_undegraded":"2023-09-30T09:46:04.027010+0000","last_fullsized":"2023-09-30T09:46:04.027010+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":550,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033497+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:46:04.033497+0000","last_peered":"2023-09-30T09:46:04.033497+0000","last_clean":"2023-09-30T09:46:04.033497+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:46:04.033497+0000","last_undegraded":"2023-09-30T09:46:04.033497+0000","last_fullsized":"2023-09-30T09:46:04.033497+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"515'7372","reported_seq":7412,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:18.819564+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:46:18.819564+0000","last_peered":"2023-09-30T09:46:18.819564+0000","last_clean":"2023-09-30T09:46:18.819564+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:46:18.819564+0000","last_undegraded":"2023-09-30T09:46:18.819564+0000","last_fullsized":"2023-09-30T09:46:18.819564+0000","mapping_epoch":483,"log_start":"496'900","ondisk_log_start":"496'900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6472,"log_dups_size":0,"ondisk_log_size":6472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324796416,"num_objects":4956,"num_object_clones":0,"num_object_copies":14868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7372,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":897,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.042690+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:46:04.042690+0000","last_peered":"2023-09-30T09:46:04.042690+0000","last_clean":"2023-09-30T09:46:04.042690+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:46:04.042690+0000","last_undegraded":"2023-09-30T09:46:04.042690+0000","last_fullsized":"2023-09-30T09:46:04.042690+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":85,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026789+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:46:04.026789+0000","last_peered":"2023-09-30T09:46:04.026789+0000","last_clean":"2023-09-30T09:46:04.026789+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:46:04.026789+0000","last_undegraded":"2023-09-30T09:46:04.026789+0000","last_fullsized":"2023-09-30T09:46:04.026789+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":90,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033766+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:46:04.033766+0000","last_peered":"2023-09-30T09:46:04.033766+0000","last_clean":"2023-09-30T09:46:04.033766+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:46:04.033766+0000","last_undegraded":"2023-09-30T09:46:04.033766+0000","last_fullsized":"2023-09-30T09:46:04.033766+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":516,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040570+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:46:04.040570+0000","last_peered":"2023-09-30T09:46:04.040570+0000","last_clean":"2023-09-30T09:46:04.040570+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:46:04.040570+0000","last_undegraded":"2023-09-30T09:46:04.040570+0000","last_fullsized":"2023-09-30T09:46:04.040570+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":834,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033205+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:46:04.033205+0000","last_peered":"2023-09-30T09:46:04.033205+0000","last_clean":"2023-09-30T09:46:04.033205+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:46:04.033205+0000","last_undegraded":"2023-09-30T09:46:04.033205+0000","last_fullsized":"2023-09-30T09:46:04.033205+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":20,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034167+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:46:04.034167+0000","last_peered":"2023-09-30T09:46:04.034167+0000","last_clean":"2023-09-30T09:46:04.034167+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:46:04.034167+0000","last_undegraded":"2023-09-30T09:46:04.034167+0000","last_fullsized":"2023-09-30T09:46:04.034167+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":220,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035515+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:46:04.035515+0000","last_peered":"2023-09-30T09:46:04.035515+0000","last_clean":"2023-09-30T09:46:04.035515+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:46:04.035515+0000","last_undegraded":"2023-09-30T09:46:04.035515+0000","last_fullsized":"2023-09-30T09:46:04.035515+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":629,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.027171+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:46:04.027171+0000","last_peered":"2023-09-30T09:46:04.027171+0000","last_clean":"2023-09-30T09:46:04.027171+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:46:04.027171+0000","last_undegraded":"2023-09-30T09:46:04.027171+0000","last_fullsized":"2023-09-30T09:46:04.027171+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":443,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035985+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:04.035985+0000","last_peered":"2023-09-30T09:46:04.035985+0000","last_clean":"2023-09-30T09:46:04.035985+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:46:04.035985+0000","last_undegraded":"2023-09-30T09:46:04.035985+0000","last_fullsized":"2023-09-30T09:46:04.035985+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":229,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041136+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:46:04.041136+0000","last_peered":"2023-09-30T09:46:04.041136+0000","last_clean":"2023-09-30T09:46:04.041136+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:46:04.041136+0000","last_undegraded":"2023-09-30T09:46:04.041136+0000","last_fullsized":"2023-09-30T09:46:04.041136+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":385,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036262+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:46:04.036262+0000","last_peered":"2023-09-30T09:46:04.036262+0000","last_clean":"2023-09-30T09:46:04.036262+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:46:04.036262+0000","last_undegraded":"2023-09-30T09:46:04.036262+0000","last_fullsized":"2023-09-30T09:46:04.036262+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":249,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040531+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:46:04.040531+0000","last_peered":"2023-09-30T09:46:04.040531+0000","last_clean":"2023-09-30T09:46:04.040531+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:46:04.040531+0000","last_undegraded":"2023-09-30T09:46:04.040531+0000","last_fullsized":"2023-09-30T09:46:04.040531+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":842,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034496+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:46:04.034496+0000","last_peered":"2023-09-30T09:46:04.034496+0000","last_clean":"2023-09-30T09:46:04.034496+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:46:04.034496+0000","last_undegraded":"2023-09-30T09:46:04.034496+0000","last_fullsized":"2023-09-30T09:46:04.034496+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":72,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026886+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:46:04.026886+0000","last_peered":"2023-09-30T09:46:04.026886+0000","last_clean":"2023-09-30T09:46:04.026886+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:46:04.026886+0000","last_undegraded":"2023-09-30T09:46:04.026886+0000","last_fullsized":"2023-09-30T09:46:04.026886+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":466,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:13.823820+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:46:13.823820+0000","last_peered":"2023-09-30T09:46:13.823820+0000","last_clean":"2023-09-30T09:46:13.823820+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:46:13.823820+0000","last_undegraded":"2023-09-30T09:46:13.823820+0000","last_fullsized":"2023-09-30T09:46:13.823820+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":885,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041538+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:46:04.041538+0000","last_peered":"2023-09-30T09:46:04.041538+0000","last_clean":"2023-09-30T09:46:04.041538+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:46:04.041538+0000","last_undegraded":"2023-09-30T09:46:04.041538+0000","last_fullsized":"2023-09-30T09:46:04.041538+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":26,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033661+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:46:04.033661+0000","last_peered":"2023-09-30T09:46:04.033661+0000","last_clean":"2023-09-30T09:46:04.033661+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:46:04.033661+0000","last_undegraded":"2023-09-30T09:46:04.033661+0000","last_fullsized":"2023-09-30T09:46:04.033661+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":530,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040436+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:46:04.040436+0000","last_peered":"2023-09-30T09:46:04.040436+0000","last_clean":"2023-09-30T09:46:04.040436+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:46:04.040436+0000","last_undegraded":"2023-09-30T09:46:04.040436+0000","last_fullsized":"2023-09-30T09:46:04.040436+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5133107224,"num_objects":78326,"num_object_clones":0,"num_object_copies":234978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3527,"num_objects_unfound":0,"num_objects_dirty":78326,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115160,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2586,"num_bytes_recovered":169476096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15574970368,"data_stored":15574958152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107460,"ondisk_log_size":107460,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234415,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3396960,"kb_used_data":2874968,"kb_used_omap":0,"kb_used_meta":521984,"kb_avail":90348192,"statfs":{"total":95995035648,"available":92516548608,"internally_reserved":0,"allocated":2943967232,"data_stored":2942243773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":534511616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:01 2023","interfaces":[{"interface":"back","average":{"1min":13.629,"5min":6.0149999999999997,"15min":4.5529999999999999},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":110.467,"5min":124.74299999999999,"15min":215.416},"last":0.32800000000000001},{"interface":"front","average":{"1min":13.702999999999999,"5min":6.1459999999999999,"15min":4.5099999999999998},"min":{"1min":0.23499999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":109.593,"5min":124.69499999999999,"15min":215.041},"last":6.4950000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:46:01 2023","interfaces":[{"interface":"back","average":{"1min":16.905000000000001,"5min":7.1210000000000004,"15min":4.1399999999999997},"min":{"1min":0.32400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":157.578,"5min":157.578,"15min":157.578},"last":0.41799999999999998},{"interface":"front","average":{"1min":17.038,"5min":6.9660000000000002,"15min":4.3380000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":157.589,"5min":157.589,"15min":157.589},"last":1.1519999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":4.3570000000000002,"15min":3.5009999999999999},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.24299999999999999},"max":{"1min":5.3789999999999996,"5min":117.14,"15min":117.14},"last":1.121},{"interface":"front","average":{"1min":2.3450000000000002,"5min":4.585,"15min":3.528},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":18.603000000000002,"5min":117.044,"15min":117.044},"last":2.476}]},{"osd":3,"last update":"Sat Sep 30 09:46:15 2023","interfaces":[{"interface":"back","average":{"1min":11.387,"5min":6.8799999999999999,"15min":8.9429999999999996},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.31},"max":{"1min":154.06700000000001,"5min":154.06700000000001,"15min":277.57799999999997},"last":5.0419999999999998},{"interface":"front","average":{"1min":11.162000000000001,"5min":6.8810000000000002,"15min":8.4610000000000003},"min":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":154.24799999999999,"5min":154.24799999999999,"15min":278.69299999999998},"last":2.5569999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":0.96699999999999997,"15min":1.6000000000000001},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":25.192,"5min":25.192,"15min":175.72},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.657,"5min":0.93100000000000005,"15min":1.526},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":24.521000000000001,"5min":24.521000000000001,"15min":179.048},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":11.993,"5min":11.993,"15min":11.993},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":197.75700000000001,"5min":197.75700000000001,"15min":197.75700000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":11.827,"5min":11.827,"15min":11.827},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":197.63300000000001,"5min":197.63300000000001,"15min":197.63300000000001},"last":4.4059999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":0.61899999999999999,"15min":2.1659999999999999},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.367,"5min":4.367,"15min":206.244},"last":0.496},{"interface":"front","average":{"1min":1.6679999999999999,"5min":0.76300000000000001,"15min":2.2000000000000002},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":202.78800000000001},"last":0.85799999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496053,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1468168,"kb_used_data":948032,"kb_used_omap":0,"kb_used_meta":520128,"kb_avail":92276984,"statfs":{"total":95995035648,"available":94491631616,"internally_reserved":0,"allocated":970784768,"data_stored":969084602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":532611072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":5.4119999999999999,"5min":3.6509999999999998,"15min":5.2279999999999998},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":79.504000000000005,"5min":95.043999999999997,"15min":187.19800000000001},"last":1.9099999999999999},{"interface":"front","average":{"1min":5.4580000000000002,"5min":3.8250000000000002,"15min":5.3479999999999999},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":78.813000000000002,"5min":94.855999999999995,"15min":187.46700000000001},"last":4.0709999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.1519999999999999,"15min":1.952},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5050000000000008,"5min":32.814,"15min":69.569999999999993},"last":1.7490000000000001},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.1850000000000001,"15min":2.1400000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":8.5169999999999995,"5min":32.856000000000002,"15min":70.471999999999994},"last":4.1539999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":3.456,"15min":3.798},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6120000000000001,"5min":207.02500000000001,"15min":207.02500000000001},"last":2.109},{"interface":"front","average":{"1min":1.661,"5min":3.8170000000000002,"15min":4.0199999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28199999999999997},"max":{"1min":6.1299999999999999,"5min":207.114,"15min":207.114},"last":6.8899999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":3.2810000000000001,"15min":4.2830000000000004},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":7.1299999999999999,"5min":107.11799999999999,"15min":212.44800000000001},"last":2.3220000000000001},{"interface":"front","average":{"1min":2.0939999999999999,"5min":3.2450000000000001,"15min":4.5759999999999996},"min":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":7.7220000000000004,"5min":106.982,"15min":210.84299999999999},"last":8.2449999999999992}]},{"osd":4,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.5740000000000001,"15min":1.3540000000000001},"min":{"1min":0.26200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.5190000000000001,"5min":63.798000000000002,"15min":63.798000000000002},"last":4.1950000000000003},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.5629999999999999,"15min":1.2490000000000001},"min":{"1min":0.34599999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.1900000000000004,"5min":63.133000000000003,"15min":63.133000000000003},"last":5.1559999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:45:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.974,"5min":14.974,"15min":14.974},"last":2.1419999999999999},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.6280000000000001,"15min":2.6280000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":16.443000000000001,"5min":16.443000000000001,"15min":16.443000000000001},"last":1.984}]},{"osd":7,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":0.78900000000000003,"15min":1.6579999999999999},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.276999999999999,"5min":11.276999999999999,"15min":138.191},"last":4.0289999999999999},{"interface":"front","average":{"1min":1.6419999999999999,"5min":0.74299999999999999,"15min":1.51},"min":{"1min":0.38100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":10.909000000000001,"5min":10.909000000000001,"15min":140.33500000000001},"last":5.7720000000000002}]}]},{"osd":1,"up_from":301,"seq":1292785156250,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3504416,"kb_used_data":2899224,"kb_used_omap":9,"kb_used_meta":605110,"kb_avail":90240736,"statfs":{"total":95995035648,"available":92406513664,"internally_reserved":0,"allocated":2968805376,"data_stored":2965876138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":619632658},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":10.134,"5min":3.8159999999999998,"15min":3.6789999999999998},"min":{"1min":0.23899999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":142.08500000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":0.252},{"interface":"front","average":{"1min":9.4260000000000002,"5min":3.5449999999999999,"15min":3.5739999999999998},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":142.137,"5min":142.137,"15min":142.137},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":5.3959999999999999,"5min":2.7629999999999999,"15min":2.5409999999999999},"min":{"1min":0.23999999999999999,"5min":0.184,"15min":0.184},"max":{"1min":62.572000000000003,"5min":62.572000000000003,"15min":62.572000000000003},"last":7.9909999999999997},{"interface":"front","average":{"1min":5.9349999999999996,"5min":2.8769999999999998,"15min":2.782},"min":{"1min":0.26800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":68.373999999999995,"5min":68.373999999999995,"15min":68.373999999999995},"last":11.138999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":4.4969999999999999,"5min":5.6699999999999999,"15min":6.4870000000000001},"min":{"1min":0.251,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":59.478000000000002,"5min":199.41,"15min":199.41},"last":0.30499999999999999},{"interface":"front","average":{"1min":5.234,"5min":5.923,"15min":6.6189999999999998},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":59.621000000000002,"5min":199.58099999999999,"15min":199.58099999999999},"last":0.28199999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":0.98199999999999998,"15min":2.1499999999999999},"min":{"1min":0.317,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.6020000000000003,"5min":5.6020000000000003,"15min":142.77099999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.508,"5min":1.244,"15min":2.3359999999999999},"min":{"1min":0.36699999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.976,"5min":8.0830000000000002,"15min":144.898},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.9339999999999999,"15min":1.9339999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.157,"5min":7.157,"15min":7.157},"last":0.5},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":5.3810000000000002,"5min":5.3810000000000002,"15min":5.3810000000000002},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":1.3100000000000001,"15min":6.5579999999999998},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":30.986999999999998,"5min":30.986999999999998,"15min":209.221},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.698,"5min":1.3009999999999999,"15min":6.4859999999999998},"min":{"1min":0.46300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":28.710999999999999,"5min":28.710999999999999,"15min":208.43600000000001},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.2789999999999999,"15min":4.6100000000000003},"min":{"1min":0.38600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.5149999999999997,"5min":9.298,"15min":165.59399999999999},"last":0.443},{"interface":"front","average":{"1min":1.641,"5min":1.3620000000000001,"15min":4.7000000000000002},"min":{"1min":0.48099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":7.5460000000000003,"5min":9.3399999999999999,"15min":165.833},"last":0.39400000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926149,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667828,"kb_used_data":1049388,"kb_used_omap":5,"kb_used_meta":618426,"kb_avail":92077324,"statfs":{"total":95995035648,"available":94287179776,"internally_reserved":0,"allocated":1074573312,"data_stored":1071402916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":633268477},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":2.1989999999999998,"15min":2.274},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":5.8079999999999998,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.839,"5min":2.4129999999999998,"15min":4.4530000000000003},"min":{"1min":0.33200000000000002,"5min":0.23300000000000001,"15min":0.214},"max":{"1min":5.7149999999999999,"5min":57.140999999999998,"15min":138.92400000000001},"last":0.91400000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":4.9539999999999997,"15min":3.1120000000000001},"min":{"1min":0.246,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.3650000000000002,"5min":127.73399999999999,"15min":127.73399999999999},"last":1.329},{"interface":"front","average":{"1min":1.9470000000000001,"5min":5.125,"15min":3.2919999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":8.407,"5min":127.69,"15min":127.69},"last":12.632}]},{"osd":3,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":4.2329999999999997,"5min":5.2009999999999996,"15min":5.2460000000000004},"min":{"1min":0.35599999999999998,"5min":0.255,"15min":0.221},"max":{"1min":28.015000000000001,"5min":125.562,"15min":218.06999999999999},"last":12.153},{"interface":"front","average":{"1min":2.2309999999999999,"5min":4.5609999999999999,"15min":4.2409999999999997},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":12.769,"5min":117.295,"15min":218.63200000000001},"last":0.628}]},{"osd":4,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.5950000000000002,"15min":1.954},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.4610000000000003,"5min":72.403999999999996,"15min":72.403999999999996},"last":0.95799999999999996},{"interface":"front","average":{"1min":3.4380000000000002,"5min":2.883,"15min":2.7480000000000002},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":11.272,"5min":72.162999999999997,"15min":136.262},"last":4.2460000000000004}]},{"osd":5,"last update":"Sat Sep 30 09:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.411,"5min":3.411,"15min":3.411},"last":4.2930000000000001},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.6099999999999999,"5min":3.6099999999999999,"15min":3.6099999999999999},"last":0.33800000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.8740000000000001,"15min":3.5089999999999999},"min":{"1min":0.38900000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.8819999999999997,"5min":57.384,"15min":214.785},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.0449999999999999,"5min":2.0390000000000001,"15min":3.7370000000000001},"min":{"1min":0.47399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.4100000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":0.98999999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.6970000000000001,"15min":2.1139999999999999},"min":{"1min":0.40300000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.8470000000000004,"5min":57.776000000000003,"15min":97.382000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.3479999999999999,"5min":2.1509999999999998,"15min":2.3380000000000001},"min":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.33100000000000002},"max":{"1min":8.4529999999999994,"5min":57.179000000000002,"15min":97.015000000000001},"last":1.0189999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599090,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2052164,"kb_used_data":1254072,"kb_used_omap":5,"kb_used_meta":798074,"kb_avail":91692988,"statfs":{"total":95995035648,"available":93893619712,"internally_reserved":0,"allocated":1284169728,"data_stored":1281275722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":817228429},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":14.532999999999999,"5min":5.5990000000000002,"15min":3.4089999999999998},"min":{"1min":0.26700000000000002,"5min":0.192,"15min":0.187},"max":{"1min":170.678,"5min":170.678,"15min":170.678},"last":0.80100000000000005},{"interface":"front","average":{"1min":14.816000000000001,"5min":5.4219999999999997,"15min":3.371},"min":{"1min":0.307,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":172.44999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":1.1100000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":13.468,"5min":4.3209999999999997,"15min":3.7850000000000001},"min":{"1min":0.27700000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":162.696,"5min":162.696,"15min":162.696},"last":0.47199999999999998},{"interface":"front","average":{"1min":13.574999999999999,"5min":4.3300000000000001,"15min":3.5249999999999999},"min":{"1min":0.23400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":162.93899999999999,"5min":162.93899999999999,"15min":162.93899999999999},"last":8.7479999999999993}]},{"osd":3,"last update":"Sat Sep 30 09:46:08 2023","interfaces":[{"interface":"back","average":{"1min":9.9629999999999992,"5min":5.625,"15min":4.9740000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":156.39400000000001,"5min":156.39400000000001,"15min":156.39400000000001},"last":3.5920000000000001},{"interface":"front","average":{"1min":8.8300000000000001,"5min":5.3040000000000003,"15min":4.8280000000000003},"min":{"1min":0.32600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":156.376,"5min":156.376,"15min":156.376},"last":3.6320000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":3.9020000000000001,"5min":1.5049999999999999,"15min":1.782},"min":{"1min":0.32800000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":47.463000000000001,"5min":47.463000000000001,"15min":83.012},"last":0.55700000000000005},{"interface":"front","average":{"1min":4.4509999999999996,"5min":1.569,"15min":1.8220000000000001},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":47.412999999999997,"5min":47.412999999999997,"15min":81.335999999999999},"last":0.58799999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.706,"5min":2.706,"15min":2.706},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.4449999999999998,"5min":3.4449999999999998,"15min":3.4449999999999998},"last":0.54100000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.1479999999999999,"15min":3.2400000000000002},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.3309999999999995,"5min":9.3309999999999995,"15min":154.417},"last":1.9039999999999999},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.4019999999999999,"15min":3.516},"min":{"1min":0.433,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.7670000000000003,"5min":9.8409999999999993,"15min":154.04400000000001},"last":2.1800000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":1.7989999999999999,"15min":2.355},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":49.816000000000003,"5min":49.816000000000003,"15min":123.294},"last":0.62},{"interface":"front","average":{"1min":4.1269999999999998,"5min":1.873,"15min":2.6440000000000001},"min":{"1min":0.40799999999999997,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":50.956000000000003,"5min":50.956000000000003,"15min":123.16},"last":2.2679999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435563,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3377500,"kb_used_data":2545300,"kb_used_omap":2,"kb_used_meta":832061,"kb_avail":90367652,"statfs":{"total":95995035648,"available":92536475648,"internally_reserved":0,"allocated":2606387200,"data_stored":2603399924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":852030823},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0190000000000001,"5min":4.3949999999999996,"15min":4.7249999999999996},"min":{"1min":0.20000000000000001,"5min":0.183,"15min":0.183},"max":{"1min":34.220999999999997,"5min":144.72,"15min":205.73099999999999},"last":2.9159999999999999},{"interface":"front","average":{"1min":4.2670000000000003,"5min":4.6139999999999999,"15min":4.6580000000000004},"min":{"1min":0.28999999999999998,"5min":0.23999999999999999,"15min":0.21199999999999999},"max":{"1min":34.253,"5min":142.36600000000001,"15min":205.495},"last":1.4650000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:45:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":3.141,"15min":8.593},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":19.725000000000001,"5min":142.434,"15min":273.99599999999998},"last":4.2590000000000003},{"interface":"front","average":{"1min":3.3340000000000001,"5min":3.1480000000000001,"15min":8.1219999999999999},"min":{"1min":0.33900000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":19.638999999999999,"5min":142.44900000000001,"15min":274.21600000000001},"last":4.4180000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":4.851,"15min":5.6269999999999998},"min":{"1min":0.36699999999999999,"5min":0.249,"15min":0.249},"max":{"1min":8.0389999999999997,"5min":146.92599999999999,"15min":206.69},"last":4.3780000000000001},{"interface":"front","average":{"1min":2.0390000000000001,"5min":4.508,"15min":5.4370000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.9649999999999999,"5min":145.75299999999999,"15min":209.22800000000001},"last":1.5109999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.976,"15min":2.0649999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.141,"5min":81.677000000000007,"15min":81.677000000000007},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.3170000000000002,"5min":2.2669999999999999,"15min":1.9330000000000001},"min":{"1min":0.35399999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0540000000000003,"5min":80.623000000000005,"15min":80.623000000000005},"last":4.1749999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.1989999999999998,"5min":9.1989999999999998,"15min":9.1989999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.2719999999999998,"5min":2.2719999999999998,"15min":2.2719999999999998},"last":4.2119999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.069,"15min":1.901},"min":{"1min":0.38800000000000001,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":5.202,"5min":5.915,"15min":60.698},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.4219999999999999,"15min":1.911},"min":{"1min":0.41899999999999998,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":6.2290000000000001,"5min":10.356999999999999,"15min":12.121},"last":4.2359999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":3.0569999999999999,"15min":2.601},"min":{"1min":0.47199999999999998,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":7.968,"5min":213.548,"15min":213.548},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.3839999999999999,"5min":2.8109999999999999,"15min":2.5129999999999999},"min":{"1min":0.438,"5min":0.41099999999999998,"15min":0.34799999999999998},"max":{"1min":5.1719999999999997,"5min":213.583,"15min":213.583},"last":0.497}]}]},{"osd":4,"up_from":30,"seq":128849019326,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1422476,"kb_used_data":949508,"kb_used_omap":0,"kb_used_meta":472960,"kb_avail":92322676,"statfs":{"total":95995035648,"available":94538420224,"internally_reserved":0,"allocated":972296192,"data_stored":970592045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":484311040},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2749999999999999,"15min":2.4950000000000001},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":34.991999999999997,"5min":73.718000000000004,"15min":105.179},"last":0.77100000000000002},{"interface":"front","average":{"1min":3.573,"5min":2.4550000000000001,"15min":2.6619999999999999},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":35.274999999999999,"5min":74.001999999999995,"15min":104.70999999999999},"last":0.75600000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.2029999999999998,"15min":4.3399999999999999},"min":{"1min":0.32000000000000001,"5min":0.25,"15min":0.25},"max":{"1min":30.538,"5min":80.712000000000003,"15min":160.69999999999999},"last":5.7290000000000001},{"interface":"front","average":{"1min":3.226,"5min":2.2730000000000001,"15min":4.2930000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":30.722000000000001,"5min":80.810000000000002,"15min":160.50700000000001},"last":0.63500000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":11.257,"5min":4.6269999999999998,"15min":3.3559999999999999},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":117.858,"5min":142.88999999999999,"15min":142.88999999999999},"last":1.55},{"interface":"front","average":{"1min":11.305999999999999,"5min":4.6950000000000003,"15min":3.403},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":117.871,"5min":142.90700000000001,"15min":142.90700000000001},"last":1.5640000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.875,"5min":1.593,"15min":2.3130000000000002},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":29.41,"5min":29.41,"15min":70.241},"last":0.377},{"interface":"front","average":{"1min":3.1219999999999999,"5min":1.4550000000000001,"15min":2.2440000000000002},"min":{"1min":0.38300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":29.401,"5min":29.401,"15min":70.433999999999997},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:45:25 2023","interfaces":[{"interface":"back","average":{"1min":10.734,"5min":10.734,"15min":10.734},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":140.11099999999999,"5min":140.11099999999999,"15min":140.11099999999999},"last":2.1579999999999999},{"interface":"front","average":{"1min":10.766,"5min":10.766,"15min":10.766},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":140.065,"5min":140.065,"15min":140.065},"last":0.29299999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.53800000000000003,"15min":2.1850000000000001},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":4.4299999999999997,"5min":4.4299999999999997,"15min":202.72499999999999},"last":2.6269999999999998},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.61099999999999999,"15min":2.169},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.249},"max":{"1min":5.7199999999999998,"5min":5.7199999999999998,"15min":202.42400000000001},"last":0.33400000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.2490000000000001,"15min":3.1819999999999999},"min":{"1min":0.36799999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.8170000000000002,"5min":161.72999999999999,"15min":230.90100000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.4490000000000001,"5min":2.2589999999999999,"15min":3.032},"min":{"1min":0.36199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.1709999999999994,"5min":162.60900000000001,"15min":229.97},"last":0.54300000000000004}]}]},{"osd":5,"up_from":489,"seq":2100239007768,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2949900,"kb_used_data":2826180,"kb_used_omap":6,"kb_used_meta":123641,"kb_avail":90795252,"statfs":{"total":95995035648,"available":92974338048,"internally_reserved":0,"allocated":2894008320,"data_stored":2892300723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":126608479},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":26.309999999999999,"5min":26.309999999999999,"15min":26.309999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":281.10599999999999,"5min":281.10599999999999,"15min":281.10599999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":26.599,"5min":26.599,"15min":26.599},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":282.07400000000001,"5min":282.07400000000001,"15min":282.07400000000001},"last":5.702}]},{"osd":1,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":14.92,"5min":14.92,"15min":14.92},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":273.48200000000003,"5min":273.48200000000003,"15min":273.48200000000003},"last":1.575},{"interface":"front","average":{"1min":14.105,"5min":14.105,"15min":14.105},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":273.50200000000001,"5min":273.50200000000001,"15min":273.50200000000001},"last":0.35899999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.0469999999999997,"5min":7.0469999999999997,"15min":7.0469999999999997},"last":4.6109999999999998},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":4.5999999999999996,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":1.631}]},{"osd":3,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":14.749000000000001,"5min":14.749000000000001,"15min":14.749000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":274.01799999999997,"5min":274.01799999999997,"15min":274.01799999999997},"last":1.548},{"interface":"front","average":{"1min":1.732,"5min":1.732,"15min":1.732},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.5830000000000002,"5min":6.5830000000000002,"15min":6.5830000000000002},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.1769999999999996,"5min":4.1769999999999996,"15min":4.1769999999999996},"last":1.448},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.9900000000000002,"5min":3.9900000000000002,"15min":3.9900000000000002},"last":1.5940000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.645,"15min":1.645},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":14.371,"5min":14.371,"15min":14.371},"last":2.3029999999999999},{"interface":"front","average":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":14.454000000000001,"5min":14.454000000000001,"15min":14.454000000000001},"last":2.3199999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.294,"15min":1.294},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.4409999999999998,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":1.4279999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.0350000000000001,"5min":3.0350000000000001,"15min":3.0350000000000001},"last":1.474}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1056964608,"data_stored":1056964608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2934771712,"data_stored":2934767640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1275461632,"data_stored":1275461632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2584346624,"data_stored":2584346624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":966524928,"data_stored":966524928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2887061504,"data_stored":2887057432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":964231168,"data_stored":964231168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2905608192,"data_stored":2905604120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:46:25.158 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:25.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:25.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:24 smithi167 ceph-mon[192880]: pgmap v1338: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:25.273 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:24 smithi019 ceph-mon[224043]: pgmap v1338: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:25.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:24 smithi019 ceph-mon[216815]: pgmap v1338: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3527/235062 objects misplaced (1.500%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:25.375 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:25.786 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:25.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:25.892 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:25.962 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:26.102 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:26.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:26.210 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:26.257 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:26.273 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:26.319 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:26.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:26.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:26.424 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:26.479 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:26.487 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:46:26.487 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:46:26.488 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:46:26.488 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:46:26.488 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:46:26.555 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:26.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:26.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:26.663 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:46:26.700 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:26.763 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:46:26.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:26.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:46:26.870 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:26.916 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:26.959 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:26.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:26.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:27.064 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:26 smithi019 ceph-mon[216815]: pgmap v1339: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3383/224403 objects misplaced (1.508%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:26 smithi019 ceph-mon[224043]: pgmap v1339: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3383/224403 objects misplaced (1.508%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:27.121 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:27.224 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:27.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:27.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:27.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:26 smithi167 ceph-mon[192880]: pgmap v1339: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3383/224403 objects misplaced (1.508%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:27.334 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:27.368 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:27.436 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:27.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:27.548 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:27.574 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:27.634 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:27.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:27.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:27.739 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:27.765 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:27.830 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:27.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:27.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:27.939 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:27.957 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:46:27.978 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:28.057 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:28.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:28.126 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:46:28.164 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:28.292 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:28.666 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:46:28.816 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:28.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:28.928 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:28.962 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:29.027 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:29.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:29.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:29.134 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:29.173 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:29.240 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:29.243 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:29.248 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:29.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:29.254 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:28 smithi167 ceph-mon[192880]: pgmap v1340: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3383/224403 objects misplaced (1.508%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:29.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:28 smithi019 ceph-mon[216815]: pgmap v1340: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3383/224403 objects misplaced (1.508%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:29.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:28 smithi019 ceph-mon[224043]: pgmap v1340: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3383/224403 objects misplaced (1.508%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:29.353 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:29.455 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:29.674 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:29.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:29.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:29.797 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:29.842 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:29.909 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:29.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:29.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:30.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:30.058 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:30.132 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:30.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:30.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:30.240 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:30.299 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:31.071 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:46:31.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:31 smithi167 ceph-mon[192880]: pgmap v1341: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 3253/213831 objects misplaced (1.521%); 2.2 MiB/s, 34 objects/s recovering 2023-09-30T09:46:31.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:31 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/611411155' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:31.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:31 smithi019 ceph-mon[216815]: pgmap v1341: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 3253/213831 objects misplaced (1.521%); 2.2 MiB/s, 34 objects/s recovering 2023-09-30T09:46:31.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:31 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/611411155' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:31.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:31 smithi019 ceph-mon[224043]: pgmap v1341: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 3253/213831 objects misplaced (1.521%); 2.2 MiB/s, 34 objects/s recovering 2023-09-30T09:46:31.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:31 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/611411155' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:32.312 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/611411155' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[216815]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[216815]: osdmap e516: 8 total, 8 up, 8 in 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[216815]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[216815]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:46:32.313 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/611411155' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:32.314 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[224043]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.314 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[224043]: osdmap e516: 8 total, 8 up, 8 in 2023-09-30T09:46:32.314 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[224043]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.314 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:32 smithi019 ceph-mon[224043]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:32 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:46:32.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:32 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:46:32.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/611411155' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:32.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:32 smithi167 ceph-mon[192880]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:32 smithi167 ceph-mon[192880]: osdmap e516: 8 total, 8 up, 8 in 2023-09-30T09:46:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:32 smithi167 ceph-mon[192880]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:32 smithi167 ceph-mon[192880]: failed to encode map e516 with expected crc 2023-09-30T09:46:32.685 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:32.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:32.790 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:32.820 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:32.902 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:32.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:33.006 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:33.034 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:33 smithi019 ceph-mon[216815]: pgmap v1343: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3253/213831 objects misplaced (1.521%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:46:33.034 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:33.034 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:33.087 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:33.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:33 smithi167 ceph-mon[192880]: pgmap v1343: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3253/213831 objects misplaced (1.521%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:46:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:33.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:33.281 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:33.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:33 smithi019 ceph-mon[224043]: pgmap v1343: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3253/213831 objects misplaced (1.521%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:46:33.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:33.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:33.301 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:33.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:33.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:33.412 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:33.505 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:33.772 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:33.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:33.878 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:33.962 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:33.971 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:46:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:46:33] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2023-09-30T09:46:34.192 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:46:34.218 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:34.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:34.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:34.261 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:46:34.312 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:46:34.328 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:46:34.353 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:46:34.371 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:34.377 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:46:34.390 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:46:34.394 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:46:34.441 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:46:34.445 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:46:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:34.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:46:34.479 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:46:34.513 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:46:34.519 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:46:34.519 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:46:34.519 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:46:34.519 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:46:34.519 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:46:34.519 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:46:34.519 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:46:34.519 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:46:34.553 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:34.585 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:34.670 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:34.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:34.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:34.721 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:34 smithi167 ceph-mon[192880]: pgmap v1344: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3253/213831 objects misplaced (1.521%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:46:34.721 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2877195990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:34.793 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:34.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:34 smithi019 ceph-mon[224043]: pgmap v1344: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3253/213831 objects misplaced (1.521%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:46:34.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2877195990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:34.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:34 smithi019 ceph-mon[216815]: pgmap v1344: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3253/213831 objects misplaced (1.521%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:46:34.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2877195990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:34.880 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:35.175 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:46:35.765 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:35.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:35.869 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:35.932 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:36.003 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:36.013 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:35 smithi167 ceph-mon[192880]: from='client.36390 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:36.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:36.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:35 smithi019 ceph-mon[216815]: from='client.36390 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:36.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:35 smithi019 ceph-mon[224043]: from='client.36390 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:36.115 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:36.169 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:36.254 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:36.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:36.361 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:36.430 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:36.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:36 smithi019 ceph-mon[216815]: pgmap v1345: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 3119/202803 objects misplaced (1.538%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:36.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:36 smithi019 ceph-mon[224043]: pgmap v1345: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 3119/202803 objects misplaced (1.538%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:36.870 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:36.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:36.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:36.975 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:37.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:36 smithi167 ceph-mon[192880]: pgmap v1345: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 3119/202803 objects misplaced (1.538%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:37.075 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:37.112 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1344,"stamp":"2023-09-30T09:46:33.807428+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4674099135,"num_objects":71277,"num_object_clones":0,"num_object_copies":213831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3253,"num_objects_unfound":0,"num_objects_dirty":71277,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":123144,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3140,"num_bytes_recovered":200038002,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116059,"ondisk_log_size":116059,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18266988,"kb_used_data":13701864,"kb_used_omap":30,"kb_used_meta":4564705,"kb_avail":731694228,"statfs":{"total":767960285184,"available":749254889472,"internally_reserved":0,"allocated":14030708736,"data_stored":14011868657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4674258274},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,2,13,5,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-463798272,"num_objects":-7077,"num_object_clones":0,"num_object_copies":-21231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-274,"num_objects_unfound":0,"num_objects_dirty":-7077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7077,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":274,"num_bytes_recovered":17956864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005324"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":938,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.037183+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:46:04.037183+0000","last_peered":"2023-09-30T09:46:04.037183+0000","last_clean":"2023-09-30T09:46:04.037183+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:46:04.037183+0000","last_undegraded":"2023-09-30T09:46:04.037183+0000","last_fullsized":"2023-09-30T09:46:04.037183+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":533,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040957+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:46:04.040957+0000","last_peered":"2023-09-30T09:46:04.040957+0000","last_clean":"2023-09-30T09:46:04.040957+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:46:04.040957+0000","last_undegraded":"2023-09-30T09:46:04.040957+0000","last_fullsized":"2023-09-30T09:46:04.040957+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":430,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041777+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:46:04.041777+0000","last_peered":"2023-09-30T09:46:04.041777+0000","last_clean":"2023-09-30T09:46:04.041777+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:46:04.041777+0000","last_undegraded":"2023-09-30T09:46:04.041777+0000","last_fullsized":"2023-09-30T09:46:04.041777+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":940,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.038276+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:46:04.038276+0000","last_peered":"2023-09-30T09:46:04.038276+0000","last_clean":"2023-09-30T09:46:04.038276+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:46:04.038276+0000","last_undegraded":"2023-09-30T09:46:04.038276+0000","last_fullsized":"2023-09-30T09:46:04.038276+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":80,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:17.911124+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:46:17.911124+0000","last_peered":"2023-09-30T09:46:17.911124+0000","last_clean":"2023-09-30T09:46:17.911124+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:46:17.911124+0000","last_undegraded":"2023-09-30T09:46:17.911124+0000","last_fullsized":"2023-09-30T09:46:17.911124+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":120,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033810+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:46:04.033810+0000","last_peered":"2023-09-30T09:46:04.033810+0000","last_clean":"2023-09-30T09:46:04.033810+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:46:04.033810+0000","last_undegraded":"2023-09-30T09:46:04.033810+0000","last_fullsized":"2023-09-30T09:46:04.033810+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":748,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040973+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:46:04.040973+0000","last_peered":"2023-09-30T09:46:04.040973+0000","last_clean":"2023-09-30T09:46:04.040973+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:46:04.040973+0000","last_undegraded":"2023-09-30T09:46:04.040973+0000","last_fullsized":"2023-09-30T09:46:04.040973+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033437+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:46:04.033437+0000","last_peered":"2023-09-30T09:46:04.033437+0000","last_clean":"2023-09-30T09:46:04.033437+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:46:04.033437+0000","last_undegraded":"2023-09-30T09:46:04.033437+0000","last_fullsized":"2023-09-30T09:46:04.033437+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":84,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026666+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:46:04.026666+0000","last_peered":"2023-09-30T09:46:04.026666+0000","last_clean":"2023-09-30T09:46:04.026666+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:46:04.026666+0000","last_undegraded":"2023-09-30T09:46:04.026666+0000","last_fullsized":"2023-09-30T09:46:04.026666+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":916,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034513+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:46:04.034513+0000","last_peered":"2023-09-30T09:46:04.034513+0000","last_clean":"2023-09-30T09:46:04.034513+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:46:04.034513+0000","last_undegraded":"2023-09-30T09:46:04.034513+0000","last_fullsized":"2023-09-30T09:46:04.034513+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":135,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034509+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:46:04.034509+0000","last_peered":"2023-09-30T09:46:04.034509+0000","last_clean":"2023-09-30T09:46:04.034509+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:46:04.034509+0000","last_undegraded":"2023-09-30T09:46:04.034509+0000","last_fullsized":"2023-09-30T09:46:04.034509+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":96,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:17.909991+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:46:17.909991+0000","last_peered":"2023-09-30T09:46:17.909991+0000","last_clean":"2023-09-30T09:46:17.909991+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:46:17.909991+0000","last_undegraded":"2023-09-30T09:46:17.909991+0000","last_fullsized":"2023-09-30T09:46:17.909991+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":704,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035729+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:46:04.035729+0000","last_peered":"2023-09-30T09:46:04.035729+0000","last_clean":"2023-09-30T09:46:04.035729+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:46:04.035729+0000","last_undegraded":"2023-09-30T09:46:04.035729+0000","last_fullsized":"2023-09-30T09:46:04.035729+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":335,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.038082+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:46:04.038082+0000","last_peered":"2023-09-30T09:46:04.038082+0000","last_clean":"2023-09-30T09:46:04.038082+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:46:04.038082+0000","last_undegraded":"2023-09-30T09:46:04.038082+0000","last_fullsized":"2023-09-30T09:46:04.038082+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":471,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033680+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:46:04.033680+0000","last_peered":"2023-09-30T09:46:04.033680+0000","last_clean":"2023-09-30T09:46:04.033680+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:46:04.033680+0000","last_undegraded":"2023-09-30T09:46:04.033680+0000","last_fullsized":"2023-09-30T09:46:04.033680+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":644,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036336+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:46:04.036336+0000","last_peered":"2023-09-30T09:46:04.036336+0000","last_clean":"2023-09-30T09:46:04.036336+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:46:04.036336+0000","last_undegraded":"2023-09-30T09:46:04.036336+0000","last_fullsized":"2023-09-30T09:46:04.036336+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":229,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034613+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:46:04.034613+0000","last_peered":"2023-09-30T09:46:04.034613+0000","last_clean":"2023-09-30T09:46:04.034613+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:46:04.034613+0000","last_undegraded":"2023-09-30T09:46:04.034613+0000","last_fullsized":"2023-09-30T09:46:04.034613+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":70,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026689+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:46:04.026689+0000","last_peered":"2023-09-30T09:46:04.026689+0000","last_clean":"2023-09-30T09:46:04.026689+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:46:04.026689+0000","last_undegraded":"2023-09-30T09:46:04.026689+0000","last_fullsized":"2023-09-30T09:46:04.026689+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":570,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033861+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:46:04.033861+0000","last_peered":"2023-09-30T09:46:04.033861+0000","last_clean":"2023-09-30T09:46:04.033861+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:46:04.033861+0000","last_undegraded":"2023-09-30T09:46:04.033861+0000","last_fullsized":"2023-09-30T09:46:04.033861+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":559,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041373+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:46:04.041373+0000","last_peered":"2023-09-30T09:46:04.041373+0000","last_clean":"2023-09-30T09:46:04.041373+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:46:04.041373+0000","last_undegraded":"2023-09-30T09:46:04.041373+0000","last_fullsized":"2023-09-30T09:46:04.041373+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":537,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041274+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:46:04.041274+0000","last_peered":"2023-09-30T09:46:04.041274+0000","last_clean":"2023-09-30T09:46:04.041274+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:46:04.041274+0000","last_undegraded":"2023-09-30T09:46:04.041274+0000","last_fullsized":"2023-09-30T09:46:04.041274+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":768,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033291+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:46:04.033291+0000","last_peered":"2023-09-30T09:46:04.033291+0000","last_clean":"2023-09-30T09:46:04.033291+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:46:04.033291+0000","last_undegraded":"2023-09-30T09:46:04.033291+0000","last_fullsized":"2023-09-30T09:46:04.033291+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":165,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033864+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:46:04.033864+0000","last_peered":"2023-09-30T09:46:04.033864+0000","last_clean":"2023-09-30T09:46:04.033864+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:46:04.033864+0000","last_undegraded":"2023-09-30T09:46:04.033864+0000","last_fullsized":"2023-09-30T09:46:04.033864+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":492,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034859+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:46:04.034859+0000","last_peered":"2023-09-30T09:46:04.034859+0000","last_clean":"2023-09-30T09:46:04.034859+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:46:04.034859+0000","last_undegraded":"2023-09-30T09:46:04.034859+0000","last_fullsized":"2023-09-30T09:46:04.034859+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":816,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036968+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:46:04.036968+0000","last_peered":"2023-09-30T09:46:04.036968+0000","last_clean":"2023-09-30T09:46:04.036968+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:46:04.036968+0000","last_undegraded":"2023-09-30T09:46:04.036968+0000","last_fullsized":"2023-09-30T09:46:04.036968+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":94,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033160+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:46:04.033160+0000","last_peered":"2023-09-30T09:46:04.033160+0000","last_clean":"2023-09-30T09:46:04.033160+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:46:04.033160+0000","last_undegraded":"2023-09-30T09:46:04.033160+0000","last_fullsized":"2023-09-30T09:46:04.033160+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":190,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035896+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:46:04.035896+0000","last_peered":"2023-09-30T09:46:04.035896+0000","last_clean":"2023-09-30T09:46:04.035896+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:46:04.035896+0000","last_undegraded":"2023-09-30T09:46:04.035896+0000","last_fullsized":"2023-09-30T09:46:04.035896+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":917,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:17.910600+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:46:17.910600+0000","last_peered":"2023-09-30T09:46:17.910600+0000","last_clean":"2023-09-30T09:46:17.910600+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:46:17.910600+0000","last_undegraded":"2023-09-30T09:46:17.910600+0000","last_fullsized":"2023-09-30T09:46:17.910600+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":199,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033798+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:46:04.033798+0000","last_peered":"2023-09-30T09:46:04.033798+0000","last_clean":"2023-09-30T09:46:04.033798+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:46:04.033798+0000","last_undegraded":"2023-09-30T09:46:04.033798+0000","last_fullsized":"2023-09-30T09:46:04.033798+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":94,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033663+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:46:04.033663+0000","last_peered":"2023-09-30T09:46:04.033663+0000","last_clean":"2023-09-30T09:46:04.033663+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:46:04.033663+0000","last_undegraded":"2023-09-30T09:46:04.033663+0000","last_fullsized":"2023-09-30T09:46:04.033663+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":925,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036168+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:46:04.036168+0000","last_peered":"2023-09-30T09:46:04.036168+0000","last_clean":"2023-09-30T09:46:04.036168+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:46:04.036168+0000","last_undegraded":"2023-09-30T09:46:04.036168+0000","last_fullsized":"2023-09-30T09:46:04.036168+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026487+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:46:04.026487+0000","last_peered":"2023-09-30T09:46:04.026487+0000","last_clean":"2023-09-30T09:46:04.026487+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:46:04.026487+0000","last_undegraded":"2023-09-30T09:46:04.026487+0000","last_fullsized":"2023-09-30T09:46:04.026487+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":80,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:17.911424+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:46:17.911424+0000","last_peered":"2023-09-30T09:46:17.911424+0000","last_clean":"2023-09-30T09:46:17.911424+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:46:17.911424+0000","last_undegraded":"2023-09-30T09:46:17.911424+0000","last_fullsized":"2023-09-30T09:46:17.911424+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"515'7564","reported_seq":7867,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:29.208577+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:46:29.208577+0000","last_peered":"2023-09-30T09:46:29.208577+0000","last_clean":"2023-09-30T09:46:29.208577+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:46:29.208577+0000","last_undegraded":"2023-09-30T09:46:29.208577+0000","last_fullsized":"2023-09-30T09:46:29.208577+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7564,"log_dups_size":0,"ondisk_log_size":7564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289013760,"num_objects":4410,"num_object_clones":0,"num_object_copies":13230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7564,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":891,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034602+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:46:04.034602+0000","last_peered":"2023-09-30T09:46:04.034602+0000","last_clean":"2023-09-30T09:46:04.034602+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:46:04.034602+0000","last_undegraded":"2023-09-30T09:46:04.034602+0000","last_fullsized":"2023-09-30T09:46:04.034602+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":191,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034750+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:46:04.034750+0000","last_peered":"2023-09-30T09:46:04.034750+0000","last_clean":"2023-09-30T09:46:04.034750+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:46:04.034750+0000","last_undegraded":"2023-09-30T09:46:04.034750+0000","last_fullsized":"2023-09-30T09:46:04.034750+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036887+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:46:04.036887+0000","last_peered":"2023-09-30T09:46:04.036887+0000","last_clean":"2023-09-30T09:46:04.036887+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:46:04.036887+0000","last_undegraded":"2023-09-30T09:46:04.036887+0000","last_fullsized":"2023-09-30T09:46:04.036887+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"515'7519","reported_seq":7559,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:28.730287+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:46:28.730287+0000","last_peered":"2023-09-30T09:46:28.730287+0000","last_clean":"2023-09-30T09:46:28.730287+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:46:28.730287+0000","last_undegraded":"2023-09-30T09:46:28.730287+0000","last_fullsized":"2023-09-30T09:46:28.730287+0000","mapping_epoch":483,"log_start":"496'1100","ondisk_log_start":"496'1100","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289996800,"num_objects":4425,"num_object_clones":0,"num_object_copies":13275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7519,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":909,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:17.913552+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:46:17.913552+0000","last_peered":"2023-09-30T09:46:17.913552+0000","last_clean":"2023-09-30T09:46:17.913552+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:46:17.913552+0000","last_undegraded":"2023-09-30T09:46:17.913552+0000","last_fullsized":"2023-09-30T09:46:17.913552+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":161,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034568+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:46:04.034568+0000","last_peered":"2023-09-30T09:46:04.034568+0000","last_clean":"2023-09-30T09:46:04.034568+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:46:04.034568+0000","last_undegraded":"2023-09-30T09:46:04.034568+0000","last_fullsized":"2023-09-30T09:46:04.034568+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":566,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040614+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:46:04.040614+0000","last_peered":"2023-09-30T09:46:04.040614+0000","last_clean":"2023-09-30T09:46:04.040614+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:46:04.040614+0000","last_undegraded":"2023-09-30T09:46:04.040614+0000","last_fullsized":"2023-09-30T09:46:04.040614+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"515'7721","reported_seq":7986,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:29.090416+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:46:29.090416+0000","last_peered":"2023-09-30T09:46:29.090416+0000","last_clean":"2023-09-30T09:46:29.090416+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:46:29.090416+0000","last_undegraded":"2023-09-30T09:46:29.090416+0000","last_fullsized":"2023-09-30T09:46:29.090416+0000","mapping_epoch":493,"log_start":"496'1000","ondisk_log_start":"496'1000","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292356096,"num_objects":4461,"num_object_clones":0,"num_object_copies":13383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7721,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":685,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026813+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:46:04.026813+0000","last_peered":"2023-09-30T09:46:04.026813+0000","last_clean":"2023-09-30T09:46:04.026813+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:46:04.026813+0000","last_undegraded":"2023-09-30T09:46:04.026813+0000","last_fullsized":"2023-09-30T09:46:04.026813+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":228,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034147+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:46:04.034147+0000","last_peered":"2023-09-30T09:46:04.034147+0000","last_clean":"2023-09-30T09:46:04.034147+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:46:04.034147+0000","last_undegraded":"2023-09-30T09:46:04.034147+0000","last_fullsized":"2023-09-30T09:46:04.034147+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":84,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026761+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:46:04.026761+0000","last_peered":"2023-09-30T09:46:04.026761+0000","last_clean":"2023-09-30T09:46:04.026761+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:46:04.026761+0000","last_undegraded":"2023-09-30T09:46:04.026761+0000","last_fullsized":"2023-09-30T09:46:04.026761+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"515'7522","reported_seq":7877,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:28.836692+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:46:28.836692+0000","last_peered":"2023-09-30T09:46:28.836692+0000","last_clean":"2023-09-30T09:46:28.836692+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:46:28.836692+0000","last_undegraded":"2023-09-30T09:46:28.836692+0000","last_fullsized":"2023-09-30T09:46:28.836692+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7522,"log_dups_size":0,"ondisk_log_size":7522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286654464,"num_objects":4374,"num_object_clones":0,"num_object_copies":13122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7522,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":750,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041652+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:46:04.041652+0000","last_peered":"2023-09-30T09:46:04.041652+0000","last_clean":"2023-09-30T09:46:04.041652+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:46:04.041652+0000","last_undegraded":"2023-09-30T09:46:04.041652+0000","last_fullsized":"2023-09-30T09:46:04.041652+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":450,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036060+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:46:04.036060+0000","last_peered":"2023-09-30T09:46:04.036060+0000","last_clean":"2023-09-30T09:46:04.036060+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:46:04.036060+0000","last_undegraded":"2023-09-30T09:46:04.036060+0000","last_fullsized":"2023-09-30T09:46:04.036060+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":114,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034078+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:46:04.034078+0000","last_peered":"2023-09-30T09:46:04.034078+0000","last_clean":"2023-09-30T09:46:04.034078+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:46:04.034078+0000","last_undegraded":"2023-09-30T09:46:04.034078+0000","last_fullsized":"2023-09-30T09:46:04.034078+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"515'7636","reported_seq":7676,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:29.297518+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:46:29.297518+0000","last_peered":"2023-09-30T09:46:29.297518+0000","last_clean":"2023-09-30T09:46:29.297518+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:46:29.297518+0000","last_undegraded":"2023-09-30T09:46:29.297518+0000","last_fullsized":"2023-09-30T09:46:29.297518+0000","mapping_epoch":483,"log_start":"496'600","ondisk_log_start":"496'600","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294649856,"num_objects":4496,"num_object_clones":0,"num_object_copies":13488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7636,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":997,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033785+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:46:04.033785+0000","last_peered":"2023-09-30T09:46:04.033785+0000","last_clean":"2023-09-30T09:46:04.033785+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:46:04.033785+0000","last_undegraded":"2023-09-30T09:46:04.033785+0000","last_fullsized":"2023-09-30T09:46:04.033785+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":73,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033899+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:46:04.033899+0000","last_peered":"2023-09-30T09:46:04.033899+0000","last_clean":"2023-09-30T09:46:04.033899+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:46:04.033899+0000","last_undegraded":"2023-09-30T09:46:04.033899+0000","last_fullsized":"2023-09-30T09:46:04.033899+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":513,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041454+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:46:04.041454+0000","last_peered":"2023-09-30T09:46:04.041454+0000","last_clean":"2023-09-30T09:46:04.041454+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:46:04.041454+0000","last_undegraded":"2023-09-30T09:46:04.041454+0000","last_fullsized":"2023-09-30T09:46:04.041454+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"515'7612","reported_seq":7652,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:29.277278+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:46:29.277278+0000","last_peered":"2023-09-30T09:46:29.277278+0000","last_clean":"2023-09-30T09:46:29.277278+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:46:29.277278+0000","last_undegraded":"2023-09-30T09:46:29.277278+0000","last_fullsized":"2023-09-30T09:46:29.277278+0000","mapping_epoch":483,"log_start":"496'600","ondisk_log_start":"496'600","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284557312,"num_objects":4342,"num_object_clones":0,"num_object_copies":13026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7612,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":996,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:13.823790+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:46:13.823790+0000","last_peered":"2023-09-30T09:46:13.823790+0000","last_clean":"2023-09-30T09:46:13.823790+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:46:13.823790+0000","last_undegraded":"2023-09-30T09:46:13.823790+0000","last_fullsized":"2023-09-30T09:46:13.823790+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026670+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:46:04.026670+0000","last_peered":"2023-09-30T09:46:04.026670+0000","last_clean":"2023-09-30T09:46:04.026670+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:46:04.026670+0000","last_undegraded":"2023-09-30T09:46:04.026670+0000","last_fullsized":"2023-09-30T09:46:04.026670+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":209,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041070+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:46:04.041070+0000","last_peered":"2023-09-30T09:46:04.041070+0000","last_clean":"2023-09-30T09:46:04.041070+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:46:04.041070+0000","last_undegraded":"2023-09-30T09:46:04.041070+0000","last_fullsized":"2023-09-30T09:46:04.041070+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"515'7806","reported_seq":7846,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:28.836509+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:46:28.836509+0000","last_peered":"2023-09-30T09:46:28.836509+0000","last_clean":"2023-09-30T09:46:28.836509+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:46:28.836509+0000","last_undegraded":"2023-09-30T09:46:28.836509+0000","last_fullsized":"2023-09-30T09:46:28.836509+0000","mapping_epoch":483,"log_start":"497'2000","ondisk_log_start":"497'2000","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292552704,"num_objects":4464,"num_object_clones":0,"num_object_copies":13392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7806,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":853,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034100+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:46:04.034100+0000","last_peered":"2023-09-30T09:46:04.034100+0000","last_clean":"2023-09-30T09:46:04.034100+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:46:04.034100+0000","last_undegraded":"2023-09-30T09:46:04.034100+0000","last_fullsized":"2023-09-30T09:46:04.034100+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":520,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040540+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:46:04.040540+0000","last_peered":"2023-09-30T09:46:04.040540+0000","last_clean":"2023-09-30T09:46:04.040540+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:46:04.040540+0000","last_undegraded":"2023-09-30T09:46:04.040540+0000","last_fullsized":"2023-09-30T09:46:04.040540+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":527,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041741+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:46:04.041741+0000","last_peered":"2023-09-30T09:46:04.041741+0000","last_clean":"2023-09-30T09:46:04.041741+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:46:04.041741+0000","last_undegraded":"2023-09-30T09:46:04.041741+0000","last_fullsized":"2023-09-30T09:46:04.041741+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"515'7544","reported_seq":7588,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:28.754592+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:46:28.754592+0000","last_peered":"2023-09-30T09:46:28.754592+0000","last_clean":"2023-09-30T09:46:28.754592+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:46:28.754592+0000","last_undegraded":"2023-09-30T09:46:28.754592+0000","last_fullsized":"2023-09-30T09:46:28.754592+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7544,"log_dups_size":0,"ondisk_log_size":7544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287834112,"num_objects":4392,"num_object_clones":0,"num_object_copies":13176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7544,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":718,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:13.823869+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:46:13.823869+0000","last_peered":"2023-09-30T09:46:13.823869+0000","last_clean":"2023-09-30T09:46:13.823869+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:46:13.823869+0000","last_undegraded":"2023-09-30T09:46:13.823869+0000","last_fullsized":"2023-09-30T09:46:13.823869+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":80,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:17.909903+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:46:17.909903+0000","last_peered":"2023-09-30T09:46:17.909903+0000","last_clean":"2023-09-30T09:46:17.909903+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:46:17.909903+0000","last_undegraded":"2023-09-30T09:46:17.909903+0000","last_fullsized":"2023-09-30T09:46:17.909903+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":200,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041844+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:46:04.041844+0000","last_peered":"2023-09-30T09:46:04.041844+0000","last_clean":"2023-09-30T09:46:04.041844+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:46:04.041844+0000","last_undegraded":"2023-09-30T09:46:04.041844+0000","last_fullsized":"2023-09-30T09:46:04.041844+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"515'7645","reported_seq":7944,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:29.104989+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:46:29.104989+0000","last_peered":"2023-09-30T09:46:29.104989+0000","last_clean":"2023-09-30T09:46:29.104989+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:46:29.104989+0000","last_undegraded":"2023-09-30T09:46:29.104989+0000","last_fullsized":"2023-09-30T09:46:29.104989+0000","mapping_epoch":494,"log_start":"496'900","ondisk_log_start":"496'900","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6745,"log_dups_size":0,"ondisk_log_size":6745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292225024,"num_objects":4459,"num_object_clones":0,"num_object_copies":13377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7645,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":757,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026691+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:46:04.026691+0000","last_peered":"2023-09-30T09:46:04.026691+0000","last_clean":"2023-09-30T09:46:04.026691+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:46:04.026691+0000","last_undegraded":"2023-09-30T09:46:04.026691+0000","last_fullsized":"2023-09-30T09:46:04.026691+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":745,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033323+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:46:04.033323+0000","last_peered":"2023-09-30T09:46:04.033323+0000","last_clean":"2023-09-30T09:46:04.033323+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:46:04.033323+0000","last_undegraded":"2023-09-30T09:46:04.033323+0000","last_fullsized":"2023-09-30T09:46:04.033323+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":443,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036045+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:46:04.036045+0000","last_peered":"2023-09-30T09:46:04.036045+0000","last_clean":"2023-09-30T09:46:04.036045+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:46:04.036045+0000","last_undegraded":"2023-09-30T09:46:04.036045+0000","last_fullsized":"2023-09-30T09:46:04.036045+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035859+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:04.035859+0000","last_peered":"2023-09-30T09:46:04.035859+0000","last_clean":"2023-09-30T09:46:04.035859+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:46:04.035859+0000","last_undegraded":"2023-09-30T09:46:04.035859+0000","last_fullsized":"2023-09-30T09:46:04.035859+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"515'7690","reported_seq":8035,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:29.199719+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:46:29.199719+0000","last_peered":"2023-09-30T09:46:29.199719+0000","last_clean":"2023-09-30T09:46:29.199719+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:46:29.199719+0000","last_undegraded":"2023-09-30T09:46:29.199719+0000","last_fullsized":"2023-09-30T09:46:29.199719+0000","mapping_epoch":493,"log_start":"488'100","ondisk_log_start":"488'100","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7590,"log_dups_size":0,"ondisk_log_size":7590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290848768,"num_objects":4438,"num_object_clones":0,"num_object_copies":13314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7690,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":956,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035781+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:46:04.035781+0000","last_peered":"2023-09-30T09:46:04.035781+0000","last_clean":"2023-09-30T09:46:04.035781+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:46:04.035781+0000","last_undegraded":"2023-09-30T09:46:04.035781+0000","last_fullsized":"2023-09-30T09:46:04.035781+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":462,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026780+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:46:04.026780+0000","last_peered":"2023-09-30T09:46:04.026780+0000","last_clean":"2023-09-30T09:46:04.026780+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:46:04.026780+0000","last_undegraded":"2023-09-30T09:46:04.026780+0000","last_fullsized":"2023-09-30T09:46:04.026780+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":534,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041312+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:46:04.041312+0000","last_peered":"2023-09-30T09:46:04.041312+0000","last_clean":"2023-09-30T09:46:04.041312+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:46:04.041312+0000","last_undegraded":"2023-09-30T09:46:04.041312+0000","last_fullsized":"2023-09-30T09:46:04.041312+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035805+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:46:04.035805+0000","last_peered":"2023-09-30T09:46:04.035805+0000","last_clean":"2023-09-30T09:46:04.035805+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:46:04.035805+0000","last_undegraded":"2023-09-30T09:46:04.035805+0000","last_fullsized":"2023-09-30T09:46:04.035805+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"515'7701","reported_seq":8046,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:29.104604+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:46:29.104604+0000","last_peered":"2023-09-30T09:46:29.104604+0000","last_clean":"2023-09-30T09:46:29.104604+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:46:29.104604+0000","last_undegraded":"2023-09-30T09:46:29.104604+0000","last_fullsized":"2023-09-30T09:46:29.104604+0000","mapping_epoch":492,"log_start":"496'1000","ondisk_log_start":"496'1000","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6701,"log_dups_size":0,"ondisk_log_size":6701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293011456,"num_objects":4471,"num_object_clones":0,"num_object_copies":13413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7701,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":934,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033791+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:46:04.033791+0000","last_peered":"2023-09-30T09:46:04.033791+0000","last_clean":"2023-09-30T09:46:04.033791+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:46:04.033791+0000","last_undegraded":"2023-09-30T09:46:04.033791+0000","last_fullsized":"2023-09-30T09:46:04.033791+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":783,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035705+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:46:04.035705+0000","last_peered":"2023-09-30T09:46:04.035705+0000","last_clean":"2023-09-30T09:46:04.035705+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:46:04.035705+0000","last_undegraded":"2023-09-30T09:46:04.035705+0000","last_fullsized":"2023-09-30T09:46:04.035705+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":277,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035325+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:46:04.035325+0000","last_peered":"2023-09-30T09:46:04.035325+0000","last_clean":"2023-09-30T09:46:04.035325+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:46:04.035325+0000","last_undegraded":"2023-09-30T09:46:04.035325+0000","last_fullsized":"2023-09-30T09:46:04.035325+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":364,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036536+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:46:04.036536+0000","last_peered":"2023-09-30T09:46:04.036536+0000","last_clean":"2023-09-30T09:46:04.036536+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:46:04.036536+0000","last_undegraded":"2023-09-30T09:46:04.036536+0000","last_fullsized":"2023-09-30T09:46:04.036536+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"515'7572","reported_seq":7877,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:29.208164+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:46:29.208164+0000","last_peered":"2023-09-30T09:46:29.208164+0000","last_clean":"2023-09-30T09:46:29.208164+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:46:29.208164+0000","last_undegraded":"2023-09-30T09:46:29.208164+0000","last_fullsized":"2023-09-30T09:46:29.208164+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7572,"log_dups_size":0,"ondisk_log_size":7572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291373056,"num_objects":4446,"num_object_clones":0,"num_object_copies":13338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7572,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1076,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.038419+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:46:04.038419+0000","last_peered":"2023-09-30T09:46:04.038419+0000","last_clean":"2023-09-30T09:46:04.038419+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:46:04.038419+0000","last_undegraded":"2023-09-30T09:46:04.038419+0000","last_fullsized":"2023-09-30T09:46:04.038419+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":877,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033577+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:46:04.033577+0000","last_peered":"2023-09-30T09:46:04.033577+0000","last_clean":"2023-09-30T09:46:04.033577+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:46:04.033577+0000","last_undegraded":"2023-09-30T09:46:04.033577+0000","last_fullsized":"2023-09-30T09:46:04.033577+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035935+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:46:04.035935+0000","last_peered":"2023-09-30T09:46:04.035935+0000","last_clean":"2023-09-30T09:46:04.035935+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:46:04.035935+0000","last_undegraded":"2023-09-30T09:46:04.035935+0000","last_fullsized":"2023-09-30T09:46:04.035935+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":70,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026777+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:46:04.026777+0000","last_peered":"2023-09-30T09:46:04.026777+0000","last_clean":"2023-09-30T09:46:04.026777+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:46:04.026777+0000","last_undegraded":"2023-09-30T09:46:04.026777+0000","last_fullsized":"2023-09-30T09:46:04.026777+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"515'7767","reported_seq":11268,"reported_epoch":515,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:46:28.843074+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:46:28.843074+0000","last_peered":"2023-09-30T09:46:28.843074+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:46:28.843074+0000","last_undegraded":"2023-09-30T09:46:28.843074+0000","last_fullsized":"2023-09-30T09:46:28.843074+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7767,"log_dups_size":0,"ondisk_log_size":7767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298778624,"num_objects":4559,"num_object_clones":0,"num_object_copies":13677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3253,"num_objects_unfound":0,"num_objects_dirty":4559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7767,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1717,"num_bytes_recovered":112525312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":1306}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":776,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.027447+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:46:04.027447+0000","last_peered":"2023-09-30T09:46:04.027447+0000","last_clean":"2023-09-30T09:46:04.027447+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:46:04.027447+0000","last_undegraded":"2023-09-30T09:46:04.027447+0000","last_fullsized":"2023-09-30T09:46:04.027447+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":923,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033525+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:46:04.033525+0000","last_peered":"2023-09-30T09:46:04.033525+0000","last_clean":"2023-09-30T09:46:04.033525+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:46:04.033525+0000","last_undegraded":"2023-09-30T09:46:04.033525+0000","last_fullsized":"2023-09-30T09:46:04.033525+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033441+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:46:04.033441+0000","last_peered":"2023-09-30T09:46:04.033441+0000","last_clean":"2023-09-30T09:46:04.033441+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:46:04.033441+0000","last_undegraded":"2023-09-30T09:46:04.033441+0000","last_fullsized":"2023-09-30T09:46:04.033441+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":20,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033598+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:46:04.033598+0000","last_peered":"2023-09-30T09:46:04.033598+0000","last_clean":"2023-09-30T09:46:04.033598+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:46:04.033598+0000","last_undegraded":"2023-09-30T09:46:04.033598+0000","last_fullsized":"2023-09-30T09:46:04.033598+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"515'7596","reported_seq":7636,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:28.845721+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:46:28.845721+0000","last_peered":"2023-09-30T09:46:28.845721+0000","last_clean":"2023-09-30T09:46:28.845721+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:46:28.845721+0000","last_undegraded":"2023-09-30T09:46:28.845721+0000","last_fullsized":"2023-09-30T09:46:28.845721+0000","mapping_epoch":483,"log_start":"497'1800","ondisk_log_start":"497'1800","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5796,"log_dups_size":0,"ondisk_log_size":5796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7596,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":767,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.027093+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:46:04.027093+0000","last_peered":"2023-09-30T09:46:04.027093+0000","last_clean":"2023-09-30T09:46:04.027093+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:46:04.027093+0000","last_undegraded":"2023-09-30T09:46:04.027093+0000","last_fullsized":"2023-09-30T09:46:04.027093+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1063,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034683+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:46:04.034683+0000","last_peered":"2023-09-30T09:46:04.034683+0000","last_clean":"2023-09-30T09:46:04.034683+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:46:04.034683+0000","last_undegraded":"2023-09-30T09:46:04.034683+0000","last_fullsized":"2023-09-30T09:46:04.034683+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":258,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035622+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:46:04.035622+0000","last_peered":"2023-09-30T09:46:04.035622+0000","last_clean":"2023-09-30T09:46:04.035622+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:46:04.035622+0000","last_undegraded":"2023-09-30T09:46:04.035622+0000","last_fullsized":"2023-09-30T09:46:04.035622+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":127,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034349+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:46:04.034349+0000","last_peered":"2023-09-30T09:46:04.034349+0000","last_clean":"2023-09-30T09:46:04.034349+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:46:04.034349+0000","last_undegraded":"2023-09-30T09:46:04.034349+0000","last_fullsized":"2023-09-30T09:46:04.034349+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"515'7498","reported_seq":7873,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:28.805538+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:46:28.805538+0000","last_peered":"2023-09-30T09:46:28.805538+0000","last_clean":"2023-09-30T09:46:28.805538+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:46:28.805538+0000","last_undegraded":"2023-09-30T09:46:28.805538+0000","last_fullsized":"2023-09-30T09:46:28.805538+0000","mapping_epoch":495,"log_start":"497'1700","ondisk_log_start":"497'1700","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295632920,"num_objects":4512,"num_object_clones":0,"num_object_copies":13536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4512,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7498,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1034,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041630+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:46:04.041630+0000","last_peered":"2023-09-30T09:46:04.041630+0000","last_clean":"2023-09-30T09:46:04.041630+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:46:04.041630+0000","last_undegraded":"2023-09-30T09:46:04.041630+0000","last_fullsized":"2023-09-30T09:46:04.041630+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":546,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034679+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:46:04.034679+0000","last_peered":"2023-09-30T09:46:04.034679+0000","last_clean":"2023-09-30T09:46:04.034679+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:46:04.034679+0000","last_undegraded":"2023-09-30T09:46:04.034679+0000","last_fullsized":"2023-09-30T09:46:04.034679+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1233,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.027010+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:46:04.027010+0000","last_peered":"2023-09-30T09:46:04.027010+0000","last_clean":"2023-09-30T09:46:04.027010+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:46:04.027010+0000","last_undegraded":"2023-09-30T09:46:04.027010+0000","last_fullsized":"2023-09-30T09:46:04.027010+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":550,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033497+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:46:04.033497+0000","last_peered":"2023-09-30T09:46:04.033497+0000","last_clean":"2023-09-30T09:46:04.033497+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:46:04.033497+0000","last_undegraded":"2023-09-30T09:46:04.033497+0000","last_fullsized":"2023-09-30T09:46:04.033497+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"515'7844","reported_seq":7884,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:28.848298+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:46:28.848298+0000","last_peered":"2023-09-30T09:46:28.848298+0000","last_clean":"2023-09-30T09:46:28.848298+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:46:28.848298+0000","last_undegraded":"2023-09-30T09:46:28.848298+0000","last_fullsized":"2023-09-30T09:46:28.848298+0000","mapping_epoch":483,"log_start":"497'1400","ondisk_log_start":"497'1400","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293863424,"num_objects":4484,"num_object_clones":0,"num_object_copies":13452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7844,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":898,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-30T09:46:17.913423+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:46:17.913423+0000","last_peered":"2023-09-30T09:46:17.913423+0000","last_clean":"2023-09-30T09:46:17.913423+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:46:17.913423+0000","last_undegraded":"2023-09-30T09:46:17.913423+0000","last_fullsized":"2023-09-30T09:46:17.913423+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":85,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026789+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:46:04.026789+0000","last_peered":"2023-09-30T09:46:04.026789+0000","last_clean":"2023-09-30T09:46:04.026789+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:46:04.026789+0000","last_undegraded":"2023-09-30T09:46:04.026789+0000","last_fullsized":"2023-09-30T09:46:04.026789+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":90,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033766+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:46:04.033766+0000","last_peered":"2023-09-30T09:46:04.033766+0000","last_clean":"2023-09-30T09:46:04.033766+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:46:04.033766+0000","last_undegraded":"2023-09-30T09:46:04.033766+0000","last_fullsized":"2023-09-30T09:46:04.033766+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":516,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040570+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:46:04.040570+0000","last_peered":"2023-09-30T09:46:04.040570+0000","last_clean":"2023-09-30T09:46:04.040570+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:46:04.040570+0000","last_undegraded":"2023-09-30T09:46:04.040570+0000","last_fullsized":"2023-09-30T09:46:04.040570+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":834,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033205+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:46:04.033205+0000","last_peered":"2023-09-30T09:46:04.033205+0000","last_clean":"2023-09-30T09:46:04.033205+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:46:04.033205+0000","last_undegraded":"2023-09-30T09:46:04.033205+0000","last_fullsized":"2023-09-30T09:46:04.033205+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":20,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034167+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:46:04.034167+0000","last_peered":"2023-09-30T09:46:04.034167+0000","last_clean":"2023-09-30T09:46:04.034167+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:46:04.034167+0000","last_undegraded":"2023-09-30T09:46:04.034167+0000","last_fullsized":"2023-09-30T09:46:04.034167+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":220,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035515+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:46:04.035515+0000","last_peered":"2023-09-30T09:46:04.035515+0000","last_clean":"2023-09-30T09:46:04.035515+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:46:04.035515+0000","last_undegraded":"2023-09-30T09:46:04.035515+0000","last_fullsized":"2023-09-30T09:46:04.035515+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":629,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.027171+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:46:04.027171+0000","last_peered":"2023-09-30T09:46:04.027171+0000","last_clean":"2023-09-30T09:46:04.027171+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:46:04.027171+0000","last_undegraded":"2023-09-30T09:46:04.027171+0000","last_fullsized":"2023-09-30T09:46:04.027171+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":443,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.035985+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:04.035985+0000","last_peered":"2023-09-30T09:46:04.035985+0000","last_clean":"2023-09-30T09:46:04.035985+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:46:04.035985+0000","last_undegraded":"2023-09-30T09:46:04.035985+0000","last_fullsized":"2023-09-30T09:46:04.035985+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":229,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041136+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:46:04.041136+0000","last_peered":"2023-09-30T09:46:04.041136+0000","last_clean":"2023-09-30T09:46:04.041136+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:46:04.041136+0000","last_undegraded":"2023-09-30T09:46:04.041136+0000","last_fullsized":"2023-09-30T09:46:04.041136+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":385,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.036262+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:46:04.036262+0000","last_peered":"2023-09-30T09:46:04.036262+0000","last_clean":"2023-09-30T09:46:04.036262+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:46:04.036262+0000","last_undegraded":"2023-09-30T09:46:04.036262+0000","last_fullsized":"2023-09-30T09:46:04.036262+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":249,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040531+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:46:04.040531+0000","last_peered":"2023-09-30T09:46:04.040531+0000","last_clean":"2023-09-30T09:46:04.040531+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:46:04.040531+0000","last_undegraded":"2023-09-30T09:46:04.040531+0000","last_fullsized":"2023-09-30T09:46:04.040531+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":842,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.034496+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:46:04.034496+0000","last_peered":"2023-09-30T09:46:04.034496+0000","last_clean":"2023-09-30T09:46:04.034496+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:46:04.034496+0000","last_undegraded":"2023-09-30T09:46:04.034496+0000","last_fullsized":"2023-09-30T09:46:04.034496+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":72,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.026886+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:46:04.026886+0000","last_peered":"2023-09-30T09:46:04.026886+0000","last_clean":"2023-09-30T09:46:04.026886+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:46:04.026886+0000","last_undegraded":"2023-09-30T09:46:04.026886+0000","last_fullsized":"2023-09-30T09:46:04.026886+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":466,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:13.823820+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:46:13.823820+0000","last_peered":"2023-09-30T09:46:13.823820+0000","last_clean":"2023-09-30T09:46:13.823820+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:46:13.823820+0000","last_undegraded":"2023-09-30T09:46:13.823820+0000","last_fullsized":"2023-09-30T09:46:13.823820+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":885,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.041538+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:46:04.041538+0000","last_peered":"2023-09-30T09:46:04.041538+0000","last_clean":"2023-09-30T09:46:04.041538+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:46:04.041538+0000","last_undegraded":"2023-09-30T09:46:04.041538+0000","last_fullsized":"2023-09-30T09:46:04.041538+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":26,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.033661+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:46:04.033661+0000","last_peered":"2023-09-30T09:46:04.033661+0000","last_clean":"2023-09-30T09:46:04.033661+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:46:04.033661+0000","last_undegraded":"2023-09-30T09:46:04.033661+0000","last_fullsized":"2023-09-30T09:46:04.033661+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":530,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-30T09:46:04.040436+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:46:04.040436+0000","last_peered":"2023-09-30T09:46:04.040436+0000","last_clean":"2023-09-30T09:46:04.040436+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:46:04.040436+0000","last_undegraded":"2023-09-30T09:46:04.040436+0000","last_fullsized":"2023-09-30T09:46:04.040436+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4669308952,"num_objects":71249,"num_object_clones":0,"num_object_copies":213747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3253,"num_objects_unfound":0,"num_objects_dirty":71249,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122237,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2860,"num_bytes_recovered":187432960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13916450816,"data_stored":13916438600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110037,"ondisk_log_size":110037,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234417,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3114720,"kb_used_data":2589912,"kb_used_omap":0,"kb_used_meta":524800,"kb_avail":90630432,"statfs":{"total":95995035648,"available":92805562368,"internally_reserved":0,"allocated":2652069888,"data_stored":2650346429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":537395200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:01 2023","interfaces":[{"interface":"back","average":{"1min":13.629,"5min":6.0149999999999997,"15min":4.5529999999999999},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":110.467,"5min":124.74299999999999,"15min":215.416},"last":95.825999999999993},{"interface":"front","average":{"1min":13.702999999999999,"5min":6.1459999999999999,"15min":4.5099999999999998},"min":{"1min":0.23499999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":109.593,"5min":124.69499999999999,"15min":215.041},"last":95.488}]},{"osd":1,"last update":"Sat Sep 30 09:46:01 2023","interfaces":[{"interface":"back","average":{"1min":16.905000000000001,"5min":7.1210000000000004,"15min":4.1399999999999997},"min":{"1min":0.32400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":157.578,"5min":157.578,"15min":157.578},"last":2.9580000000000002},{"interface":"front","average":{"1min":17.038,"5min":6.9660000000000002,"15min":4.3380000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":157.589,"5min":157.589,"15min":157.589},"last":5.048}]},{"osd":2,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":4.3570000000000002,"15min":3.5009999999999999},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.24299999999999999},"max":{"1min":5.3789999999999996,"5min":117.14,"15min":117.14},"last":5.0170000000000003},{"interface":"front","average":{"1min":2.3450000000000002,"5min":4.585,"15min":3.528},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":18.603000000000002,"5min":117.044,"15min":117.044},"last":5.2359999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:46:15 2023","interfaces":[{"interface":"back","average":{"1min":11.387,"5min":6.8799999999999999,"15min":8.9429999999999996},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.31},"max":{"1min":154.06700000000001,"5min":154.06700000000001,"15min":277.57799999999997},"last":2.9889999999999999},{"interface":"front","average":{"1min":11.162000000000001,"5min":6.8810000000000002,"15min":8.4610000000000003},"min":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":154.24799999999999,"5min":154.24799999999999,"15min":278.69299999999998},"last":5.202}]},{"osd":4,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":0.96699999999999997,"15min":1.6000000000000001},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":25.192,"5min":25.192,"15min":175.72},"last":3.161},{"interface":"front","average":{"1min":2.657,"5min":0.93100000000000005,"15min":1.526},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":24.521000000000001,"5min":24.521000000000001,"15min":179.048},"last":5.1509999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":11.993,"5min":11.993,"15min":11.993},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":197.75700000000001,"5min":197.75700000000001,"15min":197.75700000000001},"last":5.2910000000000004},{"interface":"front","average":{"1min":11.827,"5min":11.827,"15min":11.827},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":197.63300000000001,"5min":197.63300000000001,"15min":197.63300000000001},"last":5.1879999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":0.61899999999999999,"15min":2.1659999999999999},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.367,"5min":4.367,"15min":206.244},"last":3.1890000000000001},{"interface":"front","average":{"1min":1.6679999999999999,"5min":0.76300000000000001,"15min":2.2000000000000002},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":202.78800000000001},"last":5.2629999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496055,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1371912,"kb_used_data":863040,"kb_used_omap":0,"kb_used_meta":508864,"kb_avail":92373240,"statfs":{"total":95995035648,"available":94590197760,"internally_reserved":0,"allocated":883752960,"data_stored":882052794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":521076736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":5.4119999999999999,"5min":3.6509999999999998,"15min":5.2279999999999998},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":79.504000000000005,"5min":95.043999999999997,"15min":187.19800000000001},"last":4.6200000000000001},{"interface":"front","average":{"1min":5.4580000000000002,"5min":3.8250000000000002,"15min":5.3479999999999999},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":78.813000000000002,"5min":94.855999999999995,"15min":187.46700000000001},"last":2.867}]},{"osd":1,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.1519999999999999,"15min":1.952},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5050000000000008,"5min":32.814,"15min":69.569999999999993},"last":2.7719999999999998},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.1850000000000001,"15min":2.1400000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":8.5169999999999995,"5min":32.856000000000002,"15min":70.471999999999994},"last":2.7829999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":3.456,"15min":3.798},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6120000000000001,"5min":207.02500000000001,"15min":207.02500000000001},"last":4.8609999999999998},{"interface":"front","average":{"1min":1.661,"5min":3.8170000000000002,"15min":4.0199999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28199999999999997},"max":{"1min":6.1299999999999999,"5min":207.114,"15min":207.114},"last":5.2190000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":3.2810000000000001,"15min":4.2830000000000004},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":7.1299999999999999,"5min":107.11799999999999,"15min":212.44800000000001},"last":6.0179999999999998},{"interface":"front","average":{"1min":2.0939999999999999,"5min":3.2450000000000001,"15min":4.5759999999999996},"min":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":7.7220000000000004,"5min":106.982,"15min":210.84299999999999},"last":0.60199999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.5740000000000001,"15min":1.3540000000000001},"min":{"1min":0.26200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.5190000000000001,"5min":63.798000000000002,"15min":63.798000000000002},"last":0.311},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.5629999999999999,"15min":1.2490000000000001},"min":{"1min":0.34599999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.1900000000000004,"5min":63.133000000000003,"15min":63.133000000000003},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:45:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.974,"5min":14.974,"15min":14.974},"last":0.501},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.6280000000000001,"15min":2.6280000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":16.443000000000001,"5min":16.443000000000001,"15min":16.443000000000001},"last":0.378}]},{"osd":7,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":0.78900000000000003,"15min":1.6579999999999999},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.276999999999999,"5min":11.276999999999999,"15min":138.191},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.6419999999999999,"5min":0.74299999999999999,"15min":1.51},"min":{"1min":0.38100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":10.909000000000001,"5min":10.909000000000001,"15min":140.33500000000001},"last":0.45400000000000001}]}]},{"osd":1,"up_from":301,"seq":1292785156253,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3139200,"kb_used_data":2516020,"kb_used_omap":9,"kb_used_meta":623158,"kb_avail":90605952,"statfs":{"total":95995035648,"available":92780494848,"internally_reserved":0,"allocated":2576404480,"data_stored":2573463345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":638113810},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":10.134,"5min":3.8159999999999998,"15min":3.6789999999999998},"min":{"1min":0.23899999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":142.08500000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":9.4260000000000002,"5min":3.5449999999999999,"15min":3.5739999999999998},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":142.137,"5min":142.137,"15min":142.137},"last":3.758}]},{"osd":2,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":5.3959999999999999,"5min":2.7629999999999999,"15min":2.5409999999999999},"min":{"1min":0.23999999999999999,"5min":0.184,"15min":0.184},"max":{"1min":62.572000000000003,"5min":62.572000000000003,"15min":62.572000000000003},"last":4.2889999999999997},{"interface":"front","average":{"1min":5.9349999999999996,"5min":2.8769999999999998,"15min":2.782},"min":{"1min":0.26800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":68.373999999999995,"5min":68.373999999999995,"15min":68.373999999999995},"last":0.63500000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":4.4969999999999999,"5min":5.6699999999999999,"15min":6.4870000000000001},"min":{"1min":0.251,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":59.478000000000002,"5min":199.41,"15min":199.41},"last":5.0970000000000004},{"interface":"front","average":{"1min":5.234,"5min":5.923,"15min":6.6189999999999998},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":59.621000000000002,"5min":199.58099999999999,"15min":199.58099999999999},"last":5.6589999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":0.98199999999999998,"15min":2.1499999999999999},"min":{"1min":0.317,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.6020000000000003,"5min":5.6020000000000003,"15min":142.77099999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.508,"5min":1.244,"15min":2.3359999999999999},"min":{"1min":0.36699999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.976,"5min":8.0830000000000002,"15min":144.898},"last":4.8650000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.0190000000000001,"15min":1.962},"min":{"1min":0.40500000000000003,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.2100000000000009,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":4.8239999999999998},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.345,"15min":1.2629999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":15.250999999999999,"5min":15.250999999999999,"15min":15.250999999999999},"last":0.439}]},{"osd":6,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":1.3100000000000001,"15min":6.5579999999999998},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":30.986999999999998,"5min":30.986999999999998,"15min":209.221},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.698,"5min":1.3009999999999999,"15min":6.4859999999999998},"min":{"1min":0.46300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":28.710999999999999,"5min":28.710999999999999,"15min":208.43600000000001},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.2789999999999999,"15min":4.6100000000000003},"min":{"1min":0.38600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.5149999999999997,"5min":9.298,"15min":165.59399999999999},"last":4.218},{"interface":"front","average":{"1min":1.641,"5min":1.3620000000000001,"15min":4.7000000000000002},"min":{"1min":0.48099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":7.5460000000000003,"5min":9.3399999999999999,"15min":165.833},"last":4.2549999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926151,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1575860,"kb_used_data":949100,"kb_used_omap":5,"kb_used_meta":626746,"kb_avail":92169292,"statfs":{"total":95995035648,"available":94381355008,"internally_reserved":0,"allocated":971878400,"data_stored":968708004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":641788157},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":2.1989999999999998,"15min":2.274},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":5.8079999999999998,"5min":57.482999999999997,"15min":57.482999999999997},"last":5.4320000000000004},{"interface":"front","average":{"1min":1.839,"5min":2.4129999999999998,"15min":4.4530000000000003},"min":{"1min":0.33200000000000002,"5min":0.23300000000000001,"15min":0.214},"max":{"1min":5.7149999999999999,"5min":57.140999999999998,"15min":138.92400000000001},"last":17.344000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":4.9539999999999997,"15min":3.1120000000000001},"min":{"1min":0.246,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.3650000000000002,"5min":127.73399999999999,"15min":127.73399999999999},"last":3.395},{"interface":"front","average":{"1min":1.9470000000000001,"5min":5.125,"15min":3.2919999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":8.407,"5min":127.69,"15min":127.69},"last":4.3220000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":4.2329999999999997,"5min":5.2009999999999996,"15min":5.2460000000000004},"min":{"1min":0.35599999999999998,"5min":0.255,"15min":0.221},"max":{"1min":28.015000000000001,"5min":125.562,"15min":218.06999999999999},"last":4.9450000000000003},{"interface":"front","average":{"1min":2.2309999999999999,"5min":4.5609999999999999,"15min":4.2409999999999997},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":12.769,"5min":117.295,"15min":218.63200000000001},"last":5.4720000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.5950000000000002,"15min":1.954},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.4610000000000003,"5min":72.403999999999996,"15min":72.403999999999996},"last":3.4649999999999999},{"interface":"front","average":{"1min":3.4380000000000002,"5min":2.883,"15min":2.7480000000000002},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":11.272,"5min":72.162999999999997,"15min":136.262},"last":4.9139999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":4.0060000000000002,"5min":1.6579999999999999,"15min":1.2669999999999999},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":11.326000000000001,"5min":11.326000000000001,"15min":11.326000000000001},"last":4.8739999999999997},{"interface":"front","average":{"1min":1.4470000000000001,"5min":1.1279999999999999,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.8200000000000003,"5min":6.8200000000000003,"15min":6.8200000000000003},"last":1.1439999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.8740000000000001,"15min":3.5089999999999999},"min":{"1min":0.38900000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.8819999999999997,"5min":57.384,"15min":214.785},"last":1.202},{"interface":"front","average":{"1min":2.0449999999999999,"5min":2.0390000000000001,"15min":3.7370000000000001},"min":{"1min":0.47399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.4100000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":3.4889999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.6970000000000001,"15min":2.1139999999999999},"min":{"1min":0.40300000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.8470000000000004,"5min":57.776000000000003,"15min":97.382000000000005},"last":1.1779999999999999},{"interface":"front","average":{"1min":2.3479999999999999,"5min":2.1509999999999998,"15min":2.3380000000000001},"min":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.33100000000000002},"max":{"1min":8.4529999999999994,"5min":57.179000000000002,"15min":97.015000000000001},"last":3.5209999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599092,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1942720,"kb_used_data":1140152,"kb_used_omap":5,"kb_used_meta":802234,"kb_avail":91802432,"statfs":{"total":95995035648,"available":94005690368,"internally_reserved":0,"allocated":1167515648,"data_stored":1164621642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":821488269},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":14.532999999999999,"5min":5.5990000000000002,"15min":3.4089999999999998},"min":{"1min":0.26700000000000002,"5min":0.192,"15min":0.187},"max":{"1min":170.678,"5min":170.678,"15min":170.678},"last":4.0949999999999998},{"interface":"front","average":{"1min":14.816000000000001,"5min":5.4219999999999997,"15min":3.371},"min":{"1min":0.307,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":172.44999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":3.2839999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":13.468,"5min":4.3209999999999997,"15min":3.7850000000000001},"min":{"1min":0.27700000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":162.696,"5min":162.696,"15min":162.696},"last":2.899},{"interface":"front","average":{"1min":13.574999999999999,"5min":4.3300000000000001,"15min":3.5249999999999999},"min":{"1min":0.23400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":162.93899999999999,"5min":162.93899999999999,"15min":162.93899999999999},"last":3.3069999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:46:08 2023","interfaces":[{"interface":"back","average":{"1min":9.9629999999999992,"5min":5.625,"15min":4.9740000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":156.39400000000001,"5min":156.39400000000001,"15min":156.39400000000001},"last":3.1699999999999999},{"interface":"front","average":{"1min":8.8300000000000001,"5min":5.3040000000000003,"15min":4.8280000000000003},"min":{"1min":0.32600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":156.376,"5min":156.376,"15min":156.376},"last":3.8540000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":3.9020000000000001,"5min":1.5049999999999999,"15min":1.782},"min":{"1min":0.32800000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":47.463000000000001,"5min":47.463000000000001,"15min":83.012},"last":3.2589999999999999},{"interface":"front","average":{"1min":4.4509999999999996,"5min":1.569,"15min":1.8220000000000001},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":47.412999999999997,"5min":47.412999999999997,"15min":81.335999999999999},"last":3.754}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":5.2030000000000003,"5min":1.901,"15min":1.351},"min":{"1min":0.378,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":47.226999999999997,"5min":47.226999999999997,"15min":47.226999999999997},"last":3.2189999999999999},{"interface":"front","average":{"1min":5.1379999999999999,"5min":2.0800000000000001,"15min":1.5700000000000001},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":47.127000000000002,"5min":47.127000000000002,"15min":47.127000000000002},"last":1.2290000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.1479999999999999,"15min":3.2400000000000002},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.3309999999999995,"5min":9.3309999999999995,"15min":154.417},"last":3.7909999999999999},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.4019999999999999,"15min":3.516},"min":{"1min":0.433,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.7670000000000003,"5min":9.8409999999999993,"15min":154.04400000000001},"last":1.282}]},{"osd":7,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":1.7989999999999999,"15min":2.355},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":49.816000000000003,"5min":49.816000000000003,"15min":123.294},"last":3.8220000000000001},{"interface":"front","average":{"1min":4.1269999999999998,"5min":1.873,"15min":2.6440000000000001},"min":{"1min":0.40799999999999997,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":50.956000000000003,"5min":50.956000000000003,"15min":123.16},"last":1.7809999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435566,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3050936,"kb_used_data":2196656,"kb_used_omap":2,"kb_used_meta":854269,"kb_avail":90694216,"statfs":{"total":95995035648,"available":92870877184,"internally_reserved":0,"allocated":2249375744,"data_stored":2246376571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":874771815},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0190000000000001,"5min":4.3949999999999996,"15min":4.7249999999999996},"min":{"1min":0.20000000000000001,"5min":0.183,"15min":0.183},"max":{"1min":34.220999999999997,"5min":144.72,"15min":205.73099999999999},"last":4.0609999999999999},{"interface":"front","average":{"1min":4.2670000000000003,"5min":4.6139999999999999,"15min":4.6580000000000004},"min":{"1min":0.28999999999999998,"5min":0.23999999999999999,"15min":0.21199999999999999},"max":{"1min":34.253,"5min":142.36600000000001,"15min":205.495},"last":0.23799999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:45:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":3.141,"15min":8.593},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":19.725000000000001,"5min":142.434,"15min":273.99599999999998},"last":0.26200000000000001},{"interface":"front","average":{"1min":3.3340000000000001,"5min":3.1480000000000001,"15min":8.1219999999999999},"min":{"1min":0.33900000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":19.638999999999999,"5min":142.44900000000001,"15min":274.21600000000001},"last":12.516999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":4.851,"15min":5.6269999999999998},"min":{"1min":0.36699999999999999,"5min":0.249,"15min":0.249},"max":{"1min":8.0389999999999997,"5min":146.92599999999999,"15min":206.69},"last":12.561999999999999},{"interface":"front","average":{"1min":2.0390000000000001,"5min":4.508,"15min":5.4370000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.9649999999999999,"5min":145.75299999999999,"15min":209.22800000000001},"last":3.5630000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.976,"15min":2.0649999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.141,"5min":81.677000000000007,"15min":81.677000000000007},"last":3.4300000000000002},{"interface":"front","average":{"1min":2.3170000000000002,"5min":2.2669999999999999,"15min":1.9330000000000001},"min":{"1min":0.35399999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0540000000000003,"5min":80.623000000000005,"15min":80.623000000000005},"last":0.35399999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":1.732,"15min":1.595},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.701000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":11.701000000000001},{"interface":"front","average":{"1min":2.2839999999999998,"5min":1.218,"15min":1.04},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":0.311}]},{"osd":6,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.069,"15min":1.901},"min":{"1min":0.38800000000000001,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":5.202,"5min":5.915,"15min":60.698},"last":3.5230000000000001},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.4219999999999999,"15min":1.911},"min":{"1min":0.41899999999999998,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":6.2290000000000001,"5min":10.356999999999999,"15min":12.121},"last":0.38300000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":3.0569999999999999,"15min":2.601},"min":{"1min":0.47199999999999998,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":7.968,"5min":213.548,"15min":213.548},"last":11.753},{"interface":"front","average":{"1min":1.3839999999999999,"5min":2.8109999999999999,"15min":2.5129999999999999},"min":{"1min":0.438,"5min":0.41099999999999998,"15min":0.34799999999999998},"max":{"1min":5.1719999999999997,"5min":213.583,"15min":213.583},"last":3.488}]}]},{"osd":4,"up_from":30,"seq":128849019328,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353676,"kb_used_data":871684,"kb_used_omap":0,"kb_used_meta":481984,"kb_avail":92391476,"statfs":{"total":95995035648,"available":94608871424,"internally_reserved":0,"allocated":892604416,"data_stored":890900269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":493551616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2749999999999999,"15min":2.4950000000000001},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":34.991999999999997,"5min":73.718000000000004,"15min":105.179},"last":8.0589999999999993},{"interface":"front","average":{"1min":3.573,"5min":2.4550000000000001,"15min":2.6619999999999999},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":35.274999999999999,"5min":74.001999999999995,"15min":104.70999999999999},"last":4.0899999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.2029999999999998,"15min":4.3399999999999999},"min":{"1min":0.32000000000000001,"5min":0.25,"15min":0.25},"max":{"1min":30.538,"5min":80.712000000000003,"15min":160.69999999999999},"last":8.0220000000000002},{"interface":"front","average":{"1min":3.226,"5min":2.2730000000000001,"15min":4.2930000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":30.722000000000001,"5min":80.810000000000002,"15min":160.50700000000001},"last":0.51500000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":11.257,"5min":4.6269999999999998,"15min":3.3559999999999999},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":117.858,"5min":142.88999999999999,"15min":142.88999999999999},"last":2.9180000000000001},{"interface":"front","average":{"1min":11.305999999999999,"5min":4.6950000000000003,"15min":3.403},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":117.871,"5min":142.90700000000001,"15min":142.90700000000001},"last":7.9809999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.875,"5min":1.593,"15min":2.3130000000000002},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":29.41,"5min":29.41,"15min":70.241},"last":0.35799999999999998},{"interface":"front","average":{"1min":3.1219999999999999,"5min":1.4550000000000001,"15min":2.2440000000000002},"min":{"1min":0.38300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":29.401,"5min":29.401,"15min":70.433999999999997},"last":4.0510000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":8.9930000000000003,"15min":10.153},"min":{"1min":0.32800000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.133,"5min":140.11099999999999,"15min":140.11099999999999},"last":6.8840000000000003},{"interface":"front","average":{"1min":1.5029999999999999,"5min":8.9130000000000003,"15min":10.148},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0350000000000001,"5min":140.065,"15min":140.065},"last":0.627}]},{"osd":6,"last update":"Sat Sep 30 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.63100000000000001,"15min":2.1539999999999999},"min":{"1min":0.33900000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":2.7970000000000002,"5min":4.4299999999999997,"15min":202.72499999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.1259999999999999,"5min":0.71899999999999997,"15min":2.165},"min":{"1min":0.33400000000000002,"5min":0.253,"15min":0.249},"max":{"1min":7.173,"5min":7.173,"15min":202.42400000000001},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.2490000000000001,"15min":3.1819999999999999},"min":{"1min":0.36799999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.8170000000000002,"5min":161.72999999999999,"15min":230.90100000000001},"last":12.771000000000001},{"interface":"front","average":{"1min":1.4490000000000001,"5min":2.2589999999999999,"15min":3.032},"min":{"1min":0.36199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.1709999999999994,"5min":162.60900000000001,"15min":229.97},"last":0.68200000000000005}]}]},{"osd":5,"up_from":489,"seq":2100239007770,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2717964,"kb_used_data":2575300,"kb_used_omap":6,"kb_used_meta":142649,"kb_avail":91027188,"statfs":{"total":95995035648,"available":93211840512,"internally_reserved":0,"allocated":2637107200,"data_stored":2635399603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":146072671},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3500000000000001,"5min":21.718,"15min":24.779},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":25.786999999999999,"5min":281.10599999999999,"15min":281.10599999999999},"last":3.1000000000000001},{"interface":"front","average":{"1min":3.851,"5min":22.048999999999999,"15min":25.082000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":25.663,"5min":282.07400000000001,"15min":282.07400000000001},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":12.692,"15min":14.177},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":24.478999999999999,"5min":273.48200000000003,"15min":273.48200000000003},"last":0.375},{"interface":"front","average":{"1min":4.3319999999999999,"5min":12.15,"15min":13.452999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":24.359000000000002,"5min":273.50200000000001,"15min":273.50200000000001},"last":4.0469999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":11.537000000000001,"5min":3.0600000000000001,"15min":1.647},"min":{"1min":0.40899999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":163.23099999999999,"5min":163.23099999999999,"15min":163.23099999999999},"last":6.4349999999999996},{"interface":"front","average":{"1min":11.898999999999999,"5min":3.6120000000000001,"15min":2.2309999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":162.71100000000001,"5min":162.71100000000001,"15min":162.71100000000001},"last":6.024}]},{"osd":3,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.113,"5min":12.420999999999999,"15min":13.973000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.271000000000001,"5min":274.01799999999997,"15min":274.01799999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":3.2709999999999999,"5min":2.0390000000000001,"15min":1.8340000000000001},"min":{"1min":0.38400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":13.259,"5min":13.259,"15min":13.259},"last":4.0720000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.88500000000000001,"15min":0.92900000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8769999999999998,"5min":4.1769999999999996,"15min":4.1769999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.262,"5min":1.169,"15min":1.153},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.7610000000000001,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.5129999999999999,"15min":1.601},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.758,"5min":14.371,"15min":14.371},"last":2.1349999999999998},{"interface":"front","average":{"1min":1.069,"5min":1.325,"15min":1.3680000000000001},"min":{"1min":0.379,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.7359999999999998,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.238,"15min":1.2749999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.1150000000000002,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":2.1760000000000002},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.90400000000000003,"15min":0.91200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.0680000000000001,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.42199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":957743104,"data_stored":957743104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2542014464,"data_stored":2542010392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1158610944,"data_stored":1158610944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2231631872,"data_stored":2231631872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":879624192,"data_stored":879624192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2628849664,"data_stored":2628845592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":876478464,"data_stored":876478464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2641498112,"data_stored":2641494040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:46:37.119 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:46:37.501 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:37.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:37.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:37.606 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:37.689 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:37.822 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:37.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:37.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:37.927 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:37.933 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:46:37.934 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:46:37.982 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:38.014 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:38.020 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:38.255 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:38.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:38.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:38.360 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:38.427 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:38.973 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:38.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:38.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:39.079 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:39.129 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:39.135 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:38 smithi167 ceph-mon[192880]: pgmap v1346: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 3119/202803 objects misplaced (1.538%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:39.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:38 smithi019 ceph-mon[216815]: pgmap v1346: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 3119/202803 objects misplaced (1.538%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:39.186 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:38 smithi019 ceph-mon[224043]: pgmap v1346: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 3119/202803 objects misplaced (1.538%); 1.6 MiB/s, 26 objects/s recovering 2023-09-30T09:46:39.197 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:46:39.208 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:39.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:39.321 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:39.354 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:39.387 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:46:39.417 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:39.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:39.523 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:39.640 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:39.943 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:39.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:39.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:40.055 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:40.114 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:40.918 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:40.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:40.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:41.026 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:41.069 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:41.165 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:41.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:41.176 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:40 smithi167 ceph-mon[192880]: pgmap v1347: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2967/192849 objects misplaced (1.539%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:41.276 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:41.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:40 smithi019 ceph-mon[216815]: pgmap v1347: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2967/192849 objects misplaced (1.539%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:41.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:40 smithi019 ceph-mon[224043]: pgmap v1347: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2967/192849 objects misplaced (1.539%); 1.8 MiB/s, 28 objects/s recovering 2023-09-30T09:46:41.340 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:41.775 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:41.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:41.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:41.889 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:42.005 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:42.269 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:42.411 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:42.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:42.431 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:42.517 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:42.616 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:42.906 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:42.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:42.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:42.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:42 smithi019 ceph-mon[216815]: pgmap v1348: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 2967/192849 objects misplaced (1.539%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:42.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:42 smithi019 ceph-mon[224043]: pgmap v1348: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 2967/192849 objects misplaced (1.539%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:43.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:43.049 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:43.107 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:43.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:43.148 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:46:43.148 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1741,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":516,"num_osds":8,"num_up_osds":8,"osd_up_since":1696067065,"num_in_osds":8,"osd_in_since":1696066981,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":118},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":64283,"data_bytes":4215740351,"bytes_used":17617698816,"bytes_avail":750342586368,"bytes_total":767960285184,"misplaced_objects":2967,"misplaced_total":192849,"misplaced_ratio":0.015385094037303797,"recovering_objects_per_sec":26,"recovering_bytes_per_sec":1734674,"recovering_keys_per_sec":0,"num_objects_recovered":286,"num_bytes_recovered":18743296,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":647},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-30T09:45:25.765551+0000","services":{}},"progress_events":{"5b5f721d-3b3c-459d-b09b-c0a83655ed5d":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-30T09:46:43.215 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:43.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:42 smithi167 ceph-mon[192880]: pgmap v1348: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 2967/192849 objects misplaced (1.539%); 1.7 MiB/s, 26 objects/s recovering 2023-09-30T09:46:43.354 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:43.966 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:46:44.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:46:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:46:43] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2023-09-30T09:46:44.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1352153563' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:46:44.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2897291765' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:46:44.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1352153563' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:46:44.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2897291765' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:46:44.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1352153563' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:46:44.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2897291765' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:46:45.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: pgmap v1349: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2967/192849 objects misplaced (1.539%); 1.5 MiB/s, 23 objects/s recovering 2023-09-30T09:46:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2897291765' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:46:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: osdmap e517: 8 total, 8 up, 8 in 2023-09-30T09:46:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:44 smithi167 ceph-mon[192880]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: pgmap v1349: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2967/192849 objects misplaced (1.539%); 1.5 MiB/s, 23 objects/s recovering 2023-09-30T09:46:45.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:45.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2897291765' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:46:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: osdmap e517: 8 total, 8 up, 8 in 2023-09-30T09:46:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[216815]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: pgmap v1349: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2967/192849 objects misplaced (1.539%); 1.5 MiB/s, 23 objects/s recovering 2023-09-30T09:46:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2897291765' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:46:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: osdmap e517: 8 total, 8 up, 8 in 2023-09-30T09:46:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: failed to encode map e517 with expected crc 2023-09-30T09:46:45.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:44 smithi019 ceph-mon[224043]: failed to encode map e517 with expected crc 2023-09-30T09:46:46.409 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:46.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:46.432 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:46:46.433 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:46:46.523 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:46.635 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:46.761 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:46.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:46.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:46.870 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:46.929 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:47.025 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:46 smithi019 ceph-mon[224043]: pgmap v1351: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 2818/181911 objects misplaced (1.549%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:47.025 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:46 smithi019 ceph-mon[216815]: pgmap v1351: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 2818/181911 objects misplaced (1.549%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:47.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:46 smithi167 ceph-mon[192880]: pgmap v1351: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 2818/181911 objects misplaced (1.549%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:47.296 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:47.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:47.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:47.406 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:47.468 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:46:47.505 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:47.516 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:46:47.518 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:46:47.518 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:46:47.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:46:47.566 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:47.612 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:46:47.670 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:46:47.707 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:46:47.707 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:47.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:47.751 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:46:47.787 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:46:47.808 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:46:47.808 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:46:47.808 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:46:47.808 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:46:47.808 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:46:47.808 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:46:47.809 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:46:47.809 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:46:47.810 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:47.849 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:47.926 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:47.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:47.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:48.032 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:46:48.085 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:48.155 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:46:48.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:48.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:46:48.162 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2620854508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:48.263 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:48.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2620854508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:48.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2620854508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:46:48.343 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:48.640 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:46:49.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:49 smithi167 ceph-mon[192880]: pgmap v1352: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 2818/181911 objects misplaced (1.549%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:49.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:49.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:49 smithi019 ceph-mon[224043]: pgmap v1352: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 2818/181911 objects misplaced (1.549%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:49 smithi019 ceph-mon[216815]: pgmap v1352: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 2818/181911 objects misplaced (1.549%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:49.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:46:49.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:46:49.411 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:49.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:49.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:49.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:46:49.524 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:46:49.574 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:49.648 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:46:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:49.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:46:49.755 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:46:49.856 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:50.085 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:46:50.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:50.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:46:50.187 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:46:50.221 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:50.287 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:46:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:50.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:46:50.395 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:50.476 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:50.584 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:50.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:50.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:50.692 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:50.719 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:50.763 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:50.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:50.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:50.866 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:50.928 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:51.035 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:51 smithi019 ceph-mon[216815]: pgmap v1353: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:51.227 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:51.235 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:51.269 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:51.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:51.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:51 smithi019 ceph-mon[224043]: pgmap v1353: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:51.388 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:51.458 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:51.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:51 smithi167 ceph-mon[192880]: pgmap v1353: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:51.674 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:51.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:51.779 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:51.824 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:51.899 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:51.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:51.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:51.918 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:46:52.009 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:52.185 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:53 smithi019 ceph-mon[216815]: pgmap v1354: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:53 smithi019 ceph-mon[224043]: pgmap v1354: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:53 smithi167 ceph-mon[192880]: pgmap v1354: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:54.020 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:54.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:54.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:54.025 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:46:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:46:53] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2023-09-30T09:46:54.033 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:46:54.043 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:46:54.125 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:54.174 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:54.237 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:54.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:54.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:54.343 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:54.390 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:54.470 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:54.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:54.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:54.575 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:54.623 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:54.692 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:54.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:54.752 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:54 smithi167 ceph-mon[192880]: pgmap v1355: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:54.752 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:54 smithi167 ceph-mon[192880]: from='client.36411 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:54.804 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:46:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:54 smithi019 ceph-mon[216815]: pgmap v1355: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:54.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:54 smithi019 ceph-mon[216815]: from='client.36411 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:54.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:54 smithi019 ceph-mon[224043]: pgmap v1355: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2675/171633 objects misplaced (1.559%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:46:54.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:54 smithi019 ceph-mon[224043]: from='client.36411 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:46:54.854 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:54.946 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:46:54.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:54.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:46:55.056 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:46:55.155 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:55.370 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1355,"stamp":"2023-09-30T09:46:53.826568+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3752269759,"num_objects":57211,"num_object_clones":0,"num_object_copies":171633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2675,"num_objects_unfound":0,"num_objects_dirty":57211,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":137210,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3718,"num_bytes_recovered":237917810,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118725,"ondisk_log_size":118725,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15631320,"kb_used_data":11017688,"kb_used_omap":30,"kb_used_meta":4613409,"kb_avail":734329896,"statfs":{"total":767960285184,"available":751953813504,"internally_reserved":0,"allocated":11282112512,"data_stored":11263207689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4724131170},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,3,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-463470592,"num_objects":-7072,"num_object_clones":0,"num_object_copies":-21216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-292,"num_objects_unfound":0,"num_objects_dirty":-7072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7072,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":292,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.013344"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":940,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012436+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:46:31.012436+0000","last_peered":"2023-09-30T09:46:31.012436+0000","last_clean":"2023-09-30T09:46:31.012436+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:46:31.012436+0000","last_undegraded":"2023-09-30T09:46:31.012436+0000","last_fullsized":"2023-09-30T09:46:31.012436+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":535,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079446+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:46:31.079446+0000","last_peered":"2023-09-30T09:46:31.079446+0000","last_clean":"2023-09-30T09:46:31.079446+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:46:31.079446+0000","last_undegraded":"2023-09-30T09:46:31.079446+0000","last_fullsized":"2023-09-30T09:46:31.079446+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":432,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.080204+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:46:31.080204+0000","last_peered":"2023-09-30T09:46:31.080204+0000","last_clean":"2023-09-30T09:46:31.080204+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:46:31.080204+0000","last_undegraded":"2023-09-30T09:46:31.080204+0000","last_fullsized":"2023-09-30T09:46:31.080204+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":942,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012329+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:46:31.012329+0000","last_peered":"2023-09-30T09:46:31.012329+0000","last_clean":"2023-09-30T09:46:31.012329+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:46:31.012329+0000","last_undegraded":"2023-09-30T09:46:31.012329+0000","last_fullsized":"2023-09-30T09:46:31.012329+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":81,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.009195+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:46:31.009195+0000","last_peered":"2023-09-30T09:46:31.009195+0000","last_clean":"2023-09-30T09:46:31.009195+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:46:31.009195+0000","last_undegraded":"2023-09-30T09:46:31.009195+0000","last_fullsized":"2023-09-30T09:46:31.009195+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":123,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:39.216045+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:46:39.216045+0000","last_peered":"2023-09-30T09:46:39.216045+0000","last_clean":"2023-09-30T09:46:39.216045+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:46:39.216045+0000","last_undegraded":"2023-09-30T09:46:39.216045+0000","last_fullsized":"2023-09-30T09:46:39.216045+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":750,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079487+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:46:31.079487+0000","last_peered":"2023-09-30T09:46:31.079487+0000","last_clean":"2023-09-30T09:46:31.079487+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:46:31.079487+0000","last_undegraded":"2023-09-30T09:46:31.079487+0000","last_fullsized":"2023-09-30T09:46:31.079487+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":22,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.133264+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:46:31.133264+0000","last_peered":"2023-09-30T09:46:31.133264+0000","last_clean":"2023-09-30T09:46:31.133264+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:46:31.133264+0000","last_undegraded":"2023-09-30T09:46:31.133264+0000","last_fullsized":"2023-09-30T09:46:31.133264+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":86,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014840+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:46:31.014840+0000","last_peered":"2023-09-30T09:46:31.014840+0000","last_clean":"2023-09-30T09:46:31.014840+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:46:31.014840+0000","last_undegraded":"2023-09-30T09:46:31.014840+0000","last_fullsized":"2023-09-30T09:46:31.014840+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":918,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016104+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:46:31.016104+0000","last_peered":"2023-09-30T09:46:31.016104+0000","last_clean":"2023-09-30T09:46:31.016104+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:46:31.016104+0000","last_undegraded":"2023-09-30T09:46:31.016104+0000","last_fullsized":"2023-09-30T09:46:31.016104+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":137,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.132692+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:46:31.132692+0000","last_peered":"2023-09-30T09:46:31.132692+0000","last_clean":"2023-09-30T09:46:31.132692+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:46:31.132692+0000","last_undegraded":"2023-09-30T09:46:31.132692+0000","last_fullsized":"2023-09-30T09:46:31.132692+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":97,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.008708+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:46:31.008708+0000","last_peered":"2023-09-30T09:46:31.008708+0000","last_clean":"2023-09-30T09:46:31.008708+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:46:31.008708+0000","last_undegraded":"2023-09-30T09:46:31.008708+0000","last_fullsized":"2023-09-30T09:46:31.008708+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":706,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018771+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:46:31.018771+0000","last_peered":"2023-09-30T09:46:31.018771+0000","last_clean":"2023-09-30T09:46:31.018771+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:46:31.018771+0000","last_undegraded":"2023-09-30T09:46:31.018771+0000","last_fullsized":"2023-09-30T09:46:31.018771+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":337,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015100+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:46:31.015100+0000","last_peered":"2023-09-30T09:46:31.015100+0000","last_clean":"2023-09-30T09:46:31.015100+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:46:31.015100+0000","last_undegraded":"2023-09-30T09:46:31.015100+0000","last_fullsized":"2023-09-30T09:46:31.015100+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":473,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015694+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:46:31.015694+0000","last_peered":"2023-09-30T09:46:31.015694+0000","last_clean":"2023-09-30T09:46:31.015694+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:46:31.015694+0000","last_undegraded":"2023-09-30T09:46:31.015694+0000","last_fullsized":"2023-09-30T09:46:31.015694+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":646,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.013116+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:46:31.013116+0000","last_peered":"2023-09-30T09:46:31.013116+0000","last_clean":"2023-09-30T09:46:31.013116+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:46:31.013116+0000","last_undegraded":"2023-09-30T09:46:31.013116+0000","last_fullsized":"2023-09-30T09:46:31.013116+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":231,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016224+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:46:31.016224+0000","last_peered":"2023-09-30T09:46:31.016224+0000","last_clean":"2023-09-30T09:46:31.016224+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:46:31.016224+0000","last_undegraded":"2023-09-30T09:46:31.016224+0000","last_fullsized":"2023-09-30T09:46:31.016224+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014642+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:46:31.014642+0000","last_peered":"2023-09-30T09:46:31.014642+0000","last_clean":"2023-09-30T09:46:31.014642+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:46:31.014642+0000","last_undegraded":"2023-09-30T09:46:31.014642+0000","last_fullsized":"2023-09-30T09:46:31.014642+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":572,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.132420+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:46:31.132420+0000","last_peered":"2023-09-30T09:46:31.132420+0000","last_clean":"2023-09-30T09:46:31.132420+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:46:31.132420+0000","last_undegraded":"2023-09-30T09:46:31.132420+0000","last_fullsized":"2023-09-30T09:46:31.132420+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":561,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079043+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:46:31.079043+0000","last_peered":"2023-09-30T09:46:31.079043+0000","last_clean":"2023-09-30T09:46:31.079043+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:46:31.079043+0000","last_undegraded":"2023-09-30T09:46:31.079043+0000","last_fullsized":"2023-09-30T09:46:31.079043+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":539,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079659+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:46:31.079659+0000","last_peered":"2023-09-30T09:46:31.079659+0000","last_clean":"2023-09-30T09:46:31.079659+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:46:31.079659+0000","last_undegraded":"2023-09-30T09:46:31.079659+0000","last_fullsized":"2023-09-30T09:46:31.079659+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":770,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.133331+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:46:31.133331+0000","last_peered":"2023-09-30T09:46:31.133331+0000","last_clean":"2023-09-30T09:46:31.133331+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:46:31.133331+0000","last_undegraded":"2023-09-30T09:46:31.133331+0000","last_fullsized":"2023-09-30T09:46:31.133331+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":167,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015867+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:46:31.015867+0000","last_peered":"2023-09-30T09:46:31.015867+0000","last_clean":"2023-09-30T09:46:31.015867+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:46:31.015867+0000","last_undegraded":"2023-09-30T09:46:31.015867+0000","last_fullsized":"2023-09-30T09:46:31.015867+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":494,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016262+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:46:31.016262+0000","last_peered":"2023-09-30T09:46:31.016262+0000","last_clean":"2023-09-30T09:46:31.016262+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:46:31.016262+0000","last_undegraded":"2023-09-30T09:46:31.016262+0000","last_fullsized":"2023-09-30T09:46:31.016262+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":818,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012238+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:46:31.012238+0000","last_peered":"2023-09-30T09:46:31.012238+0000","last_clean":"2023-09-30T09:46:31.012238+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:46:31.012238+0000","last_undegraded":"2023-09-30T09:46:31.012238+0000","last_fullsized":"2023-09-30T09:46:31.012238+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":96,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.132078+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:46:31.132078+0000","last_peered":"2023-09-30T09:46:31.132078+0000","last_clean":"2023-09-30T09:46:31.132078+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:46:31.132078+0000","last_undegraded":"2023-09-30T09:46:31.132078+0000","last_fullsized":"2023-09-30T09:46:31.132078+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":192,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012759+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:46:31.012759+0000","last_peered":"2023-09-30T09:46:31.012759+0000","last_clean":"2023-09-30T09:46:31.012759+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:46:31.012759+0000","last_undegraded":"2023-09-30T09:46:31.012759+0000","last_fullsized":"2023-09-30T09:46:31.012759+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":918,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.008812+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:46:31.008812+0000","last_peered":"2023-09-30T09:46:31.008812+0000","last_clean":"2023-09-30T09:46:31.008812+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:46:31.008812+0000","last_undegraded":"2023-09-30T09:46:31.008812+0000","last_fullsized":"2023-09-30T09:46:31.008812+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":201,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015976+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:46:31.015976+0000","last_peered":"2023-09-30T09:46:31.015976+0000","last_clean":"2023-09-30T09:46:31.015976+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:46:31.015976+0000","last_undegraded":"2023-09-30T09:46:31.015976+0000","last_fullsized":"2023-09-30T09:46:31.015976+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":96,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.132061+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:46:31.132061+0000","last_peered":"2023-09-30T09:46:31.132061+0000","last_clean":"2023-09-30T09:46:31.132061+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:46:31.132061+0000","last_undegraded":"2023-09-30T09:46:31.132061+0000","last_fullsized":"2023-09-30T09:46:31.132061+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":927,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012824+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:46:31.012824+0000","last_peered":"2023-09-30T09:46:31.012824+0000","last_clean":"2023-09-30T09:46:31.012824+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:46:31.012824+0000","last_undegraded":"2023-09-30T09:46:31.012824+0000","last_fullsized":"2023-09-30T09:46:31.012824+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":113,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014301+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:46:31.014301+0000","last_peered":"2023-09-30T09:46:31.014301+0000","last_clean":"2023-09-30T09:46:31.014301+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:46:31.014301+0000","last_undegraded":"2023-09-30T09:46:31.014301+0000","last_fullsized":"2023-09-30T09:46:31.014301+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":81,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.009405+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:46:31.009405+0000","last_peered":"2023-09-30T09:46:31.009405+0000","last_clean":"2023-09-30T09:46:31.009405+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:46:31.009405+0000","last_undegraded":"2023-09-30T09:46:31.009405+0000","last_fullsized":"2023-09-30T09:46:31.009405+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"517'8485","reported_seq":8790,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.313783+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:46:49.313783+0000","last_peered":"2023-09-30T09:46:49.313783+0000","last_clean":"2023-09-30T09:46:49.313783+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:46:49.313783+0000","last_undegraded":"2023-09-30T09:46:49.313783+0000","last_fullsized":"2023-09-30T09:46:49.313783+0000","mapping_epoch":495,"log_start":"496'900","ondisk_log_start":"496'900","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7585,"log_dups_size":0,"ondisk_log_size":7585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228655104,"num_objects":3489,"num_object_clones":0,"num_object_copies":10467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8485,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":893,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016196+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:46:31.016196+0000","last_peered":"2023-09-30T09:46:31.016196+0000","last_clean":"2023-09-30T09:46:31.016196+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:46:31.016196+0000","last_undegraded":"2023-09-30T09:46:31.016196+0000","last_fullsized":"2023-09-30T09:46:31.016196+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":193,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016217+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:46:31.016217+0000","last_peered":"2023-09-30T09:46:31.016217+0000","last_clean":"2023-09-30T09:46:31.016217+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:46:31.016217+0000","last_undegraded":"2023-09-30T09:46:31.016217+0000","last_fullsized":"2023-09-30T09:46:31.016217+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":138,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012147+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:46:31.012147+0000","last_peered":"2023-09-30T09:46:31.012147+0000","last_clean":"2023-09-30T09:46:31.012147+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:46:31.012147+0000","last_undegraded":"2023-09-30T09:46:31.012147+0000","last_fullsized":"2023-09-30T09:46:31.012147+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"517'8369","reported_seq":8411,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:48.767095+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:46:48.767095+0000","last_peered":"2023-09-30T09:46:48.767095+0000","last_clean":"2023-09-30T09:46:48.767095+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:46:48.767095+0000","last_undegraded":"2023-09-30T09:46:48.767095+0000","last_fullsized":"2023-09-30T09:46:48.767095+0000","mapping_epoch":483,"log_start":"497'1900","ondisk_log_start":"497'1900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234291200,"num_objects":3575,"num_object_clones":0,"num_object_copies":10725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8369,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":910,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.009567+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:46:31.009567+0000","last_peered":"2023-09-30T09:46:31.009567+0000","last_clean":"2023-09-30T09:46:31.009567+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:46:31.009567+0000","last_undegraded":"2023-09-30T09:46:31.009567+0000","last_fullsized":"2023-09-30T09:46:31.009567+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":163,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016072+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:46:31.016072+0000","last_peered":"2023-09-30T09:46:31.016072+0000","last_clean":"2023-09-30T09:46:31.016072+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:46:31.016072+0000","last_undegraded":"2023-09-30T09:46:31.016072+0000","last_fullsized":"2023-09-30T09:46:31.016072+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":568,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079323+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:46:31.079323+0000","last_peered":"2023-09-30T09:46:31.079323+0000","last_clean":"2023-09-30T09:46:31.079323+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:46:31.079323+0000","last_undegraded":"2023-09-30T09:46:31.079323+0000","last_fullsized":"2023-09-30T09:46:31.079323+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"517'8605","reported_seq":8872,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.123213+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:46:49.123213+0000","last_peered":"2023-09-30T09:46:49.123213+0000","last_clean":"2023-09-30T09:46:49.123213+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:46:49.123213+0000","last_undegraded":"2023-09-30T09:46:49.123213+0000","last_fullsized":"2023-09-30T09:46:49.123213+0000","mapping_epoch":493,"log_start":"497'1900","ondisk_log_start":"497'1900","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6705,"log_dups_size":0,"ondisk_log_size":6705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234422272,"num_objects":3577,"num_object_clones":0,"num_object_copies":10731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8605,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":687,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015055+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:46:31.015055+0000","last_peered":"2023-09-30T09:46:31.015055+0000","last_clean":"2023-09-30T09:46:31.015055+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:46:31.015055+0000","last_undegraded":"2023-09-30T09:46:31.015055+0000","last_fullsized":"2023-09-30T09:46:31.015055+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":230,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015989+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:46:31.015989+0000","last_peered":"2023-09-30T09:46:31.015989+0000","last_clean":"2023-09-30T09:46:31.015989+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:46:31.015989+0000","last_undegraded":"2023-09-30T09:46:31.015989+0000","last_fullsized":"2023-09-30T09:46:31.015989+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":86,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014714+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:46:31.014714+0000","last_peered":"2023-09-30T09:46:31.014714+0000","last_clean":"2023-09-30T09:46:31.014714+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:46:31.014714+0000","last_undegraded":"2023-09-30T09:46:31.014714+0000","last_fullsized":"2023-09-30T09:46:31.014714+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"517'8347","reported_seq":8705,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:48.870867+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:46:48.870867+0000","last_peered":"2023-09-30T09:46:48.870867+0000","last_clean":"2023-09-30T09:46:48.870867+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:46:48.870867+0000","last_undegraded":"2023-09-30T09:46:48.870867+0000","last_fullsized":"2023-09-30T09:46:48.870867+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":8347,"log_dups_size":0,"ondisk_log_size":8347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232587264,"num_objects":3549,"num_object_clones":0,"num_object_copies":10647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8347,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":752,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079946+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:46:31.079946+0000","last_peered":"2023-09-30T09:46:31.079946+0000","last_clean":"2023-09-30T09:46:31.079946+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:46:31.079946+0000","last_undegraded":"2023-09-30T09:46:31.079946+0000","last_fullsized":"2023-09-30T09:46:31.079946+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":452,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012699+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:46:31.012699+0000","last_peered":"2023-09-30T09:46:31.012699+0000","last_clean":"2023-09-30T09:46:31.012699+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:46:31.012699+0000","last_undegraded":"2023-09-30T09:46:31.012699+0000","last_fullsized":"2023-09-30T09:46:31.012699+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":116,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.142021+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:46:31.142021+0000","last_peered":"2023-09-30T09:46:31.142021+0000","last_clean":"2023-09-30T09:46:31.142021+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:46:31.142021+0000","last_undegraded":"2023-09-30T09:46:31.142021+0000","last_fullsized":"2023-09-30T09:46:31.142021+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"517'8533","reported_seq":8574,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.302817+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:46:49.302817+0000","last_peered":"2023-09-30T09:46:49.302817+0000","last_clean":"2023-09-30T09:46:49.302817+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:46:49.302817+0000","last_undegraded":"2023-09-30T09:46:49.302817+0000","last_fullsized":"2023-09-30T09:46:49.302817+0000","mapping_epoch":483,"log_start":"497'1500","ondisk_log_start":"497'1500","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7033,"log_dups_size":0,"ondisk_log_size":7033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235864064,"num_objects":3599,"num_object_clones":0,"num_object_copies":10797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8533,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":999,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015874+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:46:31.015874+0000","last_peered":"2023-09-30T09:46:31.015874+0000","last_clean":"2023-09-30T09:46:31.015874+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:46:31.015874+0000","last_undegraded":"2023-09-30T09:46:31.015874+0000","last_fullsized":"2023-09-30T09:46:31.015874+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":75,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.141851+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:46:31.141851+0000","last_peered":"2023-09-30T09:46:31.141851+0000","last_clean":"2023-09-30T09:46:31.141851+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:46:31.141851+0000","last_undegraded":"2023-09-30T09:46:31.141851+0000","last_fullsized":"2023-09-30T09:46:31.141851+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":515,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079020+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:46:31.079020+0000","last_peered":"2023-09-30T09:46:31.079020+0000","last_clean":"2023-09-30T09:46:31.079020+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:46:31.079020+0000","last_undegraded":"2023-09-30T09:46:31.079020+0000","last_fullsized":"2023-09-30T09:46:31.079020+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"517'8458","reported_seq":8500,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.191051+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:46:49.191051+0000","last_peered":"2023-09-30T09:46:49.191051+0000","last_clean":"2023-09-30T09:46:49.191051+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:46:49.191051+0000","last_undegraded":"2023-09-30T09:46:49.191051+0000","last_fullsized":"2023-09-30T09:46:49.191051+0000","mapping_epoch":483,"log_start":"497'1400","ondisk_log_start":"497'1400","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229113856,"num_objects":3496,"num_object_clones":0,"num_object_copies":10488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8458,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":998,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018543+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:46:31.018543+0000","last_peered":"2023-09-30T09:46:31.018543+0000","last_clean":"2023-09-30T09:46:31.018543+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:46:31.018543+0000","last_undegraded":"2023-09-30T09:46:31.018543+0000","last_fullsized":"2023-09-30T09:46:31.018543+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":72,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014570+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:46:31.014570+0000","last_peered":"2023-09-30T09:46:31.014570+0000","last_clean":"2023-09-30T09:46:31.014570+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:46:31.014570+0000","last_undegraded":"2023-09-30T09:46:31.014570+0000","last_fullsized":"2023-09-30T09:46:31.014570+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":211,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079836+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:46:31.079836+0000","last_peered":"2023-09-30T09:46:31.079836+0000","last_clean":"2023-09-30T09:46:31.079836+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:46:31.079836+0000","last_undegraded":"2023-09-30T09:46:31.079836+0000","last_fullsized":"2023-09-30T09:46:31.079836+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"517'8722","reported_seq":8763,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:48.871808+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:46:48.871808+0000","last_peered":"2023-09-30T09:46:48.871808+0000","last_clean":"2023-09-30T09:46:48.871808+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:46:48.871808+0000","last_undegraded":"2023-09-30T09:46:48.871808+0000","last_fullsized":"2023-09-30T09:46:48.871808+0000","mapping_epoch":483,"log_start":"499'2900","ondisk_log_start":"499'2900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5822,"log_dups_size":0,"ondisk_log_size":5822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232521728,"num_objects":3548,"num_object_clones":0,"num_object_copies":10644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8722,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":855,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.142046+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:46:31.142046+0000","last_peered":"2023-09-30T09:46:31.142046+0000","last_clean":"2023-09-30T09:46:31.142046+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:46:31.142046+0000","last_undegraded":"2023-09-30T09:46:31.142046+0000","last_fullsized":"2023-09-30T09:46:31.142046+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":522,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079195+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:46:31.079195+0000","last_peered":"2023-09-30T09:46:31.079195+0000","last_clean":"2023-09-30T09:46:31.079195+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:46:31.079195+0000","last_undegraded":"2023-09-30T09:46:31.079195+0000","last_fullsized":"2023-09-30T09:46:31.079195+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":529,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.080035+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:46:31.080035+0000","last_peered":"2023-09-30T09:46:31.080035+0000","last_clean":"2023-09-30T09:46:31.080035+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:46:31.080035+0000","last_undegraded":"2023-09-30T09:46:31.080035+0000","last_fullsized":"2023-09-30T09:46:31.080035+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"517'8397","reported_seq":8441,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:48.886723+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:46:48.886723+0000","last_peered":"2023-09-30T09:46:48.886723+0000","last_clean":"2023-09-30T09:46:48.886723+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:46:48.886723+0000","last_undegraded":"2023-09-30T09:46:48.886723+0000","last_fullsized":"2023-09-30T09:46:48.886723+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":8397,"log_dups_size":0,"ondisk_log_size":8397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231931904,"num_objects":3539,"num_object_clones":0,"num_object_copies":10617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8397,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":720,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018653+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:46:31.018653+0000","last_peered":"2023-09-30T09:46:31.018653+0000","last_clean":"2023-09-30T09:46:31.018653+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:46:31.018653+0000","last_undegraded":"2023-09-30T09:46:31.018653+0000","last_fullsized":"2023-09-30T09:46:31.018653+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":81,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.008679+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:46:31.008679+0000","last_peered":"2023-09-30T09:46:31.008679+0000","last_clean":"2023-09-30T09:46:31.008679+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:46:31.008679+0000","last_undegraded":"2023-09-30T09:46:31.008679+0000","last_fullsized":"2023-09-30T09:46:31.008679+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":202,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.080124+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:46:31.080124+0000","last_peered":"2023-09-30T09:46:31.080124+0000","last_clean":"2023-09-30T09:46:31.080124+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:46:31.080124+0000","last_undegraded":"2023-09-30T09:46:31.080124+0000","last_fullsized":"2023-09-30T09:46:31.080124+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"517'8549","reported_seq":8850,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.035791+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:46:49.035791+0000","last_peered":"2023-09-30T09:46:49.035791+0000","last_clean":"2023-09-30T09:46:49.035791+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:46:49.035791+0000","last_undegraded":"2023-09-30T09:46:49.035791+0000","last_fullsized":"2023-09-30T09:46:49.035791+0000","mapping_epoch":494,"log_start":"497'1800","ondisk_log_start":"497'1800","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232980480,"num_objects":3555,"num_object_clones":0,"num_object_copies":10665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8549,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":759,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014761+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:46:31.014761+0000","last_peered":"2023-09-30T09:46:31.014761+0000","last_clean":"2023-09-30T09:46:31.014761+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:46:31.014761+0000","last_undegraded":"2023-09-30T09:46:31.014761+0000","last_fullsized":"2023-09-30T09:46:31.014761+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":747,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.131826+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:46:31.131826+0000","last_peered":"2023-09-30T09:46:31.131826+0000","last_clean":"2023-09-30T09:46:31.131826+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:46:31.131826+0000","last_undegraded":"2023-09-30T09:46:31.131826+0000","last_fullsized":"2023-09-30T09:46:31.131826+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":445,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012838+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:46:31.012838+0000","last_peered":"2023-09-30T09:46:31.012838+0000","last_clean":"2023-09-30T09:46:31.012838+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:46:31.012838+0000","last_undegraded":"2023-09-30T09:46:31.012838+0000","last_fullsized":"2023-09-30T09:46:31.012838+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":134,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012550+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:31.012550+0000","last_peered":"2023-09-30T09:46:31.012550+0000","last_clean":"2023-09-30T09:46:31.012550+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:46:31.012550+0000","last_undegraded":"2023-09-30T09:46:31.012550+0000","last_fullsized":"2023-09-30T09:46:31.012550+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"517'8569","reported_seq":8915,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.298654+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:46:49.298654+0000","last_peered":"2023-09-30T09:46:49.298654+0000","last_clean":"2023-09-30T09:46:49.298654+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:46:49.298654+0000","last_undegraded":"2023-09-30T09:46:49.298654+0000","last_fullsized":"2023-09-30T09:46:49.298654+0000","mapping_epoch":493,"log_start":"496'1000","ondisk_log_start":"496'1000","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7569,"log_dups_size":0,"ondisk_log_size":7569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233242624,"num_objects":3559,"num_object_clones":0,"num_object_copies":10677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8569,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":958,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012469+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:46:31.012469+0000","last_peered":"2023-09-30T09:46:31.012469+0000","last_clean":"2023-09-30T09:46:31.012469+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:46:31.012469+0000","last_undegraded":"2023-09-30T09:46:31.012469+0000","last_fullsized":"2023-09-30T09:46:31.012469+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":464,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014790+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:46:31.014790+0000","last_peered":"2023-09-30T09:46:31.014790+0000","last_clean":"2023-09-30T09:46:31.014790+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:46:31.014790+0000","last_undegraded":"2023-09-30T09:46:31.014790+0000","last_fullsized":"2023-09-30T09:46:31.014790+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":536,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079666+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:46:31.079666+0000","last_peered":"2023-09-30T09:46:31.079666+0000","last_clean":"2023-09-30T09:46:31.079666+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:46:31.079666+0000","last_undegraded":"2023-09-30T09:46:31.079666+0000","last_fullsized":"2023-09-30T09:46:31.079666+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":276,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018814+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:46:31.018814+0000","last_peered":"2023-09-30T09:46:31.018814+0000","last_clean":"2023-09-30T09:46:31.018814+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:46:31.018814+0000","last_undegraded":"2023-09-30T09:46:31.018814+0000","last_fullsized":"2023-09-30T09:46:31.018814+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"517'8578","reported_seq":8924,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.128115+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:46:49.128115+0000","last_peered":"2023-09-30T09:46:49.128115+0000","last_clean":"2023-09-30T09:46:49.128115+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:46:49.128115+0000","last_undegraded":"2023-09-30T09:46:49.128115+0000","last_fullsized":"2023-09-30T09:46:49.128115+0000","mapping_epoch":492,"log_start":"497'1900","ondisk_log_start":"497'1900","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6678,"log_dups_size":0,"ondisk_log_size":6678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235536384,"num_objects":3594,"num_object_clones":0,"num_object_copies":10782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8578,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":936,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015802+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:46:31.015802+0000","last_peered":"2023-09-30T09:46:31.015802+0000","last_clean":"2023-09-30T09:46:31.015802+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:46:31.015802+0000","last_undegraded":"2023-09-30T09:46:31.015802+0000","last_fullsized":"2023-09-30T09:46:31.015802+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":785,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018682+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:46:31.018682+0000","last_peered":"2023-09-30T09:46:31.018682+0000","last_clean":"2023-09-30T09:46:31.018682+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:46:31.018682+0000","last_undegraded":"2023-09-30T09:46:31.018682+0000","last_fullsized":"2023-09-30T09:46:31.018682+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":279,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018448+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:46:31.018448+0000","last_peered":"2023-09-30T09:46:31.018448+0000","last_clean":"2023-09-30T09:46:31.018448+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:46:31.018448+0000","last_undegraded":"2023-09-30T09:46:31.018448+0000","last_fullsized":"2023-09-30T09:46:31.018448+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":366,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.013343+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:46:31.013343+0000","last_peered":"2023-09-30T09:46:31.013343+0000","last_clean":"2023-09-30T09:46:31.013343+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:46:31.013343+0000","last_undegraded":"2023-09-30T09:46:31.013343+0000","last_fullsized":"2023-09-30T09:46:31.013343+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"517'8456","reported_seq":8763,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.289305+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:46:49.289305+0000","last_peered":"2023-09-30T09:46:49.289305+0000","last_clean":"2023-09-30T09:46:49.289305+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:46:49.289305+0000","last_undegraded":"2023-09-30T09:46:49.289305+0000","last_fullsized":"2023-09-30T09:46:49.289305+0000","mapping_epoch":495,"log_start":"496'900","ondisk_log_start":"496'900","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233439232,"num_objects":3562,"num_object_clones":0,"num_object_copies":10686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8456,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1078,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012472+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:46:31.012472+0000","last_peered":"2023-09-30T09:46:31.012472+0000","last_clean":"2023-09-30T09:46:31.012472+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:46:31.012472+0000","last_undegraded":"2023-09-30T09:46:31.012472+0000","last_fullsized":"2023-09-30T09:46:31.012472+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":879,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015678+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:46:31.015678+0000","last_peered":"2023-09-30T09:46:31.015678+0000","last_clean":"2023-09-30T09:46:31.015678+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:46:31.015678+0000","last_undegraded":"2023-09-30T09:46:31.015678+0000","last_fullsized":"2023-09-30T09:46:31.015678+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":93,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012622+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:46:31.012622+0000","last_peered":"2023-09-30T09:46:31.012622+0000","last_clean":"2023-09-30T09:46:31.012622+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:46:31.012622+0000","last_undegraded":"2023-09-30T09:46:31.012622+0000","last_fullsized":"2023-09-30T09:46:31.012622+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":72,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014591+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:46:31.014591+0000","last_peered":"2023-09-30T09:46:31.014591+0000","last_clean":"2023-09-30T09:46:31.014591+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:46:31.014591+0000","last_undegraded":"2023-09-30T09:46:31.014591+0000","last_fullsized":"2023-09-30T09:46:31.014591+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"517'8704","reported_seq":13362,"reported_epoch":517,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:46:48.901420+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:46:48.901420+0000","last_peered":"2023-09-30T09:46:48.901420+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:46:48.901420+0000","last_undegraded":"2023-09-30T09:46:48.901420+0000","last_fullsized":"2023-09-30T09:46:48.901420+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":8704,"log_dups_size":0,"ondisk_log_size":8704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237371392,"num_objects":3622,"num_object_clones":0,"num_object_copies":10866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2675,"num_objects_unfound":0,"num_objects_dirty":3622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8704,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2295,"num_bytes_recovered":150405120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":947}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":778,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015120+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:46:31.015120+0000","last_peered":"2023-09-30T09:46:31.015120+0000","last_clean":"2023-09-30T09:46:31.015120+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:46:31.015120+0000","last_undegraded":"2023-09-30T09:46:31.015120+0000","last_fullsized":"2023-09-30T09:46:31.015120+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":925,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015608+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:46:31.015608+0000","last_peered":"2023-09-30T09:46:31.015608+0000","last_clean":"2023-09-30T09:46:31.015608+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:46:31.015608+0000","last_undegraded":"2023-09-30T09:46:31.015608+0000","last_fullsized":"2023-09-30T09:46:31.015608+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":130,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956313+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:46:43.956313+0000","last_peered":"2023-09-30T09:46:43.956313+0000","last_clean":"2023-09-30T09:46:43.956313+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:46:43.956313+0000","last_undegraded":"2023-09-30T09:46:43.956313+0000","last_fullsized":"2023-09-30T09:46:43.956313+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.131974+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:46:31.131974+0000","last_peered":"2023-09-30T09:46:31.131974+0000","last_clean":"2023-09-30T09:46:31.131974+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:46:31.131974+0000","last_undegraded":"2023-09-30T09:46:31.131974+0000","last_fullsized":"2023-09-30T09:46:31.131974+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"517'8468","reported_seq":8509,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:48.870311+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:46:48.870311+0000","last_peered":"2023-09-30T09:46:48.870311+0000","last_clean":"2023-09-30T09:46:48.870311+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:46:48.870311+0000","last_undegraded":"2023-09-30T09:46:48.870311+0000","last_fullsized":"2023-09-30T09:46:48.870311+0000","mapping_epoch":483,"log_start":"499'2600","ondisk_log_start":"499'2600","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5868,"log_dups_size":0,"ondisk_log_size":5868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238813184,"num_objects":3644,"num_object_clones":0,"num_object_copies":10932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8468,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":769,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014729+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:46:31.014729+0000","last_peered":"2023-09-30T09:46:31.014729+0000","last_clean":"2023-09-30T09:46:31.014729+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:46:31.014729+0000","last_undegraded":"2023-09-30T09:46:31.014729+0000","last_fullsized":"2023-09-30T09:46:31.014729+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1065,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016265+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:46:31.016265+0000","last_peered":"2023-09-30T09:46:31.016265+0000","last_clean":"2023-09-30T09:46:31.016265+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:46:31.016265+0000","last_undegraded":"2023-09-30T09:46:31.016265+0000","last_fullsized":"2023-09-30T09:46:31.016265+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":260,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018741+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:46:31.018741+0000","last_peered":"2023-09-30T09:46:31.018741+0000","last_clean":"2023-09-30T09:46:31.018741+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:46:31.018741+0000","last_undegraded":"2023-09-30T09:46:31.018741+0000","last_fullsized":"2023-09-30T09:46:31.018741+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":129,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.142124+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:46:31.142124+0000","last_peered":"2023-09-30T09:46:31.142124+0000","last_clean":"2023-09-30T09:46:31.142124+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:46:31.142124+0000","last_undegraded":"2023-09-30T09:46:31.142124+0000","last_fullsized":"2023-09-30T09:46:31.142124+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"517'8370","reported_seq":8747,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:48.826852+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:46:48.826852+0000","last_peered":"2023-09-30T09:46:48.826852+0000","last_clean":"2023-09-30T09:46:48.826852+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:46:48.826852+0000","last_undegraded":"2023-09-30T09:46:48.826852+0000","last_fullsized":"2023-09-30T09:46:48.826852+0000","mapping_epoch":495,"log_start":"499'2600","ondisk_log_start":"499'2600","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238485528,"num_objects":3640,"num_object_clones":0,"num_object_copies":10920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3640,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8370,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1036,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079931+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:46:31.079931+0000","last_peered":"2023-09-30T09:46:31.079931+0000","last_clean":"2023-09-30T09:46:31.079931+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:46:31.079931+0000","last_undegraded":"2023-09-30T09:46:31.079931+0000","last_fullsized":"2023-09-30T09:46:31.079931+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":548,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016134+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:46:31.016134+0000","last_peered":"2023-09-30T09:46:31.016134+0000","last_clean":"2023-09-30T09:46:31.016134+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:46:31.016134+0000","last_undegraded":"2023-09-30T09:46:31.016134+0000","last_fullsized":"2023-09-30T09:46:31.016134+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1235,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014647+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:46:31.014647+0000","last_peered":"2023-09-30T09:46:31.014647+0000","last_clean":"2023-09-30T09:46:31.014647+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:46:31.014647+0000","last_undegraded":"2023-09-30T09:46:31.014647+0000","last_fullsized":"2023-09-30T09:46:31.014647+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":552,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.015582+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:46:31.015582+0000","last_peered":"2023-09-30T09:46:31.015582+0000","last_clean":"2023-09-30T09:46:31.015582+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:46:31.015582+0000","last_undegraded":"2023-09-30T09:46:31.015582+0000","last_fullsized":"2023-09-30T09:46:31.015582+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"517'8693","reported_seq":8735,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:48.832039+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:46:48.832039+0000","last_peered":"2023-09-30T09:46:48.832039+0000","last_clean":"2023-09-30T09:46:48.832039+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:46:48.832039+0000","last_undegraded":"2023-09-30T09:46:48.832039+0000","last_fullsized":"2023-09-30T09:46:48.832039+0000","mapping_epoch":483,"log_start":"498'2300","ondisk_log_start":"498'2300","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238223360,"num_objects":3635,"num_object_clones":0,"num_object_copies":10905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8693,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":899,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.009420+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:46:31.009420+0000","last_peered":"2023-09-30T09:46:31.009420+0000","last_clean":"2023-09-30T09:46:31.009420+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:46:31.009420+0000","last_undegraded":"2023-09-30T09:46:31.009420+0000","last_fullsized":"2023-09-30T09:46:31.009420+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":87,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014976+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:46:31.014976+0000","last_peered":"2023-09-30T09:46:31.014976+0000","last_clean":"2023-09-30T09:46:31.014976+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:46:31.014976+0000","last_undegraded":"2023-09-30T09:46:31.014976+0000","last_fullsized":"2023-09-30T09:46:31.014976+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":93,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957061+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:46:43.957061+0000","last_peered":"2023-09-30T09:46:43.957061+0000","last_clean":"2023-09-30T09:46:43.957061+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:46:43.957061+0000","last_undegraded":"2023-09-30T09:46:43.957061+0000","last_fullsized":"2023-09-30T09:46:43.957061+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":518,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079276+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:46:31.079276+0000","last_peered":"2023-09-30T09:46:31.079276+0000","last_clean":"2023-09-30T09:46:31.079276+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:46:31.079276+0000","last_undegraded":"2023-09-30T09:46:31.079276+0000","last_fullsized":"2023-09-30T09:46:31.079276+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":836,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.133242+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:46:31.133242+0000","last_peered":"2023-09-30T09:46:31.133242+0000","last_clean":"2023-09-30T09:46:31.133242+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:46:31.133242+0000","last_undegraded":"2023-09-30T09:46:31.133242+0000","last_fullsized":"2023-09-30T09:46:31.133242+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":22,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.142118+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:46:31.142118+0000","last_peered":"2023-09-30T09:46:31.142118+0000","last_clean":"2023-09-30T09:46:31.142118+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:46:31.142118+0000","last_undegraded":"2023-09-30T09:46:31.142118+0000","last_fullsized":"2023-09-30T09:46:31.142118+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":222,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018678+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:46:31.018678+0000","last_peered":"2023-09-30T09:46:31.018678+0000","last_clean":"2023-09-30T09:46:31.018678+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:46:31.018678+0000","last_undegraded":"2023-09-30T09:46:31.018678+0000","last_fullsized":"2023-09-30T09:46:31.018678+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":631,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014739+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:46:31.014739+0000","last_peered":"2023-09-30T09:46:31.014739+0000","last_clean":"2023-09-30T09:46:31.014739+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:46:31.014739+0000","last_undegraded":"2023-09-30T09:46:31.014739+0000","last_fullsized":"2023-09-30T09:46:31.014739+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":445,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012641+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:31.012641+0000","last_peered":"2023-09-30T09:46:31.012641+0000","last_clean":"2023-09-30T09:46:31.012641+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:46:31.012641+0000","last_undegraded":"2023-09-30T09:46:31.012641+0000","last_fullsized":"2023-09-30T09:46:31.012641+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":231,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079217+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:46:31.079217+0000","last_peered":"2023-09-30T09:46:31.079217+0000","last_clean":"2023-09-30T09:46:31.079217+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:46:31.079217+0000","last_undegraded":"2023-09-30T09:46:31.079217+0000","last_fullsized":"2023-09-30T09:46:31.079217+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":387,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.012927+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:46:31.012927+0000","last_peered":"2023-09-30T09:46:31.012927+0000","last_clean":"2023-09-30T09:46:31.012927+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:46:31.012927+0000","last_undegraded":"2023-09-30T09:46:31.012927+0000","last_fullsized":"2023-09-30T09:46:31.012927+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":251,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079345+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:46:31.079345+0000","last_peered":"2023-09-30T09:46:31.079345+0000","last_clean":"2023-09-30T09:46:31.079345+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:46:31.079345+0000","last_undegraded":"2023-09-30T09:46:31.079345+0000","last_fullsized":"2023-09-30T09:46:31.079345+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":844,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.016179+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:46:31.016179+0000","last_peered":"2023-09-30T09:46:31.016179+0000","last_clean":"2023-09-30T09:46:31.016179+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:46:31.016179+0000","last_undegraded":"2023-09-30T09:46:31.016179+0000","last_fullsized":"2023-09-30T09:46:31.016179+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":74,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.014636+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:46:31.014636+0000","last_peered":"2023-09-30T09:46:31.014636+0000","last_clean":"2023-09-30T09:46:31.014636+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:46:31.014636+0000","last_undegraded":"2023-09-30T09:46:31.014636+0000","last_fullsized":"2023-09-30T09:46:31.014636+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":468,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.018610+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:46:31.018610+0000","last_peered":"2023-09-30T09:46:31.018610+0000","last_clean":"2023-09-30T09:46:31.018610+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:46:31.018610+0000","last_undegraded":"2023-09-30T09:46:31.018610+0000","last_fullsized":"2023-09-30T09:46:31.018610+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":887,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079845+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:46:31.079845+0000","last_peered":"2023-09-30T09:46:31.079845+0000","last_clean":"2023-09-30T09:46:31.079845+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:46:31.079845+0000","last_undegraded":"2023-09-30T09:46:31.079845+0000","last_fullsized":"2023-09-30T09:46:31.079845+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":28,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.132490+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:46:31.132490+0000","last_peered":"2023-09-30T09:46:31.132490+0000","last_clean":"2023-09-30T09:46:31.132490+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:46:31.132490+0000","last_undegraded":"2023-09-30T09:46:31.132490+0000","last_fullsized":"2023-09-30T09:46:31.132490+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":532,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-30T09:46:31.079176+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:46:31.079176+0000","last_peered":"2023-09-30T09:46:31.079176+0000","last_clean":"2023-09-30T09:46:31.079176+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:46:31.079176+0000","last_undegraded":"2023-09-30T09:46:31.079176+0000","last_fullsized":"2023-09-30T09:46:31.079176+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3747479576,"num_objects":57183,"num_object_clones":0,"num_object_copies":171549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2675,"num_objects_unfound":0,"num_objects_dirty":57183,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136303,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3438,"num_bytes_recovered":225312768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11163283456,"data_stored":11163271240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112703,"ondisk_log_size":112703,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234421,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2623736,"kb_used_data":2081584,"kb_used_omap":0,"kb_used_meta":542144,"kb_avail":91121416,"statfs":{"total":95995035648,"available":93308329984,"internally_reserved":0,"allocated":2131542016,"data_stored":2129810367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":555155456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:01 2023","interfaces":[{"interface":"back","average":{"1min":13.629,"5min":6.0149999999999997,"15min":4.5529999999999999},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":110.467,"5min":124.74299999999999,"15min":215.416},"last":2.7919999999999998},{"interface":"front","average":{"1min":13.702999999999999,"5min":6.1459999999999999,"15min":4.5099999999999998},"min":{"1min":0.23499999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":109.593,"5min":124.69499999999999,"15min":215.041},"last":0.55400000000000005}]},{"osd":1,"last update":"Sat Sep 30 09:46:01 2023","interfaces":[{"interface":"back","average":{"1min":16.905000000000001,"5min":7.1210000000000004,"15min":4.1399999999999997},"min":{"1min":0.32400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":157.578,"5min":157.578,"15min":157.578},"last":7.6559999999999997},{"interface":"front","average":{"1min":17.038,"5min":6.9660000000000002,"15min":4.3380000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":157.589,"5min":157.589,"15min":157.589},"last":4.2320000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":2.9809999999999999,"15min":3.5510000000000002},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.24299999999999999},"max":{"1min":10.292999999999999,"5min":104.88500000000001,"15min":117.14},"last":3.883},{"interface":"front","average":{"1min":1.9430000000000001,"5min":3.1629999999999998,"15min":3.5409999999999999},"min":{"1min":0.29599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.4420000000000002,"5min":104.867,"15min":117.044},"last":3.3519999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:46:15 2023","interfaces":[{"interface":"back","average":{"1min":11.387,"5min":6.8799999999999999,"15min":8.9429999999999996},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.31},"max":{"1min":154.06700000000001,"5min":154.06700000000001,"15min":277.57799999999997},"last":2.8260000000000001},{"interface":"front","average":{"1min":11.162000000000001,"5min":6.8810000000000002,"15min":8.4610000000000003},"min":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":154.24799999999999,"5min":154.24799999999999,"15min":278.69299999999998},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.1399999999999999,"15min":1.6259999999999999},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":7.5499999999999998,"5min":25.192,"15min":175.72},"last":7.6260000000000003},{"interface":"front","average":{"1min":1.214,"5min":1.0660000000000001,"15min":1.536},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.252},"max":{"1min":5.6120000000000001,"5min":24.521000000000001,"15min":179.048},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":9.875,"15min":11.287000000000001},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.641,"5min":197.75700000000001,"15min":197.75700000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.8919999999999999,"5min":9.8399999999999999,"15min":11.164},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.7149999999999999,"5min":197.63300000000001,"15min":197.63300000000001},"last":0.42599999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.72299999999999998,"15min":2.1760000000000002},"min":{"1min":0.32900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.202,"5min":6.202,"15min":206.244},"last":7.5830000000000002},{"interface":"front","average":{"1min":1.069,"5min":0.84399999999999997,"15min":2.1850000000000001},"min":{"1min":0.34699999999999998,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":5.5800000000000001,"5min":5.8520000000000003,"15min":202.78800000000001},"last":2.3220000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496059,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1214496,"kb_used_data":691096,"kb_used_omap":0,"kb_used_meta":523392,"kb_avail":92530656,"statfs":{"total":95995035648,"available":94751391744,"internally_reserved":0,"allocated":707682304,"data_stored":705973948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":535953408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":5.4119999999999999,"5min":3.6509999999999998,"15min":5.2279999999999998},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":79.504000000000005,"5min":95.043999999999997,"15min":187.19800000000001},"last":5.5259999999999998},{"interface":"front","average":{"1min":5.4580000000000002,"5min":3.8250000000000002,"15min":5.3479999999999999},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":78.813000000000002,"5min":94.855999999999995,"15min":187.46700000000001},"last":5.2569999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.1519999999999999,"15min":1.952},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5050000000000008,"5min":32.814,"15min":69.569999999999993},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.1850000000000001,"15min":2.1400000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":8.5169999999999995,"5min":32.856000000000002,"15min":70.471999999999994},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":3.456,"15min":3.798},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6120000000000001,"5min":207.02500000000001,"15min":207.02500000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.661,"5min":3.8170000000000002,"15min":4.0199999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28199999999999997},"max":{"1min":6.1299999999999999,"5min":207.114,"15min":207.114},"last":5.9299999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":3.2810000000000001,"15min":4.2830000000000004},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":7.1299999999999999,"5min":107.11799999999999,"15min":212.44800000000001},"last":5.9669999999999996},{"interface":"front","average":{"1min":2.0939999999999999,"5min":3.2450000000000001,"15min":4.5759999999999996},"min":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":7.7220000000000004,"5min":106.982,"15min":210.84299999999999},"last":0.74099999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.5740000000000001,"15min":1.3540000000000001},"min":{"1min":0.26200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.5190000000000001,"5min":63.798000000000002,"15min":63.798000000000002},"last":0.315},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.5629999999999999,"15min":1.2490000000000001},"min":{"1min":0.34599999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.1900000000000004,"5min":63.133000000000003,"15min":63.133000000000003},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8740000000000001,"15min":1.99},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.6840000000000002,"5min":14.974,"15min":14.974},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.88400000000000001,"5min":2.2789999999999999,"15min":2.5110000000000001},"min":{"1min":0.37,"5min":0.316,"15min":0.316},"max":{"1min":3.6190000000000002,"5min":16.443000000000001,"15min":16.443000000000001},"last":0.26900000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:45:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":0.78900000000000003,"15min":1.6579999999999999},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.276999999999999,"5min":11.276999999999999,"15min":138.191},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.6419999999999999,"5min":0.74299999999999999,"15min":1.51},"min":{"1min":0.38100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":10.909000000000001,"5min":10.909000000000001,"15min":140.33500000000001},"last":0.379}]}]},{"osd":1,"up_from":301,"seq":1292785156257,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2649608,"kb_used_data":2000064,"kb_used_omap":9,"kb_used_meta":649526,"kb_avail":91095544,"statfs":{"total":95995035648,"available":93281837056,"internally_reserved":0,"allocated":2048065536,"data_stored":2045120306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":665114642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":10.134,"5min":3.8159999999999998,"15min":3.6789999999999998},"min":{"1min":0.23899999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":142.08500000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":2.456},{"interface":"front","average":{"1min":9.4260000000000002,"5min":3.5449999999999999,"15min":3.5739999999999998},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":142.137,"5min":142.137,"15min":142.137},"last":0.75900000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":5.3959999999999999,"5min":2.7629999999999999,"15min":2.5409999999999999},"min":{"1min":0.23999999999999999,"5min":0.184,"15min":0.184},"max":{"1min":62.572000000000003,"5min":62.572000000000003,"15min":62.572000000000003},"last":1.639},{"interface":"front","average":{"1min":5.9349999999999996,"5min":2.8769999999999998,"15min":2.782},"min":{"1min":0.26800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":68.373999999999995,"5min":68.373999999999995,"15min":68.373999999999995},"last":0.22500000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":4.4969999999999999,"5min":5.6699999999999999,"15min":6.4870000000000001},"min":{"1min":0.251,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":59.478000000000002,"5min":199.41,"15min":199.41},"last":0.52400000000000002},{"interface":"front","average":{"1min":5.234,"5min":5.923,"15min":6.6189999999999998},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":59.621000000000002,"5min":199.58099999999999,"15min":199.58099999999999},"last":2.677}]},{"osd":4,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":0.98199999999999998,"15min":2.1499999999999999},"min":{"1min":0.317,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.6020000000000003,"5min":5.6020000000000003,"15min":142.77099999999999},"last":2.4889999999999999},{"interface":"front","average":{"1min":1.508,"5min":1.244,"15min":2.3359999999999999},"min":{"1min":0.36699999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.976,"5min":8.0830000000000002,"15min":144.898},"last":2.3439999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.0190000000000001,"15min":1.962},"min":{"1min":0.40500000000000003,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.2100000000000009,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.345,"15min":1.2629999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":15.250999999999999,"5min":15.250999999999999,"15min":15.250999999999999},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":1.3100000000000001,"15min":6.5579999999999998},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":30.986999999999998,"5min":30.986999999999998,"15min":209.221},"last":0.442},{"interface":"front","average":{"1min":2.698,"5min":1.3009999999999999,"15min":6.4859999999999998},"min":{"1min":0.46300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":28.710999999999999,"5min":28.710999999999999,"15min":208.43600000000001},"last":2.403}]},{"osd":7,"last update":"Sat Sep 30 09:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.2789999999999999,"15min":4.6100000000000003},"min":{"1min":0.38600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.5149999999999997,"5min":9.298,"15min":165.59399999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.641,"5min":1.3620000000000001,"15min":4.7000000000000002},"min":{"1min":0.48099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":7.5460000000000003,"5min":9.3399999999999999,"15min":165.833},"last":2.4260000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926155,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1401680,"kb_used_data":779016,"kb_used_omap":5,"kb_used_meta":622586,"kb_avail":92343472,"statfs":{"total":95995035648,"available":94559715328,"internally_reserved":0,"allocated":797712384,"data_stored":794530091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":637528317},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":2.1989999999999998,"15min":2.274},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":5.8079999999999998,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.839,"5min":2.4129999999999998,"15min":4.4530000000000003},"min":{"1min":0.33200000000000002,"5min":0.23300000000000001,"15min":0.214},"max":{"1min":5.7149999999999999,"5min":57.140999999999998,"15min":138.92400000000001},"last":3.0979999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6640000000000001,"5min":4.8010000000000002,"15min":3.2829999999999999},"min":{"1min":0.28899999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":31.521000000000001,"5min":127.73399999999999,"15min":127.73399999999999},"last":2.4049999999999998},{"interface":"front","average":{"1min":4.7649999999999997,"5min":5.1299999999999999,"15min":3.5470000000000002},"min":{"1min":0.38,"5min":0.27600000000000002,"15min":0.26200000000000001},"max":{"1min":31.478999999999999,"5min":127.69,"15min":127.69},"last":5.3090000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":4.2329999999999997,"5min":5.2009999999999996,"15min":5.2460000000000004},"min":{"1min":0.35599999999999998,"5min":0.255,"15min":0.221},"max":{"1min":28.015000000000001,"5min":125.562,"15min":218.06999999999999},"last":11.055},{"interface":"front","average":{"1min":2.2309999999999999,"5min":4.5609999999999999,"15min":4.2409999999999997},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":12.769,"5min":117.295,"15min":218.63200000000001},"last":8.3019999999999996}]},{"osd":4,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.5950000000000002,"15min":1.954},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.4610000000000003,"5min":72.403999999999996,"15min":72.403999999999996},"last":2.4340000000000002},{"interface":"front","average":{"1min":3.4380000000000002,"5min":2.883,"15min":2.7480000000000002},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":11.272,"5min":72.162999999999997,"15min":136.262},"last":2.0710000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":4.0060000000000002,"5min":1.6579999999999999,"15min":1.2669999999999999},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":11.326000000000001,"5min":11.326000000000001,"15min":11.326000000000001},"last":2.117},{"interface":"front","average":{"1min":1.4470000000000001,"5min":1.1279999999999999,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.8200000000000003,"5min":6.8200000000000003,"15min":6.8200000000000003},"last":0.30499999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.8740000000000001,"15min":3.5089999999999999},"min":{"1min":0.38900000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.8819999999999997,"5min":57.384,"15min":214.785},"last":2.1930000000000001},{"interface":"front","average":{"1min":2.0449999999999999,"5min":2.0390000000000001,"15min":3.7370000000000001},"min":{"1min":0.47399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.4100000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":2.464}]},{"osd":7,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.6970000000000001,"15min":2.1139999999999999},"min":{"1min":0.40300000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.8470000000000004,"5min":57.776000000000003,"15min":97.382000000000005},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.3479999999999999,"5min":2.1509999999999998,"15min":2.3380000000000001},"min":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.33100000000000002},"max":{"1min":8.4529999999999994,"5min":57.179000000000002,"15min":97.015000000000001},"last":5.2649999999999997}]}]},{"osd":2,"up_from":104,"seq":446676599096,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1718104,"kb_used_data":920016,"kb_used_omap":5,"kb_used_meta":798074,"kb_avail":92027048,"statfs":{"total":95995035648,"available":94235697152,"internally_reserved":0,"allocated":942096384,"data_stored":939194188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":817228429},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":14.532999999999999,"5min":5.5990000000000002,"15min":3.4089999999999998},"min":{"1min":0.26700000000000002,"5min":0.192,"15min":0.187},"max":{"1min":170.678,"5min":170.678,"15min":170.678},"last":3.4329999999999998},{"interface":"front","average":{"1min":14.816000000000001,"5min":5.4219999999999997,"15min":3.371},"min":{"1min":0.307,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":172.44999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":2.5550000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":13.468,"5min":4.3209999999999997,"15min":3.7850000000000001},"min":{"1min":0.27700000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":162.696,"5min":162.696,"15min":162.696},"last":0.42999999999999999},{"interface":"front","average":{"1min":13.574999999999999,"5min":4.3300000000000001,"15min":3.5249999999999999},"min":{"1min":0.23400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":162.93899999999999,"5min":162.93899999999999,"15min":162.93899999999999},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:46:08 2023","interfaces":[{"interface":"back","average":{"1min":9.9629999999999992,"5min":5.625,"15min":4.9740000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":156.39400000000001,"5min":156.39400000000001,"15min":156.39400000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":8.8300000000000001,"5min":5.3040000000000003,"15min":4.8280000000000003},"min":{"1min":0.32600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":156.376,"5min":156.376,"15min":156.376},"last":0.67300000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":3.9020000000000001,"5min":1.5049999999999999,"15min":1.782},"min":{"1min":0.32800000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":47.463000000000001,"5min":47.463000000000001,"15min":83.012},"last":0.33600000000000002},{"interface":"front","average":{"1min":4.4509999999999996,"5min":1.569,"15min":1.8220000000000001},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":47.412999999999997,"5min":47.412999999999997,"15min":81.335999999999999},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":5.2030000000000003,"5min":1.901,"15min":1.351},"min":{"1min":0.378,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":47.226999999999997,"5min":47.226999999999997,"15min":47.226999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":5.1379999999999999,"5min":2.0800000000000001,"15min":1.5700000000000001},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":47.127000000000002,"5min":47.127000000000002,"15min":47.127000000000002},"last":0.64200000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.1479999999999999,"15min":3.2400000000000002},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.3309999999999995,"5min":9.3309999999999995,"15min":154.417},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.4019999999999999,"15min":3.516},"min":{"1min":0.433,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.7670000000000003,"5min":9.8409999999999993,"15min":154.04400000000001},"last":0.69899999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:45:46 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":1.7989999999999999,"15min":2.355},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":49.816000000000003,"5min":49.816000000000003,"15min":123.294},"last":0.72499999999999998},{"interface":"front","average":{"1min":4.1269999999999998,"5min":1.873,"15min":2.6440000000000001},"min":{"1min":0.40799999999999997,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":50.956000000000003,"5min":50.956000000000003,"15min":123.16},"last":0.73799999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435570,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2611976,"kb_used_data":1753536,"kb_used_omap":2,"kb_used_meta":858429,"kb_avail":91133176,"statfs":{"total":95995035648,"available":93320372224,"internally_reserved":0,"allocated":1795620864,"data_stored":1792613889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":879031655},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0190000000000001,"5min":4.3949999999999996,"15min":4.7249999999999996},"min":{"1min":0.20000000000000001,"5min":0.183,"15min":0.183},"max":{"1min":34.220999999999997,"5min":144.72,"15min":205.73099999999999},"last":45.957999999999998},{"interface":"front","average":{"1min":4.2670000000000003,"5min":4.6139999999999999,"15min":4.6580000000000004},"min":{"1min":0.28999999999999998,"5min":0.23999999999999999,"15min":0.21199999999999999},"max":{"1min":34.253,"5min":142.36600000000001,"15min":205.495},"last":44.298999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:46:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0869999999999997,"5min":3.7599999999999998,"15min":7.4470000000000001},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":19.059000000000001,"5min":142.434,"15min":273.99599999999998},"last":1.0640000000000001},{"interface":"front","average":{"1min":4.1550000000000002,"5min":3.7879999999999998,"15min":7.1299999999999999},"min":{"1min":0.28199999999999997,"5min":0.26800000000000002,"15min":0.20699999999999999},"max":{"1min":13.183,"5min":142.44900000000001,"15min":274.21600000000001},"last":0.66300000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":4.851,"15min":5.6269999999999998},"min":{"1min":0.36699999999999999,"5min":0.249,"15min":0.249},"max":{"1min":8.0389999999999997,"5min":146.92599999999999,"15min":206.69},"last":0.63},{"interface":"front","average":{"1min":2.0390000000000001,"5min":4.508,"15min":5.4370000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.9649999999999999,"5min":145.75299999999999,"15min":209.22800000000001},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.976,"15min":2.0649999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.141,"5min":81.677000000000007,"15min":81.677000000000007},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.3170000000000002,"5min":2.2669999999999999,"15min":1.9330000000000001},"min":{"1min":0.35399999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0540000000000003,"5min":80.623000000000005,"15min":80.623000000000005},"last":1.153}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":1.732,"15min":1.595},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.701000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":2.2839999999999998,"5min":1.218,"15min":1.04},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":1.2290000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.069,"15min":1.901},"min":{"1min":0.38800000000000001,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":5.202,"5min":5.915,"15min":60.698},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.4219999999999999,"15min":1.911},"min":{"1min":0.41899999999999998,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":6.2290000000000001,"5min":10.356999999999999,"15min":12.121},"last":1.1970000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":3.0569999999999999,"15min":2.601},"min":{"1min":0.47199999999999998,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":7.968,"5min":213.548,"15min":213.548},"last":3.6869999999999998},{"interface":"front","average":{"1min":1.3839999999999999,"5min":2.8109999999999999,"15min":2.5129999999999999},"min":{"1min":0.438,"5min":0.41099999999999998,"15min":0.34799999999999998},"max":{"1min":5.1719999999999997,"5min":213.583,"15min":213.583},"last":1.276}]}]},{"osd":4,"up_from":30,"seq":128849019332,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1183780,"kb_used_data":708316,"kb_used_omap":0,"kb_used_meta":475392,"kb_avail":92561372,"statfs":{"total":95995035648,"available":94782844928,"internally_reserved":0,"allocated":725315584,"data_stored":723603247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":486801408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2749999999999999,"15min":2.4950000000000001},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":34.991999999999997,"5min":73.718000000000004,"15min":105.179},"last":1.7070000000000001},{"interface":"front","average":{"1min":3.573,"5min":2.4550000000000001,"15min":2.6619999999999999},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":35.274999999999999,"5min":74.001999999999995,"15min":104.70999999999999},"last":0.57999999999999996}]},{"osd":1,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.2029999999999998,"15min":4.3399999999999999},"min":{"1min":0.32000000000000001,"5min":0.25,"15min":0.25},"max":{"1min":30.538,"5min":80.712000000000003,"15min":160.69999999999999},"last":4.2619999999999996},{"interface":"front","average":{"1min":3.226,"5min":2.2730000000000001,"15min":4.2930000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":30.722000000000001,"5min":80.810000000000002,"15min":160.50700000000001},"last":1.7869999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":11.257,"5min":4.6269999999999998,"15min":3.3559999999999999},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":117.858,"5min":142.88999999999999,"15min":142.88999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":11.305999999999999,"5min":4.6950000000000003,"15min":3.403},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":117.871,"5min":142.90700000000001,"15min":142.90700000000001},"last":1.7549999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.875,"5min":1.593,"15min":2.3130000000000002},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":29.41,"5min":29.41,"15min":70.241},"last":9.0600000000000005},{"interface":"front","average":{"1min":3.1219999999999999,"5min":1.4550000000000001,"15min":2.2440000000000002},"min":{"1min":0.38300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":29.401,"5min":29.401,"15min":70.433999999999997},"last":9.0709999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":8.9930000000000003,"15min":10.153},"min":{"1min":0.32800000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.133,"5min":140.11099999999999,"15min":140.11099999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.5029999999999999,"5min":8.9130000000000003,"15min":10.148},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0350000000000001,"5min":140.065,"15min":140.065},"last":0.27200000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.63100000000000001,"15min":2.1539999999999999},"min":{"1min":0.33900000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":2.7970000000000002,"5min":4.4299999999999997,"15min":202.72499999999999},"last":1.669},{"interface":"front","average":{"1min":1.1259999999999999,"5min":0.71899999999999997,"15min":2.165},"min":{"1min":0.33400000000000002,"5min":0.253,"15min":0.249},"max":{"1min":7.173,"5min":7.173,"15min":202.42400000000001},"last":0.35799999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:45:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.2490000000000001,"15min":3.1819999999999999},"min":{"1min":0.36799999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.8170000000000002,"5min":161.72999999999999,"15min":230.90100000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.4490000000000001,"5min":2.2589999999999999,"15min":3.032},"min":{"1min":0.36199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.1709999999999994,"5min":162.60900000000001,"15min":229.97},"last":3.665}]}]},{"osd":5,"up_from":489,"seq":2100239007774,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2227940,"kb_used_data":2084060,"kb_used_omap":6,"kb_used_meta":143865,"kb_avail":91517212,"statfs":{"total":95995035648,"available":93713625088,"internally_reserved":0,"allocated":2134077440,"data_stored":2132361653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":147317855},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3500000000000001,"5min":21.718,"15min":24.779},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":25.786999999999999,"5min":281.10599999999999,"15min":281.10599999999999},"last":4.2779999999999996},{"interface":"front","average":{"1min":3.851,"5min":22.048999999999999,"15min":25.082000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":25.663,"5min":282.07400000000001,"15min":282.07400000000001},"last":5.9909999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":12.692,"15min":14.177},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":24.478999999999999,"5min":273.48200000000003,"15min":273.48200000000003},"last":4.6479999999999997},{"interface":"front","average":{"1min":4.3319999999999999,"5min":12.15,"15min":13.452999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":24.359000000000002,"5min":273.50200000000001,"15min":273.50200000000001},"last":4.3010000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":11.537000000000001,"5min":3.0600000000000001,"15min":1.647},"min":{"1min":0.40899999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":163.23099999999999,"5min":163.23099999999999,"15min":163.23099999999999},"last":6.7380000000000004},{"interface":"front","average":{"1min":11.898999999999999,"5min":3.6120000000000001,"15min":2.2309999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":162.71100000000001,"5min":162.71100000000001,"15min":162.71100000000001},"last":0.44900000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.113,"5min":12.420999999999999,"15min":13.973000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.271000000000001,"5min":274.01799999999997,"15min":274.01799999999997},"last":4.601},{"interface":"front","average":{"1min":3.2709999999999999,"5min":2.0390000000000001,"15min":1.8340000000000001},"min":{"1min":0.38400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":13.259,"5min":13.259,"15min":13.259},"last":4.2439999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.88500000000000001,"15min":0.92900000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8769999999999998,"5min":4.1769999999999996,"15min":4.1769999999999996},"last":4.7699999999999996},{"interface":"front","average":{"1min":1.262,"5min":1.169,"15min":1.153},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.7610000000000001,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.5129999999999999,"15min":1.601},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.758,"5min":14.371,"15min":14.371},"last":4.2119999999999997},{"interface":"front","average":{"1min":1.069,"5min":1.325,"15min":1.3680000000000001},"min":{"1min":0.379,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.7359999999999998,"5min":14.454000000000001,"15min":14.454000000000001},"last":3.0459999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.238,"15min":1.2749999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.1150000000000002,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":4.2629999999999999},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.90400000000000003,"15min":0.91200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.0680000000000001,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":4.6840000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":781910016,"data_stored":781910016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2031620096,"data_stored":2031616024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":931725312,"data_stored":931725312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1779236864,"data_stored":1779236864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":711393280,"data_stored":711393280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2105872384,"data_stored":2105868312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":701366272,"data_stored":701366272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2120159232,"data_stored":2120155160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:46:55.466 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:46:55.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:55.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:46:55.571 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:55.611 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:55.681 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:55.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:55.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:55.786 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:55.831 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:55.897 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:55.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:55.959 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:56.005 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:56.043 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:56.113 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:56.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:56.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:56.122 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:46:56.122 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:46:56.122 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:46:56.122 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:46:56.122 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:46:56.222 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:46:56.262 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:56.328 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:46:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:56.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:46:56.437 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:46:56.535 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:56.938 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:46:56.986 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:46:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:56.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:46:57.091 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:46:57.130 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:57.172 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:46:57.178 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:56 smithi167 ceph-mon[192880]: pgmap v1356: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2517/161409 objects misplaced (1.559%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:57.178 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1824579559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:57.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:57.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:46:57.280 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:57.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:56 smithi019 ceph-mon[216815]: pgmap v1356: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2517/161409 objects misplaced (1.559%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:57.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1824579559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:57.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:56 smithi019 ceph-mon[224043]: pgmap v1356: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2517/161409 objects misplaced (1.559%); 1.6 MiB/s, 25 objects/s recovering 2023-09-30T09:46:57.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1824579559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:46:57.390 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:58.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:58.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1824579559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:58.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[224043]: failed to encode map e518 with expected crc 2023-09-30T09:46:58.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[224043]: osdmap e518: 8 total, 8 up, 8 in 2023-09-30T09:46:58.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[224043]: failed to encode map e518 with expected crc 2023-09-30T09:46:58.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:58.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1824579559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:58.089 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[216815]: failed to encode map e518 with expected crc 2023-09-30T09:46:58.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[216815]: osdmap e518: 8 total, 8 up, 8 in 2023-09-30T09:46:58.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:57 smithi019 ceph-mon[216815]: failed to encode map e518 with expected crc 2023-09-30T09:46:58.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:57 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:46:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:57 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1824579559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:46:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:57 smithi167 ceph-mon[192880]: failed to encode map e518 with expected crc 2023-09-30T09:46:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:57 smithi167 ceph-mon[192880]: osdmap e518: 8 total, 8 up, 8 in 2023-09-30T09:46:58.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:57 smithi167 ceph-mon[192880]: failed to encode map e518 with expected crc 2023-09-30T09:46:58.377 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:58.379 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:46:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:58.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:58.489 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:58.544 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:59.176 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:46:59.258 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:46:59.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:46:58 smithi019 ceph-mon[216815]: pgmap v1358: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 2517/161409 objects misplaced (1.559%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:59.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:46:58 smithi019 ceph-mon[224043]: pgmap v1358: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 2517/161409 objects misplaced (1.559%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:59.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:59.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:46:59.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:46:58 smithi167 ceph-mon[192880]: pgmap v1358: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 2517/161409 objects misplaced (1.559%); 1.9 MiB/s, 30 objects/s recovering 2023-09-30T09:46:59.363 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:46:59.395 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:46:59.467 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:46:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:46:59.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:46:59.499 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:46:59.576 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:46:59.651 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:00.029 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:00.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:00.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:00.141 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:00.184 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:00.255 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:00.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:00.352 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:47:00.361 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:00.404 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:47:00.452 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:00.460 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:47:00.484 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:47:00.521 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:47:00.562 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:47:00.598 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:47:00.608 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:00.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:00.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:00.642 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:47:00.653 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:47:00.653 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:47:00.654 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:47:00.654 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:47:00.654 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:47:00.654 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:47:00.654 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:47:00.654 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:47:00.723 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:00.778 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:00.853 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:00.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:00.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:00.949 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:00 smithi019 ceph-mon[224043]: pgmap v1359: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:00.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4098641628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:00.950 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:00 smithi019 ceph-mon[216815]: pgmap v1359: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:00.950 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4098641628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:00.965 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:01.051 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:01.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:00 smithi167 ceph-mon[192880]: pgmap v1359: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:01.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4098641628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:01.306 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:47:02.268 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:02.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:02.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:02.374 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:02.375 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:02.491 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:02.623 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:02.692 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:02.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:02.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:02.810 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:02.898 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:02.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:02 smithi019 ceph-mon[216815]: pgmap v1360: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:02.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:02 smithi019 ceph-mon[224043]: pgmap v1360: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:03.116 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:03.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:03.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:03.227 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:03.261 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:02 smithi167 ceph-mon[192880]: pgmap v1360: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:03.264 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:03.367 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:03.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:03.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:03.477 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:03.571 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:03.835 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:47:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:47:03] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-09-30T09:47:03.965 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:47:03.982 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:47:04.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:04.425 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:47:05.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:04 smithi167 ceph-mon[192880]: pgmap v1361: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:04 smithi167 ceph-mon[192880]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:04 smithi019 ceph-mon[216815]: pgmap v1361: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:05.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:04 smithi019 ceph-mon[216815]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:04 smithi019 ceph-mon[224043]: pgmap v1361: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2352/150657 objects misplaced (1.561%); 2.0 MiB/s, 32 objects/s recovering 2023-09-30T09:47:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:04 smithi019 ceph-mon[224043]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:05.440 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:05.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:05.549 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:05.636 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:06.143 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1361,"stamp":"2023-09-30T09:47:03.836897+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3294042047,"num_objects":50219,"num_object_clones":0,"num_object_copies":150657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2352,"num_objects_unfound":0,"num_objects_dirty":50219,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":144202,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4041,"num_bytes_recovered":259085938,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120117,"ondisk_log_size":120117,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14287872,"kb_used_data":9691520,"kb_used_omap":30,"kb_used_meta":4595745,"kb_avail":735673344,"statfs":{"total":767960285184,"available":753329504256,"internally_reserved":0,"allocated":9924116480,"data_stored":9905171483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4706043234},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,3,3,9,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-458227712,"num_objects":-6992,"num_object_clones":0,"num_object_copies":-20976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-323,"num_objects_unfound":0,"num_objects_dirty":-6992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6992,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":323,"num_bytes_recovered":21168128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.010328"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":941,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955080+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:46:43.955080+0000","last_peered":"2023-09-30T09:46:43.955080+0000","last_clean":"2023-09-30T09:46:43.955080+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:46:43.955080+0000","last_undegraded":"2023-09-30T09:46:43.955080+0000","last_fullsized":"2023-09-30T09:46:43.955080+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":536,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957173+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:46:43.957173+0000","last_peered":"2023-09-30T09:46:43.957173+0000","last_clean":"2023-09-30T09:46:43.957173+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:46:43.957173+0000","last_undegraded":"2023-09-30T09:46:43.957173+0000","last_fullsized":"2023-09-30T09:46:43.957173+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":433,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957516+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:46:43.957516+0000","last_peered":"2023-09-30T09:46:43.957516+0000","last_clean":"2023-09-30T09:46:43.957516+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:46:43.957516+0000","last_undegraded":"2023-09-30T09:46:43.957516+0000","last_fullsized":"2023-09-30T09:46:43.957516+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":943,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.950227+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:46:43.950227+0000","last_peered":"2023-09-30T09:46:43.950227+0000","last_clean":"2023-09-30T09:46:43.950227+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:46:43.950227+0000","last_undegraded":"2023-09-30T09:46:43.950227+0000","last_fullsized":"2023-09-30T09:46:43.950227+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.941776+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:46:43.941776+0000","last_peered":"2023-09-30T09:46:43.941776+0000","last_clean":"2023-09-30T09:46:43.941776+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:46:43.941776+0000","last_undegraded":"2023-09-30T09:46:43.941776+0000","last_fullsized":"2023-09-30T09:46:43.941776+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":124,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957143+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:46:43.957143+0000","last_peered":"2023-09-30T09:46:43.957143+0000","last_clean":"2023-09-30T09:46:43.957143+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:46:43.957143+0000","last_undegraded":"2023-09-30T09:46:43.957143+0000","last_fullsized":"2023-09-30T09:46:43.957143+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":751,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957569+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:46:43.957569+0000","last_peered":"2023-09-30T09:46:43.957569+0000","last_clean":"2023-09-30T09:46:43.957569+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:46:43.957569+0000","last_undegraded":"2023-09-30T09:46:43.957569+0000","last_fullsized":"2023-09-30T09:46:43.957569+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":23,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957141+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:46:43.957141+0000","last_peered":"2023-09-30T09:46:43.957141+0000","last_clean":"2023-09-30T09:46:43.957141+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:46:43.957141+0000","last_undegraded":"2023-09-30T09:46:43.957141+0000","last_fullsized":"2023-09-30T09:46:43.957141+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970759+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:46:43.970759+0000","last_peered":"2023-09-30T09:46:43.970759+0000","last_clean":"2023-09-30T09:46:43.970759+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:46:43.970759+0000","last_undegraded":"2023-09-30T09:46:43.970759+0000","last_fullsized":"2023-09-30T09:46:43.970759+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":919,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955721+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:46:43.955721+0000","last_peered":"2023-09-30T09:46:43.955721+0000","last_clean":"2023-09-30T09:46:43.955721+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:46:43.955721+0000","last_undegraded":"2023-09-30T09:46:43.955721+0000","last_fullsized":"2023-09-30T09:46:43.955721+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":138,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956981+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:46:43.956981+0000","last_peered":"2023-09-30T09:46:43.956981+0000","last_clean":"2023-09-30T09:46:43.956981+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:46:43.956981+0000","last_undegraded":"2023-09-30T09:46:43.956981+0000","last_fullsized":"2023-09-30T09:46:43.956981+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":98,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.944898+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:46:43.944898+0000","last_peered":"2023-09-30T09:46:43.944898+0000","last_clean":"2023-09-30T09:46:43.944898+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:46:43.944898+0000","last_undegraded":"2023-09-30T09:46:43.944898+0000","last_fullsized":"2023-09-30T09:46:43.944898+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":707,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.972481+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:46:43.972481+0000","last_peered":"2023-09-30T09:46:43.972481+0000","last_clean":"2023-09-30T09:46:43.972481+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:46:43.972481+0000","last_undegraded":"2023-09-30T09:46:43.972481+0000","last_fullsized":"2023-09-30T09:46:43.972481+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":338,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.951266+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:46:43.951266+0000","last_peered":"2023-09-30T09:46:43.951266+0000","last_clean":"2023-09-30T09:46:43.951266+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:46:43.951266+0000","last_undegraded":"2023-09-30T09:46:43.951266+0000","last_fullsized":"2023-09-30T09:46:43.951266+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":474,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955599+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:46:43.955599+0000","last_peered":"2023-09-30T09:46:43.955599+0000","last_clean":"2023-09-30T09:46:43.955599+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:46:43.955599+0000","last_undegraded":"2023-09-30T09:46:43.955599+0000","last_fullsized":"2023-09-30T09:46:43.955599+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":647,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.950371+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:46:43.950371+0000","last_peered":"2023-09-30T09:46:43.950371+0000","last_clean":"2023-09-30T09:46:43.950371+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:46:43.950371+0000","last_undegraded":"2023-09-30T09:46:43.950371+0000","last_fullsized":"2023-09-30T09:46:43.950371+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":232,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955878+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:46:43.955878+0000","last_peered":"2023-09-30T09:46:43.955878+0000","last_clean":"2023-09-30T09:46:43.955878+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:46:43.955878+0000","last_undegraded":"2023-09-30T09:46:43.955878+0000","last_fullsized":"2023-09-30T09:46:43.955878+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970563+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:46:43.970563+0000","last_peered":"2023-09-30T09:46:43.970563+0000","last_clean":"2023-09-30T09:46:43.970563+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:46:43.970563+0000","last_undegraded":"2023-09-30T09:46:43.970563+0000","last_fullsized":"2023-09-30T09:46:43.970563+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":573,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957246+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:46:43.957246+0000","last_peered":"2023-09-30T09:46:43.957246+0000","last_clean":"2023-09-30T09:46:43.957246+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:46:43.957246+0000","last_undegraded":"2023-09-30T09:46:43.957246+0000","last_fullsized":"2023-09-30T09:46:43.957246+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":562,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.958686+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:46:43.958686+0000","last_peered":"2023-09-30T09:46:43.958686+0000","last_clean":"2023-09-30T09:46:43.958686+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:46:43.958686+0000","last_undegraded":"2023-09-30T09:46:43.958686+0000","last_fullsized":"2023-09-30T09:46:43.958686+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":540,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957790+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:46:43.957790+0000","last_peered":"2023-09-30T09:46:43.957790+0000","last_clean":"2023-09-30T09:46:43.957790+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:46:43.957790+0000","last_undegraded":"2023-09-30T09:46:43.957790+0000","last_fullsized":"2023-09-30T09:46:43.957790+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":771,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956147+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:46:43.956147+0000","last_peered":"2023-09-30T09:46:43.956147+0000","last_clean":"2023-09-30T09:46:43.956147+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:46:43.956147+0000","last_undegraded":"2023-09-30T09:46:43.956147+0000","last_fullsized":"2023-09-30T09:46:43.956147+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":168,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.953571+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:46:43.953571+0000","last_peered":"2023-09-30T09:46:43.953571+0000","last_clean":"2023-09-30T09:46:43.953571+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:46:43.953571+0000","last_undegraded":"2023-09-30T09:46:43.953571+0000","last_fullsized":"2023-09-30T09:46:43.953571+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":495,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.954363+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:46:43.954363+0000","last_peered":"2023-09-30T09:46:43.954363+0000","last_clean":"2023-09-30T09:46:43.954363+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:46:43.954363+0000","last_undegraded":"2023-09-30T09:46:43.954363+0000","last_fullsized":"2023-09-30T09:46:43.954363+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":819,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.954870+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:46:43.954870+0000","last_peered":"2023-09-30T09:46:43.954870+0000","last_clean":"2023-09-30T09:46:43.954870+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:46:43.954870+0000","last_undegraded":"2023-09-30T09:46:43.954870+0000","last_fullsized":"2023-09-30T09:46:43.954870+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":97,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956450+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:46:43.956450+0000","last_peered":"2023-09-30T09:46:43.956450+0000","last_clean":"2023-09-30T09:46:43.956450+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:46:43.956450+0000","last_undegraded":"2023-09-30T09:46:43.956450+0000","last_fullsized":"2023-09-30T09:46:43.956450+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":193,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.951047+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:46:43.951047+0000","last_peered":"2023-09-30T09:46:43.951047+0000","last_clean":"2023-09-30T09:46:43.951047+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:46:43.951047+0000","last_undegraded":"2023-09-30T09:46:43.951047+0000","last_fullsized":"2023-09-30T09:46:43.951047+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":919,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.941604+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:46:43.941604+0000","last_peered":"2023-09-30T09:46:43.941604+0000","last_clean":"2023-09-30T09:46:43.941604+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:46:43.941604+0000","last_undegraded":"2023-09-30T09:46:43.941604+0000","last_fullsized":"2023-09-30T09:46:43.941604+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":202,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955833+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:46:43.955833+0000","last_peered":"2023-09-30T09:46:43.955833+0000","last_clean":"2023-09-30T09:46:43.955833+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:46:43.955833+0000","last_undegraded":"2023-09-30T09:46:43.955833+0000","last_fullsized":"2023-09-30T09:46:43.955833+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":97,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956885+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:46:43.956885+0000","last_peered":"2023-09-30T09:46:43.956885+0000","last_clean":"2023-09-30T09:46:43.956885+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:46:43.956885+0000","last_undegraded":"2023-09-30T09:46:43.956885+0000","last_fullsized":"2023-09-30T09:46:43.956885+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":928,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.951819+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:46:43.951819+0000","last_peered":"2023-09-30T09:46:43.951819+0000","last_clean":"2023-09-30T09:46:43.951819+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:46:43.951819+0000","last_undegraded":"2023-09-30T09:46:43.951819+0000","last_fullsized":"2023-09-30T09:46:43.951819+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":114,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970409+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:46:43.970409+0000","last_peered":"2023-09-30T09:46:43.970409+0000","last_clean":"2023-09-30T09:46:43.970409+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:46:43.970409+0000","last_undegraded":"2023-09-30T09:46:43.970409+0000","last_fullsized":"2023-09-30T09:46:43.970409+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":82,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.945920+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:46:43.945920+0000","last_peered":"2023-09-30T09:46:43.945920+0000","last_clean":"2023-09-30T09:46:43.945920+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:46:43.945920+0000","last_undegraded":"2023-09-30T09:46:43.945920+0000","last_fullsized":"2023-09-30T09:46:43.945920+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"518'8893","reported_seq":9201,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:59.244843+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:46:59.244843+0000","last_peered":"2023-09-30T09:46:59.244843+0000","last_clean":"2023-09-30T09:46:59.244843+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:46:59.244843+0000","last_undegraded":"2023-09-30T09:46:59.244843+0000","last_fullsized":"2023-09-30T09:46:59.244843+0000","mapping_epoch":495,"log_start":"497'1300","ondisk_log_start":"497'1300","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7593,"log_dups_size":0,"ondisk_log_size":7593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201916416,"num_objects":3081,"num_object_clones":0,"num_object_copies":9243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8893,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":894,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955799+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:46:43.955799+0000","last_peered":"2023-09-30T09:46:43.955799+0000","last_clean":"2023-09-30T09:46:43.955799+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:46:43.955799+0000","last_undegraded":"2023-09-30T09:46:43.955799+0000","last_fullsized":"2023-09-30T09:46:43.955799+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":194,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.954019+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:46:43.954019+0000","last_peered":"2023-09-30T09:46:43.954019+0000","last_clean":"2023-09-30T09:46:43.954019+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:46:43.954019+0000","last_undegraded":"2023-09-30T09:46:43.954019+0000","last_fullsized":"2023-09-30T09:46:43.954019+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":139,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.954737+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:46:43.954737+0000","last_peered":"2023-09-30T09:46:43.954737+0000","last_clean":"2023-09-30T09:46:43.954737+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:46:43.954737+0000","last_undegraded":"2023-09-30T09:46:43.954737+0000","last_fullsized":"2023-09-30T09:46:43.954737+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"518'8769","reported_seq":8812,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:58.828734+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:46:58.828734+0000","last_peered":"2023-09-30T09:46:58.828734+0000","last_clean":"2023-09-30T09:46:58.828734+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:46:58.828734+0000","last_undegraded":"2023-09-30T09:46:58.828734+0000","last_fullsized":"2023-09-30T09:46:58.828734+0000","mapping_epoch":483,"log_start":"499'2300","ondisk_log_start":"499'2300","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208076800,"num_objects":3175,"num_object_clones":0,"num_object_copies":9525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8769,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":911,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.948728+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:46:43.948728+0000","last_peered":"2023-09-30T09:46:43.948728+0000","last_clean":"2023-09-30T09:46:43.948728+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:46:43.948728+0000","last_undegraded":"2023-09-30T09:46:43.948728+0000","last_fullsized":"2023-09-30T09:46:43.948728+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":164,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.953590+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:46:43.953590+0000","last_peered":"2023-09-30T09:46:43.953590+0000","last_clean":"2023-09-30T09:46:43.953590+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:46:43.953590+0000","last_undegraded":"2023-09-30T09:46:43.953590+0000","last_fullsized":"2023-09-30T09:46:43.953590+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":569,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956938+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:46:43.956938+0000","last_peered":"2023-09-30T09:46:43.956938+0000","last_clean":"2023-09-30T09:46:43.956938+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:46:43.956938+0000","last_undegraded":"2023-09-30T09:46:43.956938+0000","last_fullsized":"2023-09-30T09:46:43.956938+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"518'9052","reported_seq":9320,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:59.074751+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:46:59.074751+0000","last_peered":"2023-09-30T09:46:59.074751+0000","last_clean":"2023-09-30T09:46:59.074751+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:46:59.074751+0000","last_undegraded":"2023-09-30T09:46:59.074751+0000","last_fullsized":"2023-09-30T09:46:59.074751+0000","mapping_epoch":493,"log_start":"498'2300","ondisk_log_start":"498'2300","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205127680,"num_objects":3130,"num_object_clones":0,"num_object_copies":9390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9052,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":688,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.971047+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:46:43.971047+0000","last_peered":"2023-09-30T09:46:43.971047+0000","last_clean":"2023-09-30T09:46:43.971047+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:46:43.971047+0000","last_undegraded":"2023-09-30T09:46:43.971047+0000","last_fullsized":"2023-09-30T09:46:43.971047+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":231,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955684+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:46:43.955684+0000","last_peered":"2023-09-30T09:46:43.955684+0000","last_clean":"2023-09-30T09:46:43.955684+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:46:43.955684+0000","last_undegraded":"2023-09-30T09:46:43.955684+0000","last_fullsized":"2023-09-30T09:46:43.955684+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970642+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:46:43.970642+0000","last_peered":"2023-09-30T09:46:43.970642+0000","last_clean":"2023-09-30T09:46:43.970642+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:46:43.970642+0000","last_undegraded":"2023-09-30T09:46:43.970642+0000","last_fullsized":"2023-09-30T09:46:43.970642+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"518'8808","reported_seq":9161,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:58.874511+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:46:58.874511+0000","last_peered":"2023-09-30T09:46:58.874511+0000","last_clean":"2023-09-30T09:46:58.874511+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:46:58.874511+0000","last_undegraded":"2023-09-30T09:46:58.874511+0000","last_fullsized":"2023-09-30T09:46:58.874511+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":8808,"log_dups_size":0,"ondisk_log_size":8808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202375168,"num_objects":3088,"num_object_clones":0,"num_object_copies":9264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8808,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":753,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957069+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:46:43.957069+0000","last_peered":"2023-09-30T09:46:43.957069+0000","last_clean":"2023-09-30T09:46:43.957069+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:46:43.957069+0000","last_undegraded":"2023-09-30T09:46:43.957069+0000","last_fullsized":"2023-09-30T09:46:43.957069+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":453,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.950474+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:46:43.950474+0000","last_peered":"2023-09-30T09:46:43.950474+0000","last_clean":"2023-09-30T09:46:43.950474+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:46:43.950474+0000","last_undegraded":"2023-09-30T09:46:43.950474+0000","last_fullsized":"2023-09-30T09:46:43.950474+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":117,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956528+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:46:43.956528+0000","last_peered":"2023-09-30T09:46:43.956528+0000","last_clean":"2023-09-30T09:46:43.956528+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:46:43.956528+0000","last_undegraded":"2023-09-30T09:46:43.956528+0000","last_fullsized":"2023-09-30T09:46:43.956528+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"518'8994","reported_seq":9037,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:59.223470+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:46:59.223470+0000","last_peered":"2023-09-30T09:46:59.223470+0000","last_clean":"2023-09-30T09:46:59.223470+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:46:59.223470+0000","last_undegraded":"2023-09-30T09:46:59.223470+0000","last_fullsized":"2023-09-30T09:46:59.223470+0000","mapping_epoch":483,"log_start":"498'2000","ondisk_log_start":"498'2000","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6994,"log_dups_size":0,"ondisk_log_size":6994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205651968,"num_objects":3138,"num_object_clones":0,"num_object_copies":9414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8994,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1000,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955747+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:46:43.955747+0000","last_peered":"2023-09-30T09:46:43.955747+0000","last_clean":"2023-09-30T09:46:43.955747+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:46:43.955747+0000","last_undegraded":"2023-09-30T09:46:43.955747+0000","last_fullsized":"2023-09-30T09:46:43.955747+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":77,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:54.318665+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:46:54.318665+0000","last_peered":"2023-09-30T09:46:54.318665+0000","last_clean":"2023-09-30T09:46:54.318665+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:46:54.318665+0000","last_undegraded":"2023-09-30T09:46:54.318665+0000","last_fullsized":"2023-09-30T09:46:54.318665+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":516,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.958767+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:46:43.958767+0000","last_peered":"2023-09-30T09:46:43.958767+0000","last_clean":"2023-09-30T09:46:43.958767+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:46:43.958767+0000","last_undegraded":"2023-09-30T09:46:43.958767+0000","last_fullsized":"2023-09-30T09:46:43.958767+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"518'8908","reported_seq":8951,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:59.305247+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:46:59.305247+0000","last_peered":"2023-09-30T09:46:59.305247+0000","last_clean":"2023-09-30T09:46:59.305247+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:46:59.305247+0000","last_undegraded":"2023-09-30T09:46:59.305247+0000","last_fullsized":"2023-09-30T09:46:59.305247+0000","mapping_epoch":483,"log_start":"497'1900","ondisk_log_start":"497'1900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199622656,"num_objects":3046,"num_object_clones":0,"num_object_copies":9138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8908,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":999,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970230+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:46:43.970230+0000","last_peered":"2023-09-30T09:46:43.970230+0000","last_clean":"2023-09-30T09:46:43.970230+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:46:43.970230+0000","last_undegraded":"2023-09-30T09:46:43.970230+0000","last_fullsized":"2023-09-30T09:46:43.970230+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970487+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:46:43.970487+0000","last_peered":"2023-09-30T09:46:43.970487+0000","last_clean":"2023-09-30T09:46:43.970487+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:46:43.970487+0000","last_undegraded":"2023-09-30T09:46:43.970487+0000","last_fullsized":"2023-09-30T09:46:43.970487+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":212,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957004+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:46:43.957004+0000","last_peered":"2023-09-30T09:46:43.957004+0000","last_clean":"2023-09-30T09:46:43.957004+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:46:43.957004+0000","last_undegraded":"2023-09-30T09:46:43.957004+0000","last_fullsized":"2023-09-30T09:46:43.957004+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"518'9157","reported_seq":9200,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:58.873234+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:46:58.873234+0000","last_peered":"2023-09-30T09:46:58.873234+0000","last_clean":"2023-09-30T09:46:58.873234+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:46:58.873234+0000","last_undegraded":"2023-09-30T09:46:58.873234+0000","last_fullsized":"2023-09-30T09:46:58.873234+0000","mapping_epoch":483,"log_start":"501'3300","ondisk_log_start":"501'3300","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5857,"log_dups_size":0,"ondisk_log_size":5857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204013568,"num_objects":3113,"num_object_clones":0,"num_object_copies":9339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9157,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":856,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956617+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:46:43.956617+0000","last_peered":"2023-09-30T09:46:43.956617+0000","last_clean":"2023-09-30T09:46:43.956617+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:46:43.956617+0000","last_undegraded":"2023-09-30T09:46:43.956617+0000","last_fullsized":"2023-09-30T09:46:43.956617+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":523,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956854+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:46:43.956854+0000","last_peered":"2023-09-30T09:46:43.956854+0000","last_clean":"2023-09-30T09:46:43.956854+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:46:43.956854+0000","last_undegraded":"2023-09-30T09:46:43.956854+0000","last_fullsized":"2023-09-30T09:46:43.956854+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":530,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957197+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:46:43.957197+0000","last_peered":"2023-09-30T09:46:43.957197+0000","last_clean":"2023-09-30T09:46:43.957197+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:46:43.957197+0000","last_undegraded":"2023-09-30T09:46:43.957197+0000","last_fullsized":"2023-09-30T09:46:43.957197+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"518'8829","reported_seq":8876,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:58.908594+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:46:58.908594+0000","last_peered":"2023-09-30T09:46:58.908594+0000","last_clean":"2023-09-30T09:46:58.908594+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:46:58.908594+0000","last_undegraded":"2023-09-30T09:46:58.908594+0000","last_fullsized":"2023-09-30T09:46:58.908594+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":8829,"log_dups_size":0,"ondisk_log_size":8829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203620352,"num_objects":3107,"num_object_clones":0,"num_object_copies":9321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8829,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":721,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.971030+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:46:43.971030+0000","last_peered":"2023-09-30T09:46:43.971030+0000","last_clean":"2023-09-30T09:46:43.971030+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:46:43.971030+0000","last_undegraded":"2023-09-30T09:46:43.971030+0000","last_fullsized":"2023-09-30T09:46:43.971030+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":82,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.944853+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:46:43.944853+0000","last_peered":"2023-09-30T09:46:43.944853+0000","last_clean":"2023-09-30T09:46:43.944853+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:46:43.944853+0000","last_undegraded":"2023-09-30T09:46:43.944853+0000","last_fullsized":"2023-09-30T09:46:43.944853+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":203,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957426+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:46:43.957426+0000","last_peered":"2023-09-30T09:46:43.957426+0000","last_clean":"2023-09-30T09:46:43.957426+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:46:43.957426+0000","last_undegraded":"2023-09-30T09:46:43.957426+0000","last_fullsized":"2023-09-30T09:46:43.957426+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"518'8975","reported_seq":9277,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:59.057634+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:46:59.057634+0000","last_peered":"2023-09-30T09:46:59.057634+0000","last_clean":"2023-09-30T09:46:59.057634+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:46:59.057634+0000","last_undegraded":"2023-09-30T09:46:59.057634+0000","last_fullsized":"2023-09-30T09:46:59.057634+0000","mapping_epoch":494,"log_start":"498'2300","ondisk_log_start":"498'2300","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205062144,"num_objects":3129,"num_object_clones":0,"num_object_copies":9387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8975,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":760,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970674+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:46:43.970674+0000","last_peered":"2023-09-30T09:46:43.970674+0000","last_clean":"2023-09-30T09:46:43.970674+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:46:43.970674+0000","last_undegraded":"2023-09-30T09:46:43.970674+0000","last_fullsized":"2023-09-30T09:46:43.970674+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":748,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957052+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:46:43.957052+0000","last_peered":"2023-09-30T09:46:43.957052+0000","last_clean":"2023-09-30T09:46:43.957052+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:46:43.957052+0000","last_undegraded":"2023-09-30T09:46:43.957052+0000","last_fullsized":"2023-09-30T09:46:43.957052+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":446,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.951193+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:46:43.951193+0000","last_peered":"2023-09-30T09:46:43.951193+0000","last_clean":"2023-09-30T09:46:43.951193+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:46:43.951193+0000","last_undegraded":"2023-09-30T09:46:43.951193+0000","last_fullsized":"2023-09-30T09:46:43.951193+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":135,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.951497+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:43.951497+0000","last_peered":"2023-09-30T09:46:43.951497+0000","last_clean":"2023-09-30T09:46:43.951497+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:46:43.951497+0000","last_undegraded":"2023-09-30T09:46:43.951497+0000","last_fullsized":"2023-09-30T09:46:43.951497+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"518'8994","reported_seq":9342,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:59.238787+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:46:59.238787+0000","last_peered":"2023-09-30T09:46:59.238787+0000","last_clean":"2023-09-30T09:46:59.238787+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:46:59.238787+0000","last_undegraded":"2023-09-30T09:46:59.238787+0000","last_fullsized":"2023-09-30T09:46:59.238787+0000","mapping_epoch":493,"log_start":"497'1400","ondisk_log_start":"497'1400","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205389824,"num_objects":3134,"num_object_clones":0,"num_object_copies":9402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8994,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":959,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.950123+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:46:43.950123+0000","last_peered":"2023-09-30T09:46:43.950123+0000","last_clean":"2023-09-30T09:46:43.950123+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:46:43.950123+0000","last_undegraded":"2023-09-30T09:46:43.950123+0000","last_fullsized":"2023-09-30T09:46:43.950123+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":465,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970958+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:46:43.970958+0000","last_peered":"2023-09-30T09:46:43.970958+0000","last_clean":"2023-09-30T09:46:43.970958+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:46:43.970958+0000","last_undegraded":"2023-09-30T09:46:43.970958+0000","last_fullsized":"2023-09-30T09:46:43.970958+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":537,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957878+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:46:43.957878+0000","last_peered":"2023-09-30T09:46:43.957878+0000","last_clean":"2023-09-30T09:46:43.957878+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:46:43.957878+0000","last_undegraded":"2023-09-30T09:46:43.957878+0000","last_fullsized":"2023-09-30T09:46:43.957878+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":277,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.972566+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:46:43.972566+0000","last_peered":"2023-09-30T09:46:43.972566+0000","last_clean":"2023-09-30T09:46:43.972566+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:46:43.972566+0000","last_undegraded":"2023-09-30T09:46:43.972566+0000","last_fullsized":"2023-09-30T09:46:43.972566+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"518'9019","reported_seq":9367,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:59.102268+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:46:59.102268+0000","last_peered":"2023-09-30T09:46:59.102268+0000","last_clean":"2023-09-30T09:46:59.102268+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:46:59.102268+0000","last_undegraded":"2023-09-30T09:46:59.102268+0000","last_fullsized":"2023-09-30T09:46:59.102268+0000","mapping_epoch":492,"log_start":"498'2300","ondisk_log_start":"498'2300","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206635008,"num_objects":3153,"num_object_clones":0,"num_object_copies":9459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9019,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":937,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955470+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:46:43.955470+0000","last_peered":"2023-09-30T09:46:43.955470+0000","last_clean":"2023-09-30T09:46:43.955470+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:46:43.955470+0000","last_undegraded":"2023-09-30T09:46:43.955470+0000","last_fullsized":"2023-09-30T09:46:43.955470+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":786,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970388+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:46:43.970388+0000","last_peered":"2023-09-30T09:46:43.970388+0000","last_clean":"2023-09-30T09:46:43.970388+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:46:43.970388+0000","last_undegraded":"2023-09-30T09:46:43.970388+0000","last_fullsized":"2023-09-30T09:46:43.970388+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":280,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970074+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:46:43.970074+0000","last_peered":"2023-09-30T09:46:43.970074+0000","last_clean":"2023-09-30T09:46:43.970074+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:46:43.970074+0000","last_undegraded":"2023-09-30T09:46:43.970074+0000","last_fullsized":"2023-09-30T09:46:43.970074+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":367,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.950585+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:46:43.950585+0000","last_peered":"2023-09-30T09:46:43.950585+0000","last_clean":"2023-09-30T09:46:43.950585+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:46:43.950585+0000","last_undegraded":"2023-09-30T09:46:43.950585+0000","last_fullsized":"2023-09-30T09:46:43.950585+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"518'8866","reported_seq":9174,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:59.238703+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:46:59.238703+0000","last_peered":"2023-09-30T09:46:59.238703+0000","last_clean":"2023-09-30T09:46:59.238703+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:46:59.238703+0000","last_undegraded":"2023-09-30T09:46:59.238703+0000","last_fullsized":"2023-09-30T09:46:59.238703+0000","mapping_epoch":495,"log_start":"497'1300","ondisk_log_start":"497'1300","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7566,"log_dups_size":0,"ondisk_log_size":7566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206569472,"num_objects":3152,"num_object_clones":0,"num_object_copies":9456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8866,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1079,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.950361+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:46:43.950361+0000","last_peered":"2023-09-30T09:46:43.950361+0000","last_clean":"2023-09-30T09:46:43.950361+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:46:43.950361+0000","last_undegraded":"2023-09-30T09:46:43.950361+0000","last_fullsized":"2023-09-30T09:46:43.950361+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":880,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.953332+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:46:43.953332+0000","last_peered":"2023-09-30T09:46:43.953332+0000","last_clean":"2023-09-30T09:46:43.953332+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:46:43.953332+0000","last_undegraded":"2023-09-30T09:46:43.953332+0000","last_fullsized":"2023-09-30T09:46:43.953332+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.951573+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:46:43.951573+0000","last_peered":"2023-09-30T09:46:43.951573+0000","last_clean":"2023-09-30T09:46:43.951573+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:46:43.951573+0000","last_undegraded":"2023-09-30T09:46:43.951573+0000","last_fullsized":"2023-09-30T09:46:43.951573+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970649+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:46:43.970649+0000","last_peered":"2023-09-30T09:46:43.970649+0000","last_clean":"2023-09-30T09:46:43.970649+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:46:43.970649+0000","last_undegraded":"2023-09-30T09:46:43.970649+0000","last_fullsized":"2023-09-30T09:46:43.970649+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"518'9132","reported_seq":14438,"reported_epoch":518,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:46:58.919331+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:46:58.919331+0000","last_peered":"2023-09-30T09:46:58.919331+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:46:58.919331+0000","last_undegraded":"2023-09-30T09:46:58.919331+0000","last_fullsized":"2023-09-30T09:46:58.919331+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":9132,"log_dups_size":0,"ondisk_log_size":9132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209321984,"num_objects":3194,"num_object_clones":0,"num_object_copies":9582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2352,"num_objects_unfound":0,"num_objects_dirty":3194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9132,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2618,"num_bytes_recovered":171573248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":842}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":779,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.971110+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:46:43.971110+0000","last_peered":"2023-09-30T09:46:43.971110+0000","last_clean":"2023-09-30T09:46:43.971110+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:46:43.971110+0000","last_undegraded":"2023-09-30T09:46:43.971110+0000","last_fullsized":"2023-09-30T09:46:43.971110+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":926,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.953259+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:46:43.953259+0000","last_peered":"2023-09-30T09:46:43.953259+0000","last_clean":"2023-09-30T09:46:43.953259+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:46:43.953259+0000","last_undegraded":"2023-09-30T09:46:43.953259+0000","last_fullsized":"2023-09-30T09:46:43.953259+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":131,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:49.313818+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:46:49.313818+0000","last_peered":"2023-09-30T09:46:49.313818+0000","last_clean":"2023-09-30T09:46:49.313818+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:46:49.313818+0000","last_undegraded":"2023-09-30T09:46:49.313818+0000","last_fullsized":"2023-09-30T09:46:49.313818+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956799+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:46:43.956799+0000","last_peered":"2023-09-30T09:46:43.956799+0000","last_clean":"2023-09-30T09:46:43.956799+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:46:43.956799+0000","last_undegraded":"2023-09-30T09:46:43.956799+0000","last_fullsized":"2023-09-30T09:46:43.956799+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"518'8924","reported_seq":8966,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:58.885558+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:46:58.885558+0000","last_peered":"2023-09-30T09:46:58.885558+0000","last_clean":"2023-09-30T09:46:58.885558+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:46:58.885558+0000","last_undegraded":"2023-09-30T09:46:58.885558+0000","last_fullsized":"2023-09-30T09:46:58.885558+0000","mapping_epoch":483,"log_start":"500'3100","ondisk_log_start":"500'3100","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208928768,"num_objects":3188,"num_object_clones":0,"num_object_copies":9564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8924,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":770,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970819+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:46:43.970819+0000","last_peered":"2023-09-30T09:46:43.970819+0000","last_clean":"2023-09-30T09:46:43.970819+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:46:43.970819+0000","last_undegraded":"2023-09-30T09:46:43.970819+0000","last_fullsized":"2023-09-30T09:46:43.970819+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1066,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.955876+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:46:43.955876+0000","last_peered":"2023-09-30T09:46:43.955876+0000","last_clean":"2023-09-30T09:46:43.955876+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:46:43.955876+0000","last_undegraded":"2023-09-30T09:46:43.955876+0000","last_fullsized":"2023-09-30T09:46:43.955876+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":261,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.972395+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:46:43.972395+0000","last_peered":"2023-09-30T09:46:43.972395+0000","last_clean":"2023-09-30T09:46:43.972395+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:46:43.972395+0000","last_undegraded":"2023-09-30T09:46:43.972395+0000","last_fullsized":"2023-09-30T09:46:43.972395+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":131,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:54.318718+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:46:54.318718+0000","last_peered":"2023-09-30T09:46:54.318718+0000","last_clean":"2023-09-30T09:46:54.318718+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:46:54.318718+0000","last_undegraded":"2023-09-30T09:46:54.318718+0000","last_fullsized":"2023-09-30T09:46:54.318718+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"518'8844","reported_seq":9222,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:58.867927+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:46:58.867927+0000","last_peered":"2023-09-30T09:46:58.867927+0000","last_clean":"2023-09-30T09:46:58.867927+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:46:58.867927+0000","last_undegraded":"2023-09-30T09:46:58.867927+0000","last_fullsized":"2023-09-30T09:46:58.867927+0000","mapping_epoch":495,"log_start":"500'3000","ondisk_log_start":"500'3000","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5844,"log_dups_size":0,"ondisk_log_size":5844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207421464,"num_objects":3166,"num_object_clones":0,"num_object_copies":9498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3166,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8844,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1037,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957214+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:46:43.957214+0000","last_peered":"2023-09-30T09:46:43.957214+0000","last_clean":"2023-09-30T09:46:43.957214+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:46:43.957214+0000","last_undegraded":"2023-09-30T09:46:43.957214+0000","last_fullsized":"2023-09-30T09:46:43.957214+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":549,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.953899+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:46:43.953899+0000","last_peered":"2023-09-30T09:46:43.953899+0000","last_clean":"2023-09-30T09:46:43.953899+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:46:43.953899+0000","last_undegraded":"2023-09-30T09:46:43.953899+0000","last_fullsized":"2023-09-30T09:46:43.953899+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1236,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970908+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:46:43.970908+0000","last_peered":"2023-09-30T09:46:43.970908+0000","last_clean":"2023-09-30T09:46:43.970908+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:46:43.970908+0000","last_undegraded":"2023-09-30T09:46:43.970908+0000","last_fullsized":"2023-09-30T09:46:43.970908+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":553,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.953178+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:46:43.953178+0000","last_peered":"2023-09-30T09:46:43.953178+0000","last_clean":"2023-09-30T09:46:43.953178+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:46:43.953178+0000","last_undegraded":"2023-09-30T09:46:43.953178+0000","last_fullsized":"2023-09-30T09:46:43.953178+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"518'9131","reported_seq":9174,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:58.825784+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:46:58.825784+0000","last_peered":"2023-09-30T09:46:58.825784+0000","last_clean":"2023-09-30T09:46:58.825784+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:46:58.825784+0000","last_undegraded":"2023-09-30T09:46:58.825784+0000","last_fullsized":"2023-09-30T09:46:58.825784+0000","mapping_epoch":483,"log_start":"499'2700","ondisk_log_start":"499'2700","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6431,"log_dups_size":0,"ondisk_log_size":6431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209518592,"num_objects":3197,"num_object_clones":0,"num_object_copies":9591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9131,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":901,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.913290+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:46:56.913290+0000","last_peered":"2023-09-30T09:46:56.913290+0000","last_clean":"2023-09-30T09:46:56.913290+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:46:56.913290+0000","last_undegraded":"2023-09-30T09:46:56.913290+0000","last_fullsized":"2023-09-30T09:46:56.913290+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":88,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970968+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:46:43.970968+0000","last_peered":"2023-09-30T09:46:43.970968+0000","last_clean":"2023-09-30T09:46:43.970968+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:46:43.970968+0000","last_undegraded":"2023-09-30T09:46:43.970968+0000","last_fullsized":"2023-09-30T09:46:43.970968+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":94,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:44.216686+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:46:44.216686+0000","last_peered":"2023-09-30T09:46:44.216686+0000","last_clean":"2023-09-30T09:46:44.216686+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:46:44.216686+0000","last_undegraded":"2023-09-30T09:46:44.216686+0000","last_fullsized":"2023-09-30T09:46:44.216686+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":519,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956869+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:46:43.956869+0000","last_peered":"2023-09-30T09:46:43.956869+0000","last_clean":"2023-09-30T09:46:43.956869+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:46:43.956869+0000","last_undegraded":"2023-09-30T09:46:43.956869+0000","last_fullsized":"2023-09-30T09:46:43.956869+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":837,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956048+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:46:43.956048+0000","last_peered":"2023-09-30T09:46:43.956048+0000","last_clean":"2023-09-30T09:46:43.956048+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:46:43.956048+0000","last_undegraded":"2023-09-30T09:46:43.956048+0000","last_fullsized":"2023-09-30T09:46:43.956048+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":23,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956807+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:46:43.956807+0000","last_peered":"2023-09-30T09:46:43.956807+0000","last_clean":"2023-09-30T09:46:43.956807+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:46:43.956807+0000","last_undegraded":"2023-09-30T09:46:43.956807+0000","last_fullsized":"2023-09-30T09:46:43.956807+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":223,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.972307+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:46:43.972307+0000","last_peered":"2023-09-30T09:46:43.972307+0000","last_clean":"2023-09-30T09:46:43.972307+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:46:43.972307+0000","last_undegraded":"2023-09-30T09:46:43.972307+0000","last_fullsized":"2023-09-30T09:46:43.972307+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":632,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970877+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:46:43.970877+0000","last_peered":"2023-09-30T09:46:43.970877+0000","last_clean":"2023-09-30T09:46:43.970877+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:46:43.970877+0000","last_undegraded":"2023-09-30T09:46:43.970877+0000","last_fullsized":"2023-09-30T09:46:43.970877+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":446,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.950400+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:43.950400+0000","last_peered":"2023-09-30T09:46:43.950400+0000","last_clean":"2023-09-30T09:46:43.950400+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:46:43.950400+0000","last_undegraded":"2023-09-30T09:46:43.950400+0000","last_fullsized":"2023-09-30T09:46:43.950400+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":232,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957517+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:46:43.957517+0000","last_peered":"2023-09-30T09:46:43.957517+0000","last_clean":"2023-09-30T09:46:43.957517+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:46:43.957517+0000","last_undegraded":"2023-09-30T09:46:43.957517+0000","last_fullsized":"2023-09-30T09:46:43.957517+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":388,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.951364+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:46:43.951364+0000","last_peered":"2023-09-30T09:46:43.951364+0000","last_clean":"2023-09-30T09:46:43.951364+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:46:43.951364+0000","last_undegraded":"2023-09-30T09:46:43.951364+0000","last_fullsized":"2023-09-30T09:46:43.951364+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":252,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957083+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:46:43.957083+0000","last_peered":"2023-09-30T09:46:43.957083+0000","last_clean":"2023-09-30T09:46:43.957083+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:46:43.957083+0000","last_undegraded":"2023-09-30T09:46:43.957083+0000","last_fullsized":"2023-09-30T09:46:43.957083+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":845,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.954298+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:46:43.954298+0000","last_peered":"2023-09-30T09:46:43.954298+0000","last_clean":"2023-09-30T09:46:43.954298+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:46:43.954298+0000","last_undegraded":"2023-09-30T09:46:43.954298+0000","last_fullsized":"2023-09-30T09:46:43.954298+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":75,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970669+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:46:43.970669+0000","last_peered":"2023-09-30T09:46:43.970669+0000","last_clean":"2023-09-30T09:46:43.970669+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:46:43.970669+0000","last_undegraded":"2023-09-30T09:46:43.970669+0000","last_fullsized":"2023-09-30T09:46:43.970669+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":469,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.970314+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:46:43.970314+0000","last_peered":"2023-09-30T09:46:43.970314+0000","last_clean":"2023-09-30T09:46:43.970314+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:46:43.970314+0000","last_undegraded":"2023-09-30T09:46:43.970314+0000","last_fullsized":"2023-09-30T09:46:43.970314+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":888,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.957083+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:46:43.957083+0000","last_peered":"2023-09-30T09:46:43.957083+0000","last_clean":"2023-09-30T09:46:43.957083+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:46:43.957083+0000","last_undegraded":"2023-09-30T09:46:43.957083+0000","last_fullsized":"2023-09-30T09:46:43.957083+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":29,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956637+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:46:43.956637+0000","last_peered":"2023-09-30T09:46:43.956637+0000","last_clean":"2023-09-30T09:46:43.956637+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:46:43.956637+0000","last_undegraded":"2023-09-30T09:46:43.956637+0000","last_fullsized":"2023-09-30T09:46:43.956637+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":533,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-30T09:46:43.956722+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:46:43.956722+0000","last_peered":"2023-09-30T09:46:43.956722+0000","last_clean":"2023-09-30T09:46:43.956722+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:46:43.956722+0000","last_undegraded":"2023-09-30T09:46:43.956722+0000","last_fullsized":"2023-09-30T09:46:43.956722+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3289251864,"num_objects":50191,"num_object_clones":0,"num_object_copies":150573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2352,"num_objects_unfound":0,"num_objects_dirty":50191,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":143295,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3761,"num_bytes_recovered":246480896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9776476160,"data_stored":9776463944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114095,"ondisk_log_size":114095,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234423,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2388228,"kb_used_data":1847804,"kb_used_omap":0,"kb_used_meta":540416,"kb_avail":91356924,"statfs":{"total":95995035648,"available":93549490176,"internally_reserved":0,"allocated":1892151296,"data_stored":1890415552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":553385984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:01 2023","interfaces":[{"interface":"back","average":{"1min":13.629,"5min":6.0149999999999997,"15min":4.5529999999999999},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":110.467,"5min":124.74299999999999,"15min":215.416},"last":2.879},{"interface":"front","average":{"1min":13.702999999999999,"5min":6.1459999999999999,"15min":4.5099999999999998},"min":{"1min":0.23499999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":109.593,"5min":124.69499999999999,"15min":215.041},"last":5.9450000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:46:01 2023","interfaces":[{"interface":"back","average":{"1min":16.905000000000001,"5min":7.1210000000000004,"15min":4.1399999999999997},"min":{"1min":0.32400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":157.578,"5min":157.578,"15min":157.578},"last":3.5720000000000001},{"interface":"front","average":{"1min":17.038,"5min":6.9660000000000002,"15min":4.3380000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":157.589,"5min":157.589,"15min":157.589},"last":4.7640000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":2.9809999999999999,"15min":3.5510000000000002},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.24299999999999999},"max":{"1min":10.292999999999999,"5min":104.88500000000001,"15min":117.14},"last":4.6699999999999999},{"interface":"front","average":{"1min":1.9430000000000001,"5min":3.1629999999999998,"15min":3.5409999999999999},"min":{"1min":0.29599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.4420000000000002,"5min":104.867,"15min":117.044},"last":5.907}]},{"osd":3,"last update":"Sat Sep 30 09:46:15 2023","interfaces":[{"interface":"back","average":{"1min":11.387,"5min":6.8799999999999999,"15min":8.9429999999999996},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.31},"max":{"1min":154.06700000000001,"5min":154.06700000000001,"15min":277.57799999999997},"last":6.9080000000000004},{"interface":"front","average":{"1min":11.162000000000001,"5min":6.8810000000000002,"15min":8.4610000000000003},"min":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":154.24799999999999,"5min":154.24799999999999,"15min":278.69299999999998},"last":0.50800000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.1399999999999999,"15min":1.6259999999999999},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":7.5499999999999998,"5min":25.192,"15min":175.72},"last":2.2610000000000001},{"interface":"front","average":{"1min":1.214,"5min":1.0660000000000001,"15min":1.536},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.252},"max":{"1min":5.6120000000000001,"5min":24.521000000000001,"15min":179.048},"last":0.28999999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":9.875,"15min":11.287000000000001},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.641,"5min":197.75700000000001,"15min":197.75700000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":9.8399999999999999,"15min":11.164},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.7149999999999999,"5min":197.63300000000001,"15min":197.63300000000001},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.72299999999999998,"15min":2.1760000000000002},"min":{"1min":0.32900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.202,"5min":6.202,"15min":206.244},"last":3.911},{"interface":"front","average":{"1min":1.069,"5min":0.84399999999999997,"15min":2.1850000000000001},"min":{"1min":0.34699999999999998,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":5.5800000000000001,"5min":5.8520000000000003,"15min":202.78800000000001},"last":0.48699999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496061,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1124332,"kb_used_data":612196,"kb_used_omap":0,"kb_used_meta":512128,"kb_avail":92620820,"statfs":{"total":95995035648,"available":94843719680,"internally_reserved":0,"allocated":626888704,"data_stored":625176253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":524419072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":5.4119999999999999,"5min":3.6509999999999998,"15min":5.2279999999999998},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":79.504000000000005,"5min":95.043999999999997,"15min":187.19800000000001},"last":11.609},{"interface":"front","average":{"1min":5.4580000000000002,"5min":3.8250000000000002,"15min":5.3479999999999999},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":78.813000000000002,"5min":94.855999999999995,"15min":187.46700000000001},"last":4.2759999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.446,"15min":1.956},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.6429999999999998,"5min":32.814,"15min":69.569999999999993},"last":13.494999999999999},{"interface":"front","average":{"1min":2.5529999999999999,"5min":1.5629999999999999,"15min":2.173},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":6.3760000000000003,"5min":32.856000000000002,"15min":70.471999999999994},"last":4.3049999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":3.9239999999999999,"5min":3.5880000000000001,"15min":4.0069999999999997},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":10.581,"5min":207.02500000000001,"15min":207.02500000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":3.3559999999999999,"5min":3.6909999999999998,"15min":4.1539999999999999},"min":{"1min":0.37,"5min":0.33500000000000002,"15min":0.28199999999999997},"max":{"1min":16.474,"5min":207.114,"15min":207.114},"last":11.585000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":3.2810000000000001,"15min":4.2830000000000004},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":7.1299999999999999,"5min":107.11799999999999,"15min":212.44800000000001},"last":11.644},{"interface":"front","average":{"1min":2.0939999999999999,"5min":3.2450000000000001,"15min":4.5759999999999996},"min":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":7.7220000000000004,"5min":106.982,"15min":210.84299999999999},"last":4.3540000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.5740000000000001,"15min":1.3540000000000001},"min":{"1min":0.26200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.5190000000000001,"5min":63.798000000000002,"15min":63.798000000000002},"last":4.2249999999999996},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.5629999999999999,"15min":1.2490000000000001},"min":{"1min":0.34599999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.1900000000000004,"5min":63.133000000000003,"15min":63.133000000000003},"last":11.558999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8740000000000001,"15min":1.99},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.6840000000000002,"5min":14.974,"15min":14.974},"last":0.374},{"interface":"front","average":{"1min":0.88400000000000001,"5min":2.2789999999999999,"15min":2.5110000000000001},"min":{"1min":0.37,"5min":0.316,"15min":0.316},"max":{"1min":3.6190000000000002,"5min":16.443000000000001,"15min":16.443000000000001},"last":11.669}]},{"osd":7,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":0.89200000000000002,"15min":1.609},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.2679999999999998,"5min":11.276999999999999,"15min":138.191},"last":4.3300000000000001},{"interface":"front","average":{"1min":1.282,"5min":0.89300000000000002,"15min":1.4410000000000001},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7720000000000002,"5min":10.909000000000001,"15min":140.33500000000001},"last":11.521000000000001}]}]},{"osd":1,"up_from":301,"seq":1292785156259,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2390932,"kb_used_data":1745420,"kb_used_omap":9,"kb_used_meta":645366,"kb_avail":91354220,"statfs":{"total":95995035648,"available":93546721280,"internally_reserved":0,"allocated":1787310080,"data_stored":1784360755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":660854802},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":4.319,"15min":3.7320000000000002},"min":{"1min":0.221,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":26.021000000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":0.317},{"interface":"front","average":{"1min":3.7639999999999998,"5min":4.1159999999999997,"15min":3.6240000000000001},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":25.971,"5min":142.137,"15min":142.137},"last":0.55100000000000005}]},{"osd":2,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.883,"15min":2.573},"min":{"1min":0.27700000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.763000000000002,"5min":62.572000000000003,"15min":62.572000000000003},"last":1.4450000000000001},{"interface":"front","average":{"1min":2.9809999999999999,"5min":2.8700000000000001,"15min":2.7999999999999998},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.138999999999999,"5min":68.373999999999995,"15min":68.373999999999995},"last":0.35899999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.484,"5min":5.9690000000000003,"15min":6.0899999999999999},"min":{"1min":0.30499999999999999,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":9.8610000000000007,"5min":199.41,"15min":199.41},"last":12.214},{"interface":"front","average":{"1min":3.149,"5min":6.2999999999999998,"15min":6.1509999999999998},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.4329999999999998,"5min":199.58099999999999,"15min":199.58099999999999},"last":0.46200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":1.091,"15min":2.1309999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":8.9399999999999995,"5min":8.9399999999999995,"15min":142.77099999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.46,"5min":1.52,"15min":2.4140000000000001},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.6319999999999997,"5min":8.6319999999999997,"15min":144.898},"last":1.9359999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.0190000000000001,"15min":1.962},"min":{"1min":0.40500000000000003,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.2100000000000009,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":1.901},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.345,"15min":1.2629999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":15.250999999999999,"5min":15.250999999999999,"15min":15.250999999999999},"last":0.622}]},{"osd":6,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.552,"15min":5.5679999999999996},"min":{"1min":0.442,"5min":0.316,"15min":0.29899999999999999},"max":{"1min":8.8559999999999999,"5min":30.986999999999998,"15min":209.221},"last":0.59199999999999997},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.4850000000000001,"15min":5.4960000000000004},"min":{"1min":0.47899999999999998,"5min":0.35399999999999998,"15min":0.33900000000000002},"max":{"1min":8.8100000000000005,"5min":28.710999999999999,"15min":208.43600000000001},"last":0.65200000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.5449999999999999,"15min":3.9500000000000002},"min":{"1min":0.36299999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":9.3879999999999999,"5min":9.3879999999999999,"15min":165.59399999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.077,"5min":1.542,"15min":3.9929999999999999},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":4.2549999999999999,"5min":9.3399999999999999,"15min":165.833},"last":0.48099999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926157,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1312476,"kb_used_data":698132,"kb_used_omap":5,"kb_used_meta":614266,"kb_avail":92432676,"statfs":{"total":95995035648,"available":94651060224,"internally_reserved":0,"allocated":714887168,"data_stored":711700780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":629008637},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.9419999999999999,"15min":2.181},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":7.2380000000000004,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.81100000000000005},{"interface":"front","average":{"1min":2.927,"5min":2.3180000000000001,"15min":4.0199999999999996},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.344000000000001,"5min":57.140999999999998,"15min":138.92400000000001},"last":2.7650000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6640000000000001,"5min":4.8010000000000002,"15min":3.2829999999999999},"min":{"1min":0.28899999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":31.521000000000001,"5min":127.73399999999999,"15min":127.73399999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":4.7649999999999997,"5min":5.1299999999999999,"15min":3.5470000000000002},"min":{"1min":0.38,"5min":0.27600000000000002,"15min":0.26200000000000001},"max":{"1min":31.478999999999999,"5min":127.69,"15min":127.69},"last":0.46400000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":4.2329999999999997,"5min":5.2009999999999996,"15min":5.2460000000000004},"min":{"1min":0.35599999999999998,"5min":0.255,"15min":0.221},"max":{"1min":28.015000000000001,"5min":125.562,"15min":218.06999999999999},"last":0.23200000000000001},{"interface":"front","average":{"1min":2.2309999999999999,"5min":4.5609999999999999,"15min":4.2409999999999997},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":12.769,"5min":117.295,"15min":218.63200000000001},"last":0.44}]},{"osd":4,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.5950000000000002,"15min":1.954},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.4610000000000003,"5min":72.403999999999996,"15min":72.403999999999996},"last":0.501},{"interface":"front","average":{"1min":3.4380000000000002,"5min":2.883,"15min":2.7480000000000002},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":11.272,"5min":72.162999999999997,"15min":136.262},"last":1.5629999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":4.0060000000000002,"5min":1.6579999999999999,"15min":1.2669999999999999},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":11.326000000000001,"5min":11.326000000000001,"15min":11.326000000000001},"last":1.5229999999999999},{"interface":"front","average":{"1min":1.4470000000000001,"5min":1.1279999999999999,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.8200000000000003,"5min":6.8200000000000003,"15min":6.8200000000000003},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.8740000000000001,"15min":3.5089999999999999},"min":{"1min":0.38900000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.8819999999999997,"5min":57.384,"15min":214.785},"last":2.7850000000000001},{"interface":"front","average":{"1min":2.0449999999999999,"5min":2.0390000000000001,"15min":3.7370000000000001},"min":{"1min":0.47399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.4100000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":2.718}]},{"osd":7,"last update":"Sat Sep 30 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.6970000000000001,"15min":2.1139999999999999},"min":{"1min":0.40300000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.8470000000000004,"5min":57.776000000000003,"15min":97.382000000000005},"last":1.0089999999999999},{"interface":"front","average":{"1min":2.3479999999999999,"5min":2.1509999999999998,"15min":2.3380000000000001},"min":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.33100000000000002},"max":{"1min":8.4529999999999994,"5min":57.179000000000002,"15min":97.015000000000001},"last":0.52700000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599098,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1614248,"kb_used_data":807840,"kb_used_omap":5,"kb_used_meta":806394,"kb_avail":92130904,"statfs":{"total":95995035648,"available":94342045696,"internally_reserved":0,"allocated":827228160,"data_stored":824318162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":825748109},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":4.0890000000000004,"5min":5.4829999999999997,"15min":3.5800000000000001},"min":{"1min":0.219,"5min":0.192,"15min":0.187},"max":{"1min":13.880000000000001,"5min":170.678,"15min":170.678},"last":0.189},{"interface":"front","average":{"1min":5.1459999999999999,"5min":5.5549999999999997,"15min":3.6320000000000001},"min":{"1min":0.32100000000000001,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":12.609999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:46:52 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":4.8449999999999998,"15min":3.7919999999999998},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":10.922000000000001,"5min":162.696,"15min":162.696},"last":0.63400000000000001},{"interface":"front","average":{"1min":4.665,"5min":5.0709999999999997,"15min":3.6320000000000001},"min":{"1min":0.27700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.705,"5min":162.93899999999999,"15min":162.93899999999999},"last":1.113}]},{"osd":3,"last update":"Sat Sep 30 09:46:08 2023","interfaces":[{"interface":"back","average":{"1min":9.9629999999999992,"5min":5.625,"15min":4.9740000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":156.39400000000001,"5min":156.39400000000001,"15min":156.39400000000001},"last":9.5999999999999996},{"interface":"front","average":{"1min":8.8300000000000001,"5min":5.3040000000000003,"15min":4.8280000000000003},"min":{"1min":0.32600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":156.376,"5min":156.376,"15min":156.376},"last":9.5090000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":1.8120000000000001,"15min":1.881},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1270000000000007,"5min":47.463000000000001,"15min":83.012},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.8520000000000001,"15min":1.879},"min":{"1min":0.42099999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.997999999999999,"5min":47.412999999999997,"15min":81.335999999999999},"last":0.317}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":5.2030000000000003,"5min":1.901,"15min":1.351},"min":{"1min":0.378,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":47.226999999999997,"5min":47.226999999999997,"15min":47.226999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":5.1379999999999999,"5min":2.0800000000000001,"15min":1.5700000000000001},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":47.127000000000002,"5min":47.127000000000002,"15min":47.127000000000002},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.4199999999999999,"15min":3.2309999999999999},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.026999999999999,"5min":11.026999999999999,"15min":154.417},"last":0.46999999999999997},{"interface":"front","average":{"1min":3.1880000000000002,"5min":1.8140000000000001,"15min":3.6379999999999999},"min":{"1min":0.46100000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.846,"5min":10.846,"15min":154.04400000000001},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":2.1240000000000001,"15min":2.4089999999999998},"min":{"1min":0.48199999999999998,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":10.928000000000001,"5min":49.816000000000003,"15min":123.294},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.3889999999999998,"15min":2.7429999999999999},"min":{"1min":0.436,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":10.885,"5min":50.956000000000003,"15min":123.16},"last":0.60499999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435572,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2377368,"kb_used_data":1522896,"kb_used_omap":2,"kb_used_meta":854269,"kb_avail":91367784,"statfs":{"total":95995035648,"available":93560610816,"internally_reserved":0,"allocated":1559445504,"data_stored":1556430727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":874771815},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0190000000000001,"5min":4.3949999999999996,"15min":4.7249999999999996},"min":{"1min":0.20000000000000001,"5min":0.183,"15min":0.183},"max":{"1min":34.220999999999997,"5min":144.72,"15min":205.73099999999999},"last":6.6189999999999998},{"interface":"front","average":{"1min":4.2670000000000003,"5min":4.6139999999999999,"15min":4.6580000000000004},"min":{"1min":0.28999999999999998,"5min":0.23999999999999999,"15min":0.21199999999999999},"max":{"1min":34.253,"5min":142.36600000000001,"15min":205.495},"last":6.1520000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:46:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0869999999999997,"5min":3.7599999999999998,"15min":7.4470000000000001},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":19.059000000000001,"5min":142.434,"15min":273.99599999999998},"last":10.131},{"interface":"front","average":{"1min":4.1550000000000002,"5min":3.7879999999999998,"15min":7.1299999999999999},"min":{"1min":0.28199999999999997,"5min":0.26800000000000002,"15min":0.20699999999999999},"max":{"1min":13.183,"5min":142.44900000000001,"15min":274.21600000000001},"last":9.8800000000000008}]},{"osd":2,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":4.851,"15min":5.6269999999999998},"min":{"1min":0.36699999999999999,"5min":0.249,"15min":0.249},"max":{"1min":8.0389999999999997,"5min":146.92599999999999,"15min":206.69},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.0390000000000001,"5min":4.508,"15min":5.4370000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.9649999999999999,"5min":145.75299999999999,"15min":209.22800000000001},"last":12.834}]},{"osd":4,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.976,"15min":2.0649999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.141,"5min":81.677000000000007,"15min":81.677000000000007},"last":6.2080000000000002},{"interface":"front","average":{"1min":2.3170000000000002,"5min":2.2669999999999999,"15min":1.9330000000000001},"min":{"1min":0.35399999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0540000000000003,"5min":80.623000000000005,"15min":80.623000000000005},"last":0.33200000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":1.732,"15min":1.595},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.701000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.2839999999999998,"5min":1.218,"15min":1.04},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.069,"15min":1.901},"min":{"1min":0.38800000000000001,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":5.202,"5min":5.915,"15min":60.698},"last":10.837999999999999},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.4219999999999999,"15min":1.911},"min":{"1min":0.41899999999999998,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":6.2290000000000001,"5min":10.356999999999999,"15min":12.121},"last":0.48399999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":3.0569999999999999,"15min":2.601},"min":{"1min":0.47199999999999998,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":7.968,"5min":213.548,"15min":213.548},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.3839999999999999,"5min":2.8109999999999999,"15min":2.5129999999999999},"min":{"1min":0.438,"5min":0.41099999999999998,"15min":0.34799999999999998},"max":{"1min":5.1719999999999997,"5min":213.583,"15min":213.583},"last":10.888999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019334,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1105392,"kb_used_data":624872,"kb_used_omap":0,"kb_used_meta":480384,"kb_avail":92639760,"statfs":{"total":95995035648,"available":94863114240,"internally_reserved":0,"allocated":639868928,"data_stored":638152496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":491913216},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2749999999999999,"15min":2.4950000000000001},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":34.991999999999997,"5min":73.718000000000004,"15min":105.179},"last":0.83699999999999997},{"interface":"front","average":{"1min":3.573,"5min":2.4550000000000001,"15min":2.6619999999999999},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":35.274999999999999,"5min":74.001999999999995,"15min":104.70999999999999},"last":2.5859999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.2029999999999998,"15min":4.3399999999999999},"min":{"1min":0.32000000000000001,"5min":0.25,"15min":0.25},"max":{"1min":30.538,"5min":80.712000000000003,"15min":160.69999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":3.226,"5min":2.2730000000000001,"15min":4.2930000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":30.722000000000001,"5min":80.810000000000002,"15min":160.50700000000001},"last":0.438}]},{"osd":2,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3069999999999999,"5min":4.9989999999999997,"15min":3.492},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":13.26,"5min":142.88999999999999,"15min":142.88999999999999},"last":4.4429999999999996},{"interface":"front","average":{"1min":3.4159999999999999,"5min":5.069,"15min":3.5249999999999999},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":8.8870000000000005,"5min":142.90700000000001,"15min":142.90700000000001},"last":2.246}]},{"osd":3,"last update":"Sat Sep 30 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":3.875,"5min":1.593,"15min":2.3130000000000002},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":29.41,"5min":29.41,"15min":70.241},"last":2.5270000000000001},{"interface":"front","average":{"1min":3.1219999999999999,"5min":1.4550000000000001,"15min":2.2440000000000002},"min":{"1min":0.38300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":29.401,"5min":29.401,"15min":70.433999999999997},"last":2.5150000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":8.9930000000000003,"15min":10.153},"min":{"1min":0.32800000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.133,"5min":140.11099999999999,"15min":140.11099999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.5029999999999999,"5min":8.9130000000000003,"15min":10.148},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0350000000000001,"5min":140.065,"15min":140.065},"last":0.498}]},{"osd":6,"last update":"Sat Sep 30 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.63100000000000001,"15min":2.1539999999999999},"min":{"1min":0.33900000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":2.7970000000000002,"5min":4.4299999999999997,"15min":202.72499999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.1259999999999999,"5min":0.71899999999999997,"15min":2.165},"min":{"1min":0.33400000000000002,"5min":0.253,"15min":0.249},"max":{"1min":7.173,"5min":7.173,"15min":202.42400000000001},"last":0.40400000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":2.5099999999999998,"15min":3.2000000000000002},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.771000000000001,"5min":161.72999999999999,"15min":230.90100000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":2.4820000000000002,"15min":3.0569999999999999},"min":{"1min":0.41999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.2970000000000006,"5min":162.60900000000001,"15min":229.97},"last":0.55600000000000005}]}]},{"osd":5,"up_from":489,"seq":2100239007776,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1974896,"kb_used_data":1832360,"kb_used_omap":6,"kb_used_meta":142521,"kb_avail":91770256,"statfs":{"total":95995035648,"available":93972742144,"internally_reserved":0,"allocated":1876336640,"data_stored":1874616758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":145941599},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3500000000000001,"5min":21.718,"15min":24.779},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":25.786999999999999,"5min":281.10599999999999,"15min":281.10599999999999},"last":8.8970000000000002},{"interface":"front","average":{"1min":3.851,"5min":22.048999999999999,"15min":25.082000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":25.663,"5min":282.07400000000001,"15min":282.07400000000001},"last":5.1459999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":12.692,"15min":14.177},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":24.478999999999999,"5min":273.48200000000003,"15min":273.48200000000003},"last":3.0379999999999998},{"interface":"front","average":{"1min":4.3319999999999999,"5min":12.15,"15min":13.452999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":24.359000000000002,"5min":273.50200000000001,"15min":273.50200000000001},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":11.537000000000001,"5min":3.0600000000000001,"15min":1.647},"min":{"1min":0.40899999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":163.23099999999999,"5min":163.23099999999999,"15min":163.23099999999999},"last":1.675},{"interface":"front","average":{"1min":11.898999999999999,"5min":3.6120000000000001,"15min":2.2309999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":162.71100000000001,"5min":162.71100000000001,"15min":162.71100000000001},"last":2.1120000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.113,"5min":12.420999999999999,"15min":13.973000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.271000000000001,"5min":274.01799999999997,"15min":274.01799999999997},"last":1.5049999999999999},{"interface":"front","average":{"1min":3.2709999999999999,"5min":2.0390000000000001,"15min":1.8340000000000001},"min":{"1min":0.38400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":13.259,"5min":13.259,"15min":13.259},"last":1.516}]},{"osd":4,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.88500000000000001,"15min":0.92900000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8769999999999998,"5min":4.1769999999999996,"15min":4.1769999999999996},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.262,"5min":1.169,"15min":1.153},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.7610000000000001,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.5129999999999999,"15min":1.601},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.758,"5min":14.371,"15min":14.371},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.069,"5min":1.325,"15min":1.3680000000000001},"min":{"1min":0.379,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.7359999999999998,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.316}]},{"osd":7,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.238,"15min":1.2749999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.1150000000000002,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.90400000000000003,"15min":0.91200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.0680000000000001,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.48199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":700252160,"data_stored":700252160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1765281792,"data_stored":1765277720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":815923200,"data_stored":815923200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1547501568,"data_stored":1547501568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":626655232,"data_stored":626655232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1848053760,"data_stored":1848049688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":613416960,"data_stored":613416960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1859391488,"data_stored":1859387416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:47:06.150 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:47:06.417 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:06.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:06.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:06.527 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:06.558 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:06.632 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:06.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:06.739 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:06.847 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:06.863 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:06.990 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:06.996 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:06 smithi167 ceph-mon[192880]: pgmap v1362: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2198/139755 objects misplaced (1.573%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:47:07.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:07.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:07.079 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:47:07.080 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:47:07.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:06 smithi019 ceph-mon[216815]: pgmap v1362: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2198/139755 objects misplaced (1.573%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:47:07.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:07 smithi019 ceph-mon[224043]: pgmap v1362: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2198/139755 objects misplaced (1.573%); 2.0 MiB/s, 31 objects/s recovering 2023-09-30T09:47:07.101 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:07.236 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:07.604 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:07.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:07.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:07.719 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:07.748 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:07.790 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:07.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:07.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:07.895 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:07.960 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:08.009 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:08.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:08.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:08.113 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:08.203 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:08.629 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:08.630 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:08.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:08.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:08.655 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:47:08.735 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:08.785 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:08.916 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:08.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:08.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:09.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:09 smithi019 ceph-mon[216815]: pgmap v1363: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 2198/139755 objects misplaced (1.573%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:47:09.018 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:09 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3823110029' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:09.019 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:09 smithi167 ceph-mon[192880]: pgmap v1363: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 2198/139755 objects misplaced (1.573%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:47:09.019 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:09 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3823110029' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:09.030 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:09.057 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:47:09.126 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:09.218 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:09.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:09 smithi019 ceph-mon[224043]: pgmap v1363: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 2198/139755 objects misplaced (1.573%); 1.8 MiB/s, 29 objects/s recovering 2023-09-30T09:47:09.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:09 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3823110029' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:10.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:10 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:10.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3823110029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:47:10.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:10 smithi167 ceph-mon[192880]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:10 smithi167 ceph-mon[192880]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:10 smithi167 ceph-mon[192880]: osdmap e519: 8 total, 8 up, 8 in 2023-09-30T09:47:10.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:10 smithi167 ceph-mon[192880]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:10 smithi167 ceph-mon[192880]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:10 smithi167 ceph-mon[192880]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:10.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3823110029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:47:10.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[216815]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[216815]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[216815]: osdmap e519: 8 total, 8 up, 8 in 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[216815]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[216815]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[216815]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3823110029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[224043]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[224043]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[224043]: osdmap e519: 8 total, 8 up, 8 in 2023-09-30T09:47:10.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[224043]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[224043]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:10 smithi019 ceph-mon[224043]: failed to encode map e519 with expected crc 2023-09-30T09:47:10.359 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:10.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:10.475 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:10.558 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:11.213 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:11.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:11.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:11.296 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:11 smithi019 ceph-mon[216815]: pgmap v1365: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:11.296 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:11 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:47:11.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:11 smithi019 ceph-mon[224043]: pgmap v1365: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:11.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:11 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:47:11.320 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:11.381 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:11.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:11 smithi167 ceph-mon[192880]: pgmap v1365: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:11.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:11 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:47:12.053 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:12.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:12.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:12.162 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:12.236 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:12.586 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:12.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:12.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:12.691 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:12.740 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:12.821 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:12.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:12.849 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:47:12.896 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:47:12.933 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:12.944 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:47:12.971 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:12.974 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:47:13.023 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:47:13.042 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:13.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:13.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:13.068 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:47:13.122 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:47:13.154 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:13.155 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:47:13.162 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:47:13.162 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:47:13.162 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:47:13.162 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:47:13.162 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:47:13.162 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:47:13.162 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:47:13.162 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:47:13.215 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:13.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:13 smithi019 ceph-mon[216815]: pgmap v1366: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:13.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3620231176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:13.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:13 smithi019 ceph-mon[224043]: pgmap v1366: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:13.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3620231176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:13.340 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:13.344 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:13 smithi167 ceph-mon[192880]: pgmap v1366: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:13.344 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3620231176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:13.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:13.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:13.347 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:47:13.347 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1771,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":519,"num_osds":8,"num_up_osds":8,"osd_up_since":1696067065,"num_in_osds":8,"osd_in_since":1696066981,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":118},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":43191,"data_bytes":2833455039,"bytes_used":13494136832,"bytes_avail":754466148352,"bytes_total":767960285184,"misplaced_objects":2041,"misplaced_total":129573,"misplaced_ratio":0.015751738402290601,"recovering_objects_per_sec":31,"recovering_bytes_per_sec":2037293,"recovering_keys_per_sec":0,"num_objects_recovered":311,"num_bytes_recovered":20381696,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":702},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-30T09:47:05.837976+0000","services":{}},"progress_events":{"5b5f721d-3b3c-459d-b09b-c0a83655ed5d":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-30T09:47:13.446 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:13.494 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:47:13.494 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:47:13.495 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:47:13.495 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:47:13.555 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:14.049 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:47:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:47:13] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-09-30T09:47:14.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/834057485' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:47:14.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/834057485' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:47:14.050 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:47:14.271 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:14.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:14.385 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:14.449 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:14.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/834057485' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:47:14.575 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:47:15.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:15 smithi019 ceph-mon[224043]: pgmap v1367: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:15.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:15 smithi019 ceph-mon[216815]: pgmap v1367: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:15.336 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:15.518 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:15.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:15 smithi167 ceph-mon[192880]: pgmap v1367: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2041/129573 objects misplaced (1.575%); 1.9 MiB/s, 31 objects/s recovering 2023-09-30T09:47:15.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:15.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:15.567 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:47:15.568 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:47:15.640 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:15.699 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:15.765 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:15.816 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:15.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:15.923 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:15.986 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:16.122 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:16.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:16.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:16.230 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:16.312 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:16.454 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:16.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:16.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:16.560 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:16.649 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:17.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:17 smithi167 ceph-mon[192880]: pgmap v1368: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1854/119487 objects misplaced (1.552%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:47:17.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:17 smithi019 ceph-mon[216815]: pgmap v1368: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1854/119487 objects misplaced (1.552%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:47:17.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:17 smithi019 ceph-mon[224043]: pgmap v1368: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1854/119487 objects misplaced (1.552%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:47:17.520 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:17.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:17.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:17.609 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:17.626 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:47:17.631 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:17.687 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:17.703 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:47:17.703 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:47:17.704 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:47:17.704 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:47:17.704 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:47:18.066 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:18.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:18.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:18.171 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:18.209 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:18.297 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:18.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:18.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:18.641 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:18.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:18.744 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:18.769 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:47:18.823 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:18.978 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:18.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:19.089 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:19.156 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:19.252 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:19.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:19.259 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:19 smithi167 ceph-mon[192880]: pgmap v1369: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 1854/119487 objects misplaced (1.552%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:47:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:19 smithi019 ceph-mon[216815]: pgmap v1369: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 1854/119487 objects misplaced (1.552%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:47:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:19 smithi019 ceph-mon[224043]: pgmap v1369: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 1854/119487 objects misplaced (1.552%); 2.1 MiB/s, 34 objects/s recovering 2023-09-30T09:47:19.360 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:19.417 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:19.512 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:19.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:19.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:19.545 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:19.618 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:19.695 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:19.786 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:19.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:19.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:19.895 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:19.955 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:20.120 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:20.127 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1608977503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:20.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:20.178 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:47:20.210 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:20.215 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1608977503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:20.215 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1608977503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:20.237 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:20.282 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:20.462 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:20.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:20.567 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:20.646 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:20.754 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:20.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:20.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:20.858 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:20.951 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[216815]: pgmap v1370: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 1649/109770 objects misplaced (1.502%); 2.3 MiB/s, 36 objects/s recovering 2023-09-30T09:47:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:47:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:47:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1608977503' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:47:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[216815]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[216815]: osdmap e520: 8 total, 8 up, 8 in 2023-09-30T09:47:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[216815]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[216815]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[224043]: pgmap v1370: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 1649/109770 objects misplaced (1.502%); 2.3 MiB/s, 36 objects/s recovering 2023-09-30T09:47:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:47:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:47:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1608977503' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:47:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[224043]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[224043]: osdmap e520: 8 total, 8 up, 8 in 2023-09-30T09:47:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[224043]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:21 smithi019 ceph-mon[224043]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:21 smithi167 ceph-mon[192880]: pgmap v1370: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 1649/109770 objects misplaced (1.502%); 2.3 MiB/s, 36 objects/s recovering 2023-09-30T09:47:21.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:21 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:47:21.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:21 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:47:21.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1608977503' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:47:21.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:21 smithi167 ceph-mon[192880]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:21 smithi167 ceph-mon[192880]: osdmap e520: 8 total, 8 up, 8 in 2023-09-30T09:47:21.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:21 smithi167 ceph-mon[192880]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:21 smithi167 ceph-mon[192880]: failed to encode map e520 with expected crc 2023-09-30T09:47:21.675 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:21.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:21.783 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:21.870 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:21.957 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:47:21.957 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:47:21.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:47:21.957 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:47:21.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.957 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54227 8.b 8:d5842c94:::benchmark_data_smithi019_478679_object54226:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:47:21.957 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.625464+0000", 2023-09-30T09:47:21.957 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 517.32594307399995, 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1320590269999999, 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54227 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625464+0000", 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625464+0000", 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625464+0000", 2023-09-30T09:47:21.959 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625491+0000", 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7118e-05 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625499+0000", 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5089999999999996e-06 2023-09-30T09:47:21.960 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625504+0000", 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0019999999999997e-06 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625559+0000", 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5186000000000003e-05 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.961 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.625568+0000", 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9740000000000008e-06 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591222+0000", 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96565369599999995 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.962 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591242+0000", 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0449e-05 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.739019+0000", 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1477772209999999 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.740584+0000", 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015645189999999999 2023-09-30T09:47:21.963 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.740738+0000", 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015377199999999999 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.746857+0000", 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006119162 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.964 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.749614+0000", 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027570709999999998 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.755509+0000", 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0058951810000000002 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.755578+0000", 2023-09-30T09:47:21.965 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9058000000000002e-05 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.757523+0000", 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019454800000000001 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54233 8.b 8:d451d48a:::benchmark_data_smithi019_478679_object54232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:47:21.966 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.632114+0000", 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 517.31929379200005, 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1284132279999999, 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54233 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.967 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632114+0000", 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632114+0000", 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632112+0000", 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:21.968 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632160+0000", 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8038999999999999e-05 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632174+0000", 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3993e-05 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.969 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632182+0000", 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8359999999999992e-06 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632248+0000", 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6484999999999998e-05 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.632258+0000", 2023-09-30T09:47:21.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4259999999999992e-06 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591274+0000", 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95901665000000003 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591282+0000", 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.464e-06 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.742560+0000", 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.151277726 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.742653+0000", 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2881999999999996e-05 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.742799+0000", 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000146193 2023-09-30T09:47:21.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.746864+0000", 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0040655650000000002 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.751138+0000", 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0042732810000000003 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.759067+0000", 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0079289270000000005 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.759128+0000", 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1440999999999997e-05 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.760527+0000", 2023-09-30T09:47:21.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001398973 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54236 8.b 8:d2fdb527:::benchmark_data_smithi019_478679_object54235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.636336+0000", 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 517.31507171999999, 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1255129159999999, 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:21.975 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54236 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636336+0000", 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:21.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636336+0000", 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636335+0000", 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636398+0000", 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3142e-05 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636412+0000", 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4188e-05 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636417+0000", 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3900000000000001e-06 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636476+0000", 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8291000000000001e-05 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.636486+0000", 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0176999999999999e-05 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591303+0000", 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95481728200000004 2023-09-30T09:47:21.979 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591311+0000", 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7249999999999997e-06 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.743981+0000", 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1526703010000001 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:21.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.744125+0000", 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000143428 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.744320+0000", 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019499900000000001 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.747871+0000", 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035515690000000001 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.752812+0000", 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049407640000000003 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.760586+0000", 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077744880000000004 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:21.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.760652+0000", 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5429e-05 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.761849+0000", 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011968269999999999 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54239 8.b 8:db6bec44:::benchmark_data_smithi019_478679_object54238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:47:21.983 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.641227+0000", 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 517.31018099899995, 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.127019604, 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54239 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.984 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641227+0000", 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641227+0000", 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641225+0000", 2023-09-30T09:47:21.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641257+0000", 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1529999999999998e-05 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641276+0000", 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.897e-05 2023-09-30T09:47:21.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641284+0000", 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7499999999999992e-06 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641338+0000", 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4008000000000002e-05 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.641349+0000", 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0942e-05 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591333+0000", 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94998327800000004 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.988 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591339+0000", 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8900000000000001e-06 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.745603+0000", 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1542637259999999 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.745764+0000", 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000160922 2023-09-30T09:47:21.989 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.745927+0000", 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016264400000000001 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.749621+0000", 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036943129999999999 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.990 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.752865+0000", 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032436539999999999 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.766700+0000", 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013835024 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.766762+0000", 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2544000000000003e-05 2023-09-30T09:47:21.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.768246+0000", 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00148379 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:54244 8.b 8:d4a2aa15:::benchmark_data_smithi019_478679_object54243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:38:44.649603+0000", 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 517.30180475500003, 2023-09-30T09:47:21.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1205748990000002, 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 54244 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649603+0000", 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649603+0000", 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649602+0000", 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649652+0000", 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0466000000000001e-05 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649663+0000", 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0319999999999999e-05 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649667+0000", 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3699999999999997e-06 2023-09-30T09:47:21.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649709+0000", 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2330000000000003e-05 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:44.649716+0000", 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7580000000000003e-06 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591360+0000", 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94164374699999998 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:45.591367+0000", 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.469e-06 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.746894+0000", 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1555268329999999 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.746982+0000", 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8103000000000007e-05 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.747072+0000", 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0148000000000005e-05 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:21.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.751012+0000", 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0039390950000000001 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.752941+0000", 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019299709999999999 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.768320+0000", 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01537876 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:21.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.768401+0000", 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0655000000000006e-05 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:38:46.770178+0000", 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001776749 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.000 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106313 8.12 8:4aa4e2a6:::benchmark_data_smithi019_478679_object13865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.811295+0000", 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 420.14011274000001, 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3249428500000002, 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106313 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.001 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811295+0000", 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811295+0000", 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.002 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811294+0000", 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811298+0000", 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1439999999999996e-06 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811305+0000", 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.396e-06 2023-09-30T09:47:22.003 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811311+0000", 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5620000000000003e-06 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811402+0000", 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0676999999999999e-05 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.004 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.811414+0000", 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2673e-05 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783552+0000", 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97213803499999996 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.005 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783562+0000", 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3579999999999992e-06 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.126079+0000", 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3425171389999999 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.126224+0000", 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000144888 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.006 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.126559+0000", 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00033534299999999999 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.132747+0000", 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0061879200000000004 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.007 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.135026+0000", 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022791180000000001 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.135032+0000", 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1120000000000001e-06 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.135063+0000", 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0518999999999997e-05 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.008 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.136238+0000", 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001174869 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106316 8.12 8:4991c4c6:::benchmark_data_smithi019_478679_object13868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.848040+0000", 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 420.10336722699998, 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.293266698, 2023-09-30T09:47:22.009 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106316 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848040+0000", 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.010 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848040+0000", 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848039+0000", 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.011 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848045+0000", 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2419999999999997e-06 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848059+0000", 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3841e-05 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848066+0000", 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9530000000000002e-06 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.012 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848117+0000", 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0683000000000002e-05 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848126+0000", 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6229999999999992e-06 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.013 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783607+0000", 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.93548119100000005 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783616+0000", 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6619999999999994e-06 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128058+0000", 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3444416809999999 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.014 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128208+0000", 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015018099999999999 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128492+0000", 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00028447699999999999 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.015 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.136246+0000", 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077533029999999996 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.136367+0000", 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000121748 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.139541+0000", 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031732449999999999 2023-09-30T09:47:22.016 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.139600+0000", 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9573e-05 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.141307+0000", 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017067 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.017 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106319 8.2 8:444e3951:::benchmark_data_smithi019_478679_object13871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.848569+0000", 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 420.10283890800002, 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2840958430000002, 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.018 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106319 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848569+0000", 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848569+0000", 2023-09-30T09:47:22.019 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848568+0000", 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848574+0000", 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6999999999999996e-06 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.020 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848589+0000", 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5568000000000001e-05 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.848596+0000", 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.708e-06 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.021 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.850432+0000", 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001836435 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.850439+0000", 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4960000000000001e-06 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.782674+0000", 2023-09-30T09:47:22.022 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.932235386 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.782685+0000", 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1199000000000001e-05 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.122402+0000", 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.339716664 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.023 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.122531+0000", 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128484 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.122847+0000", 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031693100000000001 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.128033+0000", 2023-09-30T09:47:22.024 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0051852039999999997 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.130479+0000", 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024462569999999999 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.130542+0000", 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2697e-05 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.025 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.130605+0000", 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3553000000000006e-05 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.132664+0000", 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020592990000000001 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.026 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106340 8.5 8:a1cdf9f5:::benchmark_data_smithi019_478679_object13892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.872284+0000", 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 420.07912362299999, 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2115014529999999, 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106340 2023-09-30T09:47:22.027 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872284+0000", 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872284+0000", 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.028 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872283+0000", 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872289+0000", 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1029999999999998e-06 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.029 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872300+0000", 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1249e-05 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872305+0000", 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.31e-06 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872362+0000", 2023-09-30T09:47:22.030 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7157999999999997e-05 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.872389+0000", 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6332000000000001e-05 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.782740+0000", 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.91035162199999997 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.031 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.782752+0000", 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2064e-05 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.070538+0000", 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.287785765 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.032 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.070689+0000", 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150889 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,7", 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.071063+0000", 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00037390200000000001 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.075881+0000", 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048179319999999996 2023-09-30T09:47:22.033 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.078655+0000", 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027745109999999999 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.082385+0000", 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037298819999999999 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.034 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.082477+0000", 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1236999999999994e-05 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.083785+0000", 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001308802 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.035 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:106345 8.2 8:40cb369a:::benchmark_data_smithi019_478679_object13897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:21.881246+0000", 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 420.07016142700002, 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2537290940000001, 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 106345 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.036 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881246+0000", 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881246+0000", 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.037 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881245+0000", 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881252+0000", 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8360000000000003e-06 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881267+0000", 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5738000000000001e-05 2023-09-30T09:47:22.038 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881274+0000", 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2920000000000001e-06 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881331+0000", 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7493e-05 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.039 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:21.881342+0000", 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1202999999999999e-05 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783591+0000", 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90224879199999997 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:22.783596+0000", 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3610000000000003e-06 2023-09-30T09:47:22.040 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.124236+0000", 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.340640659 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.124372+0000", 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000135401 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:47:22.041 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.124649+0000", 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027731700000000001 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.130512+0000", 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0058629190000000003 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.132711+0000", 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021989370000000002 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.042 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.132994+0000", 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00028354200000000002 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.133053+0000", 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9083000000000003e-05 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.043 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.045 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:24.134975+0000", 2023-09-30T09:47:22.045 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001921732 2023-09-30T09:47:22.045 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.045 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.045 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.045 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.045 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.045 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126839 8.12 8:4fd025f5:::benchmark_data_smithi019_478679_object34391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.691976+0000", 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 336.25943150799998, 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1084521220000001, 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126839 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.046 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.691976+0000", 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.691976+0000", 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.691974+0000", 2023-09-30T09:47:22.047 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.691984+0000", 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7200000000000001e-06 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.692031+0000", 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6649e-05 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.048 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.692047+0000", 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5835999999999999e-05 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.692148+0000", 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010145300000000001 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.049 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.692157+0000", 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6340000000000007e-06 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.645316+0000", 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95315957200000001 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.645327+0000", 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0403e-05 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.050 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.771021+0000", 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1256942649999999 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.771647+0000", 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00062577200000000003 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.051 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.781336+0000", 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0096893259999999998 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.781399+0000", 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2848000000000005e-05 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2", 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.781710+0000", 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031120099999999999 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.052 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.787637+0000", 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0059273160000000002 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.787689+0000", 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2036999999999997e-05 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.053 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.799390+0000", 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011700782999999999 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.799446+0000", 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6263e-05 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.800428+0000", 2023-09-30T09:47:22.054 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00098174400000000011 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126849 8.12 8:4f788868:::benchmark_data_smithi019_478679_object34401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.835790+0000", 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 336.11561782299998, 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9694733129999999, 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.055 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126849 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835790+0000", 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.056 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835790+0000", 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835789+0000", 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835793+0000", 2023-09-30T09:47:22.057 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4889999999999997e-06 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835823+0000", 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9592999999999999e-05 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835829+0000", 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6370000000000004e-06 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.058 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835878+0000", 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9682000000000003e-05 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.835887+0000", 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6279999999999994e-06 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.059 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.651662+0000", 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81577498599999998 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.651671+0000", 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2869999999999993e-06 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.772815+0000", 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1211436830000001 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.060 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.773518+0000", 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00070358300000000003 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.784213+0000", 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010694082000000001 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.061 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.784273+0000", 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0921000000000002e-05 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2", 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.784456+0000", 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018218600000000001 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.788906+0000", 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044500310000000001 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.062 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.791070+0000", 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002164666 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.804459+0000", 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013388954 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.804514+0000", 2023-09-30T09:47:22.063 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4344e-05 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805263+0000", 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00074934700000000003 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126863 8.2 8:43927707:::benchmark_data_smithi019_478679_object34415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:47:22.064 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:46.049126+0000", 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 335.902281571, 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.757116868, 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126863 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.065 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049126+0000", 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049126+0000", 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049125+0000", 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:22.066 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049130+0000", 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6219999999999997e-06 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049144+0000", 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3954999999999999e-05 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.067 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049167+0000", 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2149000000000001e-05 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049208+0000", 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1764000000000002e-05 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.049215+0000", 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6250000000000001e-06 2023-09-30T09:47:22.068 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.646071+0000", 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.59685646699999995 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.646079+0000", 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6159999999999996e-06 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.069 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.774139+0000", 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1280599739999999 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.774740+0000", 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00060072900000000004 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.070 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.789386+0000", 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014646632 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.789447+0000", 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1136000000000006e-05 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2", 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.789762+0000", 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031450800000000002 2023-09-30T09:47:22.071 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.797360+0000", 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075981970000000001 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.800485+0000", 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031251069999999998 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.072 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805314+0000", 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004828494 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.805367+0000", 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3581000000000001e-05 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:47.806243+0000", 2023-09-30T09:47:22.073 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00087546499999999997 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:153578 8.9 8:90d1d7bb:::benchmark_data_smithi019_478679_object61130:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:02.966549+0000", 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 258.98485815200002, 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0650080519999996, 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:47:22.074 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 153578 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966549+0000", 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.075 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966549+0000", 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966548+0000", 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966554+0000", 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9290000000000003e-06 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.076 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966567+0000", 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2621e-05 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966575+0000", 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5099999999999998e-06 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967360+0000", 2023-09-30T09:47:22.077 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00078461500000000001 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967368+0000", 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1240000000000005e-06 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.637823+0000", 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6704550610000002 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.078 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for missing object", 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.640292+0000", 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024689769999999998 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.029951+0000", 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.389658828 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.079 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031557+0000", 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016064709999999999 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:153601 8.9 8:96e6158f:::benchmark_data_smithi019_478679_object61153:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:02.966582+0000", 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 258.98482565199998, 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1046012760000004, 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.080 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 153601 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966582+0000", 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.081 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966582+0000", 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966581+0000", 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.082 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966584+0000", 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9730000000000002e-06 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966591+0000", 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5799999999999997e-06 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966595+0000", 2023-09-30T09:47:22.083 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3209999999999998e-06 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967390+0000", 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00079515100000000002 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967396+0000", 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3090000000000003e-06 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.084 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.040459+0000", 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0730634190000004 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.042937+0000", 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024781439999999998 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,6", 2023-09-30T09:47:22.085 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.047464+0000", 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045271729999999998 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.064276+0000", 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016811621999999998 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.064915+0000", 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00063849399999999998 2023-09-30T09:47:22.086 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.065536+0000", 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00062178699999999995 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.066784+0000", 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012479609999999999 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.087 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.071183+0000", 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043987760000000001 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:153614 8.9 8:92103489:::benchmark_data_smithi019_478679_object61166:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:02.966600+0000", 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 258.98480760500001, 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1120596819999999, 2023-09-30T09:47:22.088 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 153614 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966600+0000", 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.089 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966600+0000", 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966599+0000", 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.090 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966602+0000", 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8600000000000001e-06 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966609+0000", 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3319999999999997e-06 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966613+0000", 2023-09-30T09:47:22.091 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0640000000000004e-06 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967413+0000", 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00080035700000000002 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967420+0000", 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4740000000000002e-06 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.092 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.053697+0000", 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0862776460000001 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.056675+0000", 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029778230000000001 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,6", 2023-09-30T09:47:22.093 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.060861+0000", 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041855140000000004 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.071889+0000", 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011028911000000001 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.072098+0000", 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000208333 2023-09-30T09:47:22.094 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.072683+0000", 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000585495 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.073852+0000", 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011684009999999999 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.095 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.078660+0000", 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048078820000000003 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGRecoveryDeleteReply(8.9 e457,455 8:90d1d7bb:::benchmark_data_smithi019_478679_object61130:head,453'7128)", 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:05.642513+0000", 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 256.30889408199999, 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3871478769999999, 2023-09-30T09:47:22.096 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642513+0000", 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642513+0000", 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.097 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642513+0000", 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642518+0000", 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7180000000000004e-06 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.098 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642530+0000", 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2391e-05 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642537+0000", 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2490000000000004e-06 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.019708+0000", 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.377170352 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.099 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.029661+0000", 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0099537389999999996 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35820.0:153603 8.19 e457/455 8:9d676f8e:::benchmark_data_smithi019_478679_object61155:head v 457'7206, mlcod=0'0)", 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:05.654142+0000", 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 256.29726598100001, 2023-09-30T09:47:22.100 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3773550720000001, 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654142+0000", 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654142+0000", 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.101 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654141+0000", 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654148+0000", 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1420000000000004e-06 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.102 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654162+0000", 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3559e-05 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654168+0000", 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0909999999999998e-06 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.019529+0000", 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.365361477 2023-09-30T09:47:22.103 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.019550+0000", 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0965e-05 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031419+0000", 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011869104 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.104 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031497+0000", 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7324999999999996e-05 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35820.0:153650 8.19 e457/455 8:9a1a13f4:::benchmark_data_smithi019_478679_object61202:head v 457'7207, mlcod=0'0)", 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:05.655233+0000", 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 256.29617469800002, 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.37629468, 2023-09-30T09:47:22.105 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655233+0000", 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655233+0000", 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.106 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655232+0000", 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655238+0000", 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5890000000000002e-06 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.107 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655249+0000", 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1357e-05 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655255+0000", 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3460000000000001e-06 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.021117+0000", 2023-09-30T09:47:22.108 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3658622629999999 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.021139+0000", 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2002e-05 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031499+0000", 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010360596999999999 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.109 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031527+0000", 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8055000000000002e-05 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35820.0:153653 8.19 e457/455 8:9f29c27f:::benchmark_data_smithi019_478679_object61205:head v 457'7208, mlcod=0'0)", 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:05.656934+0000", 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 256.29447372200002, 2023-09-30T09:47:22.110 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3746209979999999, 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656934+0000", 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656934+0000", 2023-09-30T09:47:22.111 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656933+0000", 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656939+0000", 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6119999999999998e-06 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.112 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656951+0000", 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2422000000000001e-05 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656957+0000", 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2290000000000002e-06 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.022470+0000", 2023-09-30T09:47:22.113 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3655130689999999 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.022489+0000", 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9623000000000002e-05 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031533+0000", 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0090435029999999996 2023-09-30T09:47:22.114 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031555+0000", 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2065e-05 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:22.115 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:47:22.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:22.116 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:22.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:22.217 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:22.268 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:22.337 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:22.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:22.440 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:22.466 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:22.506 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:22.507 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:22 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:22.507 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:22 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:22.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:22.568 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:47:22.571 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:47:22.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:22 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:22.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:22 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:22.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:22 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:22.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:22 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:22.609 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:22.650 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:23.178 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:23.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:23.202 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:47:23.301 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:23.511 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:23 smithi167 ceph-mon[192880]: pgmap v1372: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1649/109770 objects misplaced (1.502%); 2.4 MiB/s, 39 objects/s recovering 2023-09-30T09:47:23.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:23 smithi019 ceph-mon[216815]: pgmap v1372: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1649/109770 objects misplaced (1.502%); 2.4 MiB/s, 39 objects/s recovering 2023-09-30T09:47:23.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:23 smithi019 ceph-mon[224043]: pgmap v1372: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1649/109770 objects misplaced (1.502%); 2.4 MiB/s, 39 objects/s recovering 2023-09-30T09:47:23.792 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:23.838 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:47:24.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:47:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:47:23] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2023-09-30T09:47:24.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:24 smithi167 ceph-mon[192880]: from='client.56293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:24.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:24 smithi019 ceph-mon[216815]: from='client.56293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:24.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:24 smithi019 ceph-mon[224043]: from='client.56293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:24.589 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:24.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:24.693 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:24.721 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:24.788 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:24.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:24.801 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1372,"stamp":"2023-09-30T09:47:21.854311+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2400851903,"num_objects":36590,"num_object_clones":0,"num_object_copies":109770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1649,"num_objects_unfound":0,"num_objects_dirty":36590,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":157831,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4744,"num_bytes_recovered":305157746,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122346,"ondisk_log_size":122346,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11901804,"kb_used_data":7279148,"kb_used_omap":30,"kb_used_meta":4622305,"kb_avail":738059412,"statfs":{"total":767960285184,"available":755772837888,"internally_reserved":0,"allocated":7453847552,"data_stored":7434858674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4733240674},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,6,3,7,7,4,0,7],"upper_bound":256},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-432603136,"num_objects":-6601,"num_object_clones":0,"num_object_copies":-19803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-392,"num_objects_unfound":0,"num_objects_dirty":-6601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6601,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":392,"num_bytes_recovered":25690112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.013545"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":942,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.910778+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:46:56.910778+0000","last_peered":"2023-09-30T09:46:56.910778+0000","last_clean":"2023-09-30T09:46:56.910778+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:46:56.910778+0000","last_undegraded":"2023-09-30T09:46:56.910778+0000","last_fullsized":"2023-09-30T09:46:56.910778+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":537,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.919893+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:46:56.919893+0000","last_peered":"2023-09-30T09:46:56.919893+0000","last_clean":"2023-09-30T09:46:56.919893+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:46:56.919893+0000","last_undegraded":"2023-09-30T09:46:56.919893+0000","last_fullsized":"2023-09-30T09:46:56.919893+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":434,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920879+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:46:56.920879+0000","last_peered":"2023-09-30T09:46:56.920879+0000","last_clean":"2023-09-30T09:46:56.920879+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:46:56.920879+0000","last_undegraded":"2023-09-30T09:46:56.920879+0000","last_fullsized":"2023-09-30T09:46:56.920879+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":944,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.910995+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:46:56.910995+0000","last_peered":"2023-09-30T09:46:56.910995+0000","last_clean":"2023-09-30T09:46:56.910995+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:46:56.910995+0000","last_undegraded":"2023-09-30T09:46:56.910995+0000","last_fullsized":"2023-09-30T09:46:56.910995+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.911246+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:46:56.911246+0000","last_peered":"2023-09-30T09:46:56.911246+0000","last_clean":"2023-09-30T09:46:56.911246+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:46:56.911246+0000","last_undegraded":"2023-09-30T09:46:56.911246+0000","last_fullsized":"2023-09-30T09:46:56.911246+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":125,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914916+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:46:56.914916+0000","last_peered":"2023-09-30T09:46:56.914916+0000","last_clean":"2023-09-30T09:46:56.914916+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:46:56.914916+0000","last_undegraded":"2023-09-30T09:46:56.914916+0000","last_fullsized":"2023-09-30T09:46:56.914916+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":752,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.919980+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:46:56.919980+0000","last_peered":"2023-09-30T09:46:56.919980+0000","last_clean":"2023-09-30T09:46:56.919980+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:46:56.919980+0000","last_undegraded":"2023-09-30T09:46:56.919980+0000","last_fullsized":"2023-09-30T09:46:56.919980+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":24,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914114+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:46:56.914114+0000","last_peered":"2023-09-30T09:46:56.914114+0000","last_clean":"2023-09-30T09:46:56.914114+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:46:56.914114+0000","last_undegraded":"2023-09-30T09:46:56.914114+0000","last_fullsized":"2023-09-30T09:46:56.914114+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":88,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924950+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:46:56.924950+0000","last_peered":"2023-09-30T09:46:56.924950+0000","last_clean":"2023-09-30T09:46:56.924950+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:46:56.924950+0000","last_undegraded":"2023-09-30T09:46:56.924950+0000","last_fullsized":"2023-09-30T09:46:56.924950+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":920,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914524+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:46:56.914524+0000","last_peered":"2023-09-30T09:46:56.914524+0000","last_clean":"2023-09-30T09:46:56.914524+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:46:56.914524+0000","last_undegraded":"2023-09-30T09:46:56.914524+0000","last_fullsized":"2023-09-30T09:46:56.914524+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":139,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915376+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:46:56.915376+0000","last_peered":"2023-09-30T09:46:56.915376+0000","last_clean":"2023-09-30T09:46:56.915376+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:46:56.915376+0000","last_undegraded":"2023-09-30T09:46:56.915376+0000","last_fullsized":"2023-09-30T09:46:56.915376+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":99,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.911012+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:46:56.911012+0000","last_peered":"2023-09-30T09:46:56.911012+0000","last_clean":"2023-09-30T09:46:56.911012+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:46:56.911012+0000","last_undegraded":"2023-09-30T09:46:56.911012+0000","last_fullsized":"2023-09-30T09:46:56.911012+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":709,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.068820+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:47:09.068820+0000","last_peered":"2023-09-30T09:47:09.068820+0000","last_clean":"2023-09-30T09:47:09.068820+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:47:09.068820+0000","last_undegraded":"2023-09-30T09:47:09.068820+0000","last_fullsized":"2023-09-30T09:47:09.068820+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":339,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.911865+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:46:56.911865+0000","last_peered":"2023-09-30T09:46:56.911865+0000","last_clean":"2023-09-30T09:46:56.911865+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:46:56.911865+0000","last_undegraded":"2023-09-30T09:46:56.911865+0000","last_fullsized":"2023-09-30T09:46:56.911865+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":475,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914739+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:46:56.914739+0000","last_peered":"2023-09-30T09:46:56.914739+0000","last_clean":"2023-09-30T09:46:56.914739+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:46:56.914739+0000","last_undegraded":"2023-09-30T09:46:56.914739+0000","last_fullsized":"2023-09-30T09:46:56.914739+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":648,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.911978+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:46:56.911978+0000","last_peered":"2023-09-30T09:46:56.911978+0000","last_clean":"2023-09-30T09:46:56.911978+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:46:56.911978+0000","last_undegraded":"2023-09-30T09:46:56.911978+0000","last_fullsized":"2023-09-30T09:46:56.911978+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":233,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915866+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:46:56.915866+0000","last_peered":"2023-09-30T09:46:56.915866+0000","last_clean":"2023-09-30T09:46:56.915866+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:46:56.915866+0000","last_undegraded":"2023-09-30T09:46:56.915866+0000","last_fullsized":"2023-09-30T09:46:56.915866+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":74,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924550+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:46:56.924550+0000","last_peered":"2023-09-30T09:46:56.924550+0000","last_clean":"2023-09-30T09:46:56.924550+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:46:56.924550+0000","last_undegraded":"2023-09-30T09:46:56.924550+0000","last_fullsized":"2023-09-30T09:46:56.924550+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":574,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914987+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:46:56.914987+0000","last_peered":"2023-09-30T09:46:56.914987+0000","last_clean":"2023-09-30T09:46:56.914987+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:46:56.914987+0000","last_undegraded":"2023-09-30T09:46:56.914987+0000","last_fullsized":"2023-09-30T09:46:56.914987+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":563,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920232+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:46:56.920232+0000","last_peered":"2023-09-30T09:46:56.920232+0000","last_clean":"2023-09-30T09:46:56.920232+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:46:56.920232+0000","last_undegraded":"2023-09-30T09:46:56.920232+0000","last_fullsized":"2023-09-30T09:46:56.920232+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":541,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920542+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:46:56.920542+0000","last_peered":"2023-09-30T09:46:56.920542+0000","last_clean":"2023-09-30T09:46:56.920542+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:46:56.920542+0000","last_undegraded":"2023-09-30T09:46:56.920542+0000","last_fullsized":"2023-09-30T09:46:56.920542+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":773,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:47:04.326058+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:47:04.326058+0000","last_peered":"2023-09-30T09:47:04.326058+0000","last_clean":"2023-09-30T09:47:04.326058+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:47:04.326058+0000","last_undegraded":"2023-09-30T09:47:04.326058+0000","last_fullsized":"2023-09-30T09:47:04.326058+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":169,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914342+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:46:56.914342+0000","last_peered":"2023-09-30T09:46:56.914342+0000","last_clean":"2023-09-30T09:46:56.914342+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:46:56.914342+0000","last_undegraded":"2023-09-30T09:46:56.914342+0000","last_fullsized":"2023-09-30T09:46:56.914342+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":496,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914561+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:46:56.914561+0000","last_peered":"2023-09-30T09:46:56.914561+0000","last_clean":"2023-09-30T09:46:56.914561+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:46:56.914561+0000","last_undegraded":"2023-09-30T09:46:56.914561+0000","last_fullsized":"2023-09-30T09:46:56.914561+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":820,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.910580+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:46:56.910580+0000","last_peered":"2023-09-30T09:46:56.910580+0000","last_clean":"2023-09-30T09:46:56.910580+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:46:56.910580+0000","last_undegraded":"2023-09-30T09:46:56.910580+0000","last_fullsized":"2023-09-30T09:46:56.910580+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":99,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:47:04.326113+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:47:04.326113+0000","last_peered":"2023-09-30T09:47:04.326113+0000","last_clean":"2023-09-30T09:47:04.326113+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:47:04.326113+0000","last_undegraded":"2023-09-30T09:47:04.326113+0000","last_fullsized":"2023-09-30T09:47:04.326113+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":194,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.911591+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:46:56.911591+0000","last_peered":"2023-09-30T09:46:56.911591+0000","last_clean":"2023-09-30T09:46:56.911591+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:46:56.911591+0000","last_undegraded":"2023-09-30T09:46:56.911591+0000","last_fullsized":"2023-09-30T09:46:56.911591+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":920,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.912258+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:46:56.912258+0000","last_peered":"2023-09-30T09:46:56.912258+0000","last_clean":"2023-09-30T09:46:56.912258+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:46:56.912258+0000","last_undegraded":"2023-09-30T09:46:56.912258+0000","last_fullsized":"2023-09-30T09:46:56.912258+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":203,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914937+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:46:56.914937+0000","last_peered":"2023-09-30T09:46:56.914937+0000","last_clean":"2023-09-30T09:46:56.914937+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:46:56.914937+0000","last_undegraded":"2023-09-30T09:46:56.914937+0000","last_fullsized":"2023-09-30T09:46:56.914937+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":99,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.048174+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:47:09.048174+0000","last_peered":"2023-09-30T09:47:09.048174+0000","last_clean":"2023-09-30T09:47:09.048174+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:47:09.048174+0000","last_undegraded":"2023-09-30T09:47:09.048174+0000","last_fullsized":"2023-09-30T09:47:09.048174+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":929,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.916053+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:46:56.916053+0000","last_peered":"2023-09-30T09:46:56.916053+0000","last_clean":"2023-09-30T09:46:56.916053+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:46:56.916053+0000","last_undegraded":"2023-09-30T09:46:56.916053+0000","last_fullsized":"2023-09-30T09:46:56.916053+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":115,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.923954+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:46:56.923954+0000","last_peered":"2023-09-30T09:46:56.923954+0000","last_clean":"2023-09-30T09:46:56.923954+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:46:56.923954+0000","last_undegraded":"2023-09-30T09:46:56.923954+0000","last_fullsized":"2023-09-30T09:46:56.923954+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.913625+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:46:56.913625+0000","last_peered":"2023-09-30T09:46:56.913625+0000","last_clean":"2023-09-30T09:46:56.913625+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:46:56.913625+0000","last_undegraded":"2023-09-30T09:46:56.913625+0000","last_fullsized":"2023-09-30T09:46:56.913625+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"519'9757","reported_seq":10066,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.338964+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:47:19.338964+0000","last_peered":"2023-09-30T09:47:19.338964+0000","last_clean":"2023-09-30T09:47:19.338964+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:47:19.338964+0000","last_undegraded":"2023-09-30T09:47:19.338964+0000","last_fullsized":"2023-09-30T09:47:19.338964+0000","mapping_epoch":495,"log_start":"498'2200","ondisk_log_start":"498'2200","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145293312,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9757,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":895,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915777+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:46:56.915777+0000","last_peered":"2023-09-30T09:46:56.915777+0000","last_clean":"2023-09-30T09:46:56.915777+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:46:56.915777+0000","last_undegraded":"2023-09-30T09:46:56.915777+0000","last_fullsized":"2023-09-30T09:46:56.915777+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":195,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914935+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:46:56.914935+0000","last_peered":"2023-09-30T09:46:56.914935+0000","last_clean":"2023-09-30T09:46:56.914935+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:46:56.914935+0000","last_undegraded":"2023-09-30T09:46:56.914935+0000","last_fullsized":"2023-09-30T09:46:56.914935+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":140,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.910499+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:46:56.910499+0000","last_peered":"2023-09-30T09:46:56.910499+0000","last_clean":"2023-09-30T09:46:56.910499+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:46:56.910499+0000","last_undegraded":"2023-09-30T09:46:56.910499+0000","last_fullsized":"2023-09-30T09:46:56.910499+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"519'9629","reported_seq":9673,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:18.806567+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:47:18.806567+0000","last_peered":"2023-09-30T09:47:18.806567+0000","last_clean":"2023-09-30T09:47:18.806567+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:47:18.806567+0000","last_undegraded":"2023-09-30T09:47:18.806567+0000","last_fullsized":"2023-09-30T09:47:18.806567+0000","mapping_epoch":483,"log_start":"501'3200","ondisk_log_start":"501'3200","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6429,"log_dups_size":0,"ondisk_log_size":6429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151715840,"num_objects":2315,"num_object_clones":0,"num_object_copies":6945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9629,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":912,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.913426+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:46:56.913426+0000","last_peered":"2023-09-30T09:46:56.913426+0000","last_clean":"2023-09-30T09:46:56.913426+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:46:56.913426+0000","last_undegraded":"2023-09-30T09:46:56.913426+0000","last_fullsized":"2023-09-30T09:46:56.913426+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":165,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915150+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:46:56.915150+0000","last_peered":"2023-09-30T09:46:56.915150+0000","last_clean":"2023-09-30T09:46:56.915150+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:46:56.915150+0000","last_undegraded":"2023-09-30T09:46:56.915150+0000","last_fullsized":"2023-09-30T09:46:56.915150+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":570,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.919623+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:46:56.919623+0000","last_peered":"2023-09-30T09:46:56.919623+0000","last_clean":"2023-09-30T09:46:56.919623+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:46:56.919623+0000","last_undegraded":"2023-09-30T09:46:56.919623+0000","last_fullsized":"2023-09-30T09:46:56.919623+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"519'9887","reported_seq":10156,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.161764+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:47:19.161764+0000","last_peered":"2023-09-30T09:47:19.161764+0000","last_clean":"2023-09-30T09:47:19.161764+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:47:19.161764+0000","last_undegraded":"2023-09-30T09:47:19.161764+0000","last_fullsized":"2023-09-30T09:47:19.161764+0000","mapping_epoch":493,"log_start":"501'3200","ondisk_log_start":"501'3200","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6687,"log_dups_size":0,"ondisk_log_size":6687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150405120,"num_objects":2295,"num_object_clones":0,"num_object_copies":6885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9887,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":689,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.925165+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:46:56.925165+0000","last_peered":"2023-09-30T09:46:56.925165+0000","last_clean":"2023-09-30T09:46:56.925165+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:46:56.925165+0000","last_undegraded":"2023-09-30T09:46:56.925165+0000","last_fullsized":"2023-09-30T09:46:56.925165+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":232,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914394+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:46:56.914394+0000","last_peered":"2023-09-30T09:46:56.914394+0000","last_clean":"2023-09-30T09:46:56.914394+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:46:56.914394+0000","last_undegraded":"2023-09-30T09:46:56.914394+0000","last_fullsized":"2023-09-30T09:46:56.914394+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":88,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924621+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:46:56.924621+0000","last_peered":"2023-09-30T09:46:56.924621+0000","last_clean":"2023-09-30T09:46:56.924621+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:46:56.924621+0000","last_undegraded":"2023-09-30T09:46:56.924621+0000","last_fullsized":"2023-09-30T09:46:56.924621+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"519'9627","reported_seq":9987,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:18.892922+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:47:18.892922+0000","last_peered":"2023-09-30T09:47:18.892922+0000","last_clean":"2023-09-30T09:47:18.892922+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:47:18.892922+0000","last_undegraded":"2023-09-30T09:47:18.892922+0000","last_fullsized":"2023-09-30T09:47:18.892922+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":9627,"log_dups_size":0,"ondisk_log_size":9627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148701184,"num_objects":2269,"num_object_clones":0,"num_object_copies":6807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9627,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":754,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920123+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:46:56.920123+0000","last_peered":"2023-09-30T09:46:56.920123+0000","last_clean":"2023-09-30T09:46:56.920123+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:46:56.920123+0000","last_undegraded":"2023-09-30T09:46:56.920123+0000","last_fullsized":"2023-09-30T09:46:56.920123+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":454,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.910882+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:46:56.910882+0000","last_peered":"2023-09-30T09:46:56.910882+0000","last_clean":"2023-09-30T09:46:56.910882+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:46:56.910882+0000","last_undegraded":"2023-09-30T09:46:56.910882+0000","last_fullsized":"2023-09-30T09:46:56.910882+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":119,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.051658+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:47:09.051658+0000","last_peered":"2023-09-30T09:47:09.051658+0000","last_clean":"2023-09-30T09:47:09.051658+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:47:09.051658+0000","last_undegraded":"2023-09-30T09:47:09.051658+0000","last_fullsized":"2023-09-30T09:47:09.051658+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"519'9821","reported_seq":9865,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.251965+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:47:19.251965+0000","last_peered":"2023-09-30T09:47:19.251965+0000","last_clean":"2023-09-30T09:47:19.251965+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:47:19.251965+0000","last_undegraded":"2023-09-30T09:47:19.251965+0000","last_fullsized":"2023-09-30T09:47:19.251965+0000","mapping_epoch":483,"log_start":"499'2800","ondisk_log_start":"499'2800","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9821,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1001,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914865+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:46:56.914865+0000","last_peered":"2023-09-30T09:46:56.914865+0000","last_clean":"2023-09-30T09:46:56.914865+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:46:56.914865+0000","last_undegraded":"2023-09-30T09:46:56.914865+0000","last_fullsized":"2023-09-30T09:46:56.914865+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":78,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915798+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:46:56.915798+0000","last_peered":"2023-09-30T09:46:56.915798+0000","last_clean":"2023-09-30T09:46:56.915798+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:46:56.915798+0000","last_undegraded":"2023-09-30T09:46:56.915798+0000","last_fullsized":"2023-09-30T09:46:56.915798+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":517,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920306+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:46:56.920306+0000","last_peered":"2023-09-30T09:46:56.920306+0000","last_clean":"2023-09-30T09:46:56.920306+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:46:56.920306+0000","last_undegraded":"2023-09-30T09:46:56.920306+0000","last_fullsized":"2023-09-30T09:46:56.920306+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"519'9690","reported_seq":9734,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.304393+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:47:19.304393+0000","last_peered":"2023-09-30T09:47:19.304393+0000","last_clean":"2023-09-30T09:47:19.304393+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:47:19.304393+0000","last_undegraded":"2023-09-30T09:47:19.304393+0000","last_fullsized":"2023-09-30T09:47:19.304393+0000","mapping_epoch":483,"log_start":"499'2700","ondisk_log_start":"499'2700","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9690,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":1000,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.916527+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:46:56.916527+0000","last_peered":"2023-09-30T09:46:56.916527+0000","last_clean":"2023-09-30T09:46:56.916527+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:46:56.916527+0000","last_undegraded":"2023-09-30T09:46:56.916527+0000","last_fullsized":"2023-09-30T09:46:56.916527+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":74,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924475+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:46:56.924475+0000","last_peered":"2023-09-30T09:46:56.924475+0000","last_clean":"2023-09-30T09:46:56.924475+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:46:56.924475+0000","last_undegraded":"2023-09-30T09:46:56.924475+0000","last_fullsized":"2023-09-30T09:46:56.924475+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920475+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:46:56.920475+0000","last_peered":"2023-09-30T09:46:56.920475+0000","last_clean":"2023-09-30T09:46:56.920475+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:46:56.920475+0000","last_undegraded":"2023-09-30T09:46:56.920475+0000","last_fullsized":"2023-09-30T09:46:56.920475+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"519'9974","reported_seq":10018,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:18.868397+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:47:18.868397+0000","last_peered":"2023-09-30T09:47:18.868397+0000","last_clean":"2023-09-30T09:47:18.868397+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:47:18.868397+0000","last_undegraded":"2023-09-30T09:47:18.868397+0000","last_fullsized":"2023-09-30T09:47:18.868397+0000","mapping_epoch":483,"log_start":"503'4200","ondisk_log_start":"503'4200","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150470656,"num_objects":2296,"num_object_clones":0,"num_object_copies":6888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9974,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":858,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.051738+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:47:09.051738+0000","last_peered":"2023-09-30T09:47:09.051738+0000","last_clean":"2023-09-30T09:47:09.051738+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:47:09.051738+0000","last_undegraded":"2023-09-30T09:47:09.051738+0000","last_fullsized":"2023-09-30T09:47:09.051738+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":524,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.919544+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:46:56.919544+0000","last_peered":"2023-09-30T09:46:56.919544+0000","last_clean":"2023-09-30T09:46:56.919544+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:46:56.919544+0000","last_undegraded":"2023-09-30T09:46:56.919544+0000","last_fullsized":"2023-09-30T09:46:56.919544+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":531,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920252+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:46:56.920252+0000","last_peered":"2023-09-30T09:46:56.920252+0000","last_clean":"2023-09-30T09:46:56.920252+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:46:56.920252+0000","last_undegraded":"2023-09-30T09:46:56.920252+0000","last_fullsized":"2023-09-30T09:46:56.920252+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"519'9675","reported_seq":9722,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:18.974606+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:47:18.974606+0000","last_peered":"2023-09-30T09:47:18.974606+0000","last_clean":"2023-09-30T09:47:18.974606+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:47:18.974606+0000","last_undegraded":"2023-09-30T09:47:18.974606+0000","last_fullsized":"2023-09-30T09:47:18.974606+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":9675,"log_dups_size":0,"ondisk_log_size":9675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148176896,"num_objects":2261,"num_object_clones":0,"num_object_copies":6783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9675,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":722,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914742+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:46:56.914742+0000","last_peered":"2023-09-30T09:46:56.914742+0000","last_clean":"2023-09-30T09:46:56.914742+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:46:56.914742+0000","last_undegraded":"2023-09-30T09:46:56.914742+0000","last_fullsized":"2023-09-30T09:46:56.914742+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":84,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037482+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:47:09.037482+0000","last_peered":"2023-09-30T09:47:09.037482+0000","last_clean":"2023-09-30T09:47:09.037482+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:47:09.037482+0000","last_undegraded":"2023-09-30T09:47:09.037482+0000","last_fullsized":"2023-09-30T09:47:09.037482+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":204,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.919809+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:46:56.919809+0000","last_peered":"2023-09-30T09:46:56.919809+0000","last_clean":"2023-09-30T09:46:56.919809+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:46:56.919809+0000","last_undegraded":"2023-09-30T09:46:56.919809+0000","last_fullsized":"2023-09-30T09:46:56.919809+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"519'9861","reported_seq":10164,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.156752+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:47:19.156752+0000","last_peered":"2023-09-30T09:47:19.156752+0000","last_clean":"2023-09-30T09:47:19.156752+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:47:19.156752+0000","last_undegraded":"2023-09-30T09:47:19.156752+0000","last_fullsized":"2023-09-30T09:47:19.156752+0000","mapping_epoch":494,"log_start":"500'3100","ondisk_log_start":"500'3100","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6761,"log_dups_size":0,"ondisk_log_size":6761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146997248,"num_objects":2243,"num_object_clones":0,"num_object_copies":6729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9861,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":761,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924874+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:46:56.924874+0000","last_peered":"2023-09-30T09:46:56.924874+0000","last_clean":"2023-09-30T09:46:56.924874+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:46:56.924874+0000","last_undegraded":"2023-09-30T09:46:56.924874+0000","last_fullsized":"2023-09-30T09:46:56.924874+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":750,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.048089+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:47:09.048089+0000","last_peered":"2023-09-30T09:47:09.048089+0000","last_clean":"2023-09-30T09:47:09.048089+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:47:09.048089+0000","last_undegraded":"2023-09-30T09:47:09.048089+0000","last_fullsized":"2023-09-30T09:47:09.048089+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":447,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.911793+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:46:56.911793+0000","last_peered":"2023-09-30T09:46:56.911793+0000","last_clean":"2023-09-30T09:46:56.911793+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:46:56.911793+0000","last_undegraded":"2023-09-30T09:46:56.911793+0000","last_fullsized":"2023-09-30T09:46:56.911793+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":136,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915726+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:56.915726+0000","last_peered":"2023-09-30T09:46:56.915726+0000","last_clean":"2023-09-30T09:46:56.915726+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:46:56.915726+0000","last_undegraded":"2023-09-30T09:46:56.915726+0000","last_fullsized":"2023-09-30T09:46:56.915726+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"519'9875","reported_seq":10224,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.301110+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:47:19.301110+0000","last_peered":"2023-09-30T09:47:19.301110+0000","last_clean":"2023-09-30T09:47:19.301110+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:47:19.301110+0000","last_undegraded":"2023-09-30T09:47:19.301110+0000","last_fullsized":"2023-09-30T09:47:19.301110+0000","mapping_epoch":493,"log_start":"498'2300","ondisk_log_start":"498'2300","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147652608,"num_objects":2253,"num_object_clones":0,"num_object_copies":6759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9875,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":960,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.910717+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:46:56.910717+0000","last_peered":"2023-09-30T09:46:56.910717+0000","last_clean":"2023-09-30T09:46:56.910717+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:46:56.910717+0000","last_undegraded":"2023-09-30T09:46:56.910717+0000","last_fullsized":"2023-09-30T09:46:56.910717+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":466,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924700+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:46:56.924700+0000","last_peered":"2023-09-30T09:46:56.924700+0000","last_clean":"2023-09-30T09:46:56.924700+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:46:56.924700+0000","last_undegraded":"2023-09-30T09:46:56.924700+0000","last_fullsized":"2023-09-30T09:46:56.924700+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":538,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920611+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:46:56.920611+0000","last_peered":"2023-09-30T09:46:56.920611+0000","last_clean":"2023-09-30T09:46:56.920611+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:46:56.920611+0000","last_undegraded":"2023-09-30T09:46:56.920611+0000","last_fullsized":"2023-09-30T09:46:56.920611+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":278,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914659+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:46:56.914659+0000","last_peered":"2023-09-30T09:46:56.914659+0000","last_clean":"2023-09-30T09:46:56.914659+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:46:56.914659+0000","last_undegraded":"2023-09-30T09:46:56.914659+0000","last_fullsized":"2023-09-30T09:46:56.914659+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"519'9910","reported_seq":10259,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.156508+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:47:19.156508+0000","last_peered":"2023-09-30T09:47:19.156508+0000","last_clean":"2023-09-30T09:47:19.156508+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:47:19.156508+0000","last_undegraded":"2023-09-30T09:47:19.156508+0000","last_fullsized":"2023-09-30T09:47:19.156508+0000","mapping_epoch":492,"log_start":"501'3200","ondisk_log_start":"501'3200","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148242432,"num_objects":2262,"num_object_clones":0,"num_object_copies":6786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9910,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":938,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.918845+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:46:56.918845+0000","last_peered":"2023-09-30T09:46:56.918845+0000","last_clean":"2023-09-30T09:46:56.918845+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:46:56.918845+0000","last_undegraded":"2023-09-30T09:46:56.918845+0000","last_fullsized":"2023-09-30T09:46:56.918845+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":787,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914955+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:46:56.914955+0000","last_peered":"2023-09-30T09:46:56.914955+0000","last_clean":"2023-09-30T09:46:56.914955+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:46:56.914955+0000","last_undegraded":"2023-09-30T09:46:56.914955+0000","last_fullsized":"2023-09-30T09:46:56.914955+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":282,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.068244+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:47:09.068244+0000","last_peered":"2023-09-30T09:47:09.068244+0000","last_clean":"2023-09-30T09:47:09.068244+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:47:09.068244+0000","last_undegraded":"2023-09-30T09:47:09.068244+0000","last_fullsized":"2023-09-30T09:47:09.068244+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":368,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.912063+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:46:56.912063+0000","last_peered":"2023-09-30T09:46:56.912063+0000","last_clean":"2023-09-30T09:46:56.912063+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:46:56.912063+0000","last_undegraded":"2023-09-30T09:46:56.912063+0000","last_fullsized":"2023-09-30T09:46:56.912063+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"519'9756","reported_seq":10065,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.225384+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:47:19.225384+0000","last_peered":"2023-09-30T09:47:19.225384+0000","last_clean":"2023-09-30T09:47:19.225384+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:47:19.225384+0000","last_undegraded":"2023-09-30T09:47:19.225384+0000","last_fullsized":"2023-09-30T09:47:19.225384+0000","mapping_epoch":495,"log_start":"498'2200","ondisk_log_start":"498'2200","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148242432,"num_objects":2262,"num_object_clones":0,"num_object_copies":6786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9756,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1080,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.911064+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:46:56.911064+0000","last_peered":"2023-09-30T09:46:56.911064+0000","last_clean":"2023-09-30T09:46:56.911064+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:46:56.911064+0000","last_undegraded":"2023-09-30T09:46:56.911064+0000","last_fullsized":"2023-09-30T09:46:56.911064+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":881,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914167+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:46:56.914167+0000","last_peered":"2023-09-30T09:46:56.914167+0000","last_clean":"2023-09-30T09:46:56.914167+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:46:56.914167+0000","last_undegraded":"2023-09-30T09:46:56.914167+0000","last_fullsized":"2023-09-30T09:46:56.914167+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":95,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915805+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:46:56.915805+0000","last_peered":"2023-09-30T09:46:56.915805+0000","last_clean":"2023-09-30T09:46:56.915805+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:46:56.915805+0000","last_undegraded":"2023-09-30T09:46:56.915805+0000","last_fullsized":"2023-09-30T09:46:56.915805+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":74,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924250+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:46:56.924250+0000","last_peered":"2023-09-30T09:46:56.924250+0000","last_clean":"2023-09-30T09:46:56.924250+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:46:56.924250+0000","last_undegraded":"2023-09-30T09:46:56.924250+0000","last_fullsized":"2023-09-30T09:46:56.924250+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"519'10005","reported_seq":16718,"reported_epoch":519,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:47:18.969854+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:47:18.969854+0000","last_peered":"2023-09-30T09:47:18.969854+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:47:18.969854+0000","last_undegraded":"2023-09-30T09:47:18.969854+0000","last_fullsized":"2023-09-30T09:47:18.969854+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10005,"log_dups_size":0,"ondisk_log_size":10005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152109056,"num_objects":2321,"num_object_clones":0,"num_object_copies":6963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1649,"num_objects_unfound":0,"num_objects_dirty":2321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10005,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":672}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":780,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.923892+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:46:56.923892+0000","last_peered":"2023-09-30T09:46:56.923892+0000","last_clean":"2023-09-30T09:46:56.923892+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:46:56.923892+0000","last_undegraded":"2023-09-30T09:46:56.923892+0000","last_fullsized":"2023-09-30T09:46:56.923892+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":927,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914118+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:46:56.914118+0000","last_peered":"2023-09-30T09:46:56.914118+0000","last_clean":"2023-09-30T09:46:56.914118+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:46:56.914118+0000","last_undegraded":"2023-09-30T09:46:56.914118+0000","last_fullsized":"2023-09-30T09:46:56.914118+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":132,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914355+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:46:56.914355+0000","last_peered":"2023-09-30T09:46:56.914355+0000","last_clean":"2023-09-30T09:46:56.914355+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:46:56.914355+0000","last_undegraded":"2023-09-30T09:46:56.914355+0000","last_fullsized":"2023-09-30T09:46:56.914355+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":24,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914648+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:46:56.914648+0000","last_peered":"2023-09-30T09:46:56.914648+0000","last_clean":"2023-09-30T09:46:56.914648+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:46:56.914648+0000","last_undegraded":"2023-09-30T09:46:56.914648+0000","last_fullsized":"2023-09-30T09:46:56.914648+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"519'9776","reported_seq":9820,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:18.869245+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:47:18.869245+0000","last_peered":"2023-09-30T09:47:18.869245+0000","last_clean":"2023-09-30T09:47:18.869245+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:47:18.869245+0000","last_undegraded":"2023-09-30T09:47:18.869245+0000","last_fullsized":"2023-09-30T09:47:18.869245+0000","mapping_epoch":483,"log_start":"503'4000","ondisk_log_start":"503'4000","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5776,"log_dups_size":0,"ondisk_log_size":5776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153092096,"num_objects":2336,"num_object_clones":0,"num_object_copies":7008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9776,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":771,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924977+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:46:56.924977+0000","last_peered":"2023-09-30T09:46:56.924977+0000","last_clean":"2023-09-30T09:46:56.924977+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:46:56.924977+0000","last_undegraded":"2023-09-30T09:46:56.924977+0000","last_fullsized":"2023-09-30T09:46:56.924977+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1067,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915951+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:46:56.915951+0000","last_peered":"2023-09-30T09:46:56.915951+0000","last_clean":"2023-09-30T09:46:56.915951+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:46:56.915951+0000","last_undegraded":"2023-09-30T09:46:56.915951+0000","last_fullsized":"2023-09-30T09:46:56.915951+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":262,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914518+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:46:56.914518+0000","last_peered":"2023-09-30T09:46:56.914518+0000","last_clean":"2023-09-30T09:46:56.914518+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:46:56.914518+0000","last_undegraded":"2023-09-30T09:46:56.914518+0000","last_fullsized":"2023-09-30T09:46:56.914518+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":132,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.917183+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:46:56.917183+0000","last_peered":"2023-09-30T09:46:56.917183+0000","last_clean":"2023-09-30T09:46:56.917183+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:46:56.917183+0000","last_undegraded":"2023-09-30T09:46:56.917183+0000","last_fullsized":"2023-09-30T09:46:56.917183+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"519'9696","reported_seq":10075,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:18.825546+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:47:18.825546+0000","last_peered":"2023-09-30T09:47:18.825546+0000","last_clean":"2023-09-30T09:47:18.825546+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:47:18.825546+0000","last_undegraded":"2023-09-30T09:47:18.825546+0000","last_fullsized":"2023-09-30T09:47:18.825546+0000","mapping_epoch":495,"log_start":"503'3900","ondisk_log_start":"503'3900","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5796,"log_dups_size":0,"ondisk_log_size":5796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151584792,"num_objects":2314,"num_object_clones":0,"num_object_copies":6942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2314,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9696,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1038,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920695+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:46:56.920695+0000","last_peered":"2023-09-30T09:46:56.920695+0000","last_clean":"2023-09-30T09:46:56.920695+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:46:56.920695+0000","last_undegraded":"2023-09-30T09:46:56.920695+0000","last_fullsized":"2023-09-30T09:46:56.920695+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":550,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914806+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:46:56.914806+0000","last_peered":"2023-09-30T09:46:56.914806+0000","last_clean":"2023-09-30T09:46:56.914806+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:46:56.914806+0000","last_undegraded":"2023-09-30T09:46:56.914806+0000","last_fullsized":"2023-09-30T09:46:56.914806+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1237,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924901+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:46:56.924901+0000","last_peered":"2023-09-30T09:46:56.924901+0000","last_clean":"2023-09-30T09:46:56.924901+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:46:56.924901+0000","last_undegraded":"2023-09-30T09:46:56.924901+0000","last_fullsized":"2023-09-30T09:46:56.924901+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":554,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914087+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:46:56.914087+0000","last_peered":"2023-09-30T09:46:56.914087+0000","last_clean":"2023-09-30T09:46:56.914087+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:46:56.914087+0000","last_undegraded":"2023-09-30T09:46:56.914087+0000","last_fullsized":"2023-09-30T09:46:56.914087+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"519'9985","reported_seq":10029,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:18.786017+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:47:18.786017+0000","last_peered":"2023-09-30T09:47:18.786017+0000","last_clean":"2023-09-30T09:47:18.786017+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:47:18.786017+0000","last_undegraded":"2023-09-30T09:47:18.786017+0000","last_fullsized":"2023-09-30T09:47:18.786017+0000","mapping_epoch":483,"log_start":"502'3600","ondisk_log_start":"502'3600","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6385,"log_dups_size":0,"ondisk_log_size":6385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153550848,"num_objects":2343,"num_object_clones":0,"num_object_copies":7029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9985,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":902,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:47:03.355971+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:47:03.355971+0000","last_peered":"2023-09-30T09:47:03.355971+0000","last_clean":"2023-09-30T09:47:03.355971+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:47:03.355971+0000","last_undegraded":"2023-09-30T09:47:03.355971+0000","last_fullsized":"2023-09-30T09:47:03.355971+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":89,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.925085+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:46:56.925085+0000","last_peered":"2023-09-30T09:46:56.925085+0000","last_clean":"2023-09-30T09:46:56.925085+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:46:56.925085+0000","last_undegraded":"2023-09-30T09:46:56.925085+0000","last_fullsized":"2023-09-30T09:46:56.925085+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":95,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914847+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:46:56.914847+0000","last_peered":"2023-09-30T09:46:56.914847+0000","last_clean":"2023-09-30T09:46:56.914847+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:46:56.914847+0000","last_undegraded":"2023-09-30T09:46:56.914847+0000","last_fullsized":"2023-09-30T09:46:56.914847+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":520,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.919597+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:46:56.919597+0000","last_peered":"2023-09-30T09:46:56.919597+0000","last_clean":"2023-09-30T09:46:56.919597+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:46:56.919597+0000","last_undegraded":"2023-09-30T09:46:56.919597+0000","last_fullsized":"2023-09-30T09:46:56.919597+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":838,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914079+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:46:56.914079+0000","last_peered":"2023-09-30T09:46:56.914079+0000","last_clean":"2023-09-30T09:46:56.914079+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:46:56.914079+0000","last_undegraded":"2023-09-30T09:46:56.914079+0000","last_fullsized":"2023-09-30T09:46:56.914079+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":24,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.917113+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:46:56.917113+0000","last_peered":"2023-09-30T09:46:56.917113+0000","last_clean":"2023-09-30T09:46:56.917113+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:46:56.917113+0000","last_undegraded":"2023-09-30T09:46:56.917113+0000","last_fullsized":"2023-09-30T09:46:56.917113+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":224,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914455+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:46:56.914455+0000","last_peered":"2023-09-30T09:46:56.914455+0000","last_clean":"2023-09-30T09:46:56.914455+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:46:56.914455+0000","last_undegraded":"2023-09-30T09:46:56.914455+0000","last_fullsized":"2023-09-30T09:46:56.914455+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":633,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.925057+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:46:56.925057+0000","last_peered":"2023-09-30T09:46:56.925057+0000","last_clean":"2023-09-30T09:46:56.925057+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:46:56.925057+0000","last_undegraded":"2023-09-30T09:46:56.925057+0000","last_fullsized":"2023-09-30T09:46:56.925057+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":447,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.910839+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:46:56.910839+0000","last_peered":"2023-09-30T09:46:56.910839+0000","last_clean":"2023-09-30T09:46:56.910839+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:46:56.910839+0000","last_undegraded":"2023-09-30T09:46:56.910839+0000","last_fullsized":"2023-09-30T09:46:56.910839+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":233,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920241+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:46:56.920241+0000","last_peered":"2023-09-30T09:46:56.920241+0000","last_clean":"2023-09-30T09:46:56.920241+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:46:56.920241+0000","last_undegraded":"2023-09-30T09:46:56.920241+0000","last_fullsized":"2023-09-30T09:46:56.920241+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":389,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.911925+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:46:56.911925+0000","last_peered":"2023-09-30T09:46:56.911925+0000","last_clean":"2023-09-30T09:46:56.911925+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:46:56.911925+0000","last_undegraded":"2023-09-30T09:46:56.911925+0000","last_fullsized":"2023-09-30T09:46:56.911925+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":253,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.919814+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:46:56.919814+0000","last_peered":"2023-09-30T09:46:56.919814+0000","last_clean":"2023-09-30T09:46:56.919814+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:46:56.919814+0000","last_undegraded":"2023-09-30T09:46:56.919814+0000","last_fullsized":"2023-09-30T09:46:56.919814+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":846,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914487+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:46:56.914487+0000","last_peered":"2023-09-30T09:46:56.914487+0000","last_clean":"2023-09-30T09:46:56.914487+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:46:56.914487+0000","last_undegraded":"2023-09-30T09:46:56.914487+0000","last_fullsized":"2023-09-30T09:46:56.914487+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":76,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.924822+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:46:56.924822+0000","last_peered":"2023-09-30T09:46:56.924822+0000","last_clean":"2023-09-30T09:46:56.924822+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:46:56.924822+0000","last_undegraded":"2023-09-30T09:46:56.924822+0000","last_fullsized":"2023-09-30T09:46:56.924822+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":470,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.914874+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:46:56.914874+0000","last_peered":"2023-09-30T09:46:56.914874+0000","last_clean":"2023-09-30T09:46:56.914874+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:46:56.914874+0000","last_undegraded":"2023-09-30T09:46:56.914874+0000","last_fullsized":"2023-09-30T09:46:56.914874+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":889,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.920560+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:46:56.920560+0000","last_peered":"2023-09-30T09:46:56.920560+0000","last_clean":"2023-09-30T09:46:56.920560+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:46:56.920560+0000","last_undegraded":"2023-09-30T09:46:56.920560+0000","last_fullsized":"2023-09-30T09:46:56.920560+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":30,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.915085+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:46:56.915085+0000","last_peered":"2023-09-30T09:46:56.915085+0000","last_clean":"2023-09-30T09:46:56.915085+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:46:56.915085+0000","last_undegraded":"2023-09-30T09:46:56.915085+0000","last_fullsized":"2023-09-30T09:46:56.915085+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":534,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-30T09:46:56.919411+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:46:56.919411+0000","last_peered":"2023-09-30T09:46:56.919411+0000","last_clean":"2023-09-30T09:46:56.919411+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:46:56.919411+0000","last_undegraded":"2023-09-30T09:46:56.919411+0000","last_fullsized":"2023-09-30T09:46:56.919411+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2396061720,"num_objects":36562,"num_object_clones":0,"num_object_copies":109686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1649,"num_objects_unfound":0,"num_objects_dirty":36562,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":156924,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4464,"num_bytes_recovered":292552704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7351840768,"data_stored":7351828552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116324,"ondisk_log_size":116324,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234427,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1884176,"kb_used_data":1342856,"kb_used_omap":0,"kb_used_meta":541312,"kb_avail":91860976,"statfs":{"total":95995035648,"available":94065639424,"internally_reserved":0,"allocated":1375084544,"data_stored":1373344705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":554303488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":8.9299999999999997,"5min":7.6980000000000004,"15min":5.0990000000000002},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":95.825999999999993,"5min":124.74299999999999,"15min":215.416},"last":3.8460000000000001},{"interface":"front","average":{"1min":8.9190000000000005,"5min":7.7690000000000001,"15min":5.0069999999999997},"min":{"1min":0.26100000000000001,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":95.488,"5min":124.69499999999999,"15min":215.041},"last":3.294}]},{"osd":1,"last update":"Sat Sep 30 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":7.4539999999999997,"15min":4.117},"min":{"1min":0.40799999999999997,"5min":0.25,"15min":0.25},"max":{"1min":7.6559999999999997,"5min":157.578,"15min":157.578},"last":9.2449999999999992},{"interface":"front","average":{"1min":2.649,"5min":7.2839999999999998,"15min":4.2679999999999998},"min":{"1min":0.41299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.8800000000000008,"5min":157.589,"15min":157.589},"last":9.2249999999999996}]},{"osd":2,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":2.9809999999999999,"15min":3.5510000000000002},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.24299999999999999},"max":{"1min":10.292999999999999,"5min":104.88500000000001,"15min":117.14},"last":3.5529999999999999},{"interface":"front","average":{"1min":1.9430000000000001,"5min":3.1629999999999998,"15min":3.5409999999999999},"min":{"1min":0.29599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.4420000000000002,"5min":104.867,"15min":117.044},"last":16.135000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":7.8529999999999998,"5min":8.1449999999999996,"15min":8.6750000000000007},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":97.037000000000006,"5min":154.06700000000001,"15min":277.57799999999997},"last":3.8929999999999998},{"interface":"front","average":{"1min":8.9039999999999999,"5min":8.4440000000000008,"15min":8.9420000000000002},"min":{"1min":0.42999999999999999,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":95.135000000000005,"5min":154.24799999999999,"15min":278.69299999999998},"last":16.190999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.1399999999999999,"15min":1.6259999999999999},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":7.5499999999999998,"5min":25.192,"15min":175.72},"last":3.194},{"interface":"front","average":{"1min":1.214,"5min":1.0660000000000001,"15min":1.536},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.252},"max":{"1min":5.6120000000000001,"5min":24.521000000000001,"15min":179.048},"last":3.3100000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":9.875,"15min":11.287000000000001},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.641,"5min":197.75700000000001,"15min":197.75700000000001},"last":3.266},{"interface":"front","average":{"1min":1.8919999999999999,"5min":9.8399999999999999,"15min":11.164},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.7149999999999999,"5min":197.63300000000001,"15min":197.63300000000001},"last":2.3450000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.72299999999999998,"15min":2.1760000000000002},"min":{"1min":0.32900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.202,"5min":6.202,"15min":206.244},"last":3.2250000000000001},{"interface":"front","average":{"1min":1.069,"5min":0.84399999999999997,"15min":2.1850000000000001},"min":{"1min":0.34699999999999998,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":5.5800000000000001,"5min":5.8520000000000003,"15min":202.78800000000001},"last":3.9420000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496065,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":969144,"kb_used_data":446704,"kb_used_omap":0,"kb_used_meta":522432,"kb_avail":92776008,"statfs":{"total":95995035648,"available":95002632192,"internally_reserved":0,"allocated":457424896,"data_stored":455708350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":534970368},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":4.1050000000000004,"15min":5.3330000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":11.609,"5min":95.043999999999997,"15min":187.19800000000001},"last":3.6030000000000002},{"interface":"front","average":{"1min":2.1600000000000001,"5min":4.1210000000000004,"15min":5.3520000000000003},"min":{"1min":0.26300000000000001,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":6.5460000000000003,"5min":94.855999999999995,"15min":187.46700000000001},"last":5.3310000000000004}]},{"osd":1,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.446,"15min":1.956},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.6429999999999998,"5min":32.814,"15min":69.569999999999993},"last":3.7599999999999998},{"interface":"front","average":{"1min":2.5529999999999999,"5min":1.5629999999999999,"15min":2.173},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":6.3760000000000003,"5min":32.856000000000002,"15min":70.471999999999994},"last":5.3730000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":3.9239999999999999,"5min":3.5880000000000001,"15min":4.0069999999999997},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":10.581,"5min":207.02500000000001,"15min":207.02500000000001},"last":5.7400000000000002},{"interface":"front","average":{"1min":3.3559999999999999,"5min":3.6909999999999998,"15min":4.1539999999999999},"min":{"1min":0.37,"5min":0.33500000000000002,"15min":0.28199999999999997},"max":{"1min":16.474,"5min":207.114,"15min":207.114},"last":0.57799999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8839999999999999,"5min":3.484,"15min":4.2560000000000002},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":11.644,"5min":107.11799999999999,"15min":212.44800000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.3239999999999998,"5min":3.4159999999999999,"15min":4.4909999999999997},"min":{"1min":0.35099999999999998,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":8.8559999999999999,"5min":106.982,"15min":210.84299999999999},"last":5.5650000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.722,"15min":1.2669999999999999},"min":{"1min":0.30199999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.4400000000000004,"5min":63.798000000000002,"15min":63.798000000000002},"last":5.4029999999999996},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.8400000000000001,"15min":1.2330000000000001},"min":{"1min":0.32900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.558999999999999,"5min":63.133000000000003,"15min":63.133000000000003},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8740000000000001,"15min":1.99},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.6840000000000002,"5min":14.974,"15min":14.974},"last":5.7030000000000003},{"interface":"front","average":{"1min":0.88400000000000001,"5min":2.2789999999999999,"15min":2.5110000000000001},"min":{"1min":0.37,"5min":0.316,"15min":0.316},"max":{"1min":3.6190000000000002,"5min":16.443000000000001,"15min":16.443000000000001},"last":2.3849999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":0.89200000000000002,"15min":1.609},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.2679999999999998,"5min":11.276999999999999,"15min":138.191},"last":5.4349999999999996},{"interface":"front","average":{"1min":1.282,"5min":0.89300000000000002,"15min":1.4410000000000001},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7720000000000002,"5min":10.909000000000001,"15min":140.33500000000001},"last":1.347}]}]},{"osd":1,"up_from":301,"seq":1292785156262,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2035360,"kb_used_data":1385816,"kb_used_omap":9,"kb_used_meta":649526,"kb_avail":91709792,"statfs":{"total":95995035648,"available":93910827008,"internally_reserved":0,"allocated":1419075584,"data_stored":1416122164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":665114642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":4.319,"15min":3.7320000000000002},"min":{"1min":0.221,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":26.021000000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":6.1589999999999998},{"interface":"front","average":{"1min":3.7639999999999998,"5min":4.1159999999999997,"15min":3.6240000000000001},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":25.971,"5min":142.137,"15min":142.137},"last":5.8150000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.883,"15min":2.573},"min":{"1min":0.27700000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.763000000000002,"5min":62.572000000000003,"15min":62.572000000000003},"last":4.7649999999999997},{"interface":"front","average":{"1min":2.9809999999999999,"5min":2.8700000000000001,"15min":2.7999999999999998},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.138999999999999,"5min":68.373999999999995,"15min":68.373999999999995},"last":2.5819999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.484,"5min":5.9690000000000003,"15min":6.0899999999999999},"min":{"1min":0.30499999999999999,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":9.8610000000000007,"5min":199.41,"15min":199.41},"last":2.6309999999999998},{"interface":"front","average":{"1min":3.149,"5min":6.2999999999999998,"15min":6.1509999999999998},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.4329999999999998,"5min":199.58099999999999,"15min":199.58099999999999},"last":1.319}]},{"osd":4,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":1.091,"15min":2.1309999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":8.9399999999999995,"5min":8.9399999999999995,"15min":142.77099999999999},"last":2.2759999999999998},{"interface":"front","average":{"1min":2.46,"5min":1.52,"15min":2.4140000000000001},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.6319999999999997,"5min":8.6319999999999997,"15min":144.898},"last":2.5030000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.0190000000000001,"15min":1.962},"min":{"1min":0.40500000000000003,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.2100000000000009,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":2.8359999999999999},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.345,"15min":1.2629999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":15.250999999999999,"5min":15.250999999999999,"15min":15.250999999999999},"last":2.8530000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.552,"15min":5.5679999999999996},"min":{"1min":0.442,"5min":0.316,"15min":0.29899999999999999},"max":{"1min":8.8559999999999999,"5min":30.986999999999998,"15min":209.221},"last":5.7469999999999999},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.4850000000000001,"15min":5.4960000000000004},"min":{"1min":0.47899999999999998,"5min":0.35399999999999998,"15min":0.33900000000000002},"max":{"1min":8.8100000000000005,"5min":28.710999999999999,"15min":208.43600000000001},"last":5.7880000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.5449999999999999,"15min":3.9500000000000002},"min":{"1min":0.36299999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":9.3879999999999999,"5min":9.3879999999999999,"15min":165.59399999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.077,"5min":1.542,"15min":3.9929999999999999},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":4.2549999999999999,"5min":9.3399999999999999,"15min":165.833},"last":4.7220000000000004}]}]},{"osd":0,"up_from":94,"seq":403726926161,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1145388,"kb_used_data":531108,"kb_used_omap":5,"kb_used_meta":614266,"kb_avail":92599764,"statfs":{"total":95995035648,"available":94822158336,"internally_reserved":0,"allocated":543854592,"data_stored":540660402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":629008637},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,7],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.9419999999999999,"15min":2.181},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":7.2380000000000004,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.927,"5min":2.3180000000000001,"15min":4.0199999999999996},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.344000000000001,"5min":57.140999999999998,"15min":138.92400000000001},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6640000000000001,"5min":4.8010000000000002,"15min":3.2829999999999999},"min":{"1min":0.28899999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":31.521000000000001,"5min":127.73399999999999,"15min":127.73399999999999},"last":0.376},{"interface":"front","average":{"1min":4.7649999999999997,"5min":5.1299999999999999,"15min":3.5470000000000002},"min":{"1min":0.38,"5min":0.27600000000000002,"15min":0.26200000000000001},"max":{"1min":31.478999999999999,"5min":127.69,"15min":127.69},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":5.5099999999999998,"5min":5.9870000000000001,"15min":5.4249999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":16.594000000000001,"5min":125.562,"15min":218.06999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":3.0680000000000001,"5min":4.984,"15min":4.2599999999999998},"min":{"1min":0.39600000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":13.361000000000001,"5min":117.295,"15min":218.63200000000001},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.7989999999999999,"15min":2.012},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.8710000000000004,"5min":72.403999999999996,"15min":72.403999999999996},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1960000000000002,"15min":2.8260000000000001},"min":{"1min":0.38600000000000001,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":9.4870000000000001,"5min":72.162999999999997,"15min":136.262},"last":0.59799999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":4.0060000000000002,"5min":1.6579999999999999,"15min":1.2669999999999999},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":11.326000000000001,"5min":11.326000000000001,"15min":11.326000000000001},"last":13.162000000000001},{"interface":"front","average":{"1min":1.4470000000000001,"5min":1.1279999999999999,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.8200000000000003,"5min":6.8200000000000003,"15min":6.8200000000000003},"last":13.422000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.958,"15min":3.508},"min":{"1min":0.38200000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":20.706,"5min":57.384,"15min":214.785},"last":20.706},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.3700000000000001,"15min":3.8370000000000002},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.684000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":20.684000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.8120000000000001,"15min":2.0720000000000001},"min":{"1min":0.42499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.6549999999999994,"5min":57.776000000000003,"15min":97.382000000000005},"last":9.6549999999999994},{"interface":"front","average":{"1min":2.6349999999999998,"5min":2.363,"15min":2.3940000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":9.7149999999999999,"5min":57.179000000000002,"15min":97.015000000000001},"last":9.6780000000000008}]}]},{"osd":2,"up_from":104,"seq":446676599102,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1399352,"kb_used_data":597104,"kb_used_omap":5,"kb_used_meta":802234,"kb_avail":92345800,"statfs":{"total":95995035648,"available":94562099200,"internally_reserved":0,"allocated":611434496,"data_stored":608516696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":821488269},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":4.0890000000000004,"5min":5.4829999999999997,"15min":3.5800000000000001},"min":{"1min":0.219,"5min":0.192,"15min":0.187},"max":{"1min":13.880000000000001,"5min":170.678,"15min":170.678},"last":0.35899999999999999},{"interface":"front","average":{"1min":5.1459999999999999,"5min":5.5549999999999997,"15min":3.6320000000000001},"min":{"1min":0.32100000000000001,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":12.609999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":4.8810000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:46:52 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":4.8449999999999998,"15min":3.7919999999999998},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":10.922000000000001,"5min":162.696,"15min":162.696},"last":2.8210000000000002},{"interface":"front","average":{"1min":4.665,"5min":5.0709999999999997,"15min":3.6320000000000001},"min":{"1min":0.27700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.705,"5min":162.93899999999999,"15min":162.93899999999999},"last":0.21099999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:47:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":6.0049999999999999,"15min":5.109},"min":{"1min":0.311,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.6609999999999996,"5min":156.39400000000001,"15min":156.39400000000001},"last":4.2240000000000002},{"interface":"front","average":{"1min":4.1980000000000004,"5min":5.9400000000000004,"15min":4.9740000000000002},"min":{"1min":0.38500000000000001,"5min":0.224,"15min":0.224},"max":{"1min":14.169,"5min":156.376,"15min":156.376},"last":4.3120000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":1.8120000000000001,"15min":1.881},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1270000000000007,"5min":47.463000000000001,"15min":83.012},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.8520000000000001,"15min":1.879},"min":{"1min":0.42099999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.997999999999999,"5min":47.412999999999997,"15min":81.335999999999999},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":5.2030000000000003,"5min":1.901,"15min":1.351},"min":{"1min":0.378,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":47.226999999999997,"5min":47.226999999999997,"15min":47.226999999999997},"last":4.2599999999999998},{"interface":"front","average":{"1min":5.1379999999999999,"5min":2.0800000000000001,"15min":1.5700000000000001},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":47.127000000000002,"5min":47.127000000000002,"15min":47.127000000000002},"last":2.8650000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.4199999999999999,"15min":3.2309999999999999},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.026999999999999,"5min":11.026999999999999,"15min":154.417},"last":0.57599999999999996},{"interface":"front","average":{"1min":3.1880000000000002,"5min":1.8140000000000001,"15min":3.6379999999999999},"min":{"1min":0.46100000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.846,"5min":10.846,"15min":154.04400000000001},"last":2.8919999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":2.1240000000000001,"15min":2.4089999999999998},"min":{"1min":0.48199999999999998,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":10.928000000000001,"5min":49.816000000000003,"15min":123.294},"last":4.3419999999999996},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.3889999999999998,"15min":2.7429999999999999},"min":{"1min":0.436,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":10.885,"5min":50.956000000000003,"15min":123.16},"last":2.9199999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435575,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2055144,"kb_used_data":1200672,"kb_used_omap":2,"kb_used_meta":854269,"kb_avail":91690008,"statfs":{"total":95995035648,"available":93890568192,"internally_reserved":0,"allocated":1229488128,"data_stored":1226465549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":874771815},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0190000000000001,"5min":4.3949999999999996,"15min":4.7249999999999996},"min":{"1min":0.20000000000000001,"5min":0.183,"15min":0.183},"max":{"1min":34.220999999999997,"5min":144.72,"15min":205.73099999999999},"last":0.23000000000000001},{"interface":"front","average":{"1min":4.2670000000000003,"5min":4.6139999999999999,"15min":4.6580000000000004},"min":{"1min":0.28999999999999998,"5min":0.23999999999999999,"15min":0.21199999999999999},"max":{"1min":34.253,"5min":142.36600000000001,"15min":205.495},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:46:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0869999999999997,"5min":3.7599999999999998,"15min":7.4470000000000001},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":19.059000000000001,"5min":142.434,"15min":273.99599999999998},"last":0.64400000000000002},{"interface":"front","average":{"1min":4.1550000000000002,"5min":3.7879999999999998,"15min":7.1299999999999999},"min":{"1min":0.28199999999999997,"5min":0.26800000000000002,"15min":0.20699999999999999},"max":{"1min":13.183,"5min":142.44900000000001,"15min":274.21600000000001},"last":0.627}]},{"osd":2,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":4.851,"15min":5.6269999999999998},"min":{"1min":0.36699999999999999,"5min":0.249,"15min":0.249},"max":{"1min":8.0389999999999997,"5min":146.92599999999999,"15min":206.69},"last":3.1200000000000001},{"interface":"front","average":{"1min":2.0390000000000001,"5min":4.508,"15min":5.4370000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.9649999999999999,"5min":145.75299999999999,"15min":209.22800000000001},"last":0.432}]},{"osd":4,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.976,"15min":2.0649999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.141,"5min":81.677000000000007,"15min":81.677000000000007},"last":2.5910000000000002},{"interface":"front","average":{"1min":2.3170000000000002,"5min":2.2669999999999999,"15min":1.9330000000000001},"min":{"1min":0.35399999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0540000000000003,"5min":80.623000000000005,"15min":80.623000000000005},"last":1.7130000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":1.732,"15min":1.595},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.701000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":1.577},{"interface":"front","average":{"1min":2.2839999999999998,"5min":1.218,"15min":1.04},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":1.5609999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.069,"15min":1.901},"min":{"1min":0.38800000000000001,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":5.202,"5min":5.915,"15min":60.698},"last":4.1230000000000002},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.4219999999999999,"15min":1.911},"min":{"1min":0.41899999999999998,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":6.2290000000000001,"5min":10.356999999999999,"15min":12.121},"last":4.141}]},{"osd":7,"last update":"Sat Sep 30 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":3.0569999999999999,"15min":2.601},"min":{"1min":0.47199999999999998,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":7.968,"5min":213.548,"15min":213.548},"last":1.05},{"interface":"front","average":{"1min":1.3839999999999999,"5min":2.8109999999999999,"15min":2.5129999999999999},"min":{"1min":0.438,"5min":0.41099999999999998,"15min":0.34799999999999998},"max":{"1min":5.1719999999999997,"5min":213.583,"15min":213.583},"last":1.0620000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019338,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":954044,"kb_used_data":459764,"kb_used_omap":0,"kb_used_meta":494272,"kb_avail":92791108,"statfs":{"total":95995035648,"available":95018094592,"internally_reserved":0,"allocated":470798336,"data_stored":469077809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":506134528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":2.7090000000000001,"15min":2.5640000000000001},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":8.7989999999999995,"5min":73.718000000000004,"15min":105.179},"last":1.1259999999999999},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.7589999999999999,"15min":2.75},"min":{"1min":0.32000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.032,"5min":74.001999999999995,"15min":104.70999999999999},"last":2.2629999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.5779999999999998,"15min":3.871},"min":{"1min":0.313,"5min":0.25,"15min":0.25},"max":{"1min":8.8620000000000001,"5min":80.712000000000003,"15min":160.69999999999999},"last":3.4510000000000001},{"interface":"front","average":{"1min":3.589,"5min":2.8490000000000002,"15min":3.8940000000000001},"min":{"1min":0.438,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":15.345000000000001,"5min":80.810000000000002,"15min":160.50700000000001},"last":3.621}]},{"osd":2,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3069999999999999,"5min":4.9989999999999997,"15min":3.492},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":13.26,"5min":142.88999999999999,"15min":142.88999999999999},"last":1.2669999999999999},{"interface":"front","average":{"1min":3.4159999999999999,"5min":5.069,"15min":3.5249999999999999},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":8.8870000000000005,"5min":142.90700000000001,"15min":142.90700000000001},"last":1.2410000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.0369999999999999,"15min":2.1800000000000002},"min":{"1min":0.35799999999999998,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":9.0600000000000005,"5min":29.41,"15min":52.762},"last":3.165},{"interface":"front","average":{"1min":3.6920000000000002,"5min":2.0099999999999998,"15min":2.1890000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.29299999999999998},"max":{"1min":9.0709999999999997,"5min":29.401,"15min":52.790999999999997},"last":3.181}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":8.9930000000000003,"15min":10.153},"min":{"1min":0.32800000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.133,"5min":140.11099999999999,"15min":140.11099999999999},"last":1.5449999999999999},{"interface":"front","average":{"1min":1.5029999999999999,"5min":8.9130000000000003,"15min":10.148},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0350000000000001,"5min":140.065,"15min":140.065},"last":0.435}]},{"osd":6,"last update":"Sat Sep 30 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.63100000000000001,"15min":2.1539999999999999},"min":{"1min":0.33900000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":2.7970000000000002,"5min":4.4299999999999997,"15min":202.72499999999999},"last":1.161},{"interface":"front","average":{"1min":1.1259999999999999,"5min":0.71899999999999997,"15min":2.165},"min":{"1min":0.33400000000000002,"5min":0.253,"15min":0.249},"max":{"1min":7.173,"5min":7.173,"15min":202.42400000000001},"last":0.39300000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":2.5099999999999998,"15min":3.2000000000000002},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.771000000000001,"5min":161.72999999999999,"15min":230.90100000000001},"last":1.5569999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":2.4820000000000002,"15min":3.0569999999999999},"min":{"1min":0.41999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.2970000000000006,"5min":162.60900000000001,"15min":229.97},"last":1.4870000000000001}]}]},{"osd":5,"up_from":489,"seq":2100239007780,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1459196,"kb_used_data":1315124,"kb_used_omap":6,"kb_used_meta":143993,"kb_avail":92285956,"statfs":{"total":95995035648,"available":94500818944,"internally_reserved":0,"allocated":1346686976,"data_stored":1344962999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":147448927},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3500000000000001,"5min":21.718,"15min":24.779},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":25.786999999999999,"5min":281.10599999999999,"15min":281.10599999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":3.851,"5min":22.048999999999999,"15min":25.082000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":25.663,"5min":282.07400000000001,"15min":282.07400000000001},"last":0.64000000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":12.692,"15min":14.177},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":24.478999999999999,"5min":273.48200000000003,"15min":273.48200000000003},"last":1.619},{"interface":"front","average":{"1min":4.3319999999999999,"5min":12.15,"15min":13.452999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":24.359000000000002,"5min":273.50200000000001,"15min":273.50200000000001},"last":0.36099999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":11.537000000000001,"5min":3.0600000000000001,"15min":1.647},"min":{"1min":0.40899999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":163.23099999999999,"5min":163.23099999999999,"15min":163.23099999999999},"last":2.1070000000000002},{"interface":"front","average":{"1min":11.898999999999999,"5min":3.6120000000000001,"15min":2.2309999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":162.71100000000001,"5min":162.71100000000001,"15min":162.71100000000001},"last":0.98799999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.113,"5min":12.420999999999999,"15min":13.973000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.271000000000001,"5min":274.01799999999997,"15min":274.01799999999997},"last":7.3179999999999996},{"interface":"front","average":{"1min":3.2709999999999999,"5min":2.0390000000000001,"15min":1.8340000000000001},"min":{"1min":0.38400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":13.259,"5min":13.259,"15min":13.259},"last":9.2469999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.88500000000000001,"15min":0.92900000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8769999999999998,"5min":4.1769999999999996,"15min":4.1769999999999996},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.262,"5min":1.169,"15min":1.153},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.7610000000000001,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.5129999999999999,"15min":1.601},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.758,"5min":14.371,"15min":14.371},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.069,"5min":1.325,"15min":1.3680000000000001},"min":{"1min":0.379,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.7359999999999998,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.238,"15min":1.2749999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.1150000000000002,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.90400000000000003,"15min":0.91200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.0680000000000001,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.48899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":529989632,"data_stored":529989632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1401098240,"data_stored":1401094168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":598278144,"data_stored":598278144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1217789952,"data_stored":1217789952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":458424320,"data_stored":458424320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1339035648,"data_stored":1339031576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":450822144,"data_stored":450822144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1356402688,"data_stored":1356398616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:47:24.892 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:24.931 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:25.088 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:25.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:25.194 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:25.240 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:25.318 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:25.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:25.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:25.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:25 smithi019 ceph-mon[216815]: pgmap v1373: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1649/109770 objects misplaced (1.502%); 2.4 MiB/s, 39 objects/s recovering 2023-09-30T09:47:25.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:25 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:25.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:25 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:25.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:25 smithi019 ceph-mon[224043]: pgmap v1373: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1649/109770 objects misplaced (1.502%); 2.4 MiB/s, 39 objects/s recovering 2023-09-30T09:47:25.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:25 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:25.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:25 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:25.422 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:25.454 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:25 smithi167 ceph-mon[192880]: pgmap v1373: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1649/109770 objects misplaced (1.502%); 2.4 MiB/s, 39 objects/s recovering 2023-09-30T09:47:25.454 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:25 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:25.454 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:25 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:25.491 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:25.530 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:47:25.533 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:25.577 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:47:25.630 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:47:25.680 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:47:25.712 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:47:25.742 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:47:25.743 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:47:25.743 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:47:25.743 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:47:25.766 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:47:25.804 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:47:25.854 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:47:25.859 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:25.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:25.873 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:47:25.873 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:47:25.873 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:47:25.873 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:47:25.873 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:47:25.873 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:47:25.873 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:47:25.873 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:47:25.972 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:26.022 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:26.097 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:26.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:26.201 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:26.266 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:26.504 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:26.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:26.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:26.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:26 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:47:26.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:26 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:47:26.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:26 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:26.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/91745671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:26.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:26 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:47:26.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:26 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:47:26.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:26 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:26.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/91745671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:26.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:26 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:47:26.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:26 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:47:26.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:26 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:47:26.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/91745671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:26.612 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:26.645 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:47:26.703 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:26.765 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:26.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:26.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:26.870 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:26.895 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:26.970 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:26.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:26.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:27.085 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:27.153 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:27.259 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:27.260 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:27.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:27.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:27.375 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:27.440 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:27.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:27 smithi167 ceph-mon[192880]: pgmap v1374: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 1413/100497 objects misplaced (1.406%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:27 smithi019 ceph-mon[224043]: pgmap v1374: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 1413/100497 objects misplaced (1.406%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:27 smithi019 ceph-mon[216815]: pgmap v1374: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 1413/100497 objects misplaced (1.406%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:27.810 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:47:28.145 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:28.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:28.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:28.267 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:28.301 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:28.336 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:28.435 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:28.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:28.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:28.538 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:28.658 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:29.242 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:29.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:29.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:29.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:29 smithi019 ceph-mon[216815]: pgmap v1375: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 1413/100497 objects misplaced (1.406%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:29.349 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:29.418 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:47:29.419 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:47:29.440 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:29 smithi167 ceph-mon[192880]: pgmap v1375: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 1413/100497 objects misplaced (1.406%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:29.447 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:29.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:29 smithi019 ceph-mon[224043]: pgmap v1375: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 1413/100497 objects misplaced (1.406%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:29.588 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:29.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:29.695 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:29.740 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:29.801 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:29.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:29.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:29.905 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:29.936 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:30.156 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:47:30.306 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:47:30.310 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:30.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:30.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:30.370 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:47:30.415 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:30.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1211878665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:47:30.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:47:30.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1211878665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:47:30.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:47:30.531 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:30.711 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:30.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1211878665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:47:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:47:30.817 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:30.878 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:30.880 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:31.000 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:31.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:31.107 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:31.159 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:31.266 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:31.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:31.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:31.372 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:31.438 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: pgmap v1376: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:31.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:31.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:47:31.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: osdmap e521: 8 total, 8 up, 8 in 2023-09-30T09:47:31.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.439 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:31 smithi167 ceph-mon[192880]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.458 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:31.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: pgmap v1376: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:31.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:31.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:47:31.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: osdmap e521: 8 total, 8 up, 8 in 2023-09-30T09:47:31.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[216815]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: pgmap v1376: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: osdmap e521: 8 total, 8 up, 8 in 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:31 smithi019 ceph-mon[224043]: failed to encode map e521 with expected crc 2023-09-30T09:47:31.638 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:31.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:31.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:31.755 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:31.782 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:31.824 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:31.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:31.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:31.857 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:31.927 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:31.960 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:32.012 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:32.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:32.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:32.117 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:32.148 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:32.237 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:47:32.254 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:47:32.393 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:32.397 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:32 smithi167 ceph-mon[192880]: pgmap v1378: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:32.397 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:32 smithi167 ceph-mon[192880]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:32.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:32.499 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:32.558 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:32 smithi019 ceph-mon[216815]: pgmap v1378: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:32.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:32 smithi019 ceph-mon[216815]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:32.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:32 smithi019 ceph-mon[224043]: pgmap v1378: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:32.559 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:32 smithi019 ceph-mon[224043]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:32.566 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:32.951 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:32.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:33.057 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:33.093 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:47:33.093 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:47:33.114 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:33.158 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:33.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:33.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:33.261 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:33.295 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:33.423 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:33.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:33.460 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:47:33.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:33.535 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:33.565 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:33.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:33.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:33.609 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:33.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:33.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:33.716 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:33.742 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:33.954 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:33.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:33.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:33.993 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:47:34.070 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:34.088 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:47:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:47:33] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-09-30T09:47:34.124 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:34.409 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:34.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:34.521 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1378,"stamp":"2023-09-30T09:47:31.862484+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1994201023,"num_objects":30385,"num_object_clones":0,"num_object_copies":91155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1192,"num_objects_unfound":0,"num_objects_dirty":30385,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":164036,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5201,"num_bytes_recovered":335107698,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123351,"ondisk_log_size":123351,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10724504,"kb_used_data":6067416,"kb_used_omap":30,"kb_used_meta":4656737,"kb_avail":739236712,"statfs":{"total":767960285184,"available":756978393088,"internally_reserved":0,"allocated":6213033984,"data_stored":6194001225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4768499042},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-406650880,"num_objects":-6205,"num_object_clones":0,"num_object_copies":-18615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-457,"num_objects_unfound":0,"num_objects_dirty":-6205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6205,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":457,"num_bytes_recovered":29949952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008172"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":943,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047148+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:47:09.047148+0000","last_peered":"2023-09-30T09:47:09.047148+0000","last_clean":"2023-09-30T09:47:09.047148+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:47:09.047148+0000","last_undegraded":"2023-09-30T09:47:09.047148+0000","last_fullsized":"2023-09-30T09:47:09.047148+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":538,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037074+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:47:09.037074+0000","last_peered":"2023-09-30T09:47:09.037074+0000","last_clean":"2023-09-30T09:47:09.037074+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:47:09.037074+0000","last_undegraded":"2023-09-30T09:47:09.037074+0000","last_fullsized":"2023-09-30T09:47:09.037074+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":435,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.040688+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:47:09.040688+0000","last_peered":"2023-09-30T09:47:09.040688+0000","last_clean":"2023-09-30T09:47:09.040688+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:47:09.040688+0000","last_undegraded":"2023-09-30T09:47:09.040688+0000","last_fullsized":"2023-09-30T09:47:09.040688+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":945,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.046824+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:47:09.046824+0000","last_peered":"2023-09-30T09:47:09.046824+0000","last_clean":"2023-09-30T09:47:09.046824+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:47:09.046824+0000","last_undegraded":"2023-09-30T09:47:09.046824+0000","last_fullsized":"2023-09-30T09:47:09.046824+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.034846+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:47:09.034846+0000","last_peered":"2023-09-30T09:47:09.034846+0000","last_clean":"2023-09-30T09:47:09.034846+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:47:09.034846+0000","last_undegraded":"2023-09-30T09:47:09.034846+0000","last_fullsized":"2023-09-30T09:47:09.034846+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":126,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.048347+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:47:09.048347+0000","last_peered":"2023-09-30T09:47:09.048347+0000","last_clean":"2023-09-30T09:47:09.048347+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:47:09.048347+0000","last_undegraded":"2023-09-30T09:47:09.048347+0000","last_fullsized":"2023-09-30T09:47:09.048347+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":753,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037134+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:47:09.037134+0000","last_peered":"2023-09-30T09:47:09.037134+0000","last_clean":"2023-09-30T09:47:09.037134+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:47:09.037134+0000","last_undegraded":"2023-09-30T09:47:09.037134+0000","last_fullsized":"2023-09-30T09:47:09.037134+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":25,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047522+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:47:09.047522+0000","last_peered":"2023-09-30T09:47:09.047522+0000","last_clean":"2023-09-30T09:47:09.047522+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:47:09.047522+0000","last_undegraded":"2023-09-30T09:47:09.047522+0000","last_fullsized":"2023-09-30T09:47:09.047522+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":89,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052780+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:47:09.052780+0000","last_peered":"2023-09-30T09:47:09.052780+0000","last_clean":"2023-09-30T09:47:09.052780+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:47:09.052780+0000","last_undegraded":"2023-09-30T09:47:09.052780+0000","last_fullsized":"2023-09-30T09:47:09.052780+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":921,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.056008+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:47:09.056008+0000","last_peered":"2023-09-30T09:47:09.056008+0000","last_clean":"2023-09-30T09:47:09.056008+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:47:09.056008+0000","last_undegraded":"2023-09-30T09:47:09.056008+0000","last_fullsized":"2023-09-30T09:47:09.056008+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":140,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052323+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:47:09.052323+0000","last_peered":"2023-09-30T09:47:09.052323+0000","last_clean":"2023-09-30T09:47:09.052323+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:47:09.052323+0000","last_undegraded":"2023-09-30T09:47:09.052323+0000","last_fullsized":"2023-09-30T09:47:09.052323+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":100,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037541+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:47:09.037541+0000","last_peered":"2023-09-30T09:47:09.037541+0000","last_clean":"2023-09-30T09:47:09.037541+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:47:09.037541+0000","last_undegraded":"2023-09-30T09:47:09.037541+0000","last_fullsized":"2023-09-30T09:47:09.037541+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":710,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:13.977721+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:47:13.977721+0000","last_peered":"2023-09-30T09:47:13.977721+0000","last_clean":"2023-09-30T09:47:13.977721+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:47:13.977721+0000","last_undegraded":"2023-09-30T09:47:13.977721+0000","last_fullsized":"2023-09-30T09:47:13.977721+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":340,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.046366+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:47:09.046366+0000","last_peered":"2023-09-30T09:47:09.046366+0000","last_clean":"2023-09-30T09:47:09.046366+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:47:09.046366+0000","last_undegraded":"2023-09-30T09:47:09.046366+0000","last_fullsized":"2023-09-30T09:47:09.046366+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":476,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055201+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:47:09.055201+0000","last_peered":"2023-09-30T09:47:09.055201+0000","last_clean":"2023-09-30T09:47:09.055201+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:47:09.055201+0000","last_undegraded":"2023-09-30T09:47:09.055201+0000","last_fullsized":"2023-09-30T09:47:09.055201+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":649,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.049700+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:47:09.049700+0000","last_peered":"2023-09-30T09:47:09.049700+0000","last_clean":"2023-09-30T09:47:09.049700+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:47:09.049700+0000","last_undegraded":"2023-09-30T09:47:09.049700+0000","last_fullsized":"2023-09-30T09:47:09.049700+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":234,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.056050+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:47:09.056050+0000","last_peered":"2023-09-30T09:47:09.056050+0000","last_clean":"2023-09-30T09:47:09.056050+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:47:09.056050+0000","last_undegraded":"2023-09-30T09:47:09.056050+0000","last_fullsized":"2023-09-30T09:47:09.056050+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052665+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:47:09.052665+0000","last_peered":"2023-09-30T09:47:09.052665+0000","last_clean":"2023-09-30T09:47:09.052665+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:47:09.052665+0000","last_undegraded":"2023-09-30T09:47:09.052665+0000","last_fullsized":"2023-09-30T09:47:09.052665+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":575,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.048397+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:47:09.048397+0000","last_peered":"2023-09-30T09:47:09.048397+0000","last_clean":"2023-09-30T09:47:09.048397+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:47:09.048397+0000","last_undegraded":"2023-09-30T09:47:09.048397+0000","last_fullsized":"2023-09-30T09:47:09.048397+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":564,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037686+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:47:09.037686+0000","last_peered":"2023-09-30T09:47:09.037686+0000","last_clean":"2023-09-30T09:47:09.037686+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:47:09.037686+0000","last_undegraded":"2023-09-30T09:47:09.037686+0000","last_fullsized":"2023-09-30T09:47:09.037686+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":542,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037737+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:47:09.037737+0000","last_peered":"2023-09-30T09:47:09.037737+0000","last_clean":"2023-09-30T09:47:09.037737+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:47:09.037737+0000","last_undegraded":"2023-09-30T09:47:09.037737+0000","last_fullsized":"2023-09-30T09:47:09.037737+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":774,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047683+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:47:09.047683+0000","last_peered":"2023-09-30T09:47:09.047683+0000","last_clean":"2023-09-30T09:47:09.047683+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:47:09.047683+0000","last_undegraded":"2023-09-30T09:47:09.047683+0000","last_fullsized":"2023-09-30T09:47:09.047683+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":170,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055485+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:47:09.055485+0000","last_peered":"2023-09-30T09:47:09.055485+0000","last_clean":"2023-09-30T09:47:09.055485+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:47:09.055485+0000","last_undegraded":"2023-09-30T09:47:09.055485+0000","last_fullsized":"2023-09-30T09:47:09.055485+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":497,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055637+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:47:09.055637+0000","last_peered":"2023-09-30T09:47:09.055637+0000","last_clean":"2023-09-30T09:47:09.055637+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:47:09.055637+0000","last_undegraded":"2023-09-30T09:47:09.055637+0000","last_fullsized":"2023-09-30T09:47:09.055637+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":821,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.046949+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:47:09.046949+0000","last_peered":"2023-09-30T09:47:09.046949+0000","last_clean":"2023-09-30T09:47:09.046949+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:47:09.046949+0000","last_undegraded":"2023-09-30T09:47:09.046949+0000","last_fullsized":"2023-09-30T09:47:09.046949+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":100,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047851+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:47:09.047851+0000","last_peered":"2023-09-30T09:47:09.047851+0000","last_clean":"2023-09-30T09:47:09.047851+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:47:09.047851+0000","last_undegraded":"2023-09-30T09:47:09.047851+0000","last_fullsized":"2023-09-30T09:47:09.047851+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047263+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:47:09.047263+0000","last_peered":"2023-09-30T09:47:09.047263+0000","last_clean":"2023-09-30T09:47:09.047263+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:47:09.047263+0000","last_undegraded":"2023-09-30T09:47:09.047263+0000","last_fullsized":"2023-09-30T09:47:09.047263+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":921,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.035302+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:47:09.035302+0000","last_peered":"2023-09-30T09:47:09.035302+0000","last_clean":"2023-09-30T09:47:09.035302+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:47:09.035302+0000","last_undegraded":"2023-09-30T09:47:09.035302+0000","last_fullsized":"2023-09-30T09:47:09.035302+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":204,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055385+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:47:09.055385+0000","last_peered":"2023-09-30T09:47:09.055385+0000","last_clean":"2023-09-30T09:47:09.055385+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:47:09.055385+0000","last_undegraded":"2023-09-30T09:47:09.055385+0000","last_fullsized":"2023-09-30T09:47:09.055385+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":100,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:14.352917+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:47:14.352917+0000","last_peered":"2023-09-30T09:47:14.352917+0000","last_clean":"2023-09-30T09:47:14.352917+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:47:14.352917+0000","last_undegraded":"2023-09-30T09:47:14.352917+0000","last_fullsized":"2023-09-30T09:47:14.352917+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":930,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.050476+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:47:09.050476+0000","last_peered":"2023-09-30T09:47:09.050476+0000","last_clean":"2023-09-30T09:47:09.050476+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:47:09.050476+0000","last_undegraded":"2023-09-30T09:47:09.050476+0000","last_fullsized":"2023-09-30T09:47:09.050476+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":116,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055501+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:47:09.055501+0000","last_peered":"2023-09-30T09:47:09.055501+0000","last_clean":"2023-09-30T09:47:09.055501+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:47:09.055501+0000","last_undegraded":"2023-09-30T09:47:09.055501+0000","last_fullsized":"2023-09-30T09:47:09.055501+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":84,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.035697+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:47:09.035697+0000","last_peered":"2023-09-30T09:47:09.035697+0000","last_clean":"2023-09-30T09:47:09.035697+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:47:09.035697+0000","last_undegraded":"2023-09-30T09:47:09.035697+0000","last_fullsized":"2023-09-30T09:47:09.035697+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"520'10149","reported_seq":10459,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:29.339189+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:47:29.339189+0000","last_peered":"2023-09-30T09:47:29.339189+0000","last_clean":"2023-09-30T09:47:29.339189+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:47:29.339189+0000","last_undegraded":"2023-09-30T09:47:29.339189+0000","last_fullsized":"2023-09-30T09:47:29.339189+0000","mapping_epoch":495,"log_start":"499'2600","ondisk_log_start":"499'2600","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7549,"log_dups_size":0,"ondisk_log_size":7549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119603200,"num_objects":1825,"num_object_clones":0,"num_object_copies":5475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10149,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":896,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055958+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:47:09.055958+0000","last_peered":"2023-09-30T09:47:09.055958+0000","last_clean":"2023-09-30T09:47:09.055958+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:47:09.055958+0000","last_undegraded":"2023-09-30T09:47:09.055958+0000","last_fullsized":"2023-09-30T09:47:09.055958+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":196,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.056307+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:47:09.056307+0000","last_peered":"2023-09-30T09:47:09.056307+0000","last_clean":"2023-09-30T09:47:09.056307+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:47:09.056307+0000","last_undegraded":"2023-09-30T09:47:09.056307+0000","last_fullsized":"2023-09-30T09:47:09.056307+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":141,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.046866+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:47:09.046866+0000","last_peered":"2023-09-30T09:47:09.046866+0000","last_clean":"2023-09-30T09:47:09.046866+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:47:09.046866+0000","last_undegraded":"2023-09-30T09:47:09.046866+0000","last_fullsized":"2023-09-30T09:47:09.046866+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"520'10044","reported_seq":10087,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:28.871721+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:47:28.871721+0000","last_peered":"2023-09-30T09:47:28.871721+0000","last_clean":"2023-09-30T09:47:28.871721+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:47:28.871721+0000","last_undegraded":"2023-09-30T09:47:28.871721+0000","last_fullsized":"2023-09-30T09:47:28.871721+0000","mapping_epoch":483,"log_start":"502'3600","ondisk_log_start":"502'3600","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124518400,"num_objects":1900,"num_object_clones":0,"num_object_copies":5700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10044,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":913,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.035320+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:47:09.035320+0000","last_peered":"2023-09-30T09:47:09.035320+0000","last_clean":"2023-09-30T09:47:09.035320+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:47:09.035320+0000","last_undegraded":"2023-09-30T09:47:09.035320+0000","last_fullsized":"2023-09-30T09:47:09.035320+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":166,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.056042+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:47:09.056042+0000","last_peered":"2023-09-30T09:47:09.056042+0000","last_clean":"2023-09-30T09:47:09.056042+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:47:09.056042+0000","last_undegraded":"2023-09-30T09:47:09.056042+0000","last_fullsized":"2023-09-30T09:47:09.056042+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":571,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037177+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:47:09.037177+0000","last_peered":"2023-09-30T09:47:09.037177+0000","last_clean":"2023-09-30T09:47:09.037177+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:47:09.037177+0000","last_undegraded":"2023-09-30T09:47:09.037177+0000","last_fullsized":"2023-09-30T09:47:09.037177+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"520'10297","reported_seq":10566,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:29.146854+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:47:29.146854+0000","last_peered":"2023-09-30T09:47:29.146854+0000","last_clean":"2023-09-30T09:47:29.146854+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:47:29.146854+0000","last_undegraded":"2023-09-30T09:47:29.146854+0000","last_fullsized":"2023-09-30T09:47:29.146854+0000","mapping_epoch":493,"log_start":"502'3600","ondisk_log_start":"502'3600","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123535360,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10297,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":690,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052896+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:47:09.052896+0000","last_peered":"2023-09-30T09:47:09.052896+0000","last_clean":"2023-09-30T09:47:09.052896+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:47:09.052896+0000","last_undegraded":"2023-09-30T09:47:09.052896+0000","last_fullsized":"2023-09-30T09:47:09.052896+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":233,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055791+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:47:09.055791+0000","last_peered":"2023-09-30T09:47:09.055791+0000","last_clean":"2023-09-30T09:47:09.055791+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:47:09.055791+0000","last_undegraded":"2023-09-30T09:47:09.055791+0000","last_fullsized":"2023-09-30T09:47:09.055791+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":89,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052272+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:47:09.052272+0000","last_peered":"2023-09-30T09:47:09.052272+0000","last_clean":"2023-09-30T09:47:09.052272+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:47:09.052272+0000","last_undegraded":"2023-09-30T09:47:09.052272+0000","last_fullsized":"2023-09-30T09:47:09.052272+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"520'10036","reported_seq":10397,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:28.931640+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:47:28.931640+0000","last_peered":"2023-09-30T09:47:28.931640+0000","last_clean":"2023-09-30T09:47:28.931640+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:47:28.931640+0000","last_undegraded":"2023-09-30T09:47:28.931640+0000","last_fullsized":"2023-09-30T09:47:28.931640+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10036,"log_dups_size":0,"ondisk_log_size":10036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121896960,"num_objects":1860,"num_object_clones":0,"num_object_copies":5580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10036,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":755,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.039441+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:47:09.039441+0000","last_peered":"2023-09-30T09:47:09.039441+0000","last_clean":"2023-09-30T09:47:09.039441+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:47:09.039441+0000","last_undegraded":"2023-09-30T09:47:09.039441+0000","last_fullsized":"2023-09-30T09:47:09.039441+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":455,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.046186+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:47:09.046186+0000","last_peered":"2023-09-30T09:47:09.046186+0000","last_clean":"2023-09-30T09:47:09.046186+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:47:09.046186+0000","last_undegraded":"2023-09-30T09:47:09.046186+0000","last_fullsized":"2023-09-30T09:47:09.046186+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":120,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.353473+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:47:19.353473+0000","last_peered":"2023-09-30T09:47:19.353473+0000","last_clean":"2023-09-30T09:47:19.353473+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:47:19.353473+0000","last_undegraded":"2023-09-30T09:47:19.353473+0000","last_fullsized":"2023-09-30T09:47:19.353473+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"520'10220","reported_seq":10265,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:29.298991+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:47:29.298991+0000","last_peered":"2023-09-30T09:47:29.298991+0000","last_clean":"2023-09-30T09:47:29.298991+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:47:29.298991+0000","last_undegraded":"2023-09-30T09:47:29.298991+0000","last_fullsized":"2023-09-30T09:47:29.298991+0000","mapping_epoch":483,"log_start":"501'3200","ondisk_log_start":"501'3200","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7020,"log_dups_size":0,"ondisk_log_size":7020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125304832,"num_objects":1912,"num_object_clones":0,"num_object_copies":5736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10220,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1002,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055297+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:47:09.055297+0000","last_peered":"2023-09-30T09:47:09.055297+0000","last_clean":"2023-09-30T09:47:09.055297+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:47:09.055297+0000","last_undegraded":"2023-09-30T09:47:09.055297+0000","last_fullsized":"2023-09-30T09:47:09.055297+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":79,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.051277+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:47:09.051277+0000","last_peered":"2023-09-30T09:47:09.051277+0000","last_clean":"2023-09-30T09:47:09.051277+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:47:09.051277+0000","last_undegraded":"2023-09-30T09:47:09.051277+0000","last_fullsized":"2023-09-30T09:47:09.051277+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":518,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.038057+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:47:09.038057+0000","last_peered":"2023-09-30T09:47:09.038057+0000","last_clean":"2023-09-30T09:47:09.038057+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:47:09.038057+0000","last_undegraded":"2023-09-30T09:47:09.038057+0000","last_fullsized":"2023-09-30T09:47:09.038057+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"520'10073","reported_seq":10118,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:29.317604+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:47:29.317604+0000","last_peered":"2023-09-30T09:47:29.317604+0000","last_clean":"2023-09-30T09:47:29.317604+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:47:29.317604+0000","last_undegraded":"2023-09-30T09:47:29.317604+0000","last_fullsized":"2023-09-30T09:47:29.317604+0000","mapping_epoch":483,"log_start":"500'3000","ondisk_log_start":"500'3000","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123273216,"num_objects":1881,"num_object_clones":0,"num_object_copies":5643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10073,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":1001,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.068385+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:47:09.068385+0000","last_peered":"2023-09-30T09:47:09.068385+0000","last_clean":"2023-09-30T09:47:09.068385+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:47:09.068385+0000","last_undegraded":"2023-09-30T09:47:09.068385+0000","last_fullsized":"2023-09-30T09:47:09.068385+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":75,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052119+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:47:09.052119+0000","last_peered":"2023-09-30T09:47:09.052119+0000","last_clean":"2023-09-30T09:47:09.052119+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:47:09.052119+0000","last_undegraded":"2023-09-30T09:47:09.052119+0000","last_fullsized":"2023-09-30T09:47:09.052119+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":214,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.040173+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:47:09.040173+0000","last_peered":"2023-09-30T09:47:09.040173+0000","last_clean":"2023-09-30T09:47:09.040173+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:47:09.040173+0000","last_undegraded":"2023-09-30T09:47:09.040173+0000","last_fullsized":"2023-09-30T09:47:09.040173+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"520'10352","reported_seq":10397,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:28.869648+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:47:28.869648+0000","last_peered":"2023-09-30T09:47:28.869648+0000","last_clean":"2023-09-30T09:47:28.869648+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:47:28.869648+0000","last_undegraded":"2023-09-30T09:47:28.869648+0000","last_fullsized":"2023-09-30T09:47:28.869648+0000","mapping_epoch":483,"log_start":"504'4500","ondisk_log_start":"504'4500","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5852,"log_dups_size":0,"ondisk_log_size":5852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125698048,"num_objects":1918,"num_object_clones":0,"num_object_copies":5754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10352,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":859,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.353502+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:47:19.353502+0000","last_peered":"2023-09-30T09:47:19.353502+0000","last_clean":"2023-09-30T09:47:19.353502+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:47:19.353502+0000","last_undegraded":"2023-09-30T09:47:19.353502+0000","last_fullsized":"2023-09-30T09:47:19.353502+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":525,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037025+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:47:09.037025+0000","last_peered":"2023-09-30T09:47:09.037025+0000","last_clean":"2023-09-30T09:47:09.037025+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:47:09.037025+0000","last_undegraded":"2023-09-30T09:47:09.037025+0000","last_fullsized":"2023-09-30T09:47:09.037025+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":532,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.039542+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:47:09.039542+0000","last_peered":"2023-09-30T09:47:09.039542+0000","last_clean":"2023-09-30T09:47:09.039542+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:47:09.039542+0000","last_undegraded":"2023-09-30T09:47:09.039542+0000","last_fullsized":"2023-09-30T09:47:09.039542+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"520'10051","reported_seq":10100,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:28.915682+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:47:28.915682+0000","last_peered":"2023-09-30T09:47:28.915682+0000","last_clean":"2023-09-30T09:47:28.915682+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:47:28.915682+0000","last_undegraded":"2023-09-30T09:47:28.915682+0000","last_fullsized":"2023-09-30T09:47:28.915682+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10051,"log_dups_size":0,"ondisk_log_size":10051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123535360,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10051,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":723,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.066332+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:47:09.066332+0000","last_peered":"2023-09-30T09:47:09.066332+0000","last_clean":"2023-09-30T09:47:09.066332+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:47:09.066332+0000","last_undegraded":"2023-09-30T09:47:09.066332+0000","last_fullsized":"2023-09-30T09:47:09.066332+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":85,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:13.370470+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:47:13.370470+0000","last_peered":"2023-09-30T09:47:13.370470+0000","last_clean":"2023-09-30T09:47:13.370470+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:47:13.370470+0000","last_undegraded":"2023-09-30T09:47:13.370470+0000","last_fullsized":"2023-09-30T09:47:13.370470+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":205,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037402+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:47:09.037402+0000","last_peered":"2023-09-30T09:47:09.037402+0000","last_clean":"2023-09-30T09:47:09.037402+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:47:09.037402+0000","last_undegraded":"2023-09-30T09:47:09.037402+0000","last_fullsized":"2023-09-30T09:47:09.037402+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"520'10216","reported_seq":10520,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:29.094676+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:47:29.094676+0000","last_peered":"2023-09-30T09:47:29.094676+0000","last_clean":"2023-09-30T09:47:29.094676+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:47:29.094676+0000","last_undegraded":"2023-09-30T09:47:29.094676+0000","last_fullsized":"2023-09-30T09:47:29.094676+0000","mapping_epoch":494,"log_start":"501'3500","ondisk_log_start":"501'3500","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6716,"log_dups_size":0,"ondisk_log_size":6716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123731968,"num_objects":1888,"num_object_clones":0,"num_object_copies":5664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10216,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":762,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052721+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:47:09.052721+0000","last_peered":"2023-09-30T09:47:09.052721+0000","last_clean":"2023-09-30T09:47:09.052721+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:47:09.052721+0000","last_undegraded":"2023-09-30T09:47:09.052721+0000","last_fullsized":"2023-09-30T09:47:09.052721+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":751,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:19.353531+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:47:19.353531+0000","last_peered":"2023-09-30T09:47:19.353531+0000","last_clean":"2023-09-30T09:47:19.353531+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:47:19.353531+0000","last_undegraded":"2023-09-30T09:47:19.353531+0000","last_fullsized":"2023-09-30T09:47:19.353531+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":448,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.048750+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:47:09.048750+0000","last_peered":"2023-09-30T09:47:09.048750+0000","last_clean":"2023-09-30T09:47:09.048750+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:47:09.048750+0000","last_undegraded":"2023-09-30T09:47:09.048750+0000","last_fullsized":"2023-09-30T09:47:09.048750+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":137,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047360+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:47:09.047360+0000","last_peered":"2023-09-30T09:47:09.047360+0000","last_clean":"2023-09-30T09:47:09.047360+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:47:09.047360+0000","last_undegraded":"2023-09-30T09:47:09.047360+0000","last_fullsized":"2023-09-30T09:47:09.047360+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"520'10238","reported_seq":10588,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:29.322359+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:47:29.322359+0000","last_peered":"2023-09-30T09:47:29.322359+0000","last_clean":"2023-09-30T09:47:29.322359+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:47:29.322359+0000","last_undegraded":"2023-09-30T09:47:29.322359+0000","last_fullsized":"2023-09-30T09:47:29.322359+0000","mapping_epoch":493,"log_start":"499'2700","ondisk_log_start":"499'2700","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123863040,"num_objects":1890,"num_object_clones":0,"num_object_copies":5670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10238,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":961,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.046672+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:47:09.046672+0000","last_peered":"2023-09-30T09:47:09.046672+0000","last_clean":"2023-09-30T09:47:09.046672+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:47:09.046672+0000","last_undegraded":"2023-09-30T09:47:09.046672+0000","last_fullsized":"2023-09-30T09:47:09.046672+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":467,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052362+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:47:09.052362+0000","last_peered":"2023-09-30T09:47:09.052362+0000","last_clean":"2023-09-30T09:47:09.052362+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:47:09.052362+0000","last_undegraded":"2023-09-30T09:47:09.052362+0000","last_fullsized":"2023-09-30T09:47:09.052362+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":539,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037810+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:47:09.037810+0000","last_peered":"2023-09-30T09:47:09.037810+0000","last_clean":"2023-09-30T09:47:09.037810+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:47:09.037810+0000","last_undegraded":"2023-09-30T09:47:09.037810+0000","last_fullsized":"2023-09-30T09:47:09.037810+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":279,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.068907+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:47:09.068907+0000","last_peered":"2023-09-30T09:47:09.068907+0000","last_clean":"2023-09-30T09:47:09.068907+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:47:09.068907+0000","last_undegraded":"2023-09-30T09:47:09.068907+0000","last_fullsized":"2023-09-30T09:47:09.068907+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"520'10291","reported_seq":10641,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:29.130468+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:47:29.130468+0000","last_peered":"2023-09-30T09:47:29.130468+0000","last_clean":"2023-09-30T09:47:29.130468+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:47:29.130468+0000","last_undegraded":"2023-09-30T09:47:29.130468+0000","last_fullsized":"2023-09-30T09:47:29.130468+0000","mapping_epoch":492,"log_start":"502'3600","ondisk_log_start":"502'3600","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123273216,"num_objects":1881,"num_object_clones":0,"num_object_copies":5643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10291,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":939,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055958+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:47:09.055958+0000","last_peered":"2023-09-30T09:47:09.055958+0000","last_clean":"2023-09-30T09:47:09.055958+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:47:09.055958+0000","last_undegraded":"2023-09-30T09:47:09.055958+0000","last_fullsized":"2023-09-30T09:47:09.055958+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":788,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.067760+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:47:09.067760+0000","last_peered":"2023-09-30T09:47:09.067760+0000","last_clean":"2023-09-30T09:47:09.067760+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:47:09.067760+0000","last_undegraded":"2023-09-30T09:47:09.067760+0000","last_fullsized":"2023-09-30T09:47:09.067760+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":283,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:13.977622+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:47:13.977622+0000","last_peered":"2023-09-30T09:47:13.977622+0000","last_clean":"2023-09-30T09:47:13.977622+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:47:13.977622+0000","last_undegraded":"2023-09-30T09:47:13.977622+0000","last_fullsized":"2023-09-30T09:47:13.977622+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":369,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.049913+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:47:09.049913+0000","last_peered":"2023-09-30T09:47:09.049913+0000","last_clean":"2023-09-30T09:47:09.049913+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:47:09.049913+0000","last_undegraded":"2023-09-30T09:47:09.049913+0000","last_fullsized":"2023-09-30T09:47:09.049913+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"520'10138","reported_seq":10448,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:29.331469+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:47:29.331469+0000","last_peered":"2023-09-30T09:47:29.331469+0000","last_clean":"2023-09-30T09:47:29.331469+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:47:29.331469+0000","last_undegraded":"2023-09-30T09:47:29.331469+0000","last_fullsized":"2023-09-30T09:47:29.331469+0000","mapping_epoch":495,"log_start":"499'2600","ondisk_log_start":"499'2600","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123207680,"num_objects":1880,"num_object_clones":0,"num_object_copies":5640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10138,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1081,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.046947+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:47:09.046947+0000","last_peered":"2023-09-30T09:47:09.046947+0000","last_clean":"2023-09-30T09:47:09.046947+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:47:09.046947+0000","last_undegraded":"2023-09-30T09:47:09.046947+0000","last_fullsized":"2023-09-30T09:47:09.046947+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":882,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055232+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:47:09.055232+0000","last_peered":"2023-09-30T09:47:09.055232+0000","last_clean":"2023-09-30T09:47:09.055232+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:47:09.055232+0000","last_undegraded":"2023-09-30T09:47:09.055232+0000","last_fullsized":"2023-09-30T09:47:09.055232+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":96,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047442+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:47:09.047442+0000","last_peered":"2023-09-30T09:47:09.047442+0000","last_clean":"2023-09-30T09:47:09.047442+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:47:09.047442+0000","last_undegraded":"2023-09-30T09:47:09.047442+0000","last_fullsized":"2023-09-30T09:47:09.047442+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":75,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052544+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:47:09.052544+0000","last_peered":"2023-09-30T09:47:09.052544+0000","last_clean":"2023-09-30T09:47:09.052544+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:47:09.052544+0000","last_undegraded":"2023-09-30T09:47:09.052544+0000","last_fullsized":"2023-09-30T09:47:09.052544+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"520'10416","reported_seq":18044,"reported_epoch":520,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:47:28.966468+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:47:28.966468+0000","last_peered":"2023-09-30T09:47:28.966468+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:47:28.966468+0000","last_undegraded":"2023-09-30T09:47:28.966468+0000","last_fullsized":"2023-09-30T09:47:28.966468+0000","mapping_epoch":506,"log_start":"496'400","ondisk_log_start":"496'400","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10016,"log_dups_size":0,"ondisk_log_size":10016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125173760,"num_objects":1910,"num_object_clones":0,"num_object_copies":5730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1192,"num_objects_unfound":0,"num_objects_dirty":1910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10416,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3778,"num_bytes_recovered":247595008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":718}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":781,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052474+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:47:09.052474+0000","last_peered":"2023-09-30T09:47:09.052474+0000","last_clean":"2023-09-30T09:47:09.052474+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:47:09.052474+0000","last_undegraded":"2023-09-30T09:47:09.052474+0000","last_fullsized":"2023-09-30T09:47:09.052474+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":928,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055145+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:47:09.055145+0000","last_peered":"2023-09-30T09:47:09.055145+0000","last_clean":"2023-09-30T09:47:09.055145+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:47:09.055145+0000","last_undegraded":"2023-09-30T09:47:09.055145+0000","last_fullsized":"2023-09-30T09:47:09.055145+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":133,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047846+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:47:09.047846+0000","last_peered":"2023-09-30T09:47:09.047846+0000","last_clean":"2023-09-30T09:47:09.047846+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:47:09.047846+0000","last_undegraded":"2023-09-30T09:47:09.047846+0000","last_fullsized":"2023-09-30T09:47:09.047846+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":25,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.048172+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:47:09.048172+0000","last_peered":"2023-09-30T09:47:09.048172+0000","last_clean":"2023-09-30T09:47:09.048172+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:47:09.048172+0000","last_undegraded":"2023-09-30T09:47:09.048172+0000","last_fullsized":"2023-09-30T09:47:09.048172+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"520'10148","reported_seq":10193,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:28.893114+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:47:28.893114+0000","last_peered":"2023-09-30T09:47:28.893114+0000","last_clean":"2023-09-30T09:47:28.893114+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:47:28.893114+0000","last_undegraded":"2023-09-30T09:47:28.893114+0000","last_fullsized":"2023-09-30T09:47:28.893114+0000","mapping_epoch":483,"log_start":"504'4300","ondisk_log_start":"504'4300","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712704,"num_objects":1964,"num_object_clones":0,"num_object_copies":5892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10148,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":772,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052721+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:47:09.052721+0000","last_peered":"2023-09-30T09:47:09.052721+0000","last_clean":"2023-09-30T09:47:09.052721+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:47:09.052721+0000","last_undegraded":"2023-09-30T09:47:09.052721+0000","last_fullsized":"2023-09-30T09:47:09.052721+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1068,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.056097+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:47:09.056097+0000","last_peered":"2023-09-30T09:47:09.056097+0000","last_clean":"2023-09-30T09:47:09.056097+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:47:09.056097+0000","last_undegraded":"2023-09-30T09:47:09.056097+0000","last_fullsized":"2023-09-30T09:47:09.056097+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":263,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.067321+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:47:09.067321+0000","last_peered":"2023-09-30T09:47:09.067321+0000","last_clean":"2023-09-30T09:47:09.067321+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:47:09.067321+0000","last_undegraded":"2023-09-30T09:47:09.067321+0000","last_fullsized":"2023-09-30T09:47:09.067321+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":133,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.051897+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:47:09.051897+0000","last_peered":"2023-09-30T09:47:09.051897+0000","last_clean":"2023-09-30T09:47:09.051897+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:47:09.051897+0000","last_undegraded":"2023-09-30T09:47:09.051897+0000","last_fullsized":"2023-09-30T09:47:09.051897+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"520'10098","reported_seq":10478,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:28.898348+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:47:28.898348+0000","last_peered":"2023-09-30T09:47:28.898348+0000","last_clean":"2023-09-30T09:47:28.898348+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:47:28.898348+0000","last_undegraded":"2023-09-30T09:47:28.898348+0000","last_fullsized":"2023-09-30T09:47:28.898348+0000","mapping_epoch":495,"log_start":"504'4300","ondisk_log_start":"504'4300","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125239320,"num_objects":1912,"num_object_clones":0,"num_object_copies":5736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1912,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10098,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1039,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.040400+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:47:09.040400+0000","last_peered":"2023-09-30T09:47:09.040400+0000","last_clean":"2023-09-30T09:47:09.040400+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:47:09.040400+0000","last_undegraded":"2023-09-30T09:47:09.040400+0000","last_fullsized":"2023-09-30T09:47:09.040400+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":551,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.056192+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:47:09.056192+0000","last_peered":"2023-09-30T09:47:09.056192+0000","last_clean":"2023-09-30T09:47:09.056192+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:47:09.056192+0000","last_undegraded":"2023-09-30T09:47:09.056192+0000","last_fullsized":"2023-09-30T09:47:09.056192+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1238,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052610+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:47:09.052610+0000","last_peered":"2023-09-30T09:47:09.052610+0000","last_clean":"2023-09-30T09:47:09.052610+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:47:09.052610+0000","last_undegraded":"2023-09-30T09:47:09.052610+0000","last_fullsized":"2023-09-30T09:47:09.052610+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":555,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055141+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:47:09.055141+0000","last_peered":"2023-09-30T09:47:09.055141+0000","last_clean":"2023-09-30T09:47:09.055141+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:47:09.055141+0000","last_undegraded":"2023-09-30T09:47:09.055141+0000","last_fullsized":"2023-09-30T09:47:09.055141+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"520'10362","reported_seq":10407,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-30T09:47:28.856522+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:47:28.856522+0000","last_peered":"2023-09-30T09:47:28.856522+0000","last_clean":"2023-09-30T09:47:28.856522+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:47:28.856522+0000","last_undegraded":"2023-09-30T09:47:28.856522+0000","last_fullsized":"2023-09-30T09:47:28.856522+0000","mapping_epoch":483,"log_start":"502'3900","ondisk_log_start":"502'3900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6462,"log_dups_size":0,"ondisk_log_size":6462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128843776,"num_objects":1966,"num_object_clones":0,"num_object_copies":5898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10362,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":903,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.035180+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:47:09.035180+0000","last_peered":"2023-09-30T09:47:09.035180+0000","last_clean":"2023-09-30T09:47:09.035180+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:47:09.035180+0000","last_undegraded":"2023-09-30T09:47:09.035180+0000","last_fullsized":"2023-09-30T09:47:09.035180+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":90,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052854+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:47:09.052854+0000","last_peered":"2023-09-30T09:47:09.052854+0000","last_clean":"2023-09-30T09:47:09.052854+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:47:09.052854+0000","last_undegraded":"2023-09-30T09:47:09.052854+0000","last_fullsized":"2023-09-30T09:47:09.052854+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.048296+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:47:09.048296+0000","last_peered":"2023-09-30T09:47:09.048296+0000","last_clean":"2023-09-30T09:47:09.048296+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:47:09.048296+0000","last_undegraded":"2023-09-30T09:47:09.048296+0000","last_fullsized":"2023-09-30T09:47:09.048296+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":521,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037101+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:47:09.037101+0000","last_peered":"2023-09-30T09:47:09.037101+0000","last_clean":"2023-09-30T09:47:09.037101+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:47:09.037101+0000","last_undegraded":"2023-09-30T09:47:09.037101+0000","last_fullsized":"2023-09-30T09:47:09.037101+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":839,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.047577+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:47:09.047577+0000","last_peered":"2023-09-30T09:47:09.047577+0000","last_clean":"2023-09-30T09:47:09.047577+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:47:09.047577+0000","last_undegraded":"2023-09-30T09:47:09.047577+0000","last_fullsized":"2023-09-30T09:47:09.047577+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":25,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.051824+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:47:09.051824+0000","last_peered":"2023-09-30T09:47:09.051824+0000","last_clean":"2023-09-30T09:47:09.051824+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:47:09.051824+0000","last_undegraded":"2023-09-30T09:47:09.051824+0000","last_fullsized":"2023-09-30T09:47:09.051824+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":225,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.068815+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:47:09.068815+0000","last_peered":"2023-09-30T09:47:09.068815+0000","last_clean":"2023-09-30T09:47:09.068815+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:47:09.068815+0000","last_undegraded":"2023-09-30T09:47:09.068815+0000","last_fullsized":"2023-09-30T09:47:09.068815+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":634,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052731+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:47:09.052731+0000","last_peered":"2023-09-30T09:47:09.052731+0000","last_clean":"2023-09-30T09:47:09.052731+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:47:09.052731+0000","last_undegraded":"2023-09-30T09:47:09.052731+0000","last_fullsized":"2023-09-30T09:47:09.052731+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":448,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.046113+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:47:09.046113+0000","last_peered":"2023-09-30T09:47:09.046113+0000","last_clean":"2023-09-30T09:47:09.046113+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:47:09.046113+0000","last_undegraded":"2023-09-30T09:47:09.046113+0000","last_fullsized":"2023-09-30T09:47:09.046113+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":234,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037402+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:47:09.037402+0000","last_peered":"2023-09-30T09:47:09.037402+0000","last_clean":"2023-09-30T09:47:09.037402+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:47:09.037402+0000","last_undegraded":"2023-09-30T09:47:09.037402+0000","last_fullsized":"2023-09-30T09:47:09.037402+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":390,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.048630+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:47:09.048630+0000","last_peered":"2023-09-30T09:47:09.048630+0000","last_clean":"2023-09-30T09:47:09.048630+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:47:09.048630+0000","last_undegraded":"2023-09-30T09:47:09.048630+0000","last_fullsized":"2023-09-30T09:47:09.048630+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":254,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.037045+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:47:09.037045+0000","last_peered":"2023-09-30T09:47:09.037045+0000","last_clean":"2023-09-30T09:47:09.037045+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:47:09.037045+0000","last_undegraded":"2023-09-30T09:47:09.037045+0000","last_fullsized":"2023-09-30T09:47:09.037045+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":847,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.055570+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:47:09.055570+0000","last_peered":"2023-09-30T09:47:09.055570+0000","last_clean":"2023-09-30T09:47:09.055570+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:47:09.055570+0000","last_undegraded":"2023-09-30T09:47:09.055570+0000","last_fullsized":"2023-09-30T09:47:09.055570+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":77,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052528+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:47:09.052528+0000","last_peered":"2023-09-30T09:47:09.052528+0000","last_clean":"2023-09-30T09:47:09.052528+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:47:09.052528+0000","last_undegraded":"2023-09-30T09:47:09.052528+0000","last_fullsized":"2023-09-30T09:47:09.052528+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":471,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.067671+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:47:09.067671+0000","last_peered":"2023-09-30T09:47:09.067671+0000","last_clean":"2023-09-30T09:47:09.067671+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:47:09.067671+0000","last_undegraded":"2023-09-30T09:47:09.067671+0000","last_fullsized":"2023-09-30T09:47:09.067671+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":890,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.040259+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:47:09.040259+0000","last_peered":"2023-09-30T09:47:09.040259+0000","last_clean":"2023-09-30T09:47:09.040259+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:47:09.040259+0000","last_undegraded":"2023-09-30T09:47:09.040259+0000","last_fullsized":"2023-09-30T09:47:09.040259+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":31,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.052000+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:47:09.052000+0000","last_peered":"2023-09-30T09:47:09.052000+0000","last_clean":"2023-09-30T09:47:09.052000+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:47:09.052000+0000","last_undegraded":"2023-09-30T09:47:09.052000+0000","last_fullsized":"2023-09-30T09:47:09.052000+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":535,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-30T09:47:09.036783+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:47:09.036783+0000","last_peered":"2023-09-30T09:47:09.036783+0000","last_clean":"2023-09-30T09:47:09.036783+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:47:09.036783+0000","last_undegraded":"2023-09-30T09:47:09.036783+0000","last_fullsized":"2023-09-30T09:47:09.036783+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1989410840,"num_objects":30357,"num_object_clones":0,"num_object_copies":91071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1192,"num_objects_unfound":0,"num_objects_dirty":30357,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163129,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4921,"num_bytes_recovered":322502656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6123499520,"data_stored":6123487304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117329,"ondisk_log_size":117329,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234429,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1656924,"kb_used_data":1117268,"kb_used_omap":0,"kb_used_meta":539520,"kb_avail":92088228,"statfs":{"total":95995035648,"available":94298345472,"internally_reserved":0,"allocated":1144082432,"data_stored":1142338498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":552468480},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":8.9299999999999997,"5min":7.6980000000000004,"15min":5.0990000000000002},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":95.825999999999993,"5min":124.74299999999999,"15min":215.416},"last":0.63800000000000001},{"interface":"front","average":{"1min":8.9190000000000005,"5min":7.7690000000000001,"15min":5.0069999999999997},"min":{"1min":0.26100000000000001,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":95.488,"5min":124.69499999999999,"15min":215.041},"last":0.78600000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":7.4539999999999997,"15min":4.117},"min":{"1min":0.40799999999999997,"5min":0.25,"15min":0.25},"max":{"1min":7.6559999999999997,"5min":157.578,"15min":157.578},"last":0.57699999999999996},{"interface":"front","average":{"1min":2.649,"5min":7.2839999999999998,"15min":4.2679999999999998},"min":{"1min":0.41299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.8800000000000008,"5min":157.589,"15min":157.589},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":2.9809999999999999,"15min":3.5510000000000002},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.24299999999999999},"max":{"1min":10.292999999999999,"5min":104.88500000000001,"15min":117.14},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.9430000000000001,"5min":3.1629999999999998,"15min":3.5409999999999999},"min":{"1min":0.29599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.4420000000000002,"5min":104.867,"15min":117.044},"last":1.585}]},{"osd":3,"last update":"Sat Sep 30 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":7.8529999999999998,"5min":8.1449999999999996,"15min":8.6750000000000007},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":97.037000000000006,"5min":154.06700000000001,"15min":277.57799999999997},"last":0.72199999999999998},{"interface":"front","average":{"1min":8.9039999999999999,"5min":8.4440000000000008,"15min":8.9420000000000002},"min":{"1min":0.42999999999999999,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":95.135000000000005,"5min":154.24799999999999,"15min":278.69299999999998},"last":0.65700000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.1399999999999999,"15min":1.6259999999999999},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":7.5499999999999998,"5min":25.192,"15min":175.72},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.214,"5min":1.0660000000000001,"15min":1.536},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.252},"max":{"1min":5.6120000000000001,"5min":24.521000000000001,"15min":179.048},"last":0.73599999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":9.875,"15min":11.287000000000001},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.641,"5min":197.75700000000001,"15min":197.75700000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.8919999999999999,"5min":9.8399999999999999,"15min":11.164},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.7149999999999999,"5min":197.63300000000001,"15min":197.63300000000001},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.72299999999999998,"15min":2.1760000000000002},"min":{"1min":0.32900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.202,"5min":6.202,"15min":206.244},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.069,"5min":0.84399999999999997,"15min":2.1850000000000001},"min":{"1min":0.34699999999999998,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":5.5800000000000001,"5min":5.8520000000000003,"15min":202.78800000000001},"last":0.76000000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496067,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903492,"kb_used_data":371772,"kb_used_omap":0,"kb_used_meta":531584,"kb_avail":92841660,"statfs":{"total":95995035648,"available":95069859840,"internally_reserved":0,"allocated":380694528,"data_stored":378973887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":544342016},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":4.1050000000000004,"15min":5.3330000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":11.609,"5min":95.043999999999997,"15min":187.19800000000001},"last":2.7919999999999998},{"interface":"front","average":{"1min":2.1600000000000001,"5min":4.1210000000000004,"15min":5.3520000000000003},"min":{"1min":0.26300000000000001,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":6.5460000000000003,"5min":94.855999999999995,"15min":187.46700000000001},"last":8.6329999999999991}]},{"osd":1,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.446,"15min":1.956},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.6429999999999998,"5min":32.814,"15min":69.569999999999993},"last":2.867},{"interface":"front","average":{"1min":2.5529999999999999,"5min":1.5629999999999999,"15min":2.173},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":6.3760000000000003,"5min":32.856000000000002,"15min":70.471999999999994},"last":8.5869999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":3.9239999999999999,"5min":3.5880000000000001,"15min":4.0069999999999997},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":10.581,"5min":207.02500000000001,"15min":207.02500000000001},"last":11.432},{"interface":"front","average":{"1min":3.3559999999999999,"5min":3.6909999999999998,"15min":4.1539999999999999},"min":{"1min":0.37,"5min":0.33500000000000002,"15min":0.28199999999999997},"max":{"1min":16.474,"5min":207.114,"15min":207.114},"last":2.8969999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8839999999999999,"5min":3.484,"15min":4.2560000000000002},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":11.644,"5min":107.11799999999999,"15min":212.44800000000001},"last":2.9180000000000001},{"interface":"front","average":{"1min":2.3239999999999998,"5min":3.4159999999999999,"15min":4.4909999999999997},"min":{"1min":0.35099999999999998,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":8.8559999999999999,"5min":106.982,"15min":210.84299999999999},"last":6.5640000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.722,"15min":1.2669999999999999},"min":{"1min":0.30199999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.4400000000000004,"5min":63.798000000000002,"15min":63.798000000000002},"last":11.499000000000001},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.8400000000000001,"15min":1.2330000000000001},"min":{"1min":0.32900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.558999999999999,"5min":63.133000000000003,"15min":63.133000000000003},"last":2.8250000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8740000000000001,"15min":1.99},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.6840000000000002,"5min":14.974,"15min":14.974},"last":11.367000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":2.2789999999999999,"15min":2.5110000000000001},"min":{"1min":0.37,"5min":0.316,"15min":0.316},"max":{"1min":3.6190000000000002,"5min":16.443000000000001,"15min":16.443000000000001},"last":10.864000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":0.89200000000000002,"15min":1.609},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.2679999999999998,"5min":11.276999999999999,"15min":138.191},"last":7.1639999999999997},{"interface":"front","average":{"1min":1.282,"5min":0.89300000000000002,"15min":1.4410000000000001},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7720000000000002,"5min":10.909000000000001,"15min":140.33500000000001},"last":10.821}]}]},{"osd":1,"up_from":301,"seq":1292785156264,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1796336,"kb_used_data":1150952,"kb_used_omap":9,"kb_used_meta":645366,"kb_avail":91948816,"statfs":{"total":95995035648,"available":94155587584,"internally_reserved":0,"allocated":1178574848,"data_stored":1175613626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":660854802},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":4.319,"15min":3.7320000000000002},"min":{"1min":0.221,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":26.021000000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":2.0510000000000002},{"interface":"front","average":{"1min":3.7639999999999998,"5min":4.1159999999999997,"15min":3.6240000000000001},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":25.971,"5min":142.137,"15min":142.137},"last":4.0110000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.883,"15min":2.573},"min":{"1min":0.27700000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.763000000000002,"5min":62.572000000000003,"15min":62.572000000000003},"last":3.27},{"interface":"front","average":{"1min":2.9809999999999999,"5min":2.8700000000000001,"15min":2.7999999999999998},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.138999999999999,"5min":68.373999999999995,"15min":68.373999999999995},"last":1.3500000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.484,"5min":5.9690000000000003,"15min":6.0899999999999999},"min":{"1min":0.30499999999999999,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":9.8610000000000007,"5min":199.41,"15min":199.41},"last":14.776},{"interface":"front","average":{"1min":3.149,"5min":6.2999999999999998,"15min":6.1509999999999998},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.4329999999999998,"5min":199.58099999999999,"15min":199.58099999999999},"last":5.21}]},{"osd":4,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":1.091,"15min":2.1309999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":8.9399999999999995,"5min":8.9399999999999995,"15min":142.77099999999999},"last":1.323},{"interface":"front","average":{"1min":2.46,"5min":1.52,"15min":2.4140000000000001},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.6319999999999997,"5min":8.6319999999999997,"15min":144.898},"last":4.1340000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.0190000000000001,"15min":1.962},"min":{"1min":0.40500000000000003,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.2100000000000009,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":4.0999999999999996},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.345,"15min":1.2629999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":15.250999999999999,"5min":15.250999999999999,"15min":15.250999999999999},"last":1.262}]},{"osd":6,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.552,"15min":5.5679999999999996},"min":{"1min":0.442,"5min":0.316,"15min":0.29899999999999999},"max":{"1min":8.8559999999999999,"5min":30.986999999999998,"15min":209.221},"last":1.3759999999999999},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.4850000000000001,"15min":5.4960000000000004},"min":{"1min":0.47899999999999998,"5min":0.35399999999999998,"15min":0.33900000000000002},"max":{"1min":8.8100000000000005,"5min":28.710999999999999,"15min":208.43600000000001},"last":1.296}]},{"osd":7,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.5449999999999999,"15min":3.9500000000000002},"min":{"1min":0.36299999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":9.3879999999999999,"5min":9.3879999999999999,"15min":165.59399999999999},"last":2.911},{"interface":"front","average":{"1min":2.077,"5min":1.542,"15min":3.9929999999999999},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":4.2549999999999999,"5min":9.3399999999999999,"15min":165.833},"last":2.9460000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926163,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1118972,"kb_used_data":460276,"kb_used_omap":5,"kb_used_meta":658682,"kb_avail":92626180,"statfs":{"total":95995035648,"available":94849208320,"internally_reserved":0,"allocated":471322624,"data_stored":468120632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":674490621},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.9419999999999999,"15min":2.181},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":7.2380000000000004,"5min":57.482999999999997,"15min":57.482999999999997},"last":1.625},{"interface":"front","average":{"1min":2.927,"5min":2.3180000000000001,"15min":4.0199999999999996},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.344000000000001,"5min":57.140999999999998,"15min":138.92400000000001},"last":3.742}]},{"osd":2,"last update":"Sat Sep 30 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6640000000000001,"5min":4.8010000000000002,"15min":3.2829999999999999},"min":{"1min":0.28899999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":31.521000000000001,"5min":127.73399999999999,"15min":127.73399999999999},"last":10.525},{"interface":"front","average":{"1min":4.7649999999999997,"5min":5.1299999999999999,"15min":3.5470000000000002},"min":{"1min":0.38,"5min":0.27600000000000002,"15min":0.26200000000000001},"max":{"1min":31.478999999999999,"5min":127.69,"15min":127.69},"last":3.774}]},{"osd":3,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":5.5099999999999998,"5min":5.9870000000000001,"15min":5.4249999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":16.594000000000001,"5min":125.562,"15min":218.06999999999999},"last":3.8570000000000002},{"interface":"front","average":{"1min":3.0680000000000001,"5min":4.984,"15min":4.2599999999999998},"min":{"1min":0.39600000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":13.361000000000001,"5min":117.295,"15min":218.63200000000001},"last":0.66400000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.7989999999999999,"15min":2.012},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.8710000000000004,"5min":72.403999999999996,"15min":72.403999999999996},"last":3.8279999999999998},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1960000000000002,"15min":2.8260000000000001},"min":{"1min":0.38600000000000001,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":9.4870000000000001,"5min":72.162999999999997,"15min":136.262},"last":3.819}]},{"osd":5,"last update":"Sat Sep 30 09:47:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2570000000000001,"5min":2.0950000000000002,"15min":1.413},"min":{"1min":0.35299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":13.162000000000001,"5min":13.162000000000001,"15min":13.162000000000001},"last":3.956},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.2150000000000001,"15min":1.1040000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":13.422000000000001,"5min":13.422000000000001,"15min":13.422000000000001},"last":0.626}]},{"osd":6,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.958,"15min":3.508},"min":{"1min":0.38200000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":20.706,"5min":57.384,"15min":214.785},"last":0.69199999999999995},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.3700000000000001,"15min":3.8370000000000002},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.684000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":3.8900000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.8120000000000001,"15min":2.0720000000000001},"min":{"1min":0.42499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.6549999999999994,"5min":57.776000000000003,"15min":97.382000000000005},"last":1.1180000000000001},{"interface":"front","average":{"1min":2.6349999999999998,"5min":2.363,"15min":2.3940000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":9.7149999999999999,"5min":57.179000000000002,"15min":97.015000000000001},"last":10.564}]}]},{"osd":2,"up_from":104,"seq":446676599104,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1303368,"kb_used_data":496960,"kb_used_omap":5,"kb_used_meta":806394,"kb_avail":92441784,"statfs":{"total":95995035648,"available":94660386816,"internally_reserved":0,"allocated":508887040,"data_stored":505961438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":825748109},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":4.0890000000000004,"5min":5.4829999999999997,"15min":3.5800000000000001},"min":{"1min":0.219,"5min":0.192,"15min":0.187},"max":{"1min":13.880000000000001,"5min":170.678,"15min":170.678},"last":0.59899999999999998},{"interface":"front","average":{"1min":5.1459999999999999,"5min":5.5549999999999997,"15min":3.6320000000000001},"min":{"1min":0.32100000000000001,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":12.609999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":2.4009999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:46:52 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":4.8449999999999998,"15min":3.7919999999999998},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":10.922000000000001,"5min":162.696,"15min":162.696},"last":2.7210000000000001},{"interface":"front","average":{"1min":4.665,"5min":5.0709999999999997,"15min":3.6320000000000001},"min":{"1min":0.27700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.705,"5min":162.93899999999999,"15min":162.93899999999999},"last":1.474}]},{"osd":3,"last update":"Sat Sep 30 09:47:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":6.0049999999999999,"15min":5.109},"min":{"1min":0.311,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.6609999999999996,"5min":156.39400000000001,"15min":156.39400000000001},"last":1.3380000000000001},{"interface":"front","average":{"1min":4.1980000000000004,"5min":5.9400000000000004,"15min":4.9740000000000002},"min":{"1min":0.38500000000000001,"5min":0.224,"15min":0.224},"max":{"1min":14.169,"5min":156.376,"15min":156.376},"last":0.63100000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":1.8120000000000001,"15min":1.881},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1270000000000007,"5min":47.463000000000001,"15min":83.012},"last":1.2789999999999999},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.8520000000000001,"15min":1.879},"min":{"1min":0.42099999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.997999999999999,"5min":47.412999999999997,"15min":81.335999999999999},"last":0.26100000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":5.2030000000000003,"5min":1.901,"15min":1.351},"min":{"1min":0.378,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":47.226999999999997,"5min":47.226999999999997,"15min":47.226999999999997},"last":1.319},{"interface":"front","average":{"1min":5.1379999999999999,"5min":2.0800000000000001,"15min":1.5700000000000001},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":47.127000000000002,"5min":47.127000000000002,"15min":47.127000000000002},"last":2.7789999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.4199999999999999,"15min":3.2309999999999999},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.026999999999999,"5min":11.026999999999999,"15min":154.417},"last":0.29399999999999998},{"interface":"front","average":{"1min":3.1880000000000002,"5min":1.8140000000000001,"15min":3.6379999999999999},"min":{"1min":0.46100000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.846,"5min":10.846,"15min":154.04400000000001},"last":2.8359999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":2.1240000000000001,"15min":2.4089999999999998},"min":{"1min":0.48199999999999998,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":10.928000000000001,"5min":49.816000000000003,"15min":123.294},"last":0.32000000000000001},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.3889999999999998,"15min":2.7429999999999999},"min":{"1min":0.436,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":10.885,"5min":50.956000000000003,"15min":123.16},"last":2.8069999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435577,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1846516,"kb_used_data":996396,"kb_used_omap":2,"kb_used_meta":850109,"kb_avail":91898636,"statfs":{"total":95995035648,"available":94104203264,"internally_reserved":0,"allocated":1020309504,"data_stored":1017282830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":870511975},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0759999999999996,"5min":5.2880000000000003,"15min":4.9829999999999997},"min":{"1min":0.22700000000000001,"5min":0.183,"15min":0.183},"max":{"1min":45.957999999999998,"5min":144.72,"15min":205.73099999999999},"last":0.23499999999999999},{"interface":"front","average":{"1min":4.8479999999999999,"5min":5.2919999999999998,"15min":4.9100000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":44.298999999999999,"5min":142.36600000000001,"15min":205.495},"last":0.68799999999999994}]},{"osd":1,"last update":"Sat Sep 30 09:46:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0869999999999997,"5min":3.7599999999999998,"15min":7.4470000000000001},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":19.059000000000001,"5min":142.434,"15min":273.99599999999998},"last":0.374},{"interface":"front","average":{"1min":4.1550000000000002,"5min":3.7879999999999998,"15min":7.1299999999999999},"min":{"1min":0.28199999999999997,"5min":0.26800000000000002,"15min":0.20699999999999999},"max":{"1min":13.183,"5min":142.44900000000001,"15min":274.21600000000001},"last":1.5509999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":3.8239999999999998,"5min":5.3940000000000001,"15min":5.7999999999999998},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":12.561999999999999,"5min":146.92599999999999,"15min":206.69},"last":5.5030000000000001},{"interface":"front","average":{"1min":3.5979999999999999,"5min":4.9619999999999997,"15min":5.5789999999999997},"min":{"1min":0.38900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.834,"5min":145.75299999999999,"15min":209.22800000000001},"last":2.5950000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.3180000000000001,"15min":2.0920000000000001},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.2919999999999998,"5min":81.677000000000007,"15min":81.677000000000007},"last":0.42699999999999999},{"interface":"front","average":{"1min":3.0499999999999998,"5min":2.6360000000000001,"15min":2.04},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":18.943000000000001,"5min":80.623000000000005,"15min":80.623000000000005},"last":0.50900000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:46:29 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":1.732,"15min":1.595},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.701000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":1.621},{"interface":"front","average":{"1min":2.2839999999999998,"5min":1.218,"15min":1.04},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":1.498,"15min":1.9790000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33200000000000002},"max":{"1min":10.837999999999999,"5min":10.837999999999999,"15min":60.698},"last":0.498},{"interface":"front","average":{"1min":2.4369999999999998,"5min":1.659,"15min":1.9650000000000001},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":8.2050000000000001,"5min":10.356999999999999,"15min":12.121},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":3.488,"15min":2.7120000000000002},"min":{"1min":0.42499999999999999,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":11.753,"5min":213.548,"15min":213.548},"last":1.661},{"interface":"front","average":{"1min":3.2559999999999998,"5min":3.3290000000000002,"15min":2.605},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.888999999999999,"5min":213.583,"15min":213.583},"last":0.96599999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019340,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":865416,"kb_used_data":382656,"kb_used_omap":0,"kb_used_meta":482752,"kb_avail":92879736,"statfs":{"total":95995035648,"available":95108849664,"internally_reserved":0,"allocated":391839744,"data_stored":390115122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":494338048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":2.7090000000000001,"15min":2.5640000000000001},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":8.7989999999999995,"5min":73.718000000000004,"15min":105.179},"last":53.241999999999997},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.7589999999999999,"15min":2.75},"min":{"1min":0.32000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.032,"5min":74.001999999999995,"15min":104.70999999999999},"last":53.340000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.5779999999999998,"15min":3.871},"min":{"1min":0.313,"5min":0.25,"15min":0.25},"max":{"1min":8.8620000000000001,"5min":80.712000000000003,"15min":160.69999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":3.589,"5min":2.8490000000000002,"15min":3.8940000000000001},"min":{"1min":0.438,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":15.345000000000001,"5min":80.810000000000002,"15min":160.50700000000001},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3069999999999999,"5min":4.9989999999999997,"15min":3.492},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":13.26,"5min":142.88999999999999,"15min":142.88999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":3.4159999999999999,"5min":5.069,"15min":3.5249999999999999},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":8.8870000000000005,"5min":142.90700000000001,"15min":142.90700000000001},"last":0.61899999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.0369999999999999,"15min":2.1800000000000002},"min":{"1min":0.35799999999999998,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":9.0600000000000005,"5min":29.41,"15min":52.762},"last":0.65000000000000002},{"interface":"front","average":{"1min":3.6920000000000002,"5min":2.0099999999999998,"15min":2.1890000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.29299999999999998},"max":{"1min":9.0709999999999997,"5min":29.401,"15min":52.790999999999997},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:47:26 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":7.0880000000000001,"15min":9.5180000000000007},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.1470000000000002,"5min":140.11099999999999,"15min":140.11099999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.2889999999999999,"5min":7.0179999999999998,"15min":9.516},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.8789999999999996,"5min":140.065,"15min":140.065},"last":0.32300000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:47:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":0.84799999999999998,"15min":2.1850000000000001},"min":{"1min":0.34499999999999997,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":9.1470000000000002,"5min":9.1470000000000002,"15min":202.72499999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.026,"5min":0.78300000000000003,"15min":2.1720000000000002},"min":{"1min":0.311,"5min":0.253,"15min":0.249},"max":{"1min":5.2539999999999996,"5min":7.173,"15min":202.42400000000001},"last":0.376}]},{"osd":7,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":2.5099999999999998,"15min":3.2000000000000002},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.771000000000001,"5min":161.72999999999999,"15min":230.90100000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":2.4820000000000002,"15min":3.0569999999999999},"min":{"1min":0.41999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.2970000000000006,"5min":162.60900000000001,"15min":229.97},"last":0.52700000000000002}]}]},{"osd":5,"up_from":489,"seq":2100239007782,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1233480,"kb_used_data":1091136,"kb_used_omap":6,"kb_used_meta":142329,"kb_avail":92511672,"statfs":{"total":95995035648,"available":94731952128,"internally_reserved":0,"allocated":1117323264,"data_stored":1115595192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":145744991},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3500000000000001,"5min":21.718,"15min":24.779},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":25.786999999999999,"5min":281.10599999999999,"15min":281.10599999999999},"last":5.6669999999999998},{"interface":"front","average":{"1min":3.851,"5min":22.048999999999999,"15min":25.082000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":25.663,"5min":282.07400000000001,"15min":282.07400000000001},"last":0.23899999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":12.692,"15min":14.177},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":24.478999999999999,"5min":273.48200000000003,"15min":273.48200000000003},"last":7.4859999999999998},{"interface":"front","average":{"1min":4.3319999999999999,"5min":12.15,"15min":13.452999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":24.359000000000002,"5min":273.50200000000001,"15min":273.50200000000001},"last":5.7889999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":11.537000000000001,"5min":3.0600000000000001,"15min":1.647},"min":{"1min":0.40899999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":163.23099999999999,"5min":163.23099999999999,"15min":163.23099999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":11.898999999999999,"5min":3.6120000000000001,"15min":2.2309999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":162.71100000000001,"5min":162.71100000000001,"15min":162.71100000000001},"last":3.069}]},{"osd":3,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":3.113,"5min":12.420999999999999,"15min":13.973000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.271000000000001,"5min":274.01799999999997,"15min":274.01799999999997},"last":7.452},{"interface":"front","average":{"1min":3.2709999999999999,"5min":2.0390000000000001,"15min":1.8340000000000001},"min":{"1min":0.38400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":13.259,"5min":13.259,"15min":13.259},"last":6.0739999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.88500000000000001,"15min":0.92900000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8769999999999998,"5min":4.1769999999999996,"15min":4.1769999999999996},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.262,"5min":1.169,"15min":1.153},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.7610000000000001,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":3.0259999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.5129999999999999,"15min":1.601},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.758,"5min":14.371,"15min":14.371},"last":5.7569999999999997},{"interface":"front","average":{"1min":1.069,"5min":1.325,"15min":1.3680000000000001},"min":{"1min":0.379,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.7359999999999998,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:46:28 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.238,"15min":1.2749999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.1150000000000002,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":5.7199999999999998},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.90400000000000003,"15min":0.91200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.0680000000000001,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.497}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":458162176,"data_stored":458162176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1164840960,"data_stored":1164836888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":495058944,"data_stored":495058944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1012662272,"data_stored":1012662272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":382205952,"data_stored":382205952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1108611072,"data_stored":1108607000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":372572160,"data_stored":372572160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1129385984,"data_stored":1129381912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:47:34.535 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-30T09:47:34.620 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:34.624 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:34.667 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:34.694 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:34.729 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:34.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:34.790 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:34 smithi019 ceph-mon[216815]: pgmap v1379: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:34.791 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:34 smithi019 ceph-mon[224043]: pgmap v1379: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:34.839 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:34.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:34 smithi167 ceph-mon[192880]: pgmap v1379: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1192/91155 objects misplaced (1.308%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:47:34.959 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:35.383 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:47:35.428 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:47:35.447 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:35.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:35.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:35.475 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:47:35.534 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:47:35.553 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:35.592 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:35.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:47:35.631 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:47:35.665 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:35.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:35.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:35.680 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:47:35.711 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:47:35.715 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:47:35.715 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:47:35.716 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:47:35.716 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:47:35.716 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:47:35.716 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:47:35.716 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:47:35.716 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:47:35.773 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:35.863 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:35.958 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:35.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:35.972 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2962356556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:35.972 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2962356556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:35.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:36.025 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2962356556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:36.073 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:36.143 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:36.209 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:36.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:36.319 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:36.384 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:36.826 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:47:36.826 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46214.0:168284 9.7 e521/493 9:e2c37f8d:::benchmark_data_smithi019_560486_object71539:head v 521'10617, mlcod=521'10614)", 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:36.817424+0000", 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0076275179999999998, 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076581770000000004, 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:36.827 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817424+0000", 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817424+0000", 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817423+0000", 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:36.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817432+0000", 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3510000000000002e-06 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817443+0000", 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1379999999999999e-05 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817448+0000", 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4689999999999996e-06 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817487+0000", 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8928999999999999e-05 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817508+0000", 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0418999999999999e-05 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46214.0:168288 9.7 e521/493 9:e1ae962c:::benchmark_data_smithi019_560486_object71543:head v 521'10618, mlcod=521'10614)", 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:36.817465+0000", 2023-09-30T09:47:36.831 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0075861360000000003, 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076928860000000003, 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817465+0000", 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817465+0000", 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817465+0000", 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817469+0000", 2023-09-30T09:47:36.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7409999999999998e-06 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817476+0000", 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.866e-06 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817480+0000", 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8829999999999999e-06 2023-09-30T09:47:36.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.820884+0000", 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034039209999999999 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.820909+0000", 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5503e-05 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:36.835 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46214.0:168293 9.7 e521/493 9:e204bfb2:::benchmark_data_smithi019_560486_object71548:head v 521'10619, mlcod=521'10615)", 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:36.817485+0000", 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0075665680000000001, 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077264839999999996, 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817485+0000", 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817485+0000", 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817485+0000", 2023-09-30T09:47:36.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967296 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817493+0000", 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8709999999999997e-06 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817498+0000", 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4249999999999997e-06 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.817502+0000", 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5159999999999999e-06 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824117+0000", 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0066153100000000001 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824135+0000", 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8590999999999999e-05 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:168302 9.d 9:b73ec8c8:::benchmark_data_smithi019_560486_object71557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:36.822091+0000", 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.002960308, 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031834789999999999, 2023-09-30T09:47:36.840 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 168302 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822091+0000", 2023-09-30T09:47:36.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822091+0000", 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822090+0000", 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:36.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822096+0000", 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0469999999999999e-06 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822106+0000", 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6169999999999996e-06 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822111+0000", 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2630000000000003e-06 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822157+0000", 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5892e-05 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822267+0000", 2023-09-30T09:47:36.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000110282 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.822388+0000", 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000121033 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46214.0:168304 9.5 e521/495 9:af2bc5fa:::benchmark_data_smithi019_560486_object71559:head v 521'10500, mlcod=521'10498)", 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:36.824137+0000", 2023-09-30T09:47:36.845 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.00091487600000000001, 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011834899999999999, 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824137+0000", 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824137+0000", 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824136+0000", 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824143+0000", 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8320000000000006e-06 2023-09-30T09:47:36.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824154+0000", 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0581000000000001e-05 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824159+0000", 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4659999999999998e-06 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824223+0000", 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3461999999999996e-05 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:36.824241+0000", 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8502999999999999e-05 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:36.849 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:47:36.850 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 5 2023-09-30T09:47:36.850 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:47:37.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:36 smithi019 ceph-mon[224043]: pgmap v1380: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 963/81522 objects misplaced (1.181%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:37.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:36 smithi019 ceph-mon[216815]: pgmap v1380: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 963/81522 objects misplaced (1.181%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:36 smithi167 ceph-mon[192880]: pgmap v1380: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 963/81522 objects misplaced (1.181%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:37.375 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:37.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:37.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:37.480 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:37.481 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:37.570 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:37.799 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:47:37.809 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:37.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:37.922 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:38.006 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:38.258 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:38.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:38.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:38.355 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:38.372 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:38.425 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:38.527 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:38.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:38.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:38.633 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:38.727 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:38.839 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:38.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:38.946 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:38 smithi019 ceph-mon[216815]: pgmap v1381: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 963/81522 objects misplaced (1.181%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/396110058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:38 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:39.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:38 smithi019 ceph-mon[224043]: pgmap v1381: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 963/81522 objects misplaced (1.181%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:39.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/396110058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:39.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:38 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:39.113 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:39.113 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:38 smithi167 ceph-mon[192880]: pgmap v1381: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 963/81522 objects misplaced (1.181%); 2.8 MiB/s, 44 objects/s recovering 2023-09-30T09:47:39.114 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/396110058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:39.114 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:38 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:47:39.462 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:47:39.474 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:39.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:39.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:39.578 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:39.676 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:39.794 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:47:39.848 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:39.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:39.933 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:39 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:39.933 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:47:39.933 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:39 smithi019 ceph-mon[216815]: osdmap e522: 8 total, 8 up, 8 in 2023-09-30T09:47:39.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:39 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:39.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:47:39.934 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:39 smithi019 ceph-mon[224043]: osdmap e522: 8 total, 8 up, 8 in 2023-09-30T09:47:39.970 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:40.033 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:40.219 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:39 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:40.219 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:47:40.219 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:39 smithi167 ceph-mon[192880]: osdmap e522: 8 total, 8 up, 8 in 2023-09-30T09:47:40.273 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:40.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:40.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:40.378 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:40.448 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:40.512 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:47:40.513 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:47:40.513 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:47:40.513 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:47:40.513 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:47:40.540 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:40.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:40.647 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:40.708 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:47:40.709 INFO:teuthology.orchestra.run.smithi019.stdout:{"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1798,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":522,"num_osds":8,"num_up_osds":8,"osd_up_since":1696067065,"num_in_osds":8,"osd_in_since":1696066981,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":118},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":27174,"data_bytes":1783764927,"bytes_used":10251800576,"bytes_avail":757708484608,"bytes_total":767960285184,"misplaced_objects":963,"misplaced_total":81522,"misplaced_ratio":0.011812762199160963,"recovering_objects_per_sec":44,"recovering_bytes_per_sec":2947151,"recovering_keys_per_sec":0,"num_objects_recovered":450,"num_bytes_recovered":29491200,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":632},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi019.kcgktg","status":"up:active","gid":14670}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-30T09:47:05.837976+0000","services":{}},"progress_events":{"5b5f721d-3b3c-459d-b09b-c0a83655ed5d":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"9f0380b9-3874-4c4c-8c58-eede54796939":{"message":"Upgrade to 18.0.0-6447-g1d10b717 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-30T09:47:40.715 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:40.796 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:40.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:40.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:40.903 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:40.950 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:41.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[224043]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[224043]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[224043]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[224043]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[224043]: pgmap v1383: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 699/71367 objects misplaced (0.979%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2272998158' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[216815]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[216815]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[216815]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[216815]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[216815]: pgmap v1383: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 699/71367 objects misplaced (0.979%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:47:41.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2272998158' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:47:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:40 smithi167 ceph-mon[192880]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:40 smithi167 ceph-mon[192880]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:40 smithi167 ceph-mon[192880]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:40 smithi167 ceph-mon[192880]: failed to encode map e522 with expected crc 2023-09-30T09:47:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:40 smithi167 ceph-mon[192880]: pgmap v1383: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 699/71367 objects misplaced (0.979%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:47:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2272998158' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-30T09:47:41.519 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:47:41.760 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:41.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:41.873 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:41.879 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-30T09:47:41.880 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:47:41.919 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:42.034 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:42.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:42.059 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:42.111 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:47:42.143 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:42.197 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:42.288 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:42.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:42.392 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:42.454 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:42.606 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:42.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:42.712 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:42.799 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:43.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:42 smithi019 ceph-mon[216815]: pgmap v1384: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 699/71367 objects misplaced (0.979%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:47:43.036 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:42 smithi019 ceph-mon[224043]: pgmap v1384: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 699/71367 objects misplaced (0.979%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:47:43.062 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:43.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:43.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:43.166 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:43.191 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:42 smithi167 ceph-mon[192880]: pgmap v1384: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 699/71367 objects misplaced (0.979%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:47:43.194 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:43.237 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:43.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:43.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:43.341 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:43.369 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:43.414 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:43.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:43.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:43.517 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:43.556 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:47:43.614 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:43.911 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:43.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:43.927 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:47:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:47:43] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-09-30T09:47:43.960 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:44.019 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:44.074 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:44.181 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:44.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:44.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:44.293 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:44.317 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:47:44.317 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:47:44.317 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:47:44.317 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:47:44.317 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.317 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101707 8.c 8:32eb269e:::benchmark_data_smithi019_478679_object9259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:47:44.317 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:47:44.317 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 460.995476365, 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.50943429299999998, 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101707 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.318 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314890+0000", 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314889+0000", 2023-09-30T09:47:44.319 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314894+0000", 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3600000000000004e-06 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314906+0000", 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1963999999999999e-05 2023-09-30T09:47:44.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.314912+0000", 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2939999999999996e-06 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.774633+0000", 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.45972106400000001 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.778979+0000", 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043463390000000003 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.785183+0000", 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062043089999999999 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.797548+0000", 2023-09-30T09:47:44.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012364841 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.798913+0000", 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013653890000000001 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.811924+0000", 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013010661999999999 2023-09-30T09:47:44.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.812345+0000", 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00042060699999999998 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.814649+0000", 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023046479999999999 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.824324+0000", 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0096749469999999997 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101711 8.c 8:342f96d5:::benchmark_data_smithi019_478679_object9263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 460.93069803399999, 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.50859565699999998, 2023-09-30T09:47:44.325 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101711 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:47:44.326 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379668+0000", 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379667+0000", 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379676+0000", 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1120000000000002e-06 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379696+0000", 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9777999999999999e-05 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.379705+0000", 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0049999999999993e-06 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.824943+0000", 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.44523816599999999 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.830542+0000", 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055991670000000004 2023-09-30T09:47:44.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.837804+0000", 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0072618020000000004 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.871495+0000", 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.033691169999999999 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.872040+0000", 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00054504200000000003 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.873778+0000", 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00173759 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.874777+0000", 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00099985400000000011 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.876989+0000", 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022113549999999999 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.888264+0000", 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011275253000000001 2023-09-30T09:47:44.332 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101717 8.c 8:35bdb3c9:::benchmark_data_smithi019_478679_object9269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 460.905450535, 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.49835376599999998, 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.333 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101717 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404916+0000", 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404914+0000", 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404923+0000", 2023-09-30T09:47:44.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4540000000000007e-06 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404937+0000", 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4056000000000001e-05 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.404944+0000", 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3300000000000001e-06 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.845543+0000", 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.44059901200000001 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.851287+0000", 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057437360000000002 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:47:44.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.859094+0000", 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0078075109999999996 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.872354+0000", 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013259979 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.889815+0000", 2023-09-30T09:47:44.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.017460304999999999 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.891113+0000", 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001298738 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.892101+0000", 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00098792200000000002 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.894073+0000", 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019714590000000001 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.903270+0000", 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091969100000000008 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101732 8.c 8:37cad26d:::benchmark_data_smithi019_478679_object9284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 460.80639702000002, 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.45659628400000002, 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101732 2023-09-30T09:47:44.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503969+0000", 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503968+0000", 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503977+0000", 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1269999999999992e-06 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.503994+0000", 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.702e-05 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.504002+0000", 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0490000000000004e-06 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.903740+0000", 2023-09-30T09:47:44.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.39973803200000002 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.908656+0000", 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049154869999999996 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.916127+0000", 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007471742 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.946021+0000", 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.029893491000000001 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.947537+0000", 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001516107 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.948747+0000", 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001209768 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.949116+0000", 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000369735 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.951057+0000", 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001940499 2023-09-30T09:47:44.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.960566+0000", 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0095088229999999996 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101748 8.c 8:3570d742:::benchmark_data_smithi019_478679_object9300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:47:44.348 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 460.77195913899999, 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.43868606199999999, 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101748 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538407+0000", 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538406+0000", 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:44.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538413+0000", 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.25e-06 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538425+0000", 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1833e-05 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.351 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.538431+0000", 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.541e-06 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.923820+0000", 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.38538876100000002 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.929942+0000", 2023-09-30T09:47:44.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0061226579999999996 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.937368+0000", 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007425783 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.961017+0000", 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.023649196000000001 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.962281+0000", 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001264285 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.963794+0000", 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015121139999999999 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.965116+0000", 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001322611 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.967433+0000", 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023167349999999999 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.977093+0000", 2023-09-30T09:47:44.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0096604380000000004 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101750 8.c 8:3710a7dd:::benchmark_data_smithi019_478679_object9302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 460.754497539, 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.47879937299999997, 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.356 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101750 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555869+0000", 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555867+0000", 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555876+0000", 2023-09-30T09:47:44.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9660000000000002e-06 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555894+0000", 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7373000000000001e-05 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.555904+0000", 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0555e-05 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.977641+0000", 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.42173738700000002 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.982678+0000", 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050369020000000002 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:47:44.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.989714+0000", 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007035683 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.018051+0000", 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.028337256000000002 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.019726+0000", 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00167417 2023-09-30T09:47:44.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.020234+0000", 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00050844099999999999 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.023410+0000", 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003176201 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.025302+0000", 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018919290000000001 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.034668+0000", 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0093661639999999997 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101764 8.c 8:3df3ae0b:::benchmark_data_smithi019_478679_object9316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 460.68267772600001, 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.421116035, 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101764 2023-09-30T09:47:44.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627689+0000", 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627687+0000", 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627696+0000", 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3260000000000002e-06 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627713+0000", 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7232999999999999e-05 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.627722+0000", 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6969999999999999e-06 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.996707+0000", 2023-09-30T09:47:44.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.36898468600000001 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.001172+0000", 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0044654580000000003 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.008935+0000", 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077631189999999997 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.020869+0000", 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011933208000000001 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.022114+0000", 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012456979999999999 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.036065+0000", 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013950393 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.037310+0000", 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001245377 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.039037+0000", 2023-09-30T09:47:44.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017264960000000001 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.048805+0000", 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097681729999999998 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:101769 8.c 8:36635324:::benchmark_data_smithi019_478679_object9321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2023-09-30T09:47:44.371 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 460.577941135, 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.374847404, 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101769 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732425+0000", 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732424+0000", 2023-09-30T09:47:44.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732431+0000", 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3520000000000003e-06 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732444+0000", 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3142000000000001e-05 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:03.732450+0000", 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9399999999999999e-06 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.049283+0000", 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.31683297199999999 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.054085+0000", 2023-09-30T09:47:44.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048020700000000003 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4,5", 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.060789+0000", 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.006704448 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.090971+0000", 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.030181990999999998 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.092186+0000", 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001215198 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.093180+0000", 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00099359499999999989 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.094984+0000", 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018042119999999999 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.097274+0000", 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022901610000000002 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:40:04.107273+0000", 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0099984359999999994 2023-09-30T09:47:44.378 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126845 8.c 8:36dee011:::benchmark_data_smithi019_478679_object34397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 358.53850778899999, 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.93607846500000003, 2023-09-30T09:47:44.379 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126845 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:47:44.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771858+0000", 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771863+0000", 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6930000000000004e-06 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771879+0000", 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6004000000000001e-05 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771886+0000", 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4760000000000003e-06 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771947+0000", 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1393e-05 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771958+0000", 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0495e-05 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.687778+0000", 2023-09-30T09:47:44.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.91581990400000002 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.687930+0000", 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015184700000000001 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.688083+0000", 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000153113 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.694570+0000", 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064871449999999997 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695773+0000", 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012035189999999999 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707761+0000", 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011987507999999999 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707817+0000", 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6196999999999997e-05 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707937+0000", 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000120059 2023-09-30T09:47:44.387 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126846 8.c 8:38abd25d:::benchmark_data_smithi019_478679_object34398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 358.53016916299998, 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92797822600000002, 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.388 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126846 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:47:44.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780196+0000", 2023-09-30T09:47:44.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780202+0000", 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0950000000000001e-06 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780217+0000", 2023-09-30T09:47:44.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4602e-05 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780223+0000", 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6860000000000003e-06 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780256+0000", 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2988999999999999e-05 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780263+0000", 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1359999999999999e-06 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691354+0000", 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.911091706 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691496+0000", 2023-09-30T09:47:44.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000142058 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691644+0000", 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014777599999999999 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695526+0000", 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003882108 2023-09-30T09:47:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695780+0000", 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000253804 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708058+0000", 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012278213 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708097+0000", 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8503e-05 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708175+0000", 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8568999999999999e-05 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.396 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:132587 8.d 8:b57b0da1:::benchmark_data_smithi019_478679_object40139:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e435)", 2023-09-30T09:47:44.397 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:47:44.397 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 336.40429768799999, 2023-09-30T09:47:44.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.74966547400000005, 2023-09-30T09:47:44.397 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.397 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.397 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.397 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.397 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 132587 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:47:44.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906068+0000", 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906071+0000", 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1879999999999998e-06 2023-09-30T09:47:44.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906078+0000", 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7440000000000001e-06 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906083+0000", 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5639999999999998e-06 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.914573+0000", 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0084897930000000007 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.914581+0000", 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8370000000000003e-06 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644509+0000", 2023-09-30T09:47:44.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.72992777600000003 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644642+0000", 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013305200000000001 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2", 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644719+0000", 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6363000000000001e-05 2023-09-30T09:47:44.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655647+0000", 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010928208 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655652+0000", 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.967e-06 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655686+0000", 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4452999999999997e-05 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655734+0000", 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7939000000000003e-05 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:176082 8.18 8:1f93b3c9:::benchmark_data_smithi019_478679_object83634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 232.028138442, 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.35076756999999997, 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 176082 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.405 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282230+0000", 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6960000000000001e-06 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282237+0000", 2023-09-30T09:47:44.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7569999999999998e-06 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282241+0000", 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9249999999999997e-06 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.284838+0000", 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002597125 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.284993+0000", 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015470999999999999 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.285408+0000", 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00041504500000000001 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.287512+0000", 2023-09-30T09:47:44.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021044219999999999 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.500557+0000", 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21304519899999999 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632421+0000", 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.13186376399999999 2023-09-30T09:47:44.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632470+0000", 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8624000000000001e-05 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632996+0000", 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052566699999999995 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.411 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:176090 8.8 8:17545422:::benchmark_data_smithi019_478679_object83642:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 232.02113028299999, 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34340390900000001, 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:47:44.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 176090 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289239+0000", 2023-09-30T09:47:44.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.264e-06 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289246+0000", 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8859999999999998e-06 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289250+0000", 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4379999999999997e-06 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289291+0000", 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1014999999999999e-05 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289420+0000", 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128823 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2,3", 2023-09-30T09:47:44.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289844+0000", 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00042338899999999998 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.292151+0000", 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023077509999999998 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.294495+0000", 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023442430000000002 2023-09-30T09:47:44.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.500522+0000", 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20602649000000001 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632458+0000", 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.13193581900000001 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632514+0000", 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5893000000000002e-05 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632640+0000", 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000126327 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2297 9.d 9:b0cd7046:::benchmark_data_smithi019_560486_object2296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 199.27888493099999, 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0633178289999998, 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2297 2023-09-30T09:47:44.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031516+0000", 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.464e-05 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031523+0000", 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8779999999999998e-06 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031528+0000", 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6970000000000001e-06 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032166+0000", 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00063765200000000005 2023-09-30T09:47:44.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032280+0000", 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000113811 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032419+0000", 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013967100000000001 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.033675+0000", 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001255363 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053674+0000", 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.019999847000000001 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053684+0000", 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9990000000000003e-06 2023-09-30T09:47:44.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.059238+0000", 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.00555335 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.059246+0000", 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4940000000000003e-06 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.086761+0000", 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.027514383 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.088028+0000", 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012668429999999999 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094723+0000", 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0066955230000000001 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094799+0000", 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6225999999999998e-05 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.428 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2293 9.a 9:552d93c3:::benchmark_data_smithi019_560486_object2292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 198.26529859300001, 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.051156384, 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2293 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045066+0000", 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045217+0000", 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150221 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045234+0000", 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7676000000000002e-05 2023-09-30T09:47:44.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045243+0000", 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0429999999999996e-06 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045361+0000", 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011739900000000001 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045370+0000", 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0049999999999993e-06 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058484+0000", 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013114657 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058494+0000", 2023-09-30T09:47:44.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3559999999999999e-06 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.085277+0000", 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.026783555 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.087303+0000", 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020253559999999999 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096095+0000", 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0087920210000000006 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096224+0000", 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129453 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2299 9.a 9:5ff10925:::benchmark_data_smithi019_560486_object2298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 198.26510944699999, 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0514241090000001, 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2299 2023-09-30T09:47:44.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045256+0000", 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045294+0000", 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7457999999999999e-05 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045304+0000", 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0271999999999999e-05 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045310+0000", 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7760000000000003e-06 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045439+0000", 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128909 2023-09-30T09:47:44.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045445+0000", 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0229999999999998e-06 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058514+0000", 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013069717999999999 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058520+0000", 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2920000000000003e-06 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.087342+0000", 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.028822465 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.088285+0000", 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00094316300000000001 2023-09-30T09:47:44.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096601+0000", 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0083155179999999992 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096681+0000", 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0162999999999998e-05 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.442 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5657 9.d 9:b0e10006:::benchmark_data_smithi019_560486_object5656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 193.63367477599999, 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.006709437, 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:47:44.443 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5657 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:47:44.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676689+0000", 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676814+0000", 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012537599999999999 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676838+0000", 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3623000000000001e-05 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676847+0000", 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2590000000000006e-06 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676916+0000", 2023-09-30T09:47:44.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8670000000000005e-05 2023-09-30T09:47:44.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676927+0000", 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0849000000000001e-05 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680187+0000", 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0032605560000001 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680317+0000", 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129453 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680486+0000", 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016957800000000001 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682922+0000", 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024357670000000001 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683200+0000", 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027740300000000001 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683261+0000", 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1591e-05 2023-09-30T09:47:44.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683346+0000", 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4541000000000003e-05 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683401+0000", 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5316999999999999e-05 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.451 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5664 9.d 9:b51be770:::benchmark_data_smithi019_560486_object5663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 193.632518619, 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.005731259, 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:47:44.452 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5664 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677847+0000", 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677904+0000", 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7519999999999998e-05 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677923+0000", 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8369e-05 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677930+0000", 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4680000000000003e-06 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677977+0000", 2023-09-30T09:47:44.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7170999999999999e-05 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677988+0000", 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0855e-05 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680921+0000", 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0029331560000001 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681034+0000", 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011288199999999999 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681184+0000", 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150094 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682930+0000", 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017458339999999999 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683222+0000", 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00029221800000000001 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683451+0000", 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00022835100000000001 2023-09-30T09:47:44.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683528+0000", 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6852000000000002e-05 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683579+0000", 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1418999999999997e-05 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.459 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5673 9.d 9:b96c3c13:::benchmark_data_smithi019_560486_object5672:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 193.63058074099999, 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0071881890000001, 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:47:44.460 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5673 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679785+0000", 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679870+0000", 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4948000000000001e-05 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679884+0000", 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4348000000000001e-05 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679891+0000", 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9589999999999998e-06 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679948+0000", 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7120999999999997e-05 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679955+0000", 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7870000000000004e-06 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681621+0000", 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.001666497 2023-09-30T09:47:44.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681730+0000", 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010867 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681870+0000", 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013995699999999999 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682935+0000", 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010647829999999999 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.685992+0000", 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030569799999999999 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686739+0000", 2023-09-30T09:47:44.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00074698799999999997 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686937+0000", 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000197858 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686974+0000", 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7280999999999998e-05 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.467 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5686 9.d 9:b2765571:::benchmark_data_smithi019_560486_object5685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 193.62506226299999, 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.002027996, 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:47:44.468 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5686 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:47:44.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685339+0000", 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5487000000000001e-05 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685347+0000", 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.072e-06 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685353+0000", 2023-09-30T09:47:44.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8860000000000001e-06 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685417+0000", 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4487999999999996e-05 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685433+0000", 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5132e-05 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682219+0000", 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99678628499999999 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682328+0000", 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109515 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:47:44.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682483+0000", 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015417100000000001 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686027+0000", 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035446449999999999 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687236+0000", 2023-09-30T09:47:44.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012086670000000001 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687247+0000", 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.136e-05 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687295+0000", 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8118999999999998e-05 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:47:44.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687332+0000", 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6742999999999997e-05 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:47:44.476 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:44.535 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:44.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:44.642 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:47:44.679 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:44.741 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:47:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:44.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:47:44.845 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:44.876 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:44 smithi019 ceph-mon[216815]: pgmap v1385: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 699/71367 objects misplaced (0.979%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:47:44.876 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:44 smithi019 ceph-mon[224043]: pgmap v1385: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 699/71367 objects misplaced (0.979%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:47:44.880 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:44.948 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:44.962 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:47:44.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:44.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:45.013 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:47:45.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:44 smithi167 ceph-mon[192880]: pgmap v1385: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 699/71367 objects misplaced (0.979%); 3.1 MiB/s, 49 objects/s recovering 2023-09-30T09:47:45.045 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:47:45.069 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:45.084 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:47:45.137 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:47:45.159 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:45.188 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:47:45.230 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:47:45.270 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:47:45.275 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:47:45.275 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:47:45.275 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:47:45.275 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:47:45.275 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:47:45.275 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:47:45.275 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:47:45.275 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:47:45.302 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:45.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:45.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:45.330 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:47:45.412 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:45.572 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:45.690 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:45.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:45.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:45.798 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:45.873 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:45.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:45 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1614033171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:45.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:45 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1614033171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:46.042 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:46.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:46.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:46.151 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:46.189 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:45 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1614033171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:46.195 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:46.292 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:46.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:46.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:46.397 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:47:46.445 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:46.535 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:47:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:46.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:47:46.643 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:46.646 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:46.675 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:46.749 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:46.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:46.857 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:46.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:46 smithi019 ceph-mon[216815]: pgmap v1386: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 334/61917 objects misplaced (0.539%); 1.7 MiB/s, 62 objects/s recovering 2023-09-30T09:47:46.896 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:46 smithi019 ceph-mon[224043]: pgmap v1386: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 334/61917 objects misplaced (0.539%); 1.7 MiB/s, 62 objects/s recovering 2023-09-30T09:47:46.942 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:47.026 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:46 smithi167 ceph-mon[192880]: pgmap v1386: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 334/61917 objects misplaced (0.539%); 1.7 MiB/s, 62 objects/s recovering 2023-09-30T09:47:47.175 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:47.201 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:47.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:47.319 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:47.376 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:47.453 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:47.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:47.562 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:47.648 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:47.810 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:47:47.842 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:47:48.681 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:47:48.775 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:48.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:48.886 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:48.913 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:49.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:48 smithi019 ceph-mon[224043]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:49.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:48 smithi019 ceph-mon[224043]: pgmap v1387: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 334/61917 objects misplaced (0.539%); 1.7 MiB/s, 62 objects/s recovering 2023-09-30T09:47:49.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:49.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:48 smithi019 ceph-mon[216815]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:49.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:48 smithi019 ceph-mon[216815]: pgmap v1387: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 334/61917 objects misplaced (0.539%); 1.7 MiB/s, 62 objects/s recovering 2023-09-30T09:47:49.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:48 smithi167 ceph-mon[192880]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:47:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:48 smithi167 ceph-mon[192880]: pgmap v1387: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 334/61917 objects misplaced (0.539%); 1.7 MiB/s, 62 objects/s recovering 2023-09-30T09:47:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:47:49.405 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:49.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:49.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:49.509 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:49.576 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1386,"stamp":"2023-09-30T09:47:45.876558+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1355487167,"num_objects":20639,"num_object_clones":0,"num_object_copies":61917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":334,"num_objects_unfound":0,"num_objects_dirty":20639,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":173782,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6059,"num_bytes_recovered":368334450,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123397,"ondisk_log_size":123397,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8854256,"kb_used_data":4199852,"kb_used_omap":30,"kb_used_meta":4654241,"kb_avail":741106960,"statfs":{"total":767960285184,"available":758893527040,"internally_reserved":0,"allocated":4300648448,"data_stored":4281531634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4765943138},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,8,4,1,1,0,6],"upper_bound":256},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-428277760,"num_objects":-6535,"num_object_clones":0,"num_object_copies":-19605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-629,"num_objects_unfound":0,"num_objects_dirty":-6535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6535,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":629,"num_bytes_recovered":18219008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.013059"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":945,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352680+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:47:30.352680+0000","last_peered":"2023-09-30T09:47:30.352680+0000","last_clean":"2023-09-30T09:47:30.352680+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:47:30.352680+0000","last_undegraded":"2023-09-30T09:47:30.352680+0000","last_fullsized":"2023-09-30T09:47:30.352680+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":540,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364638+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:47:30.364638+0000","last_peered":"2023-09-30T09:47:30.364638+0000","last_clean":"2023-09-30T09:47:30.364638+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:47:30.364638+0000","last_undegraded":"2023-09-30T09:47:30.364638+0000","last_fullsized":"2023-09-30T09:47:30.364638+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":437,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.365192+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:47:30.365192+0000","last_peered":"2023-09-30T09:47:30.365192+0000","last_clean":"2023-09-30T09:47:30.365192+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:47:30.365192+0000","last_undegraded":"2023-09-30T09:47:30.365192+0000","last_fullsized":"2023-09-30T09:47:30.365192+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":947,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354033+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:47:30.354033+0000","last_peered":"2023-09-30T09:47:30.354033+0000","last_clean":"2023-09-30T09:47:30.354033+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:47:30.354033+0000","last_undegraded":"2023-09-30T09:47:30.354033+0000","last_fullsized":"2023-09-30T09:47:30.354033+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":87,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:39.778009+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:47:39.778009+0000","last_peered":"2023-09-30T09:47:39.778009+0000","last_clean":"2023-09-30T09:47:39.778009+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:47:39.778009+0000","last_undegraded":"2023-09-30T09:47:39.778009+0000","last_fullsized":"2023-09-30T09:47:39.778009+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":128,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.360862+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:47:30.360862+0000","last_peered":"2023-09-30T09:47:30.360862+0000","last_clean":"2023-09-30T09:47:30.360862+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:47:30.360862+0000","last_undegraded":"2023-09-30T09:47:30.360862+0000","last_fullsized":"2023-09-30T09:47:30.360862+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":755,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364724+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:47:30.364724+0000","last_peered":"2023-09-30T09:47:30.364724+0000","last_clean":"2023-09-30T09:47:30.364724+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:47:30.364724+0000","last_undegraded":"2023-09-30T09:47:30.364724+0000","last_fullsized":"2023-09-30T09:47:30.364724+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":27,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.361934+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:47:30.361934+0000","last_peered":"2023-09-30T09:47:30.361934+0000","last_clean":"2023-09-30T09:47:30.361934+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:47:30.361934+0000","last_undegraded":"2023-09-30T09:47:30.361934+0000","last_fullsized":"2023-09-30T09:47:30.361934+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.347605+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:47:30.347605+0000","last_peered":"2023-09-30T09:47:30.347605+0000","last_clean":"2023-09-30T09:47:30.347605+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:47:30.347605+0000","last_undegraded":"2023-09-30T09:47:30.347605+0000","last_fullsized":"2023-09-30T09:47:30.347605+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":923,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354395+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:47:30.354395+0000","last_peered":"2023-09-30T09:47:30.354395+0000","last_clean":"2023-09-30T09:47:30.354395+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:47:30.354395+0000","last_undegraded":"2023-09-30T09:47:30.354395+0000","last_fullsized":"2023-09-30T09:47:30.354395+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":142,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.361341+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:47:30.361341+0000","last_peered":"2023-09-30T09:47:30.361341+0000","last_clean":"2023-09-30T09:47:30.361341+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:47:30.361341+0000","last_undegraded":"2023-09-30T09:47:30.361341+0000","last_fullsized":"2023-09-30T09:47:30.361341+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":103,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:33.386265+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:47:33.386265+0000","last_peered":"2023-09-30T09:47:33.386265+0000","last_clean":"2023-09-30T09:47:33.386265+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:47:33.386265+0000","last_undegraded":"2023-09-30T09:47:33.386265+0000","last_fullsized":"2023-09-30T09:47:33.386265+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":712,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.358646+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:47:30.358646+0000","last_peered":"2023-09-30T09:47:30.358646+0000","last_clean":"2023-09-30T09:47:30.358646+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:47:30.358646+0000","last_undegraded":"2023-09-30T09:47:30.358646+0000","last_fullsized":"2023-09-30T09:47:30.358646+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":342,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352454+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:47:30.352454+0000","last_peered":"2023-09-30T09:47:30.352454+0000","last_clean":"2023-09-30T09:47:30.352454+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:47:30.352454+0000","last_undegraded":"2023-09-30T09:47:30.352454+0000","last_fullsized":"2023-09-30T09:47:30.352454+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":478,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354128+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:47:30.354128+0000","last_peered":"2023-09-30T09:47:30.354128+0000","last_clean":"2023-09-30T09:47:30.354128+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:47:30.354128+0000","last_undegraded":"2023-09-30T09:47:30.354128+0000","last_fullsized":"2023-09-30T09:47:30.354128+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":651,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352779+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:47:30.352779+0000","last_peered":"2023-09-30T09:47:30.352779+0000","last_clean":"2023-09-30T09:47:30.352779+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:47:30.352779+0000","last_undegraded":"2023-09-30T09:47:30.352779+0000","last_fullsized":"2023-09-30T09:47:30.352779+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":236,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354587+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:47:30.354587+0000","last_peered":"2023-09-30T09:47:30.354587+0000","last_clean":"2023-09-30T09:47:30.354587+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:47:30.354587+0000","last_undegraded":"2023-09-30T09:47:30.354587+0000","last_fullsized":"2023-09-30T09:47:30.354587+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.346709+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:47:30.346709+0000","last_peered":"2023-09-30T09:47:30.346709+0000","last_clean":"2023-09-30T09:47:30.346709+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:47:30.346709+0000","last_undegraded":"2023-09-30T09:47:30.346709+0000","last_fullsized":"2023-09-30T09:47:30.346709+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":577,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.360937+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:47:30.360937+0000","last_peered":"2023-09-30T09:47:30.360937+0000","last_clean":"2023-09-30T09:47:30.360937+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:47:30.360937+0000","last_undegraded":"2023-09-30T09:47:30.360937+0000","last_fullsized":"2023-09-30T09:47:30.360937+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":566,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364842+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:47:30.364842+0000","last_peered":"2023-09-30T09:47:30.364842+0000","last_clean":"2023-09-30T09:47:30.364842+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:47:30.364842+0000","last_undegraded":"2023-09-30T09:47:30.364842+0000","last_fullsized":"2023-09-30T09:47:30.364842+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":544,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.365019+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:47:30.365019+0000","last_peered":"2023-09-30T09:47:30.365019+0000","last_clean":"2023-09-30T09:47:30.365019+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:47:30.365019+0000","last_undegraded":"2023-09-30T09:47:30.365019+0000","last_fullsized":"2023-09-30T09:47:30.365019+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":776,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.361566+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:47:30.361566+0000","last_peered":"2023-09-30T09:47:30.361566+0000","last_clean":"2023-09-30T09:47:30.361566+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:47:30.361566+0000","last_undegraded":"2023-09-30T09:47:30.361566+0000","last_fullsized":"2023-09-30T09:47:30.361566+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":172,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354438+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:47:30.354438+0000","last_peered":"2023-09-30T09:47:30.354438+0000","last_clean":"2023-09-30T09:47:30.354438+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:47:30.354438+0000","last_undegraded":"2023-09-30T09:47:30.354438+0000","last_fullsized":"2023-09-30T09:47:30.354438+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":499,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352527+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:47:30.352527+0000","last_peered":"2023-09-30T09:47:30.352527+0000","last_clean":"2023-09-30T09:47:30.352527+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:47:30.352527+0000","last_undegraded":"2023-09-30T09:47:30.352527+0000","last_fullsized":"2023-09-30T09:47:30.352527+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":823,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352445+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:47:30.352445+0000","last_peered":"2023-09-30T09:47:30.352445+0000","last_clean":"2023-09-30T09:47:30.352445+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:47:30.352445+0000","last_undegraded":"2023-09-30T09:47:30.352445+0000","last_fullsized":"2023-09-30T09:47:30.352445+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":102,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.360333+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:47:30.360333+0000","last_peered":"2023-09-30T09:47:30.360333+0000","last_clean":"2023-09-30T09:47:30.360333+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:47:30.360333+0000","last_undegraded":"2023-09-30T09:47:30.360333+0000","last_fullsized":"2023-09-30T09:47:30.360333+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":197,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353885+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:47:30.353885+0000","last_peered":"2023-09-30T09:47:30.353885+0000","last_clean":"2023-09-30T09:47:30.353885+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:47:30.353885+0000","last_undegraded":"2023-09-30T09:47:30.353885+0000","last_fullsized":"2023-09-30T09:47:30.353885+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":924,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:33.386299+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:47:33.386299+0000","last_peered":"2023-09-30T09:47:33.386299+0000","last_clean":"2023-09-30T09:47:33.386299+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:47:33.386299+0000","last_undegraded":"2023-09-30T09:47:33.386299+0000","last_fullsized":"2023-09-30T09:47:33.386299+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354439+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:47:30.354439+0000","last_peered":"2023-09-30T09:47:30.354439+0000","last_clean":"2023-09-30T09:47:30.354439+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:47:30.354439+0000","last_undegraded":"2023-09-30T09:47:30.354439+0000","last_fullsized":"2023-09-30T09:47:30.354439+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":102,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.360643+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:47:30.360643+0000","last_peered":"2023-09-30T09:47:30.360643+0000","last_clean":"2023-09-30T09:47:30.360643+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:47:30.360643+0000","last_undegraded":"2023-09-30T09:47:30.360643+0000","last_fullsized":"2023-09-30T09:47:30.360643+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":932,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353632+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:47:30.353632+0000","last_peered":"2023-09-30T09:47:30.353632+0000","last_clean":"2023-09-30T09:47:30.353632+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:47:30.353632+0000","last_undegraded":"2023-09-30T09:47:30.353632+0000","last_fullsized":"2023-09-30T09:47:30.353632+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":118,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.345298+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:47:30.345298+0000","last_peered":"2023-09-30T09:47:30.345298+0000","last_clean":"2023-09-30T09:47:30.345298+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:47:30.345298+0000","last_undegraded":"2023-09-30T09:47:30.345298+0000","last_fullsized":"2023-09-30T09:47:30.345298+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":87,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:39.778770+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:47:39.778770+0000","last_peered":"2023-09-30T09:47:39.778770+0000","last_clean":"2023-09-30T09:47:39.778770+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:47:39.778770+0000","last_undegraded":"2023-09-30T09:47:39.778770+0000","last_fullsized":"2023-09-30T09:47:39.778770+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"522'10765","reported_seq":11077,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.318676+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:47:44.318676+0000","last_peered":"2023-09-30T09:47:44.318676+0000","last_clean":"2023-09-30T09:47:44.318676+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:47:44.318676+0000","last_undegraded":"2023-09-30T09:47:44.318676+0000","last_fullsized":"2023-09-30T09:47:44.318676+0000","mapping_epoch":495,"log_start":"501'3200","ondisk_log_start":"501'3200","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79233024,"num_objects":1209,"num_object_clones":0,"num_object_copies":3627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10765,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":898,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354525+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:47:30.354525+0000","last_peered":"2023-09-30T09:47:30.354525+0000","last_clean":"2023-09-30T09:47:30.354525+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:47:30.354525+0000","last_undegraded":"2023-09-30T09:47:30.354525+0000","last_fullsized":"2023-09-30T09:47:30.354525+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":198,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353503+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:47:30.353503+0000","last_peered":"2023-09-30T09:47:30.353503+0000","last_clean":"2023-09-30T09:47:30.353503+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:47:30.353503+0000","last_undegraded":"2023-09-30T09:47:30.353503+0000","last_fullsized":"2023-09-30T09:47:30.353503+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":143,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352344+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:47:30.352344+0000","last_peered":"2023-09-30T09:47:30.352344+0000","last_clean":"2023-09-30T09:47:30.352344+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:47:30.352344+0000","last_undegraded":"2023-09-30T09:47:30.352344+0000","last_fullsized":"2023-09-30T09:47:30.352344+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"522'10627","reported_seq":10674,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:43.792539+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:47:43.792539+0000","last_peered":"2023-09-30T09:47:43.792539+0000","last_clean":"2023-09-30T09:47:43.792539+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:47:43.792539+0000","last_undegraded":"2023-09-30T09:47:43.792539+0000","last_fullsized":"2023-09-30T09:47:43.792539+0000","mapping_epoch":483,"log_start":"504'4200","ondisk_log_start":"504'4200","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6427,"log_dups_size":0,"ondisk_log_size":6427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86310912,"num_objects":1317,"num_object_clones":0,"num_object_copies":3951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10627,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":916,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:38.397483+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:47:38.397483+0000","last_peered":"2023-09-30T09:47:38.397483+0000","last_clean":"2023-09-30T09:47:38.397483+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:47:38.397483+0000","last_undegraded":"2023-09-30T09:47:38.397483+0000","last_fullsized":"2023-09-30T09:47:38.397483+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":168,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353069+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:47:30.353069+0000","last_peered":"2023-09-30T09:47:30.353069+0000","last_clean":"2023-09-30T09:47:30.353069+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:47:30.353069+0000","last_undegraded":"2023-09-30T09:47:30.353069+0000","last_fullsized":"2023-09-30T09:47:30.353069+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":573,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364483+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:47:30.364483+0000","last_peered":"2023-09-30T09:47:30.364483+0000","last_clean":"2023-09-30T09:47:30.364483+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:47:30.364483+0000","last_undegraded":"2023-09-30T09:47:30.364483+0000","last_fullsized":"2023-09-30T09:47:30.364483+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"522'10920","reported_seq":11192,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.205864+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:47:44.205864+0000","last_peered":"2023-09-30T09:47:44.205864+0000","last_clean":"2023-09-30T09:47:44.205864+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:47:44.205864+0000","last_undegraded":"2023-09-30T09:47:44.205864+0000","last_fullsized":"2023-09-30T09:47:44.205864+0000","mapping_epoch":493,"log_start":"503'4200","ondisk_log_start":"503'4200","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82706432,"num_objects":1262,"num_object_clones":0,"num_object_copies":3786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10920,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":692,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.347742+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:47:30.347742+0000","last_peered":"2023-09-30T09:47:30.347742+0000","last_clean":"2023-09-30T09:47:30.347742+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:47:30.347742+0000","last_undegraded":"2023-09-30T09:47:30.347742+0000","last_fullsized":"2023-09-30T09:47:30.347742+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":235,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354271+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:47:30.354271+0000","last_peered":"2023-09-30T09:47:30.354271+0000","last_clean":"2023-09-30T09:47:30.354271+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:47:30.354271+0000","last_undegraded":"2023-09-30T09:47:30.354271+0000","last_fullsized":"2023-09-30T09:47:30.354271+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.346783+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:47:30.346783+0000","last_peered":"2023-09-30T09:47:30.346783+0000","last_clean":"2023-09-30T09:47:30.346783+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:47:30.346783+0000","last_undegraded":"2023-09-30T09:47:30.346783+0000","last_fullsized":"2023-09-30T09:47:30.346783+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"522'10647","reported_seq":11010,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:43.977857+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:47:43.977857+0000","last_peered":"2023-09-30T09:47:43.977857+0000","last_clean":"2023-09-30T09:47:43.977857+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:47:43.977857+0000","last_undegraded":"2023-09-30T09:47:43.977857+0000","last_fullsized":"2023-09-30T09:47:43.977857+0000","mapping_epoch":495,"log_start":"496'600","ondisk_log_start":"496'600","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10047,"log_dups_size":0,"ondisk_log_size":10047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81854464,"num_objects":1249,"num_object_clones":0,"num_object_copies":3747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10647,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":757,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.365530+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:47:30.365530+0000","last_peered":"2023-09-30T09:47:30.365530+0000","last_clean":"2023-09-30T09:47:30.365530+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:47:30.365530+0000","last_undegraded":"2023-09-30T09:47:30.365530+0000","last_fullsized":"2023-09-30T09:47:30.365530+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":457,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.355432+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:47:30.355432+0000","last_peered":"2023-09-30T09:47:30.355432+0000","last_clean":"2023-09-30T09:47:30.355432+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:47:30.355432+0000","last_undegraded":"2023-09-30T09:47:30.355432+0000","last_fullsized":"2023-09-30T09:47:30.355432+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":122,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.363089+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:47:30.363089+0000","last_peered":"2023-09-30T09:47:30.363089+0000","last_clean":"2023-09-30T09:47:30.363089+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:47:30.363089+0000","last_undegraded":"2023-09-30T09:47:30.363089+0000","last_fullsized":"2023-09-30T09:47:30.363089+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"522'10863","reported_seq":10910,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.289774+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:47:44.289774+0000","last_peered":"2023-09-30T09:47:44.289774+0000","last_clean":"2023-09-30T09:47:44.289774+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:47:44.289774+0000","last_undegraded":"2023-09-30T09:47:44.289774+0000","last_fullsized":"2023-09-30T09:47:44.289774+0000","mapping_epoch":483,"log_start":"502'3800","ondisk_log_start":"502'3800","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7063,"log_dups_size":0,"ondisk_log_size":7063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10863,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1004,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354276+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:47:30.354276+0000","last_peered":"2023-09-30T09:47:30.354276+0000","last_clean":"2023-09-30T09:47:30.354276+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:47:30.354276+0000","last_undegraded":"2023-09-30T09:47:30.354276+0000","last_fullsized":"2023-09-30T09:47:30.354276+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":81,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.362923+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:47:30.362923+0000","last_peered":"2023-09-30T09:47:30.362923+0000","last_clean":"2023-09-30T09:47:30.362923+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:47:30.362923+0000","last_undegraded":"2023-09-30T09:47:30.362923+0000","last_fullsized":"2023-09-30T09:47:30.362923+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":520,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364913+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:47:30.364913+0000","last_peered":"2023-09-30T09:47:30.364913+0000","last_clean":"2023-09-30T09:47:30.364913+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:47:30.364913+0000","last_undegraded":"2023-09-30T09:47:30.364913+0000","last_fullsized":"2023-09-30T09:47:30.364913+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"522'10650","reported_seq":10697,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.303224+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:47:44.303224+0000","last_peered":"2023-09-30T09:47:44.303224+0000","last_clean":"2023-09-30T09:47:44.303224+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:47:44.303224+0000","last_undegraded":"2023-09-30T09:47:44.303224+0000","last_fullsized":"2023-09-30T09:47:44.303224+0000","mapping_epoch":483,"log_start":"502'3600","ondisk_log_start":"502'3600","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7050,"log_dups_size":0,"ondisk_log_size":7050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85458944,"num_objects":1304,"num_object_clones":0,"num_object_copies":3912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10650,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":1003,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354346+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:47:30.354346+0000","last_peered":"2023-09-30T09:47:30.354346+0000","last_clean":"2023-09-30T09:47:30.354346+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:47:30.354346+0000","last_undegraded":"2023-09-30T09:47:30.354346+0000","last_fullsized":"2023-09-30T09:47:30.354346+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":77,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.346631+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:47:30.346631+0000","last_peered":"2023-09-30T09:47:30.346631+0000","last_clean":"2023-09-30T09:47:30.346631+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:47:30.346631+0000","last_undegraded":"2023-09-30T09:47:30.346631+0000","last_fullsized":"2023-09-30T09:47:30.346631+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":216,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364767+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:47:30.364767+0000","last_peered":"2023-09-30T09:47:30.364767+0000","last_clean":"2023-09-30T09:47:30.364767+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:47:30.364767+0000","last_undegraded":"2023-09-30T09:47:30.364767+0000","last_fullsized":"2023-09-30T09:47:30.364767+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"522'10946","reported_seq":10993,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:43.907915+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:47:43.907915+0000","last_peered":"2023-09-30T09:47:43.907915+0000","last_clean":"2023-09-30T09:47:43.907915+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:47:43.907915+0000","last_undegraded":"2023-09-30T09:47:43.907915+0000","last_fullsized":"2023-09-30T09:47:43.907915+0000","mapping_epoch":483,"log_start":"510'5100","ondisk_log_start":"510'5100","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86769664,"num_objects":1324,"num_object_clones":0,"num_object_copies":3972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10946,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":861,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.363104+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:47:30.363104+0000","last_peered":"2023-09-30T09:47:30.363104+0000","last_clean":"2023-09-30T09:47:30.363104+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:47:30.363104+0000","last_undegraded":"2023-09-30T09:47:30.363104+0000","last_fullsized":"2023-09-30T09:47:30.363104+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":527,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.363159+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:47:30.363159+0000","last_peered":"2023-09-30T09:47:30.363159+0000","last_clean":"2023-09-30T09:47:30.363159+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:47:30.363159+0000","last_undegraded":"2023-09-30T09:47:30.363159+0000","last_fullsized":"2023-09-30T09:47:30.363159+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":534,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.365601+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:47:30.365601+0000","last_peered":"2023-09-30T09:47:30.365601+0000","last_clean":"2023-09-30T09:47:30.365601+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:47:30.365601+0000","last_undegraded":"2023-09-30T09:47:30.365601+0000","last_fullsized":"2023-09-30T09:47:30.365601+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"522'10656","reported_seq":10707,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.022789+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:47:44.022789+0000","last_peered":"2023-09-30T09:47:44.022789+0000","last_clean":"2023-09-30T09:47:44.022789+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:47:44.022789+0000","last_undegraded":"2023-09-30T09:47:44.022789+0000","last_fullsized":"2023-09-30T09:47:44.022789+0000","mapping_epoch":483,"log_start":"496'600","ondisk_log_start":"496'600","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10056,"log_dups_size":0,"ondisk_log_size":10056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83886080,"num_objects":1280,"num_object_clones":0,"num_object_copies":3840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10656,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":725,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.359232+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:47:30.359232+0000","last_peered":"2023-09-30T09:47:30.359232+0000","last_clean":"2023-09-30T09:47:30.359232+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:47:30.359232+0000","last_undegraded":"2023-09-30T09:47:30.359232+0000","last_fullsized":"2023-09-30T09:47:30.359232+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.361244+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:47:30.361244+0000","last_peered":"2023-09-30T09:47:30.361244+0000","last_clean":"2023-09-30T09:47:30.361244+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:47:30.361244+0000","last_undegraded":"2023-09-30T09:47:30.361244+0000","last_fullsized":"2023-09-30T09:47:30.361244+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":207,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.365307+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:47:30.365307+0000","last_peered":"2023-09-30T09:47:30.365307+0000","last_clean":"2023-09-30T09:47:30.365307+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:47:30.365307+0000","last_undegraded":"2023-09-30T09:47:30.365307+0000","last_fullsized":"2023-09-30T09:47:30.365307+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"522'10831","reported_seq":11137,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.180642+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:47:44.180642+0000","last_peered":"2023-09-30T09:47:44.180642+0000","last_clean":"2023-09-30T09:47:44.180642+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:47:44.180642+0000","last_undegraded":"2023-09-30T09:47:44.180642+0000","last_fullsized":"2023-09-30T09:47:44.180642+0000","mapping_epoch":494,"log_start":"503'4100","ondisk_log_start":"503'4100","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83427328,"num_objects":1273,"num_object_clones":0,"num_object_copies":3819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10831,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":764,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.347529+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:47:30.347529+0000","last_peered":"2023-09-30T09:47:30.347529+0000","last_clean":"2023-09-30T09:47:30.347529+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:47:30.347529+0000","last_undegraded":"2023-09-30T09:47:30.347529+0000","last_fullsized":"2023-09-30T09:47:30.347529+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":753,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.360483+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:47:30.360483+0000","last_peered":"2023-09-30T09:47:30.360483+0000","last_clean":"2023-09-30T09:47:30.360483+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:47:30.360483+0000","last_undegraded":"2023-09-30T09:47:30.360483+0000","last_fullsized":"2023-09-30T09:47:30.360483+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":450,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354038+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:47:30.354038+0000","last_peered":"2023-09-30T09:47:30.354038+0000","last_clean":"2023-09-30T09:47:30.354038+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:47:30.354038+0000","last_undegraded":"2023-09-30T09:47:30.354038+0000","last_fullsized":"2023-09-30T09:47:30.354038+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":139,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353290+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:47:30.353290+0000","last_peered":"2023-09-30T09:47:30.353290+0000","last_clean":"2023-09-30T09:47:30.353290+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:47:30.353290+0000","last_undegraded":"2023-09-30T09:47:30.353290+0000","last_fullsized":"2023-09-30T09:47:30.353290+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"522'10836","reported_seq":11187,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.380513+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:47:44.380513+0000","last_peered":"2023-09-30T09:47:44.380513+0000","last_clean":"2023-09-30T09:47:44.380513+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:47:44.380513+0000","last_undegraded":"2023-09-30T09:47:44.380513+0000","last_fullsized":"2023-09-30T09:47:44.380513+0000","mapping_epoch":493,"log_start":"501'3300","ondisk_log_start":"501'3300","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7536,"log_dups_size":0,"ondisk_log_size":7536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84672512,"num_objects":1292,"num_object_clones":0,"num_object_copies":3876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10836,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":963,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354639+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:47:30.354639+0000","last_peered":"2023-09-30T09:47:30.354639+0000","last_clean":"2023-09-30T09:47:30.354639+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:47:30.354639+0000","last_undegraded":"2023-09-30T09:47:30.354639+0000","last_fullsized":"2023-09-30T09:47:30.354639+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":469,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.346864+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:47:30.346864+0000","last_peered":"2023-09-30T09:47:30.346864+0000","last_clean":"2023-09-30T09:47:30.346864+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:47:30.346864+0000","last_undegraded":"2023-09-30T09:47:30.346864+0000","last_fullsized":"2023-09-30T09:47:30.346864+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":541,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.365119+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:47:30.365119+0000","last_peered":"2023-09-30T09:47:30.365119+0000","last_clean":"2023-09-30T09:47:30.365119+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:47:30.365119+0000","last_undegraded":"2023-09-30T09:47:30.365119+0000","last_fullsized":"2023-09-30T09:47:30.365119+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":281,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.358727+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:47:30.358727+0000","last_peered":"2023-09-30T09:47:30.358727+0000","last_clean":"2023-09-30T09:47:30.358727+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:47:30.358727+0000","last_undegraded":"2023-09-30T09:47:30.358727+0000","last_fullsized":"2023-09-30T09:47:30.358727+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"522'10891","reported_seq":11243,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.020151+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:47:44.020151+0000","last_peered":"2023-09-30T09:47:44.020151+0000","last_clean":"2023-09-30T09:47:44.020151+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:47:44.020151+0000","last_undegraded":"2023-09-30T09:47:44.020151+0000","last_fullsized":"2023-09-30T09:47:44.020151+0000","mapping_epoch":492,"log_start":"503'4200","ondisk_log_start":"503'4200","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83951616,"num_objects":1281,"num_object_clones":0,"num_object_copies":3843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10891,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":941,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353504+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:47:30.353504+0000","last_peered":"2023-09-30T09:47:30.353504+0000","last_clean":"2023-09-30T09:47:30.353504+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:47:30.353504+0000","last_undegraded":"2023-09-30T09:47:30.353504+0000","last_fullsized":"2023-09-30T09:47:30.353504+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":791,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:39.011943+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:47:39.011943+0000","last_peered":"2023-09-30T09:47:39.011943+0000","last_clean":"2023-09-30T09:47:39.011943+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:47:39.011943+0000","last_undegraded":"2023-09-30T09:47:39.011943+0000","last_fullsized":"2023-09-30T09:47:39.011943+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":285,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354170+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:47:30.354170+0000","last_peered":"2023-09-30T09:47:30.354170+0000","last_clean":"2023-09-30T09:47:30.354170+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:47:30.354170+0000","last_undegraded":"2023-09-30T09:47:30.354170+0000","last_fullsized":"2023-09-30T09:47:30.354170+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":371,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353004+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:47:30.353004+0000","last_peered":"2023-09-30T09:47:30.353004+0000","last_clean":"2023-09-30T09:47:30.353004+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:47:30.353004+0000","last_undegraded":"2023-09-30T09:47:30.353004+0000","last_fullsized":"2023-09-30T09:47:30.353004+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"522'10809","reported_seq":11121,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:44.342929+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:47:44.342929+0000","last_peered":"2023-09-30T09:47:44.342929+0000","last_clean":"2023-09-30T09:47:44.342929+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:47:44.342929+0000","last_undegraded":"2023-09-30T09:47:44.342929+0000","last_fullsized":"2023-09-30T09:47:44.342929+0000","mapping_epoch":495,"log_start":"501'3300","ondisk_log_start":"501'3300","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7509,"log_dups_size":0,"ondisk_log_size":7509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79233024,"num_objects":1209,"num_object_clones":0,"num_object_copies":3627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10809,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1083,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354173+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:47:30.354173+0000","last_peered":"2023-09-30T09:47:30.354173+0000","last_clean":"2023-09-30T09:47:30.354173+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:47:30.354173+0000","last_undegraded":"2023-09-30T09:47:30.354173+0000","last_fullsized":"2023-09-30T09:47:30.354173+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":884,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352513+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:47:30.352513+0000","last_peered":"2023-09-30T09:47:30.352513+0000","last_clean":"2023-09-30T09:47:30.352513+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:47:30.352513+0000","last_undegraded":"2023-09-30T09:47:30.352513+0000","last_fullsized":"2023-09-30T09:47:30.352513+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":98,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353371+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:47:30.353371+0000","last_peered":"2023-09-30T09:47:30.353371+0000","last_clean":"2023-09-30T09:47:30.353371+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:47:30.353371+0000","last_undegraded":"2023-09-30T09:47:30.353371+0000","last_fullsized":"2023-09-30T09:47:30.353371+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.347362+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:47:30.347362+0000","last_peered":"2023-09-30T09:47:30.347362+0000","last_clean":"2023-09-30T09:47:30.347362+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:47:30.347362+0000","last_undegraded":"2023-09-30T09:47:30.347362+0000","last_fullsized":"2023-09-30T09:47:30.347362+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"522'11015","reported_seq":20361,"reported_epoch":522,"state":"active+recovering+remapped","last_fresh":"2023-09-30T09:47:44.023493+0000","last_change":"2023-09-30T09:45:26.893218+0000","last_active":"2023-09-30T09:47:44.023493+0000","last_peered":"2023-09-30T09:47:44.023493+0000","last_clean":"2023-09-30T09:45:24.672070+0000","last_became_active":"2023-09-30T09:45:26.847415+0000","last_became_peered":"2023-09-30T09:45:26.847415+0000","last_unstale":"2023-09-30T09:47:44.023493+0000","last_undegraded":"2023-09-30T09:47:44.023493+0000","last_fullsized":"2023-09-30T09:47:44.023493+0000","mapping_epoch":506,"log_start":"496'1000","ondisk_log_start":"496'1000","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10015,"log_dups_size":0,"ondisk_log_size":10015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:37.107436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":334,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11015,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4636,"num_bytes_recovered":280821760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":977}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":783,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.348009+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:47:30.348009+0000","last_peered":"2023-09-30T09:47:30.348009+0000","last_clean":"2023-09-30T09:47:30.348009+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:47:30.348009+0000","last_undegraded":"2023-09-30T09:47:30.348009+0000","last_fullsized":"2023-09-30T09:47:30.348009+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":930,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352426+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:47:30.352426+0000","last_peered":"2023-09-30T09:47:30.352426+0000","last_clean":"2023-09-30T09:47:30.352426+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:47:30.352426+0000","last_undegraded":"2023-09-30T09:47:30.352426+0000","last_fullsized":"2023-09-30T09:47:30.352426+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":135,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.361696+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:47:30.361696+0000","last_peered":"2023-09-30T09:47:30.361696+0000","last_clean":"2023-09-30T09:47:30.361696+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:47:30.361696+0000","last_undegraded":"2023-09-30T09:47:30.361696+0000","last_fullsized":"2023-09-30T09:47:30.361696+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":28,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:39.380367+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:47:39.380367+0000","last_peered":"2023-09-30T09:47:39.380367+0000","last_clean":"2023-09-30T09:47:39.380367+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:47:39.380367+0000","last_undegraded":"2023-09-30T09:47:39.380367+0000","last_fullsized":"2023-09-30T09:47:39.380367+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"522'10757","reported_seq":10804,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:43.855311+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:47:43.855311+0000","last_peered":"2023-09-30T09:47:43.855311+0000","last_clean":"2023-09-30T09:47:43.855311+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:47:43.855311+0000","last_undegraded":"2023-09-30T09:47:43.855311+0000","last_fullsized":"2023-09-30T09:47:43.855311+0000","mapping_epoch":483,"log_start":"510'4900","ondisk_log_start":"510'4900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5857,"log_dups_size":0,"ondisk_log_size":5857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88801280,"num_objects":1355,"num_object_clones":0,"num_object_copies":4065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10757,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":774,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.346834+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:47:30.346834+0000","last_peered":"2023-09-30T09:47:30.346834+0000","last_clean":"2023-09-30T09:47:30.346834+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:47:30.346834+0000","last_undegraded":"2023-09-30T09:47:30.346834+0000","last_fullsized":"2023-09-30T09:47:30.346834+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1070,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354614+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:47:30.354614+0000","last_peered":"2023-09-30T09:47:30.354614+0000","last_clean":"2023-09-30T09:47:30.354614+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:47:30.354614+0000","last_undegraded":"2023-09-30T09:47:30.354614+0000","last_fullsized":"2023-09-30T09:47:30.354614+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":265,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.358561+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:47:30.358561+0000","last_peered":"2023-09-30T09:47:30.358561+0000","last_clean":"2023-09-30T09:47:30.358561+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:47:30.358561+0000","last_undegraded":"2023-09-30T09:47:30.358561+0000","last_fullsized":"2023-09-30T09:47:30.358561+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":135,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.363209+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:47:30.363209+0000","last_peered":"2023-09-30T09:47:30.363209+0000","last_clean":"2023-09-30T09:47:30.363209+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:47:30.363209+0000","last_undegraded":"2023-09-30T09:47:30.363209+0000","last_fullsized":"2023-09-30T09:47:30.363209+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"522'10713","reported_seq":11095,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:43.907990+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:47:43.907990+0000","last_peered":"2023-09-30T09:47:43.907990+0000","last_clean":"2023-09-30T09:47:43.907990+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:47:43.907990+0000","last_undegraded":"2023-09-30T09:47:43.907990+0000","last_fullsized":"2023-09-30T09:47:43.907990+0000","mapping_epoch":495,"log_start":"510'4900","ondisk_log_start":"510'4900","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5813,"log_dups_size":0,"ondisk_log_size":5813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84934680,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10713,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1041,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364970+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:47:30.364970+0000","last_peered":"2023-09-30T09:47:30.364970+0000","last_clean":"2023-09-30T09:47:30.364970+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:47:30.364970+0000","last_undegraded":"2023-09-30T09:47:30.364970+0000","last_fullsized":"2023-09-30T09:47:30.364970+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":553,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.353370+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:47:30.353370+0000","last_peered":"2023-09-30T09:47:30.353370+0000","last_clean":"2023-09-30T09:47:30.353370+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:47:30.353370+0000","last_undegraded":"2023-09-30T09:47:30.353370+0000","last_fullsized":"2023-09-30T09:47:30.353370+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1240,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.346757+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:47:30.346757+0000","last_peered":"2023-09-30T09:47:30.346757+0000","last_clean":"2023-09-30T09:47:30.346757+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:47:30.346757+0000","last_undegraded":"2023-09-30T09:47:30.346757+0000","last_fullsized":"2023-09-30T09:47:30.346757+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":557,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352333+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:47:30.352333+0000","last_peered":"2023-09-30T09:47:30.352333+0000","last_clean":"2023-09-30T09:47:30.352333+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:47:30.352333+0000","last_undegraded":"2023-09-30T09:47:30.352333+0000","last_fullsized":"2023-09-30T09:47:30.352333+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"522'10949","reported_seq":10996,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-30T09:47:43.834060+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:47:43.834060+0000","last_peered":"2023-09-30T09:47:43.834060+0000","last_clean":"2023-09-30T09:47:43.834060+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:47:43.834060+0000","last_undegraded":"2023-09-30T09:47:43.834060+0000","last_fullsized":"2023-09-30T09:47:43.834060+0000","mapping_epoch":483,"log_start":"504'4500","ondisk_log_start":"504'4500","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6449,"log_dups_size":0,"ondisk_log_size":6449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90374144,"num_objects":1379,"num_object_clones":0,"num_object_copies":4137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10949,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":905,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.363765+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:47:30.363765+0000","last_peered":"2023-09-30T09:47:30.363765+0000","last_clean":"2023-09-30T09:47:30.363765+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:47:30.363765+0000","last_undegraded":"2023-09-30T09:47:30.363765+0000","last_fullsized":"2023-09-30T09:47:30.363765+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":92,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.347724+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:47:30.347724+0000","last_peered":"2023-09-30T09:47:30.347724+0000","last_clean":"2023-09-30T09:47:30.347724+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:47:30.347724+0000","last_undegraded":"2023-09-30T09:47:30.347724+0000","last_fullsized":"2023-09-30T09:47:30.347724+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":98,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.360790+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:47:30.360790+0000","last_peered":"2023-09-30T09:47:30.360790+0000","last_clean":"2023-09-30T09:47:30.360790+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:47:30.360790+0000","last_undegraded":"2023-09-30T09:47:30.360790+0000","last_fullsized":"2023-09-30T09:47:30.360790+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":523,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.363184+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:47:30.363184+0000","last_peered":"2023-09-30T09:47:30.363184+0000","last_clean":"2023-09-30T09:47:30.363184+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:47:30.363184+0000","last_undegraded":"2023-09-30T09:47:30.363184+0000","last_fullsized":"2023-09-30T09:47:30.363184+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":841,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.361477+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:47:30.361477+0000","last_peered":"2023-09-30T09:47:30.361477+0000","last_clean":"2023-09-30T09:47:30.361477+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:47:30.361477+0000","last_undegraded":"2023-09-30T09:47:30.361477+0000","last_fullsized":"2023-09-30T09:47:30.361477+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":27,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.363212+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:47:30.363212+0000","last_peered":"2023-09-30T09:47:30.363212+0000","last_clean":"2023-09-30T09:47:30.363212+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:47:30.363212+0000","last_undegraded":"2023-09-30T09:47:30.363212+0000","last_fullsized":"2023-09-30T09:47:30.363212+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":227,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.358472+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:47:30.358472+0000","last_peered":"2023-09-30T09:47:30.358472+0000","last_clean":"2023-09-30T09:47:30.358472+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:47:30.358472+0000","last_undegraded":"2023-09-30T09:47:30.358472+0000","last_fullsized":"2023-09-30T09:47:30.358472+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":636,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.346908+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:47:30.346908+0000","last_peered":"2023-09-30T09:47:30.346908+0000","last_clean":"2023-09-30T09:47:30.346908+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:47:30.346908+0000","last_undegraded":"2023-09-30T09:47:30.346908+0000","last_fullsized":"2023-09-30T09:47:30.346908+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":450,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.355347+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:47:30.355347+0000","last_peered":"2023-09-30T09:47:30.355347+0000","last_clean":"2023-09-30T09:47:30.355347+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:47:30.355347+0000","last_undegraded":"2023-09-30T09:47:30.355347+0000","last_fullsized":"2023-09-30T09:47:30.355347+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":236,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364696+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:47:30.364696+0000","last_peered":"2023-09-30T09:47:30.364696+0000","last_clean":"2023-09-30T09:47:30.364696+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:47:30.364696+0000","last_undegraded":"2023-09-30T09:47:30.364696+0000","last_fullsized":"2023-09-30T09:47:30.364696+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":392,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354247+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:47:30.354247+0000","last_peered":"2023-09-30T09:47:30.354247+0000","last_clean":"2023-09-30T09:47:30.354247+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:47:30.354247+0000","last_undegraded":"2023-09-30T09:47:30.354247+0000","last_fullsized":"2023-09-30T09:47:30.354247+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":256,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364551+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:47:30.364551+0000","last_peered":"2023-09-30T09:47:30.364551+0000","last_clean":"2023-09-30T09:47:30.364551+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:47:30.364551+0000","last_undegraded":"2023-09-30T09:47:30.364551+0000","last_fullsized":"2023-09-30T09:47:30.364551+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":849,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.352458+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:47:30.352458+0000","last_peered":"2023-09-30T09:47:30.352458+0000","last_clean":"2023-09-30T09:47:30.352458+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:47:30.352458+0000","last_undegraded":"2023-09-30T09:47:30.352458+0000","last_fullsized":"2023-09-30T09:47:30.352458+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":79,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.346679+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:47:30.346679+0000","last_peered":"2023-09-30T09:47:30.346679+0000","last_clean":"2023-09-30T09:47:30.346679+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:47:30.346679+0000","last_undegraded":"2023-09-30T09:47:30.346679+0000","last_fullsized":"2023-09-30T09:47:30.346679+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":473,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.354383+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:47:30.354383+0000","last_peered":"2023-09-30T09:47:30.354383+0000","last_clean":"2023-09-30T09:47:30.354383+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:47:30.354383+0000","last_undegraded":"2023-09-30T09:47:30.354383+0000","last_fullsized":"2023-09-30T09:47:30.354383+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":892,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.364887+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:47:30.364887+0000","last_peered":"2023-09-30T09:47:30.364887+0000","last_clean":"2023-09-30T09:47:30.364887+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:47:30.364887+0000","last_undegraded":"2023-09-30T09:47:30.364887+0000","last_fullsized":"2023-09-30T09:47:30.364887+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":34,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:34.370352+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:47:34.370352+0000","last_peered":"2023-09-30T09:47:34.370352+0000","last_clean":"2023-09-30T09:47:34.370352+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:47:34.370352+0000","last_undegraded":"2023-09-30T09:47:34.370352+0000","last_fullsized":"2023-09-30T09:47:34.370352+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":537,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-30T09:47:30.363044+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:47:30.363044+0000","last_peered":"2023-09-30T09:47:30.363044+0000","last_clean":"2023-09-30T09:47:30.363044+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:47:30.363044+0000","last_undegraded":"2023-09-30T09:47:30.363044+0000","last_fullsized":"2023-09-30T09:47:30.363044+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1350696984,"num_objects":20611,"num_object_clones":0,"num_object_copies":61833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":334,"num_objects_unfound":0,"num_objects_dirty":20611,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172875,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5779,"num_bytes_recovered":355729408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4196675584,"data_stored":4196663368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117375,"ondisk_log_size":117375,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234432,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1321844,"kb_used_data":774828,"kb_used_omap":0,"kb_used_meta":547008,"kb_avail":92423308,"statfs":{"total":95995035648,"available":94641467392,"internally_reserved":0,"allocated":793423872,"data_stored":791671748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":560136192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":8.9299999999999997,"5min":7.6980000000000004,"15min":5.0990000000000002},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":95.825999999999993,"5min":124.74299999999999,"15min":215.416},"last":4.1109999999999998},{"interface":"front","average":{"1min":8.9190000000000005,"5min":7.7690000000000001,"15min":5.0069999999999997},"min":{"1min":0.26100000000000001,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":95.488,"5min":124.69499999999999,"15min":215.041},"last":2.173}]},{"osd":1,"last update":"Sat Sep 30 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":7.4539999999999997,"15min":4.117},"min":{"1min":0.40799999999999997,"5min":0.25,"15min":0.25},"max":{"1min":7.6559999999999997,"5min":157.578,"15min":157.578},"last":1.7190000000000001},{"interface":"front","average":{"1min":2.649,"5min":7.2839999999999998,"15min":4.2679999999999998},"min":{"1min":0.41299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.8800000000000008,"5min":157.589,"15min":157.589},"last":12.013999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":3.9289999999999998,"5min":3.1749999999999998,"15min":3.762},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.24299999999999999},"max":{"1min":10.173999999999999,"5min":104.88500000000001,"15min":117.14},"last":2.7749999999999999},{"interface":"front","average":{"1min":3.5270000000000001,"5min":3.1680000000000001,"15min":3.7149999999999999},"min":{"1min":0.436,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":16.135000000000002,"5min":104.867,"15min":117.044},"last":2.29}]},{"osd":3,"last update":"Sat Sep 30 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":7.8529999999999998,"5min":8.1449999999999996,"15min":8.6750000000000007},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":97.037000000000006,"5min":154.06700000000001,"15min":277.57799999999997},"last":0.46400000000000002},{"interface":"front","average":{"1min":8.9039999999999999,"5min":8.4440000000000008,"15min":8.9420000000000002},"min":{"1min":0.42999999999999999,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":95.135000000000005,"5min":154.24799999999999,"15min":278.69299999999998},"last":4.7869999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.3779999999999999,"15min":1.6839999999999999},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":7.6260000000000003,"5min":25.192,"15min":175.72},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.2290000000000001,"15min":1.5389999999999999},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.252},"max":{"1min":3.3100000000000001,"5min":24.521000000000001,"15min":179.048},"last":2.7429999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:47:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":7.6619999999999999,"15min":10.548999999999999},"min":{"1min":0.32700000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.266,"5min":197.75700000000001,"15min":197.75700000000001},"last":2.3180000000000001},{"interface":"front","average":{"1min":1.5780000000000001,"5min":7.79,"15min":10.481},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.7089999999999996,"5min":197.63300000000001,"15min":197.63300000000001},"last":2.149}]},{"osd":6,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.046,"15min":2.2589999999999999},"min":{"1min":0.41499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5830000000000002,"5min":7.5830000000000002,"15min":206.244},"last":1.357},{"interface":"front","average":{"1min":1.5009999999999999,"5min":1.05,"15min":2.2189999999999999},"min":{"1min":0.46700000000000003,"5min":0.31900000000000001,"15min":0.29199999999999998},"max":{"1min":3.9420000000000002,"5min":5.8520000000000003,"15min":202.78800000000001},"last":2.2170000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496070,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":774172,"kb_used_data":253268,"kb_used_omap":0,"kb_used_meta":520832,"kb_avail":92970980,"statfs":{"total":95995035648,"available":95202283520,"internally_reserved":0,"allocated":259346432,"data_stored":257617601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":533331968},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":4.1050000000000004,"15min":5.3330000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":11.609,"5min":95.043999999999997,"15min":187.19800000000001},"last":0.433},{"interface":"front","average":{"1min":2.1600000000000001,"5min":4.1210000000000004,"15min":5.3520000000000003},"min":{"1min":0.26300000000000001,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":6.5460000000000003,"5min":94.855999999999995,"15min":187.46700000000001},"last":2.0009999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.446,"15min":1.956},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.6429999999999998,"5min":32.814,"15min":69.569999999999993},"last":5.1920000000000002},{"interface":"front","average":{"1min":2.5529999999999999,"5min":1.5629999999999999,"15min":2.173},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":6.3760000000000003,"5min":32.856000000000002,"15min":70.471999999999994},"last":4.9950000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":3.9239999999999999,"5min":3.5880000000000001,"15min":4.0069999999999997},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":10.581,"5min":207.02500000000001,"15min":207.02500000000001},"last":3.4249999999999998},{"interface":"front","average":{"1min":3.3559999999999999,"5min":3.6909999999999998,"15min":4.1539999999999999},"min":{"1min":0.37,"5min":0.33500000000000002,"15min":0.28199999999999997},"max":{"1min":16.474,"5min":207.114,"15min":207.114},"last":0.53800000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8839999999999999,"5min":3.484,"15min":4.2560000000000002},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":11.644,"5min":107.11799999999999,"15min":212.44800000000001},"last":1.357},{"interface":"front","average":{"1min":2.3239999999999998,"5min":3.4159999999999999,"15min":4.4909999999999997},"min":{"1min":0.35099999999999998,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":8.8559999999999999,"5min":106.982,"15min":210.84299999999999},"last":6.3559999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.722,"15min":1.2669999999999999},"min":{"1min":0.30199999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.4400000000000004,"5min":63.798000000000002,"15min":63.798000000000002},"last":2.27},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.8400000000000001,"15min":1.2330000000000001},"min":{"1min":0.32900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.558999999999999,"5min":63.133000000000003,"15min":63.133000000000003},"last":0.503}]},{"osd":5,"last update":"Sat Sep 30 09:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":1.825,"15min":1.974},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.367000000000001,"5min":14.974,"15min":14.974},"last":2.4399999999999999},{"interface":"front","average":{"1min":2.1739999999999999,"5min":2.1880000000000002,"15min":2.4809999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.669,"5min":16.443000000000001,"15min":16.443000000000001},"last":0.67500000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:46:51 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":0.89200000000000002,"15min":1.609},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.2679999999999998,"5min":11.276999999999999,"15min":138.191},"last":4.298},{"interface":"front","average":{"1min":1.282,"5min":0.89300000000000002,"15min":1.4410000000000001},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7720000000000002,"5min":10.909000000000001,"15min":140.33500000000001},"last":5.1479999999999997}]}]},{"osd":1,"up_from":301,"seq":1292785156267,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1449812,"kb_used_data":804424,"kb_used_omap":9,"kb_used_meta":645366,"kb_avail":92295340,"statfs":{"total":95995035648,"available":94510428160,"internally_reserved":0,"allocated":823730176,"data_stored":820753350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":660854802},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":4.319,"15min":3.7320000000000002},"min":{"1min":0.221,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":26.021000000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":8.6379999999999999},{"interface":"front","average":{"1min":3.7639999999999998,"5min":4.1159999999999997,"15min":3.6240000000000001},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":25.971,"5min":142.137,"15min":142.137},"last":8.6799999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.883,"15min":2.573},"min":{"1min":0.27700000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.763000000000002,"5min":62.572000000000003,"15min":62.572000000000003},"last":5.6529999999999996},{"interface":"front","average":{"1min":2.9809999999999999,"5min":2.8700000000000001,"15min":2.7999999999999998},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.138999999999999,"5min":68.373999999999995,"15min":68.373999999999995},"last":12.635999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.484,"5min":5.9690000000000003,"15min":6.0899999999999999},"min":{"1min":0.30499999999999999,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":9.8610000000000007,"5min":199.41,"15min":199.41},"last":4.2720000000000002},{"interface":"front","average":{"1min":3.149,"5min":6.2999999999999998,"15min":6.1509999999999998},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.4329999999999998,"5min":199.58099999999999,"15min":199.58099999999999},"last":0.48299999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":1.091,"15min":2.1309999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":8.9399999999999995,"5min":8.9399999999999995,"15min":142.77099999999999},"last":6.2560000000000002},{"interface":"front","average":{"1min":2.46,"5min":1.52,"15min":2.4140000000000001},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.6319999999999997,"5min":8.6319999999999997,"15min":144.898},"last":2.8450000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":2.206,"15min":2.024},"min":{"1min":0.41099999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":8.6790000000000003,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":2.8769999999999998},{"interface":"front","average":{"1min":2.198,"5min":1.54,"15min":1.3280000000000001},"min":{"1min":0.39100000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.891,"5min":15.250999999999999,"15min":15.250999999999999},"last":6.2130000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.552,"15min":5.5679999999999996},"min":{"1min":0.442,"5min":0.316,"15min":0.29899999999999999},"max":{"1min":8.8559999999999999,"5min":30.986999999999998,"15min":209.221},"last":6.2889999999999997},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.4850000000000001,"15min":5.4960000000000004},"min":{"1min":0.47899999999999998,"5min":0.35399999999999998,"15min":0.33900000000000002},"max":{"1min":8.8100000000000005,"5min":28.710999999999999,"15min":208.43600000000001},"last":10.266999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.5449999999999999,"15min":3.9500000000000002},"min":{"1min":0.36299999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":9.3879999999999999,"5min":9.3879999999999999,"15min":165.59399999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.077,"5min":1.542,"15min":3.9929999999999999},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":4.2549999999999999,"5min":9.3399999999999999,"15min":165.833},"last":0.60099999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926166,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":988052,"kb_used_data":333516,"kb_used_omap":5,"kb_used_meta":654522,"kb_avail":92757100,"statfs":{"total":95995035648,"available":94983270400,"internally_reserved":0,"allocated":341520384,"data_stored":338310202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":670230781},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,6],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.9419999999999999,"15min":2.181},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":7.2380000000000004,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":2.927,"5min":2.3180000000000001,"15min":4.0199999999999996},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.344000000000001,"5min":57.140999999999998,"15min":138.92400000000001},"last":0.317}]},{"osd":2,"last update":"Sat Sep 30 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6640000000000001,"5min":4.8010000000000002,"15min":3.2829999999999999},"min":{"1min":0.28899999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":31.521000000000001,"5min":127.73399999999999,"15min":127.73399999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":4.7649999999999997,"5min":5.1299999999999999,"15min":3.5470000000000002},"min":{"1min":0.38,"5min":0.27600000000000002,"15min":0.26200000000000001},"max":{"1min":31.478999999999999,"5min":127.69,"15min":127.69},"last":0.27800000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":5.5099999999999998,"5min":5.9870000000000001,"15min":5.4249999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":16.594000000000001,"5min":125.562,"15min":218.06999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":3.0680000000000001,"5min":4.984,"15min":4.2599999999999998},"min":{"1min":0.39600000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":13.361000000000001,"5min":117.295,"15min":218.63200000000001},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.7989999999999999,"15min":2.012},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.8710000000000004,"5min":72.403999999999996,"15min":72.403999999999996},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1960000000000002,"15min":2.8260000000000001},"min":{"1min":0.38600000000000001,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":9.4870000000000001,"5min":72.162999999999997,"15min":136.262},"last":0.5}]},{"osd":5,"last update":"Sat Sep 30 09:47:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2570000000000001,"5min":2.0950000000000002,"15min":1.413},"min":{"1min":0.35299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":13.162000000000001,"5min":13.162000000000001,"15min":13.162000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.2150000000000001,"15min":1.1040000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":13.422000000000001,"5min":13.422000000000001,"15min":13.422000000000001},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.958,"15min":3.508},"min":{"1min":0.38200000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":20.706,"5min":57.384,"15min":214.785},"last":0.621},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.3700000000000001,"15min":3.8370000000000002},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.684000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.8120000000000001,"15min":2.0720000000000001},"min":{"1min":0.42499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.6549999999999994,"5min":57.776000000000003,"15min":97.382000000000005},"last":146.828},{"interface":"front","average":{"1min":2.6349999999999998,"5min":2.363,"15min":2.3940000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":9.7149999999999999,"5min":57.179000000000002,"15min":97.015000000000001},"last":146.876}]}]},{"osd":2,"up_from":104,"seq":446676599107,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1140008,"kb_used_data":341920,"kb_used_omap":5,"kb_used_meta":798074,"kb_avail":92605144,"statfs":{"total":95995035648,"available":94827667456,"internally_reserved":0,"allocated":350126080,"data_stored":347184874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":817228429},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":4.0890000000000004,"5min":5.4829999999999997,"15min":3.5800000000000001},"min":{"1min":0.219,"5min":0.192,"15min":0.187},"max":{"1min":13.880000000000001,"5min":170.678,"15min":170.678},"last":202.40899999999999},{"interface":"front","average":{"1min":5.1459999999999999,"5min":5.5549999999999997,"15min":3.6320000000000001},"min":{"1min":0.32100000000000001,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":12.609999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":202.38800000000001}]},{"osd":1,"last update":"Sat Sep 30 09:46:52 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":4.8449999999999998,"15min":3.7919999999999998},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":10.922000000000001,"5min":162.696,"15min":162.696},"last":202.369},{"interface":"front","average":{"1min":4.665,"5min":5.0709999999999997,"15min":3.6320000000000001},"min":{"1min":0.27700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.705,"5min":162.93899999999999,"15min":162.93899999999999},"last":202.54499999999999}]},{"osd":3,"last update":"Sat Sep 30 09:47:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":6.0049999999999999,"15min":5.109},"min":{"1min":0.311,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.6609999999999996,"5min":156.39400000000001,"15min":156.39400000000001},"last":202.59},{"interface":"front","average":{"1min":4.1980000000000004,"5min":5.9400000000000004,"15min":4.9740000000000002},"min":{"1min":0.38500000000000001,"5min":0.224,"15min":0.224},"max":{"1min":14.169,"5min":156.376,"15min":156.376},"last":202.34200000000001}]},{"osd":4,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":1.8120000000000001,"15min":1.881},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1270000000000007,"5min":47.463000000000001,"15min":83.012},"last":202.62299999999999},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.8520000000000001,"15min":1.879},"min":{"1min":0.42099999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.997999999999999,"5min":47.412999999999997,"15min":81.335999999999999},"last":202.50800000000001}]},{"osd":5,"last update":"Sat Sep 30 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":11.856,"5min":4.0570000000000004,"15min":2.069},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":243.86500000000001,"5min":243.86500000000001,"15min":243.86500000000001},"last":202.65600000000001},{"interface":"front","average":{"1min":12.890000000000001,"5min":4.3949999999999996,"15min":2.3420000000000001},"min":{"1min":0.44,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":244.02199999999999,"5min":244.02199999999999,"15min":244.02199999999999},"last":202.86799999999999}]},{"osd":6,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.4199999999999999,"15min":3.2309999999999999},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.026999999999999,"5min":11.026999999999999,"15min":154.417},"last":202.70599999999999},{"interface":"front","average":{"1min":3.1880000000000002,"5min":1.8140000000000001,"15min":3.6379999999999999},"min":{"1min":0.46100000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.846,"5min":10.846,"15min":154.04400000000001},"last":202.916}]},{"osd":7,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":2.1240000000000001,"15min":2.4089999999999998},"min":{"1min":0.48199999999999998,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":10.928000000000001,"5min":49.816000000000003,"15min":123.294},"last":202.75},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.3889999999999998,"15min":2.7429999999999999},"min":{"1min":0.436,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":10.885,"5min":50.956000000000003,"15min":123.16},"last":202.89599999999999}]}]},{"osd":3,"up_from":109,"seq":468151435580,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533840,"kb_used_data":683720,"kb_used_omap":2,"kb_used_meta":850109,"kb_avail":92211312,"statfs":{"total":95995035648,"available":94424383488,"internally_reserved":0,"allocated":700129280,"data_stored":697090709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":870511975},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0759999999999996,"5min":5.2880000000000003,"15min":4.9829999999999997},"min":{"1min":0.22700000000000001,"5min":0.183,"15min":0.183},"max":{"1min":45.957999999999998,"5min":144.72,"15min":205.73099999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":4.8479999999999999,"5min":5.2919999999999998,"15min":4.9100000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":44.298999999999999,"5min":142.36600000000001,"15min":205.495},"last":5.7160000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:46:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0869999999999997,"5min":3.7599999999999998,"15min":7.4470000000000001},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":19.059000000000001,"5min":142.434,"15min":273.99599999999998},"last":0.36299999999999999},{"interface":"front","average":{"1min":4.1550000000000002,"5min":3.7879999999999998,"15min":7.1299999999999999},"min":{"1min":0.28199999999999997,"5min":0.26800000000000002,"15min":0.20699999999999999},"max":{"1min":13.183,"5min":142.44900000000001,"15min":274.21600000000001},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":3.8239999999999998,"5min":5.3940000000000001,"15min":5.7999999999999998},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":12.561999999999999,"5min":146.92599999999999,"15min":206.69},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.5979999999999999,"5min":4.9619999999999997,"15min":5.5789999999999997},"min":{"1min":0.38900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.834,"5min":145.75299999999999,"15min":209.22800000000001},"last":0.55400000000000005}]},{"osd":4,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.3180000000000001,"15min":2.0920000000000001},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.2919999999999998,"5min":81.677000000000007,"15min":81.677000000000007},"last":0.46800000000000003},{"interface":"front","average":{"1min":3.0499999999999998,"5min":2.6360000000000001,"15min":2.04},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":18.943000000000001,"5min":80.623000000000005,"15min":80.623000000000005},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:47:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5609999999999999,"5min":1.9390000000000001,"15min":1.6639999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.194000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.5430000000000001,"5min":1.536,"15min":1.1459999999999999},"min":{"1min":0.33700000000000002,"5min":0.311,"15min":0.311},"max":{"1min":8.0399999999999991,"5min":8.0399999999999991,"15min":8.0399999999999991},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":1.498,"15min":1.9790000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33200000000000002},"max":{"1min":10.837999999999999,"5min":10.837999999999999,"15min":60.698},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.4369999999999998,"5min":1.659,"15min":1.9650000000000001},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":8.2050000000000001,"5min":10.356999999999999,"15min":12.121},"last":9.8079999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":3.488,"15min":2.7120000000000002},"min":{"1min":0.42499999999999999,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":11.753,"5min":213.548,"15min":213.548},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.2559999999999998,"5min":3.3290000000000002,"15min":2.605},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.888999999999999,"5min":213.583,"15min":213.583},"last":0.70199999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019343,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":757024,"kb_used_data":268952,"kb_used_omap":0,"kb_used_meta":488064,"kb_avail":92988128,"statfs":{"total":95995035648,"available":95219843072,"internally_reserved":0,"allocated":275406848,"data_stored":273674036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":499777536},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":2.7090000000000001,"15min":2.5640000000000001},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":8.7989999999999995,"5min":73.718000000000004,"15min":105.179},"last":7.516},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.7589999999999999,"15min":2.75},"min":{"1min":0.32000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.032,"5min":74.001999999999995,"15min":104.70999999999999},"last":1.74}]},{"osd":1,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.5779999999999998,"15min":3.871},"min":{"1min":0.313,"5min":0.25,"15min":0.25},"max":{"1min":8.8620000000000001,"5min":80.712000000000003,"15min":160.69999999999999},"last":7.5679999999999996},{"interface":"front","average":{"1min":3.589,"5min":2.8490000000000002,"15min":3.8940000000000001},"min":{"1min":0.438,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":15.345000000000001,"5min":80.810000000000002,"15min":160.50700000000001},"last":8.0150000000000006}]},{"osd":2,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3069999999999999,"5min":4.9989999999999997,"15min":3.492},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":13.26,"5min":142.88999999999999,"15min":142.88999999999999},"last":1.4850000000000001},{"interface":"front","average":{"1min":3.4159999999999999,"5min":5.069,"15min":3.5249999999999999},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":8.8870000000000005,"5min":142.90700000000001,"15min":142.90700000000001},"last":7.5430000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.0369999999999999,"15min":2.1800000000000002},"min":{"1min":0.35799999999999998,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":9.0600000000000005,"5min":29.41,"15min":52.762},"last":7.6159999999999997},{"interface":"front","average":{"1min":3.6920000000000002,"5min":2.0099999999999998,"15min":2.1890000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.29299999999999998},"max":{"1min":9.0709999999999997,"5min":29.401,"15min":52.790999999999997},"last":0.57899999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:47:26 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":7.0880000000000001,"15min":9.5180000000000007},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.1470000000000002,"5min":140.11099999999999,"15min":140.11099999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.2889999999999999,"5min":7.0179999999999998,"15min":9.516},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.8789999999999996,"5min":140.065,"15min":140.065},"last":0.89400000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:47:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":0.84799999999999998,"15min":2.1850000000000001},"min":{"1min":0.34499999999999997,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":9.1470000000000002,"5min":9.1470000000000002,"15min":202.72499999999999},"last":7.4749999999999996},{"interface":"front","average":{"1min":1.026,"5min":0.78300000000000003,"15min":2.1720000000000002},"min":{"1min":0.311,"5min":0.253,"15min":0.249},"max":{"1min":5.2539999999999996,"5min":7.173,"15min":202.42400000000001},"last":3.0390000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":2.5099999999999998,"15min":3.2000000000000002},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.771000000000001,"5min":161.72999999999999,"15min":230.90100000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":2.4820000000000002,"15min":3.0569999999999999},"min":{"1min":0.41999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.2970000000000006,"5min":162.60900000000001,"15min":229.97},"last":7.5919999999999996}]}]},{"osd":5,"up_from":489,"seq":2100239007785,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889504,"kb_used_data":739224,"kb_used_omap":6,"kb_used_meta":150265,"kb_avail":92855648,"statfs":{"total":95995035648,"available":95084183552,"internally_reserved":0,"allocated":756965376,"data_stored":755229114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":153871455},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":6.7590000000000003,"5min":17.806999999999999,"15min":23.475000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":89.805999999999997,"5min":281.10599999999999,"15min":281.10599999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":6.4459999999999997,"5min":18.018000000000001,"15min":23.738},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":88.313999999999993,"5min":282.07400000000001,"15min":282.07400000000001},"last":2.75}]},{"osd":1,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":6.7160000000000002,"5min":11.051,"15min":13.630000000000001},"min":{"1min":0.31900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":76.828999999999994,"5min":273.48200000000003,"15min":273.48200000000003},"last":0.68500000000000005},{"interface":"front","average":{"1min":5.3710000000000004,"5min":10.403,"15min":12.871},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":76.805000000000007,"5min":273.50200000000001,"15min":273.50200000000001},"last":1.823}]},{"osd":2,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":7.258,"5min":4.3230000000000004,"15min":2.0680000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":95.641000000000005,"5min":163.23099999999999,"15min":163.23099999999999},"last":5.5309999999999997},{"interface":"front","average":{"1min":6.7030000000000003,"5min":4.6449999999999996,"15min":2.5750000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":95.484999999999999,"5min":162.71100000000001,"15min":162.71100000000001},"last":1.583}]},{"osd":3,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":7.5179999999999998,"5min":10.975,"15min":13.491},"min":{"1min":0.45900000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":79.421999999999997,"5min":274.01799999999997,"15min":274.01799999999997},"last":2.0619999999999998},{"interface":"front","average":{"1min":7.2709999999999999,"5min":3.1469999999999998,"15min":2.2029999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":79.459000000000003,"5min":79.459000000000003,"15min":79.459000000000003},"last":2.0499999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1,"15min":0.96799999999999997},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.46,"5min":5.46,"15min":5.46},"last":2.665},{"interface":"front","average":{"1min":1.554,"5min":1.25,"15min":1.1799999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.8239999999999998,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.516,"15min":1.6020000000000001},"min":{"1min":0.33000000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7569999999999997,"5min":14.371,"15min":14.371},"last":1.038},{"interface":"front","average":{"1min":1.778,"5min":1.403,"15min":1.3939999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.8979999999999997,"5min":14.454000000000001,"15min":14.454000000000001},"last":2.714}]},{"osd":7,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.397,"15min":1.3280000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.7560000000000002,"5min":9.7560000000000002,"15min":9.7560000000000002},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.744,"5min":1.069,"15min":0.96699999999999997},"min":{"1min":0.34899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.8529999999999998,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":2.7759999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327352320,"data_stored":327352320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":809046016,"data_stored":809041944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":338034688,"data_stored":338034688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":690683904,"data_stored":690683904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":265224192,"data_stored":265224192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":743903232,"data_stored":743899160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":253034496,"data_stored":253034496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":769396736,"data_stored":769392664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:47:49.621 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:49.888 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:49.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:49.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:49.995 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:50.036 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:50.965 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:50.998 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:51.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:51.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[216815]: pgmap v1388: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 20/51231 objects misplaced (0.039%); 89 KiB/s, 67 objects/s recovering 2023-09-30T09:47:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[216815]: osdmap e523: 8 total, 8 up, 8 in 2023-09-30T09:47:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[216815]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[216815]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[216815]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[216815]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[224043]: pgmap v1388: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 20/51231 objects misplaced (0.039%); 89 KiB/s, 67 objects/s recovering 2023-09-30T09:47:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[224043]: osdmap e523: 8 total, 8 up, 8 in 2023-09-30T09:47:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[224043]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[224043]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[224043]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:50 smithi019 ceph-mon[224043]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.103 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:47:51.177 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:50 smithi167 ceph-mon[192880]: pgmap v1388: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 20/51231 objects misplaced (0.039%); 89 KiB/s, 67 objects/s recovering 2023-09-30T09:47:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:50 smithi167 ceph-mon[192880]: osdmap e523: 8 total, 8 up, 8 in 2023-09-30T09:47:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:50 smithi167 ceph-mon[192880]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:50 smithi167 ceph-mon[192880]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:50 smithi167 ceph-mon[192880]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:50 smithi167 ceph-mon[192880]: failed to encode map e523 with expected crc 2023-09-30T09:47:51.624 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:47:52.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:52.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:47:52.191 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:47:52.191 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:47:52.193 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:47:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:52 smithi167 ceph-mon[192880]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:52 smithi167 ceph-mon[192880]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:52 smithi167 ceph-mon[192880]: osdmap e524: 8 total, 8 up, 8 in 2023-09-30T09:47:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:52 smithi167 ceph-mon[192880]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1864209023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:52.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:52.292 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:52.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[216815]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[216815]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[216815]: osdmap e524: 8 total, 8 up, 8 in 2023-09-30T09:47:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[216815]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1864209023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:52.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[224043]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[224043]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[224043]: osdmap e524: 8 total, 8 up, 8 in 2023-09-30T09:47:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[224043]: failed to encode map e524 with expected crc 2023-09-30T09:47:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1864209023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:47:52.595 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:52.595 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:47:53.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: pgmap v1391: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 20/51231 objects misplaced (0.039%); 112 KiB/s, 84 objects/s recovering 2023-09-30T09:47:53.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: osdmap e525: 8 total, 8 up, 8 in 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[216815]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: pgmap v1391: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 20/51231 objects misplaced (0.039%); 112 KiB/s, 84 objects/s recovering 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: osdmap e525: 8 total, 8 up, 8 in 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:53 smithi167 ceph-mon[192880]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: pgmap v1391: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 20/51231 objects misplaced (0.039%); 112 KiB/s, 84 objects/s recovering 2023-09-30T09:47:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:47:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:47:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: osdmap e525: 8 total, 8 up, 8 in 2023-09-30T09:47:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:53 smithi019 ceph-mon[224043]: failed to encode map e525 with expected crc 2023-09-30T09:47:53.558 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:53.571 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:47:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:53.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:53.673 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:53.700 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:54.088 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:47:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:47:53] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-09-30T09:47:54.298 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:54.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:54.454 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:47:54.584 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:54.685 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:47:54.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:54.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:47:54.750 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:54.756 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:54.811 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:54.901 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:54.974 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:47:55.015 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:47:55.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:54 smithi167 ceph-mon[192880]: pgmap v1393: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 20/51231 objects misplaced (0.039%); 0 B/s, 52 objects/s recovering 2023-09-30T09:47:55.073 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:47:55.083 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:54 smithi019 ceph-mon[216815]: pgmap v1393: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 20/51231 objects misplaced (0.039%); 0 B/s, 52 objects/s recovering 2023-09-30T09:47:55.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:54 smithi019 ceph-mon[224043]: pgmap v1393: 119 pgs: 1 active+recovering+remapped, 118 active+clean; 1.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 20/51231 objects misplaced (0.039%); 0 B/s, 52 objects/s recovering 2023-09-30T09:47:55.124 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:47:55.166 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:47:55.197 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:47:55.235 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:47:55.270 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:55.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:55.297 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:47:55.305 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:47:55.306 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:47:55.306 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:47:55.306 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:47:55.306 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:47:55.306 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:47:55.306 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:47:55.306 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:47:55.391 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:55.471 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:56.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2558806214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:56.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2558806214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:56.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2558806214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:47:56.354 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:47:56.438 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:56.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:56.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:56.547 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:47:56.680 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:56.796 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:47:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:56.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:47:56.906 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:47:56.942 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:57.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:56 smithi019 ceph-mon[216815]: pgmap v1394: 119 pgs: 119 active+clean; 875 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 0 B/s, 3 objects/s recovering 2023-09-30T09:47:57.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:56 smithi019 ceph-mon[224043]: pgmap v1394: 119 pgs: 119 active+clean; 875 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 0 B/s, 3 objects/s recovering 2023-09-30T09:47:57.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:56 smithi167 ceph-mon[192880]: pgmap v1394: 119 pgs: 119 active+clean; 875 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 0 B/s, 3 objects/s recovering 2023-09-30T09:47:57.860 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:47:57.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:57.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:47:57.968 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:58.061 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:58.465 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:47:58.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:58.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:47:58.574 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:47:58.612 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:47:58.612 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:47:58.612 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:47:58.612 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:47:58.612 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:47:58.620 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:58.693 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:47:58.698 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:47:58.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:47:58.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:47:58.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:47:58.888 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:47:59.191 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:47:59.194 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:47:59.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:58 smithi019 ceph-mon[224043]: pgmap v1395: 119 pgs: 119 active+clean; 875 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:47:59.227 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:58 smithi019 ceph-mon[216815]: pgmap v1395: 119 pgs: 119 active+clean; 875 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:47:59.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:58 smithi167 ceph-mon[192880]: pgmap v1395: 119 pgs: 119 active+clean; 875 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:47:59.840 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:48:00.012 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:00.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:00.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:00.118 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:00.177 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:47:59 smithi019 ceph-mon[216815]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:48:00.177 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:47:59 smithi019 ceph-mon[224043]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:48:00.180 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:47:59 smithi167 ceph-mon[192880]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:48:00.511 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:48:00.803 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:00.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:00.907 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1395,"stamp":"2023-09-30T09:47:57.884822+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":917510079,"num_objects":13956,"num_object_clones":0,"num_object_copies":41868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13956,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":180465,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6394,"num_bytes_recovered":368334450,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120802,"ondisk_log_size":120802,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7495184,"kb_used_data":2786512,"kb_used_omap":30,"kb_used_meta":4708385,"kb_avail":742466032,"statfs":{"total":767960285184,"available":760285216768,"internally_reserved":0,"allocated":2853388288,"data_stored":2834237425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4821386594},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-204537856,"num_objects":-3121,"num_object_clones":0,"num_object_copies":-9363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-20,"num_objects_unfound":0,"num_objects_dirty":-3121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3121,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.913831"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":948,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976007+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:47:50.976007+0000","last_peered":"2023-09-30T09:47:50.976007+0000","last_clean":"2023-09-30T09:47:50.976007+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:47:50.976007+0000","last_undegraded":"2023-09-30T09:47:50.976007+0000","last_fullsized":"2023-09-30T09:47:50.976007+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":543,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.072603+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:47:51.072603+0000","last_peered":"2023-09-30T09:47:51.072603+0000","last_clean":"2023-09-30T09:47:51.072603+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:47:51.072603+0000","last_undegraded":"2023-09-30T09:47:51.072603+0000","last_fullsized":"2023-09-30T09:47:51.072603+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":440,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.076061+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:47:51.076061+0000","last_peered":"2023-09-30T09:47:51.076061+0000","last_clean":"2023-09-30T09:47:51.076061+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:47:51.076061+0000","last_undegraded":"2023-09-30T09:47:51.076061+0000","last_fullsized":"2023-09-30T09:47:51.076061+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":950,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976191+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:47:50.976191+0000","last_peered":"2023-09-30T09:47:50.976191+0000","last_clean":"2023-09-30T09:47:50.976191+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:47:50.976191+0000","last_undegraded":"2023-09-30T09:47:50.976191+0000","last_fullsized":"2023-09-30T09:47:50.976191+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":90,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.067425+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:47:51.067425+0000","last_peered":"2023-09-30T09:47:51.067425+0000","last_clean":"2023-09-30T09:47:51.067425+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:47:51.067425+0000","last_undegraded":"2023-09-30T09:47:51.067425+0000","last_fullsized":"2023-09-30T09:47:51.067425+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":131,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.979757+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:47:50.979757+0000","last_peered":"2023-09-30T09:47:50.979757+0000","last_clean":"2023-09-30T09:47:50.979757+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:47:50.979757+0000","last_undegraded":"2023-09-30T09:47:50.979757+0000","last_fullsized":"2023-09-30T09:47:50.979757+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":758,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.072670+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:47:51.072670+0000","last_peered":"2023-09-30T09:47:51.072670+0000","last_clean":"2023-09-30T09:47:51.072670+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:47:51.072670+0000","last_undegraded":"2023-09-30T09:47:51.072670+0000","last_fullsized":"2023-09-30T09:47:51.072670+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":31,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:52.015251+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:47:52.015251+0000","last_peered":"2023-09-30T09:47:52.015251+0000","last_clean":"2023-09-30T09:47:52.015251+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:47:52.015251+0000","last_undegraded":"2023-09-30T09:47:52.015251+0000","last_fullsized":"2023-09-30T09:47:52.015251+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":94,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977551+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:47:50.977551+0000","last_peered":"2023-09-30T09:47:50.977551+0000","last_clean":"2023-09-30T09:47:50.977551+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:47:50.977551+0000","last_undegraded":"2023-09-30T09:47:50.977551+0000","last_fullsized":"2023-09-30T09:47:50.977551+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":926,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.065035+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:47:51.065035+0000","last_peered":"2023-09-30T09:47:51.065035+0000","last_clean":"2023-09-30T09:47:51.065035+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:47:51.065035+0000","last_undegraded":"2023-09-30T09:47:51.065035+0000","last_fullsized":"2023-09-30T09:47:51.065035+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:52.015899+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:47:52.015899+0000","last_peered":"2023-09-30T09:47:52.015899+0000","last_clean":"2023-09-30T09:47:52.015899+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:47:52.015899+0000","last_undegraded":"2023-09-30T09:47:52.015899+0000","last_fullsized":"2023-09-30T09:47:52.015899+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":106,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.067001+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:47:51.067001+0000","last_peered":"2023-09-30T09:47:51.067001+0000","last_clean":"2023-09-30T09:47:51.067001+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:47:51.067001+0000","last_undegraded":"2023-09-30T09:47:51.067001+0000","last_fullsized":"2023-09-30T09:47:51.067001+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":715,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.096219+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:47:51.096219+0000","last_peered":"2023-09-30T09:47:51.096219+0000","last_clean":"2023-09-30T09:47:51.096219+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:47:51.096219+0000","last_undegraded":"2023-09-30T09:47:51.096219+0000","last_fullsized":"2023-09-30T09:47:51.096219+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":345,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976599+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:47:50.976599+0000","last_peered":"2023-09-30T09:47:50.976599+0000","last_clean":"2023-09-30T09:47:50.976599+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:47:50.976599+0000","last_undegraded":"2023-09-30T09:47:50.976599+0000","last_fullsized":"2023-09-30T09:47:50.976599+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":481,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.064070+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:47:51.064070+0000","last_peered":"2023-09-30T09:47:51.064070+0000","last_clean":"2023-09-30T09:47:51.064070+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:47:51.064070+0000","last_undegraded":"2023-09-30T09:47:51.064070+0000","last_fullsized":"2023-09-30T09:47:51.064070+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":654,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976720+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:47:50.976720+0000","last_peered":"2023-09-30T09:47:50.976720+0000","last_clean":"2023-09-30T09:47:50.976720+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:47:50.976720+0000","last_undegraded":"2023-09-30T09:47:50.976720+0000","last_fullsized":"2023-09-30T09:47:50.976720+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":239,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.065178+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:47:51.065178+0000","last_peered":"2023-09-30T09:47:51.065178+0000","last_clean":"2023-09-30T09:47:51.065178+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:47:51.065178+0000","last_undegraded":"2023-09-30T09:47:51.065178+0000","last_fullsized":"2023-09-30T09:47:51.065178+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":80,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976844+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:47:50.976844+0000","last_peered":"2023-09-30T09:47:50.976844+0000","last_clean":"2023-09-30T09:47:50.976844+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:47:50.976844+0000","last_undegraded":"2023-09-30T09:47:50.976844+0000","last_fullsized":"2023-09-30T09:47:50.976844+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":581,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.980814+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:47:50.980814+0000","last_peered":"2023-09-30T09:47:50.980814+0000","last_clean":"2023-09-30T09:47:50.980814+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:47:50.980814+0000","last_undegraded":"2023-09-30T09:47:50.980814+0000","last_fullsized":"2023-09-30T09:47:50.980814+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":569,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.071886+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:47:51.071886+0000","last_peered":"2023-09-30T09:47:51.071886+0000","last_clean":"2023-09-30T09:47:51.071886+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:47:51.071886+0000","last_undegraded":"2023-09-30T09:47:51.071886+0000","last_fullsized":"2023-09-30T09:47:51.071886+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":547,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.072441+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:47:51.072441+0000","last_peered":"2023-09-30T09:47:51.072441+0000","last_clean":"2023-09-30T09:47:51.072441+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:47:51.072441+0000","last_undegraded":"2023-09-30T09:47:51.072441+0000","last_fullsized":"2023-09-30T09:47:51.072441+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":779,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976526+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:47:50.976526+0000","last_peered":"2023-09-30T09:47:50.976526+0000","last_clean":"2023-09-30T09:47:50.976526+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:47:50.976526+0000","last_undegraded":"2023-09-30T09:47:50.976526+0000","last_fullsized":"2023-09-30T09:47:50.976526+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":175,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.063372+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:47:51.063372+0000","last_peered":"2023-09-30T09:47:51.063372+0000","last_clean":"2023-09-30T09:47:51.063372+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:47:51.063372+0000","last_undegraded":"2023-09-30T09:47:51.063372+0000","last_fullsized":"2023-09-30T09:47:51.063372+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":502,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.063758+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:47:51.063758+0000","last_peered":"2023-09-30T09:47:51.063758+0000","last_clean":"2023-09-30T09:47:51.063758+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:47:51.063758+0000","last_undegraded":"2023-09-30T09:47:51.063758+0000","last_fullsized":"2023-09-30T09:47:51.063758+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":826,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.975888+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:47:50.975888+0000","last_peered":"2023-09-30T09:47:50.975888+0000","last_clean":"2023-09-30T09:47:50.975888+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:47:50.975888+0000","last_undegraded":"2023-09-30T09:47:50.975888+0000","last_fullsized":"2023-09-30T09:47:50.975888+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":105,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976543+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:47:50.976543+0000","last_peered":"2023-09-30T09:47:50.976543+0000","last_clean":"2023-09-30T09:47:50.976543+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:47:50.976543+0000","last_undegraded":"2023-09-30T09:47:50.976543+0000","last_fullsized":"2023-09-30T09:47:50.976543+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":200,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976476+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:47:50.976476+0000","last_peered":"2023-09-30T09:47:50.976476+0000","last_clean":"2023-09-30T09:47:50.976476+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:47:50.976476+0000","last_undegraded":"2023-09-30T09:47:50.976476+0000","last_fullsized":"2023-09-30T09:47:50.976476+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":927,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.067166+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:47:51.067166+0000","last_peered":"2023-09-30T09:47:51.067166+0000","last_clean":"2023-09-30T09:47:51.067166+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:47:51.067166+0000","last_undegraded":"2023-09-30T09:47:51.067166+0000","last_fullsized":"2023-09-30T09:47:51.067166+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":209,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.064269+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:47:51.064269+0000","last_peered":"2023-09-30T09:47:51.064269+0000","last_clean":"2023-09-30T09:47:51.064269+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:47:51.064269+0000","last_undegraded":"2023-09-30T09:47:51.064269+0000","last_fullsized":"2023-09-30T09:47:51.064269+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":105,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.979488+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:47:50.979488+0000","last_peered":"2023-09-30T09:47:50.979488+0000","last_clean":"2023-09-30T09:47:50.979488+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:47:50.979488+0000","last_undegraded":"2023-09-30T09:47:50.979488+0000","last_fullsized":"2023-09-30T09:47:50.979488+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":935,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976275+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:47:50.976275+0000","last_peered":"2023-09-30T09:47:50.976275+0000","last_clean":"2023-09-30T09:47:50.976275+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:47:50.976275+0000","last_undegraded":"2023-09-30T09:47:50.976275+0000","last_fullsized":"2023-09-30T09:47:50.976275+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":121,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976317+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:47:50.976317+0000","last_peered":"2023-09-30T09:47:50.976317+0000","last_clean":"2023-09-30T09:47:50.976317+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:47:50.976317+0000","last_undegraded":"2023-09-30T09:47:50.976317+0000","last_fullsized":"2023-09-30T09:47:50.976317+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.067749+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:47:51.067749+0000","last_peered":"2023-09-30T09:47:51.067749+0000","last_clean":"2023-09-30T09:47:51.067749+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:47:51.067749+0000","last_undegraded":"2023-09-30T09:47:51.067749+0000","last_fullsized":"2023-09-30T09:47:51.067749+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"525'11155","reported_seq":11470,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.366318+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:47:54.366318+0000","last_peered":"2023-09-30T09:47:54.366318+0000","last_clean":"2023-09-30T09:47:54.366318+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:47:54.366318+0000","last_undegraded":"2023-09-30T09:47:54.366318+0000","last_fullsized":"2023-09-30T09:47:54.366318+0000","mapping_epoch":495,"log_start":"502'3600","ondisk_log_start":"502'3600","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7555,"log_dups_size":0,"ondisk_log_size":7555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53673984,"num_objects":819,"num_object_clones":0,"num_object_copies":2457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11155,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":901,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.067129+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:47:51.067129+0000","last_peered":"2023-09-30T09:47:51.067129+0000","last_clean":"2023-09-30T09:47:51.067129+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:47:51.067129+0000","last_undegraded":"2023-09-30T09:47:51.067129+0000","last_fullsized":"2023-09-30T09:47:51.067129+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":201,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.065863+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:47:51.065863+0000","last_peered":"2023-09-30T09:47:51.065863+0000","last_clean":"2023-09-30T09:47:51.065863+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:47:51.065863+0000","last_undegraded":"2023-09-30T09:47:51.065863+0000","last_fullsized":"2023-09-30T09:47:51.065863+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":146,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.975817+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:47:50.975817+0000","last_peered":"2023-09-30T09:47:50.975817+0000","last_clean":"2023-09-30T09:47:50.975817+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:47:50.975817+0000","last_undegraded":"2023-09-30T09:47:50.975817+0000","last_fullsized":"2023-09-30T09:47:50.975817+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"525'11044","reported_seq":11094,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:53.636036+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:47:53.636036+0000","last_peered":"2023-09-30T09:47:53.636036+0000","last_clean":"2023-09-30T09:47:53.636036+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:47:53.636036+0000","last_undegraded":"2023-09-30T09:47:53.636036+0000","last_fullsized":"2023-09-30T09:47:53.636036+0000","mapping_epoch":483,"log_start":"504'4600","ondisk_log_start":"504'4600","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58982400,"num_objects":900,"num_object_clones":0,"num_object_copies":2700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11044,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":919,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.069500+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:47:51.069500+0000","last_peered":"2023-09-30T09:47:51.069500+0000","last_clean":"2023-09-30T09:47:51.069500+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:47:51.069500+0000","last_undegraded":"2023-09-30T09:47:51.069500+0000","last_fullsized":"2023-09-30T09:47:51.069500+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":171,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.065585+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:47:51.065585+0000","last_peered":"2023-09-30T09:47:51.065585+0000","last_clean":"2023-09-30T09:47:51.065585+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:47:51.065585+0000","last_undegraded":"2023-09-30T09:47:51.065585+0000","last_fullsized":"2023-09-30T09:47:51.065585+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":576,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.072070+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:47:51.072070+0000","last_peered":"2023-09-30T09:47:51.072070+0000","last_clean":"2023-09-30T09:47:51.072070+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:47:51.072070+0000","last_undegraded":"2023-09-30T09:47:51.072070+0000","last_fullsized":"2023-09-30T09:47:51.072070+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"525'11332","reported_seq":11607,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.220913+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:47:54.220913+0000","last_peered":"2023-09-30T09:47:54.220913+0000","last_clean":"2023-09-30T09:47:54.220913+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:47:54.220913+0000","last_undegraded":"2023-09-30T09:47:54.220913+0000","last_fullsized":"2023-09-30T09:47:54.220913+0000","mapping_epoch":493,"log_start":"504'4600","ondisk_log_start":"504'4600","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6732,"log_dups_size":0,"ondisk_log_size":6732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55705600,"num_objects":850,"num_object_clones":0,"num_object_copies":2550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11332,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":695,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977705+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:47:50.977705+0000","last_peered":"2023-09-30T09:47:50.977705+0000","last_clean":"2023-09-30T09:47:50.977705+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:47:50.977705+0000","last_undegraded":"2023-09-30T09:47:50.977705+0000","last_fullsized":"2023-09-30T09:47:50.977705+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":238,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.064930+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:47:51.064930+0000","last_peered":"2023-09-30T09:47:51.064930+0000","last_clean":"2023-09-30T09:47:51.064930+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:47:51.064930+0000","last_undegraded":"2023-09-30T09:47:51.064930+0000","last_fullsized":"2023-09-30T09:47:51.064930+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":94,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976879+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:47:50.976879+0000","last_peered":"2023-09-30T09:47:50.976879+0000","last_clean":"2023-09-30T09:47:50.976879+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:47:50.976879+0000","last_undegraded":"2023-09-30T09:47:50.976879+0000","last_fullsized":"2023-09-30T09:47:50.976879+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"525'11063","reported_seq":11427,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.003543+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:47:54.003543+0000","last_peered":"2023-09-30T09:47:54.003543+0000","last_clean":"2023-09-30T09:47:54.003543+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:47:54.003543+0000","last_undegraded":"2023-09-30T09:47:54.003543+0000","last_fullsized":"2023-09-30T09:47:54.003543+0000","mapping_epoch":495,"log_start":"496'1000","ondisk_log_start":"496'1000","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10063,"log_dups_size":0,"ondisk_log_size":10063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54591488,"num_objects":833,"num_object_clones":0,"num_object_copies":2499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11063,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":760,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.078473+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:47:51.078473+0000","last_peered":"2023-09-30T09:47:51.078473+0000","last_clean":"2023-09-30T09:47:51.078473+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:47:51.078473+0000","last_undegraded":"2023-09-30T09:47:51.078473+0000","last_fullsized":"2023-09-30T09:47:51.078473+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":460,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977266+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:47:50.977266+0000","last_peered":"2023-09-30T09:47:50.977266+0000","last_clean":"2023-09-30T09:47:50.977266+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:47:50.977266+0000","last_undegraded":"2023-09-30T09:47:50.977266+0000","last_fullsized":"2023-09-30T09:47:50.977266+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":125,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977225+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:47:50.977225+0000","last_peered":"2023-09-30T09:47:50.977225+0000","last_clean":"2023-09-30T09:47:50.977225+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:47:50.977225+0000","last_undegraded":"2023-09-30T09:47:50.977225+0000","last_fullsized":"2023-09-30T09:47:50.977225+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"525'11311","reported_seq":11361,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.284076+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:47:54.284076+0000","last_peered":"2023-09-30T09:47:54.284076+0000","last_clean":"2023-09-30T09:47:54.284076+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:47:54.284076+0000","last_undegraded":"2023-09-30T09:47:54.284076+0000","last_fullsized":"2023-09-30T09:47:54.284076+0000","mapping_epoch":483,"log_start":"504'4300","ondisk_log_start":"504'4300","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53805056,"num_objects":821,"num_object_clones":0,"num_object_copies":2463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11311,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1007,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.064198+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:47:51.064198+0000","last_peered":"2023-09-30T09:47:51.064198+0000","last_clean":"2023-09-30T09:47:51.064198+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:47:51.064198+0000","last_undegraded":"2023-09-30T09:47:51.064198+0000","last_fullsized":"2023-09-30T09:47:51.064198+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":84,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977007+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:47:50.977007+0000","last_peered":"2023-09-30T09:47:50.977007+0000","last_clean":"2023-09-30T09:47:50.977007+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:47:50.977007+0000","last_undegraded":"2023-09-30T09:47:50.977007+0000","last_fullsized":"2023-09-30T09:47:50.977007+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":523,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.071781+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:47:51.071781+0000","last_peered":"2023-09-30T09:47:51.071781+0000","last_clean":"2023-09-30T09:47:51.071781+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:47:51.071781+0000","last_undegraded":"2023-09-30T09:47:51.071781+0000","last_fullsized":"2023-09-30T09:47:51.071781+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"525'11059","reported_seq":11109,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.287645+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:47:54.287645+0000","last_peered":"2023-09-30T09:47:54.287645+0000","last_clean":"2023-09-30T09:47:54.287645+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:47:54.287645+0000","last_undegraded":"2023-09-30T09:47:54.287645+0000","last_fullsized":"2023-09-30T09:47:54.287645+0000","mapping_epoch":483,"log_start":"503'4000","ondisk_log_start":"503'4000","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7059,"log_dups_size":0,"ondisk_log_size":7059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58654720,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11059,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":1006,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.095934+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:47:51.095934+0000","last_peered":"2023-09-30T09:47:51.095934+0000","last_clean":"2023-09-30T09:47:51.095934+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:47:51.095934+0000","last_undegraded":"2023-09-30T09:47:51.095934+0000","last_fullsized":"2023-09-30T09:47:51.095934+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":80,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976801+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:47:50.976801+0000","last_peered":"2023-09-30T09:47:50.976801+0000","last_clean":"2023-09-30T09:47:50.976801+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:47:50.976801+0000","last_undegraded":"2023-09-30T09:47:50.976801+0000","last_fullsized":"2023-09-30T09:47:50.976801+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":219,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.075687+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:47:51.075687+0000","last_peered":"2023-09-30T09:47:51.075687+0000","last_clean":"2023-09-30T09:47:51.075687+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:47:51.075687+0000","last_undegraded":"2023-09-30T09:47:51.075687+0000","last_fullsized":"2023-09-30T09:47:51.075687+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"525'11347","reported_seq":11397,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:53.669373+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:47:53.669373+0000","last_peered":"2023-09-30T09:47:53.669373+0000","last_clean":"2023-09-30T09:47:53.669373+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:47:53.669373+0000","last_undegraded":"2023-09-30T09:47:53.669373+0000","last_fullsized":"2023-09-30T09:47:53.669373+0000","mapping_epoch":483,"log_start":"512'5500","ondisk_log_start":"512'5500","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5847,"log_dups_size":0,"ondisk_log_size":5847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60489728,"num_objects":923,"num_object_clones":0,"num_object_copies":2769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11347,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":864,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977318+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:47:50.977318+0000","last_peered":"2023-09-30T09:47:50.977318+0000","last_clean":"2023-09-30T09:47:50.977318+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:47:50.977318+0000","last_undegraded":"2023-09-30T09:47:50.977318+0000","last_fullsized":"2023-09-30T09:47:50.977318+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":530,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.071935+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:47:51.071935+0000","last_peered":"2023-09-30T09:47:51.071935+0000","last_clean":"2023-09-30T09:47:51.071935+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:47:51.071935+0000","last_undegraded":"2023-09-30T09:47:51.071935+0000","last_fullsized":"2023-09-30T09:47:51.071935+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":537,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.072564+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:47:51.072564+0000","last_peered":"2023-09-30T09:47:51.072564+0000","last_clean":"2023-09-30T09:47:51.072564+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:47:51.072564+0000","last_undegraded":"2023-09-30T09:47:51.072564+0000","last_fullsized":"2023-09-30T09:47:51.072564+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"525'11082","reported_seq":11136,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.025375+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:47:54.025375+0000","last_peered":"2023-09-30T09:47:54.025375+0000","last_clean":"2023-09-30T09:47:54.025375+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:47:54.025375+0000","last_undegraded":"2023-09-30T09:47:54.025375+0000","last_fullsized":"2023-09-30T09:47:54.025375+0000","mapping_epoch":483,"log_start":"496'1000","ondisk_log_start":"496'1000","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10082,"log_dups_size":0,"ondisk_log_size":10082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55967744,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11082,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":728,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.096459+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:47:51.096459+0000","last_peered":"2023-09-30T09:47:51.096459+0000","last_clean":"2023-09-30T09:47:51.096459+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:47:51.096459+0000","last_undegraded":"2023-09-30T09:47:51.096459+0000","last_fullsized":"2023-09-30T09:47:51.096459+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.066988+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:47:51.066988+0000","last_peered":"2023-09-30T09:47:51.066988+0000","last_clean":"2023-09-30T09:47:51.066988+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:47:51.066988+0000","last_undegraded":"2023-09-30T09:47:51.066988+0000","last_fullsized":"2023-09-30T09:47:51.066988+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":210,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.076574+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:47:51.076574+0000","last_peered":"2023-09-30T09:47:51.076574+0000","last_clean":"2023-09-30T09:47:51.076574+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:47:51.076574+0000","last_undegraded":"2023-09-30T09:47:51.076574+0000","last_fullsized":"2023-09-30T09:47:51.076574+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"525'11270","reported_seq":11579,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.232276+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:47:54.232276+0000","last_peered":"2023-09-30T09:47:54.232276+0000","last_clean":"2023-09-30T09:47:54.232276+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:47:54.232276+0000","last_undegraded":"2023-09-30T09:47:54.232276+0000","last_fullsized":"2023-09-30T09:47:54.232276+0000","mapping_epoch":494,"log_start":"504'4600","ondisk_log_start":"504'4600","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54657024,"num_objects":834,"num_object_clones":0,"num_object_copies":2502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11270,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":767,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977467+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:47:50.977467+0000","last_peered":"2023-09-30T09:47:50.977467+0000","last_clean":"2023-09-30T09:47:50.977467+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:47:50.977467+0000","last_undegraded":"2023-09-30T09:47:50.977467+0000","last_fullsized":"2023-09-30T09:47:50.977467+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":756,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.979337+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:47:50.979337+0000","last_peered":"2023-09-30T09:47:50.979337+0000","last_clean":"2023-09-30T09:47:50.979337+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:47:50.979337+0000","last_undegraded":"2023-09-30T09:47:50.979337+0000","last_fullsized":"2023-09-30T09:47:50.979337+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":453,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976577+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:47:50.976577+0000","last_peered":"2023-09-30T09:47:50.976577+0000","last_clean":"2023-09-30T09:47:50.976577+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:47:50.976577+0000","last_undegraded":"2023-09-30T09:47:50.976577+0000","last_fullsized":"2023-09-30T09:47:50.976577+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":142,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976057+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:47:50.976057+0000","last_peered":"2023-09-30T09:47:50.976057+0000","last_clean":"2023-09-30T09:47:50.976057+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:47:50.976057+0000","last_undegraded":"2023-09-30T09:47:50.976057+0000","last_fullsized":"2023-09-30T09:47:50.976057+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"525'11250","reported_seq":11605,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.381713+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:47:54.381713+0000","last_peered":"2023-09-30T09:47:54.381713+0000","last_clean":"2023-09-30T09:47:54.381713+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:47:54.381713+0000","last_undegraded":"2023-09-30T09:47:54.381713+0000","last_fullsized":"2023-09-30T09:47:54.381713+0000","mapping_epoch":493,"log_start":"502'3700","ondisk_log_start":"502'3700","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57540608,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11250,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":966,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.975989+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:47:50.975989+0000","last_peered":"2023-09-30T09:47:50.975989+0000","last_clean":"2023-09-30T09:47:50.975989+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:47:50.975989+0000","last_undegraded":"2023-09-30T09:47:50.975989+0000","last_fullsized":"2023-09-30T09:47:50.975989+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":472,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976933+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:47:50.976933+0000","last_peered":"2023-09-30T09:47:50.976933+0000","last_clean":"2023-09-30T09:47:50.976933+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:47:50.976933+0000","last_undegraded":"2023-09-30T09:47:50.976933+0000","last_fullsized":"2023-09-30T09:47:50.976933+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":544,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.072316+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:47:51.072316+0000","last_peered":"2023-09-30T09:47:51.072316+0000","last_clean":"2023-09-30T09:47:51.072316+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:47:51.072316+0000","last_undegraded":"2023-09-30T09:47:51.072316+0000","last_fullsized":"2023-09-30T09:47:51.072316+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":285,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.096299+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:47:51.096299+0000","last_peered":"2023-09-30T09:47:51.096299+0000","last_clean":"2023-09-30T09:47:51.096299+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:47:51.096299+0000","last_undegraded":"2023-09-30T09:47:51.096299+0000","last_fullsized":"2023-09-30T09:47:51.096299+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"525'11298","reported_seq":11653,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.231368+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:47:54.231368+0000","last_peered":"2023-09-30T09:47:54.231368+0000","last_clean":"2023-09-30T09:47:54.231368+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:47:54.231368+0000","last_undegraded":"2023-09-30T09:47:54.231368+0000","last_fullsized":"2023-09-30T09:47:54.231368+0000","mapping_epoch":492,"log_start":"504'4600","ondisk_log_start":"504'4600","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57278464,"num_objects":874,"num_object_clones":0,"num_object_copies":2622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11298,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":944,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.065016+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:47:51.065016+0000","last_peered":"2023-09-30T09:47:51.065016+0000","last_clean":"2023-09-30T09:47:51.065016+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:47:51.065016+0000","last_undegraded":"2023-09-30T09:47:51.065016+0000","last_fullsized":"2023-09-30T09:47:51.065016+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":794,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.096114+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:47:51.096114+0000","last_peered":"2023-09-30T09:47:51.096114+0000","last_clean":"2023-09-30T09:47:51.096114+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:47:51.096114+0000","last_undegraded":"2023-09-30T09:47:51.096114+0000","last_fullsized":"2023-09-30T09:47:51.096114+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":288,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.095593+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:47:51.095593+0000","last_peered":"2023-09-30T09:47:51.095593+0000","last_clean":"2023-09-30T09:47:51.095593+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:47:51.095593+0000","last_undegraded":"2023-09-30T09:47:51.095593+0000","last_fullsized":"2023-09-30T09:47:51.095593+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":374,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976821+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:47:50.976821+0000","last_peered":"2023-09-30T09:47:50.976821+0000","last_clean":"2023-09-30T09:47:50.976821+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:47:50.976821+0000","last_undegraded":"2023-09-30T09:47:50.976821+0000","last_fullsized":"2023-09-30T09:47:50.976821+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"525'11208","reported_seq":11523,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.342604+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:47:54.342604+0000","last_peered":"2023-09-30T09:47:54.342604+0000","last_clean":"2023-09-30T09:47:54.342604+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:47:54.342604+0000","last_undegraded":"2023-09-30T09:47:54.342604+0000","last_fullsized":"2023-09-30T09:47:54.342604+0000","mapping_epoch":495,"log_start":"502'3700","ondisk_log_start":"502'3700","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7508,"log_dups_size":0,"ondisk_log_size":7508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53084160,"num_objects":810,"num_object_clones":0,"num_object_copies":2430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11208,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1086,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976322+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:47:50.976322+0000","last_peered":"2023-09-30T09:47:50.976322+0000","last_clean":"2023-09-30T09:47:50.976322+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:47:50.976322+0000","last_undegraded":"2023-09-30T09:47:50.976322+0000","last_fullsized":"2023-09-30T09:47:50.976322+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":887,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.063233+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:47:51.063233+0000","last_peered":"2023-09-30T09:47:51.063233+0000","last_clean":"2023-09-30T09:47:51.063233+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:47:51.063233+0000","last_undegraded":"2023-09-30T09:47:51.063233+0000","last_fullsized":"2023-09-30T09:47:51.063233+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":101,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976160+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:47:50.976160+0000","last_peered":"2023-09-30T09:47:50.976160+0000","last_clean":"2023-09-30T09:47:50.976160+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:47:50.976160+0000","last_undegraded":"2023-09-30T09:47:50.976160+0000","last_fullsized":"2023-09-30T09:47:50.976160+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977229+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:47:50.977229+0000","last_peered":"2023-09-30T09:47:50.977229+0000","last_clean":"2023-09-30T09:47:50.977229+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:47:50.977229+0000","last_undegraded":"2023-09-30T09:47:50.977229+0000","last_fullsized":"2023-09-30T09:47:50.977229+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"525'11440","reported_seq":21474,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:54.372315+0000","last_change":"2023-09-30T09:47:51.102725+0000","last_active":"2023-09-30T09:47:54.372315+0000","last_peered":"2023-09-30T09:47:54.372315+0000","last_clean":"2023-09-30T09:47:54.372315+0000","last_became_active":"2023-09-30T09:47:50.995967+0000","last_became_peered":"2023-09-30T09:47:50.995967+0000","last_unstale":"2023-09-30T09:47:54.372315+0000","last_undegraded":"2023-09-30T09:47:54.372315+0000","last_fullsized":"2023-09-30T09:47:54.372315+0000","mapping_epoch":523,"log_start":"502'3878","ondisk_log_start":"502'3878","created":483,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7562,"log_dups_size":0,"ondisk_log_size":7562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:15.442443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58064896,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11440,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4971,"num_bytes_recovered":280821760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":786,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977283+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:47:50.977283+0000","last_peered":"2023-09-30T09:47:50.977283+0000","last_clean":"2023-09-30T09:47:50.977283+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:47:50.977283+0000","last_undegraded":"2023-09-30T09:47:50.977283+0000","last_fullsized":"2023-09-30T09:47:50.977283+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":933,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.063192+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:47:51.063192+0000","last_peered":"2023-09-30T09:47:51.063192+0000","last_clean":"2023-09-30T09:47:51.063192+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:47:51.063192+0000","last_undegraded":"2023-09-30T09:47:51.063192+0000","last_fullsized":"2023-09-30T09:47:51.063192+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":138,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976691+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:47:50.976691+0000","last_peered":"2023-09-30T09:47:50.976691+0000","last_clean":"2023-09-30T09:47:50.976691+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:47:50.976691+0000","last_undegraded":"2023-09-30T09:47:50.976691+0000","last_fullsized":"2023-09-30T09:47:50.976691+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":31,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.979396+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:47:50.979396+0000","last_peered":"2023-09-30T09:47:50.979396+0000","last_clean":"2023-09-30T09:47:50.979396+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:47:50.979396+0000","last_undegraded":"2023-09-30T09:47:50.979396+0000","last_fullsized":"2023-09-30T09:47:50.979396+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"525'11199","reported_seq":11249,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:53.651451+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:47:53.651451+0000","last_peered":"2023-09-30T09:47:53.651451+0000","last_clean":"2023-09-30T09:47:53.651451+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:47:53.651451+0000","last_undegraded":"2023-09-30T09:47:53.651451+0000","last_fullsized":"2023-09-30T09:47:53.651451+0000","mapping_epoch":483,"log_start":"512'5400","ondisk_log_start":"512'5400","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59834368,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11199,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":777,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977560+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:47:50.977560+0000","last_peered":"2023-09-30T09:47:50.977560+0000","last_clean":"2023-09-30T09:47:50.977560+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:47:50.977560+0000","last_undegraded":"2023-09-30T09:47:50.977560+0000","last_fullsized":"2023-09-30T09:47:50.977560+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1073,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.065250+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:47:51.065250+0000","last_peered":"2023-09-30T09:47:51.065250+0000","last_clean":"2023-09-30T09:47:51.065250+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:47:51.065250+0000","last_undegraded":"2023-09-30T09:47:51.065250+0000","last_fullsized":"2023-09-30T09:47:51.065250+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":269,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.096128+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:47:51.096128+0000","last_peered":"2023-09-30T09:47:51.096128+0000","last_clean":"2023-09-30T09:47:51.096128+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:47:51.096128+0000","last_undegraded":"2023-09-30T09:47:51.096128+0000","last_fullsized":"2023-09-30T09:47:51.096128+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":138,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977504+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:47:50.977504+0000","last_peered":"2023-09-30T09:47:50.977504+0000","last_clean":"2023-09-30T09:47:50.977504+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:47:50.977504+0000","last_undegraded":"2023-09-30T09:47:50.977504+0000","last_fullsized":"2023-09-30T09:47:50.977504+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"525'11111","reported_seq":11496,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:53.902909+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:47:53.902909+0000","last_peered":"2023-09-30T09:47:53.902909+0000","last_clean":"2023-09-30T09:47:53.902909+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:47:53.902909+0000","last_undegraded":"2023-09-30T09:47:53.902909+0000","last_fullsized":"2023-09-30T09:47:53.902909+0000","mapping_epoch":495,"log_start":"511'5300","ondisk_log_start":"511'5300","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5811,"log_dups_size":0,"ondisk_log_size":5811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58851352,"num_objects":899,"num_object_clones":0,"num_object_copies":2697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":899,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11111,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1044,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.075850+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:47:51.075850+0000","last_peered":"2023-09-30T09:47:51.075850+0000","last_clean":"2023-09-30T09:47:51.075850+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:47:51.075850+0000","last_undegraded":"2023-09-30T09:47:51.075850+0000","last_fullsized":"2023-09-30T09:47:51.075850+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":556,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.065817+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:47:51.065817+0000","last_peered":"2023-09-30T09:47:51.065817+0000","last_clean":"2023-09-30T09:47:51.065817+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:47:51.065817+0000","last_undegraded":"2023-09-30T09:47:51.065817+0000","last_fullsized":"2023-09-30T09:47:51.065817+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1243,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977514+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:47:50.977514+0000","last_peered":"2023-09-30T09:47:50.977514+0000","last_clean":"2023-09-30T09:47:50.977514+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:47:50.977514+0000","last_undegraded":"2023-09-30T09:47:50.977514+0000","last_fullsized":"2023-09-30T09:47:50.977514+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":560,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.063149+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:47:51.063149+0000","last_peered":"2023-09-30T09:47:51.063149+0000","last_clean":"2023-09-30T09:47:51.063149+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:47:51.063149+0000","last_undegraded":"2023-09-30T09:47:51.063149+0000","last_fullsized":"2023-09-30T09:47:51.063149+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"525'11389","reported_seq":11439,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:53.872575+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:47:53.872575+0000","last_peered":"2023-09-30T09:47:53.872575+0000","last_clean":"2023-09-30T09:47:53.872575+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:47:53.872575+0000","last_undegraded":"2023-09-30T09:47:53.872575+0000","last_fullsized":"2023-09-30T09:47:53.872575+0000","mapping_epoch":483,"log_start":"510'5000","ondisk_log_start":"510'5000","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6389,"log_dups_size":0,"ondisk_log_size":6389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11389,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":908,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.069416+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:47:51.069416+0000","last_peered":"2023-09-30T09:47:51.069416+0000","last_clean":"2023-09-30T09:47:51.069416+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:47:51.069416+0000","last_undegraded":"2023-09-30T09:47:51.069416+0000","last_fullsized":"2023-09-30T09:47:51.069416+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":95,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977663+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:47:50.977663+0000","last_peered":"2023-09-30T09:47:50.977663+0000","last_clean":"2023-09-30T09:47:50.977663+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:47:50.977663+0000","last_undegraded":"2023-09-30T09:47:50.977663+0000","last_fullsized":"2023-09-30T09:47:50.977663+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":101,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.979664+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:47:50.979664+0000","last_peered":"2023-09-30T09:47:50.979664+0000","last_clean":"2023-09-30T09:47:50.979664+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:47:50.979664+0000","last_undegraded":"2023-09-30T09:47:50.979664+0000","last_fullsized":"2023-09-30T09:47:50.979664+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":526,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.072007+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:47:51.072007+0000","last_peered":"2023-09-30T09:47:51.072007+0000","last_clean":"2023-09-30T09:47:51.072007+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:47:51.072007+0000","last_undegraded":"2023-09-30T09:47:51.072007+0000","last_fullsized":"2023-09-30T09:47:51.072007+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":845,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-30T09:47:52.018360+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:47:52.018360+0000","last_peered":"2023-09-30T09:47:52.018360+0000","last_clean":"2023-09-30T09:47:52.018360+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:47:52.018360+0000","last_undegraded":"2023-09-30T09:47:52.018360+0000","last_fullsized":"2023-09-30T09:47:52.018360+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":30,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977418+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:47:50.977418+0000","last_peered":"2023-09-30T09:47:50.977418+0000","last_clean":"2023-09-30T09:47:50.977418+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:47:50.977418+0000","last_undegraded":"2023-09-30T09:47:50.977418+0000","last_fullsized":"2023-09-30T09:47:50.977418+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":230,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.096045+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:47:51.096045+0000","last_peered":"2023-09-30T09:47:51.096045+0000","last_clean":"2023-09-30T09:47:51.096045+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:47:51.096045+0000","last_undegraded":"2023-09-30T09:47:51.096045+0000","last_fullsized":"2023-09-30T09:47:51.096045+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":639,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977608+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:47:50.977608+0000","last_peered":"2023-09-30T09:47:50.977608+0000","last_clean":"2023-09-30T09:47:50.977608+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:47:50.977608+0000","last_undegraded":"2023-09-30T09:47:50.977608+0000","last_fullsized":"2023-09-30T09:47:50.977608+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":453,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977172+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:47:50.977172+0000","last_peered":"2023-09-30T09:47:50.977172+0000","last_clean":"2023-09-30T09:47:50.977172+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:47:50.977172+0000","last_undegraded":"2023-09-30T09:47:50.977172+0000","last_fullsized":"2023-09-30T09:47:50.977172+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":239,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.072801+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:47:51.072801+0000","last_peered":"2023-09-30T09:47:51.072801+0000","last_clean":"2023-09-30T09:47:51.072801+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:47:51.072801+0000","last_undegraded":"2023-09-30T09:47:51.072801+0000","last_fullsized":"2023-09-30T09:47:51.072801+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":395,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.976669+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:47:50.976669+0000","last_peered":"2023-09-30T09:47:50.976669+0000","last_clean":"2023-09-30T09:47:50.976669+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:47:50.976669+0000","last_undegraded":"2023-09-30T09:47:50.976669+0000","last_fullsized":"2023-09-30T09:47:50.976669+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":259,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.071749+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:47:51.071749+0000","last_peered":"2023-09-30T09:47:51.071749+0000","last_clean":"2023-09-30T09:47:51.071749+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:47:51.071749+0000","last_undegraded":"2023-09-30T09:47:51.071749+0000","last_fullsized":"2023-09-30T09:47:51.071749+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":852,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.063685+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:47:51.063685+0000","last_peered":"2023-09-30T09:47:51.063685+0000","last_clean":"2023-09-30T09:47:51.063685+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:47:51.063685+0000","last_undegraded":"2023-09-30T09:47:51.063685+0000","last_fullsized":"2023-09-30T09:47:51.063685+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":82,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.977464+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:47:50.977464+0000","last_peered":"2023-09-30T09:47:50.977464+0000","last_clean":"2023-09-30T09:47:50.977464+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:47:50.977464+0000","last_undegraded":"2023-09-30T09:47:50.977464+0000","last_fullsized":"2023-09-30T09:47:50.977464+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":476,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.096034+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:47:51.096034+0000","last_peered":"2023-09-30T09:47:51.096034+0000","last_clean":"2023-09-30T09:47:51.096034+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:47:51.096034+0000","last_undegraded":"2023-09-30T09:47:51.096034+0000","last_fullsized":"2023-09-30T09:47:51.096034+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":895,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.075771+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:47:51.075771+0000","last_peered":"2023-09-30T09:47:51.075771+0000","last_clean":"2023-09-30T09:47:51.075771+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:47:51.075771+0000","last_undegraded":"2023-09-30T09:47:51.075771+0000","last_fullsized":"2023-09-30T09:47:51.075771+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":37,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:50.982285+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:47:50.982285+0000","last_peered":"2023-09-30T09:47:50.982285+0000","last_clean":"2023-09-30T09:47:50.982285+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:47:50.982285+0000","last_undegraded":"2023-09-30T09:47:50.982285+0000","last_fullsized":"2023-09-30T09:47:50.982285+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":540,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-30T09:47:51.071710+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:47:51.071710+0000","last_peered":"2023-09-30T09:47:51.071710+0000","last_clean":"2023-09-30T09:47:51.071710+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:47:51.071710+0000","last_undegraded":"2023-09-30T09:47:51.071710+0000","last_fullsized":"2023-09-30T09:47:51.071710+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":912719896,"num_objects":13928,"num_object_clones":0,"num_object_copies":41784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13928,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179558,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6114,"num_bytes_recovered":355729408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2742431744,"data_stored":2742419528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114780,"ondisk_log_size":114780,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234434,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1080012,"kb_used_data":534532,"kb_used_omap":0,"kb_used_meta":545344,"kb_avail":92665140,"statfs":{"total":95995035648,"available":94889103360,"internally_reserved":0,"allocated":547360768,"data_stored":545608548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":558432256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":8.9299999999999997,"5min":7.6980000000000004,"15min":5.0990000000000002},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":95.825999999999993,"5min":124.74299999999999,"15min":215.416},"last":4.4820000000000002},{"interface":"front","average":{"1min":8.9190000000000005,"5min":7.7690000000000001,"15min":5.0069999999999997},"min":{"1min":0.26100000000000001,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":95.488,"5min":124.69499999999999,"15min":215.041},"last":4.1840000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":7.4539999999999997,"15min":4.117},"min":{"1min":0.40799999999999997,"5min":0.25,"15min":0.25},"max":{"1min":7.6559999999999997,"5min":157.578,"15min":157.578},"last":10.749000000000001},{"interface":"front","average":{"1min":2.649,"5min":7.2839999999999998,"15min":4.2679999999999998},"min":{"1min":0.41299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.8800000000000008,"5min":157.589,"15min":157.589},"last":6.8209999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":3.9289999999999998,"5min":3.1749999999999998,"15min":3.762},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.24299999999999999},"max":{"1min":10.173999999999999,"5min":104.88500000000001,"15min":117.14},"last":1.444},{"interface":"front","average":{"1min":3.5270000000000001,"5min":3.1680000000000001,"15min":3.7149999999999999},"min":{"1min":0.436,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":16.135000000000002,"5min":104.867,"15min":117.044},"last":8.1419999999999995}]},{"osd":3,"last update":"Sat Sep 30 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":7.8529999999999998,"5min":8.1449999999999996,"15min":8.6750000000000007},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":97.037000000000006,"5min":154.06700000000001,"15min":277.57799999999997},"last":2.7519999999999998},{"interface":"front","average":{"1min":8.9039999999999999,"5min":8.4440000000000008,"15min":8.9420000000000002},"min":{"1min":0.42999999999999999,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":95.135000000000005,"5min":154.24799999999999,"15min":278.69299999999998},"last":3.2530000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.3779999999999999,"15min":1.6839999999999999},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":7.6260000000000003,"5min":25.192,"15min":175.72},"last":2.6909999999999998},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.2290000000000001,"15min":1.5389999999999999},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.252},"max":{"1min":3.3100000000000001,"5min":24.521000000000001,"15min":179.048},"last":3.1819999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:47:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":7.6619999999999999,"15min":10.548999999999999},"min":{"1min":0.32700000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.266,"5min":197.75700000000001,"15min":197.75700000000001},"last":3.4929999999999999},{"interface":"front","average":{"1min":1.5780000000000001,"5min":7.79,"15min":10.481},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.7089999999999996,"5min":197.63300000000001,"15min":197.63300000000001},"last":0.309}]},{"osd":6,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.046,"15min":2.2589999999999999},"min":{"1min":0.41499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5830000000000002,"5min":7.5830000000000002,"15min":206.244},"last":2.7240000000000002},{"interface":"front","average":{"1min":1.5009999999999999,"5min":1.05,"15min":2.2189999999999999},"min":{"1min":0.46700000000000003,"5min":0.31900000000000001,"15min":0.29199999999999998},"max":{"1min":3.9420000000000002,"5min":5.8520000000000003,"15min":202.78800000000001},"last":3.46}]}]},{"osd":6,"up_from":46,"seq":197568496072,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":706164,"kb_used_data":180204,"kb_used_omap":0,"kb_used_meta":525952,"kb_avail":93038988,"statfs":{"total":95995035648,"available":95271923712,"internally_reserved":0,"allocated":184528896,"data_stored":182799969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":538574848},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":4.1050000000000004,"15min":5.3330000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":11.609,"5min":95.043999999999997,"15min":187.19800000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.1600000000000001,"5min":4.1210000000000004,"15min":5.3520000000000003},"min":{"1min":0.26300000000000001,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":6.5460000000000003,"5min":94.855999999999995,"15min":187.46700000000001},"last":1.1890000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":1.8300000000000001,"15min":2.0139999999999998},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":13.494999999999999,"5min":32.814,"15min":69.569999999999993},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.6480000000000001,"5min":1.9710000000000001,"15min":2.2120000000000002},"min":{"1min":0.34999999999999998,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":8.5869999999999997,"5min":32.856000000000002,"15min":70.471999999999994},"last":1.1499999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":3.859,"15min":4.069},"min":{"1min":0.28899999999999998,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":11.432,"5min":207.02500000000001,"15min":207.02500000000001},"last":4.2240000000000002},{"interface":"front","average":{"1min":2.234,"5min":3.758,"15min":4.1609999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":11.585000000000001,"5min":207.114,"15min":207.114},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8839999999999999,"5min":3.484,"15min":4.2560000000000002},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":11.644,"5min":107.11799999999999,"15min":212.44800000000001},"last":2.7210000000000001},{"interface":"front","average":{"1min":2.3239999999999998,"5min":3.4159999999999999,"15min":4.4909999999999997},"min":{"1min":0.35099999999999998,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":8.8559999999999999,"5min":106.982,"15min":210.84299999999999},"last":3.5760000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:47:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.722,"15min":1.2669999999999999},"min":{"1min":0.30199999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.4400000000000004,"5min":63.798000000000002,"15min":63.798000000000002},"last":3.613},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.8400000000000001,"15min":1.2330000000000001},"min":{"1min":0.32900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.558999999999999,"5min":63.133000000000003,"15min":63.133000000000003},"last":4.4870000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":1.825,"15min":1.974},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.367000000000001,"5min":14.974,"15min":14.974},"last":4.2949999999999999},{"interface":"front","average":{"1min":2.1739999999999999,"5min":2.1880000000000002,"15min":2.4809999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.669,"5min":16.443000000000001,"15min":16.443000000000001},"last":2.6749999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.2090000000000001,"15min":1.633},"min":{"1min":0.378,"5min":0.28599999999999998,"15min":0.26500000000000001},"max":{"1min":7.1639999999999997,"5min":11.276999999999999,"15min":138.191},"last":5.6239999999999997},{"interface":"front","average":{"1min":2.3519999999999999,"5min":1.27,"15min":1.528},"min":{"1min":0.36899999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.521000000000001,"5min":11.521000000000001,"15min":140.33500000000001},"last":2.7040000000000002}]}]},{"osd":1,"up_from":301,"seq":1292785156270,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1085872,"kb_used_data":440488,"kb_used_omap":9,"kb_used_meta":645366,"kb_avail":92659280,"statfs":{"total":95995035648,"available":94883102720,"internally_reserved":0,"allocated":451059712,"data_stored":448075384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":660854802},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":4.319,"15min":3.7320000000000002},"min":{"1min":0.221,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":26.021000000000001,"5min":142.08500000000001,"15min":142.08500000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":3.7639999999999998,"5min":4.1159999999999997,"15min":3.6240000000000001},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":25.971,"5min":142.137,"15min":142.137},"last":3.7839999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.883,"15min":2.573},"min":{"1min":0.27700000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.763000000000002,"5min":62.572000000000003,"15min":62.572000000000003},"last":1.2350000000000001},{"interface":"front","average":{"1min":2.9809999999999999,"5min":2.8700000000000001,"15min":2.7999999999999998},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.138999999999999,"5min":68.373999999999995,"15min":68.373999999999995},"last":0.59599999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":3.484,"5min":5.9690000000000003,"15min":6.0899999999999999},"min":{"1min":0.30499999999999999,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":9.8610000000000007,"5min":199.41,"15min":199.41},"last":6.9950000000000001},{"interface":"front","average":{"1min":3.149,"5min":6.2999999999999998,"15min":6.1509999999999998},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.4329999999999998,"5min":199.58099999999999,"15min":199.58099999999999},"last":7.8879999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":1.091,"15min":2.1309999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":8.9399999999999995,"5min":8.9399999999999995,"15min":142.77099999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.46,"5min":1.52,"15min":2.4140000000000001},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.6319999999999997,"5min":8.6319999999999997,"15min":144.898},"last":2.6110000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":2.206,"15min":2.024},"min":{"1min":0.41099999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":8.6790000000000003,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":2.6469999999999998},{"interface":"front","average":{"1min":2.198,"5min":1.54,"15min":1.3280000000000001},"min":{"1min":0.39100000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.891,"5min":15.250999999999999,"15min":15.250999999999999},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.552,"15min":5.5679999999999996},"min":{"1min":0.442,"5min":0.316,"15min":0.29899999999999999},"max":{"1min":8.8559999999999999,"5min":30.986999999999998,"15min":209.221},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.4850000000000001,"15min":5.4960000000000004},"min":{"1min":0.47899999999999998,"5min":0.35399999999999998,"15min":0.33900000000000002},"max":{"1min":8.8100000000000005,"5min":28.710999999999999,"15min":208.43600000000001},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:46:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.5449999999999999,"15min":3.9500000000000002},"min":{"1min":0.36299999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":9.3879999999999999,"5min":9.3879999999999999,"15min":165.59399999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.077,"5min":1.542,"15min":3.9929999999999999},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":4.2549999999999999,"5min":9.3399999999999999,"15min":165.833},"last":0.42399999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926168,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":894708,"kb_used_data":231852,"kb_used_omap":5,"kb_used_meta":662842,"kb_avail":92850444,"statfs":{"total":95995035648,"available":95078854656,"internally_reserved":0,"allocated":237416448,"data_stored":234198764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":678750461},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.9419999999999999,"15min":2.181},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":7.2380000000000004,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.927,"5min":2.3180000000000001,"15min":4.0199999999999996},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.344000000000001,"5min":57.140999999999998,"15min":138.92400000000001},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":6.0810000000000004,"5min":4.0350000000000001,"15min":3.6200000000000001},"min":{"1min":0.376,"5min":0.23799999999999999,"15min":0.20799999999999999},"max":{"1min":19.504000000000001,"5min":55.137,"15min":127.73399999999999},"last":1.073},{"interface":"front","average":{"1min":5.7590000000000003,"5min":4.3019999999999996,"15min":3.8860000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":19.042999999999999,"5min":56.963999999999999,"15min":127.69},"last":0.307}]},{"osd":3,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":5.5099999999999998,"5min":5.9870000000000001,"15min":5.4249999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":16.594000000000001,"5min":125.562,"15min":218.06999999999999},"last":1.665},{"interface":"front","average":{"1min":3.0680000000000001,"5min":4.984,"15min":4.2599999999999998},"min":{"1min":0.39600000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":13.361000000000001,"5min":117.295,"15min":218.63200000000001},"last":0.89700000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.7989999999999999,"15min":2.012},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.8710000000000004,"5min":72.403999999999996,"15min":72.403999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1960000000000002,"15min":2.8260000000000001},"min":{"1min":0.38600000000000001,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":9.4870000000000001,"5min":72.162999999999997,"15min":136.262},"last":1.623}]},{"osd":5,"last update":"Sat Sep 30 09:47:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2570000000000001,"5min":2.0950000000000002,"15min":1.413},"min":{"1min":0.35299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":13.162000000000001,"5min":13.162000000000001,"15min":13.162000000000001},"last":1.6930000000000001},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.2150000000000001,"15min":1.1040000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":13.422000000000001,"5min":13.422000000000001,"15min":13.422000000000001},"last":2.024}]},{"osd":6,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.958,"15min":3.508},"min":{"1min":0.38200000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":20.706,"5min":57.384,"15min":214.785},"last":0.68600000000000005},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.3700000000000001,"15min":3.8370000000000002},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.684000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.8120000000000001,"15min":2.0720000000000001},"min":{"1min":0.42499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.6549999999999994,"5min":57.776000000000003,"15min":97.382000000000005},"last":0.73199999999999998},{"interface":"front","average":{"1min":2.6349999999999998,"5min":2.363,"15min":2.3940000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":9.7149999999999999,"5min":57.179000000000002,"15min":97.015000000000001},"last":0.46300000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599109,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1042764,"kb_used_data":236356,"kb_used_omap":5,"kb_used_meta":806394,"kb_avail":92702388,"statfs":{"total":95995035648,"available":94927245312,"internally_reserved":0,"allocated":242028544,"data_stored":239076158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":825748109},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":4.0890000000000004,"5min":5.4829999999999997,"15min":3.5800000000000001},"min":{"1min":0.219,"5min":0.192,"15min":0.187},"max":{"1min":13.880000000000001,"5min":170.678,"15min":170.678},"last":0.53100000000000003},{"interface":"front","average":{"1min":5.1459999999999999,"5min":5.5549999999999997,"15min":3.6320000000000001},"min":{"1min":0.32100000000000001,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":12.609999999999999,"5min":172.44999999999999,"15min":172.44999999999999},"last":7.1950000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:46:52 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":4.8449999999999998,"15min":3.7919999999999998},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":10.922000000000001,"5min":162.696,"15min":162.696},"last":6.8620000000000001},{"interface":"front","average":{"1min":4.665,"5min":5.0709999999999997,"15min":3.6320000000000001},"min":{"1min":0.27700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.705,"5min":162.93899999999999,"15min":162.93899999999999},"last":7.3170000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:47:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":6.0049999999999999,"15min":5.109},"min":{"1min":0.311,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.6609999999999996,"5min":156.39400000000001,"15min":156.39400000000001},"last":1.611},{"interface":"front","average":{"1min":4.1980000000000004,"5min":5.9400000000000004,"15min":4.9740000000000002},"min":{"1min":0.38500000000000001,"5min":0.224,"15min":0.224},"max":{"1min":14.169,"5min":156.376,"15min":156.376},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":1.8120000000000001,"15min":1.881},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1270000000000007,"5min":47.463000000000001,"15min":83.012},"last":1.3979999999999999},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.8520000000000001,"15min":1.879},"min":{"1min":0.42099999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.997999999999999,"5min":47.412999999999997,"15min":81.335999999999999},"last":0.39800000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":11.856,"5min":4.0570000000000004,"15min":2.069},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":243.86500000000001,"5min":243.86500000000001,"15min":243.86500000000001},"last":1.4339999999999999},{"interface":"front","average":{"1min":12.890000000000001,"5min":4.3949999999999996,"15min":2.3420000000000001},"min":{"1min":0.44,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":244.02199999999999,"5min":244.02199999999999,"15min":244.02199999999999},"last":1.375}]},{"osd":6,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.4199999999999999,"15min":3.2309999999999999},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.026999999999999,"5min":11.026999999999999,"15min":154.417},"last":0.44500000000000001},{"interface":"front","average":{"1min":3.1880000000000002,"5min":1.8140000000000001,"15min":3.6379999999999999},"min":{"1min":0.46100000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.846,"5min":10.846,"15min":154.04400000000001},"last":1.29}]},{"osd":7,"last update":"Sat Sep 30 09:46:50 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":2.1240000000000001,"15min":2.4089999999999998},"min":{"1min":0.48199999999999998,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":10.928000000000001,"5min":49.816000000000003,"15min":123.294},"last":0.5},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.3889999999999998,"15min":2.7429999999999999},"min":{"1min":0.436,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":10.885,"5min":50.956000000000003,"15min":123.16},"last":1.337}]}]},{"osd":3,"up_from":109,"seq":468151435582,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353648,"kb_used_data":477096,"kb_used_omap":2,"kb_used_meta":876477,"kb_avail":92391504,"statfs":{"total":95995035648,"available":94608900096,"internally_reserved":0,"allocated":488546304,"data_stored":485500252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":897512807},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0759999999999996,"5min":5.2880000000000003,"15min":4.9829999999999997},"min":{"1min":0.22700000000000001,"5min":0.183,"15min":0.183},"max":{"1min":45.957999999999998,"5min":144.72,"15min":205.73099999999999},"last":1.8080000000000001},{"interface":"front","average":{"1min":4.8479999999999999,"5min":5.2919999999999998,"15min":4.9100000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":44.298999999999999,"5min":142.36600000000001,"15min":205.495},"last":5.4370000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":4.0839999999999996,"15min":6.2290000000000001},"min":{"1min":0.27600000000000002,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":14.055,"5min":142.434,"15min":273.99599999999998},"last":5.6799999999999997},{"interface":"front","average":{"1min":3.4329999999999998,"5min":4.3170000000000002,"15min":6.0910000000000002},"min":{"1min":0.38300000000000001,"5min":0.28199999999999997,"15min":0.20699999999999999},"max":{"1min":12.470000000000001,"5min":142.44900000000001,"15min":274.21600000000001},"last":0.5}]},{"osd":2,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":3.8239999999999998,"5min":5.3940000000000001,"15min":5.7999999999999998},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":12.561999999999999,"5min":146.92599999999999,"15min":206.69},"last":0.53600000000000003},{"interface":"front","average":{"1min":3.5979999999999999,"5min":4.9619999999999997,"15min":5.5789999999999997},"min":{"1min":0.38900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.834,"5min":145.75299999999999,"15min":209.22800000000001},"last":0.625}]},{"osd":4,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.3180000000000001,"15min":2.0920000000000001},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.2919999999999998,"5min":81.677000000000007,"15min":81.677000000000007},"last":0.67900000000000005},{"interface":"front","average":{"1min":3.0499999999999998,"5min":2.6360000000000001,"15min":2.04},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":18.943000000000001,"5min":80.623000000000005,"15min":80.623000000000005},"last":5.5380000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:47:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5609999999999999,"5min":1.9390000000000001,"15min":1.6639999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.194000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.83899999999999997},{"interface":"front","average":{"1min":2.5430000000000001,"5min":1.536,"15min":1.1459999999999999},"min":{"1min":0.33700000000000002,"5min":0.311,"15min":0.311},"max":{"1min":8.0399999999999991,"5min":8.0399999999999991,"15min":8.0399999999999991},"last":5.5010000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":1.498,"15min":1.9790000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33200000000000002},"max":{"1min":10.837999999999999,"5min":10.837999999999999,"15min":60.698},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.4369999999999998,"5min":1.659,"15min":1.9650000000000001},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":8.2050000000000001,"5min":10.356999999999999,"15min":12.121},"last":5.7530000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":3.488,"15min":2.7120000000000002},"min":{"1min":0.42499999999999999,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":11.753,"5min":213.548,"15min":213.548},"last":1.401},{"interface":"front","average":{"1min":3.2559999999999998,"5min":3.3290000000000002,"15min":2.605},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.888999999999999,"5min":213.583,"15min":213.583},"last":1.3859999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019345,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":683448,"kb_used_data":186352,"kb_used_omap":0,"kb_used_meta":497088,"kb_avail":93061704,"statfs":{"total":95995035648,"available":95295184896,"internally_reserved":0,"allocated":190824448,"data_stored":189091540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":509018112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":2.7090000000000001,"15min":2.5640000000000001},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":8.7989999999999995,"5min":73.718000000000004,"15min":105.179},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.7589999999999999,"15min":2.75},"min":{"1min":0.32000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.032,"5min":74.001999999999995,"15min":104.70999999999999},"last":0.32100000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.5779999999999998,"15min":3.871},"min":{"1min":0.313,"5min":0.25,"15min":0.25},"max":{"1min":8.8620000000000001,"5min":80.712000000000003,"15min":160.69999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":3.589,"5min":2.8490000000000002,"15min":3.8940000000000001},"min":{"1min":0.438,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":15.345000000000001,"5min":80.810000000000002,"15min":160.50700000000001},"last":0.55500000000000005}]},{"osd":2,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3069999999999999,"5min":4.9989999999999997,"15min":3.492},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":13.26,"5min":142.88999999999999,"15min":142.88999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":3.4159999999999999,"5min":5.069,"15min":3.5249999999999999},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":8.8870000000000005,"5min":142.90700000000001,"15min":142.90700000000001},"last":0.60199999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.0369999999999999,"15min":2.1800000000000002},"min":{"1min":0.35799999999999998,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":9.0600000000000005,"5min":29.41,"15min":52.762},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.6920000000000002,"5min":2.0099999999999998,"15min":2.1890000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.29299999999999998},"max":{"1min":9.0709999999999997,"5min":29.401,"15min":52.790999999999997},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:47:26 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":7.0880000000000001,"15min":9.5180000000000007},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.1470000000000002,"5min":140.11099999999999,"15min":140.11099999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.2889999999999999,"5min":7.0179999999999998,"15min":9.516},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.8789999999999996,"5min":140.065,"15min":140.065},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:47:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":0.84799999999999998,"15min":2.1850000000000001},"min":{"1min":0.34499999999999997,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":9.1470000000000002,"5min":9.1470000000000002,"15min":202.72499999999999},"last":0.621},{"interface":"front","average":{"1min":1.026,"5min":0.78300000000000003,"15min":2.1720000000000002},"min":{"1min":0.311,"5min":0.253,"15min":0.249},"max":{"1min":5.2539999999999996,"5min":7.173,"15min":202.42400000000001},"last":0.39900000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:46:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":2.5099999999999998,"15min":3.2000000000000002},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.771000000000001,"5min":161.72999999999999,"15min":230.90100000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.7270000000000001,"5min":2.4820000000000002,"15min":3.0569999999999999},"min":{"1min":0.41999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.2970000000000006,"5min":162.60900000000001,"15min":229.97},"last":0.63900000000000001}]}]},{"osd":5,"up_from":489,"seq":2100239007787,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":648568,"kb_used_data":499632,"kb_used_omap":6,"kb_used_meta":148921,"kb_avail":93096584,"statfs":{"total":95995035648,"available":95330902016,"internally_reserved":0,"allocated":511623168,"data_stored":509886810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":152495199},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":6.7590000000000003,"5min":17.806999999999999,"15min":23.475000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":89.805999999999997,"5min":281.10599999999999,"15min":281.10599999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":6.4459999999999997,"5min":18.018000000000001,"15min":23.738},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":88.313999999999993,"5min":282.07400000000001,"15min":282.07400000000001},"last":0.51300000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":6.7160000000000002,"5min":11.051,"15min":13.630000000000001},"min":{"1min":0.31900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":76.828999999999994,"5min":273.48200000000003,"15min":273.48200000000003},"last":0.46200000000000002},{"interface":"front","average":{"1min":5.3710000000000004,"5min":10.403,"15min":12.871},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":76.805000000000007,"5min":273.50200000000001,"15min":273.50200000000001},"last":0.498}]},{"osd":2,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":7.258,"5min":4.3230000000000004,"15min":2.0680000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":95.641000000000005,"5min":163.23099999999999,"15min":163.23099999999999},"last":12.920999999999999},{"interface":"front","average":{"1min":6.7030000000000003,"5min":4.6449999999999996,"15min":2.5750000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":95.484999999999999,"5min":162.71100000000001,"15min":162.71100000000001},"last":12.869999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":7.5179999999999998,"5min":10.975,"15min":13.491},"min":{"1min":0.45900000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":79.421999999999997,"5min":274.01799999999997,"15min":274.01799999999997},"last":10.468},{"interface":"front","average":{"1min":7.2709999999999999,"5min":3.1469999999999998,"15min":2.2029999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":79.459000000000003,"5min":79.459000000000003,"15min":79.459000000000003},"last":10.458}]},{"osd":4,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1,"15min":0.96799999999999997},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.46,"5min":5.46,"15min":5.46},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.554,"5min":1.25,"15min":1.1799999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.8239999999999998,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.33400000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.516,"15min":1.6020000000000001},"min":{"1min":0.33000000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7569999999999997,"5min":14.371,"15min":14.371},"last":0.433},{"interface":"front","average":{"1min":1.778,"5min":1.403,"15min":1.3939999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.8979999999999997,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.39100000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.397,"15min":1.3280000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.7560000000000002,"5min":9.7560000000000002,"15min":9.7560000000000002},"last":0.373},{"interface":"front","average":{"1min":1.744,"5min":1.069,"15min":0.96699999999999997},"min":{"1min":0.34899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.8529999999999998,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":0.35099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":222363648,"data_stored":222363648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":438833152,"data_stored":438829080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":233111552,"data_stored":233111552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":469106688,"data_stored":469106688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":180420608,"data_stored":180420608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":502730752,"data_stored":502726680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":170655744,"data_stored":170655744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":525209600,"data_stored":525205528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:48:00.916 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-30T09:48:00.916 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:48:00.916 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-30T09:48:00.916 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:48:00.916 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:48:00.917 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-30T09:48:00.920 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:01.028 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:01.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:00 smithi019 ceph-mon[216815]: pgmap v1396: 119 pgs: 119 active+clean; 641 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:48:01.065 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:00 smithi019 ceph-mon[224043]: pgmap v1396: 119 pgs: 119 active+clean; 641 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:48:01.117 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:01.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:01.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:01.224 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:01.265 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:00 smithi167 ceph-mon[192880]: pgmap v1396: 119 pgs: 119 active+clean; 641 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:48:01.281 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:01.363 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:01.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:01.473 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:48:01.474 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:01.501 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:01.732 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:01.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:01.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:01.837 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:01.898 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:01.991 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:01.992 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:02.087 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:02.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:02.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:02.204 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:02.314 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:02.480 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:02.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:02.587 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:02.680 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:02.976 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:02 smithi019 ceph-mon[216815]: pgmap v1397: 119 pgs: 119 active+clean; 641 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:48:03.231 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:02 smithi019 ceph-mon[224043]: pgmap v1397: 119 pgs: 119 active+clean; 641 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:48:03.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:02 smithi167 ceph-mon[192880]: pgmap v1397: 119 pgs: 119 active+clean; 641 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 0 B/s, 2 objects/s recovering 2023-09-30T09:48:03.615 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:03.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:03.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:03.733 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:03.869 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:03.978 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:03.978 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3598708804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:03.978 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:03.978 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:03.979 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:48:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:48:03] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.43.0" 2023-09-30T09:48:03.979 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:03.979 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3598708804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:03.979 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:03.980 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:03.986 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:03.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:03.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:04.004 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:48:04.092 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:04.204 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3598708804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:04.992 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:05.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:05.102 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:05.179 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:04 smithi167 ceph-mon[192880]: pgmap v1398: 119 pgs: 119 active+clean; 611 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 0 B/s, 1 objects/s recovering 2023-09-30T09:48:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:04 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:48:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:04 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:48:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:04 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:04 smithi167 ceph-mon[192880]: osdmap e526: 8 total, 8 up, 8 in 2023-09-30T09:48:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:04 smithi167 ceph-mon[192880]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:04 smithi167 ceph-mon[192880]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:04 smithi167 ceph-mon[192880]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[224043]: pgmap v1398: 119 pgs: 119 active+clean; 611 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 0 B/s, 1 objects/s recovering 2023-09-30T09:48:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:48:05.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:48:05.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:05.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[224043]: osdmap e526: 8 total, 8 up, 8 in 2023-09-30T09:48:05.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[224043]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[224043]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[224043]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[216815]: pgmap v1398: 119 pgs: 119 active+clean; 611 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 0 B/s, 1 objects/s recovering 2023-09-30T09:48:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:48:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:48:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[216815]: osdmap e526: 8 total, 8 up, 8 in 2023-09-30T09:48:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[216815]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[216815]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:04 smithi019 ceph-mon[216815]: failed to encode map e526 with expected crc 2023-09-30T09:48:05.512 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:05.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:05.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:05.617 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:05.672 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:05.808 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:05.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:05.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:05.834 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:48:05.834 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "" 2023-09-30T09:48:05.834 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:48:05.916 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:05.991 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:06.079 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:06.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:06.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:06.185 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:06.266 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153598 8.13 8:ca06820b:::benchmark_data_smithi019_478679_object61150:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.972895+0000", 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 303.285403395, 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1570760339999999, 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.267 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153598 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972895+0000", 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.268 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972895+0000", 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972897+0000", 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.576e-06 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972901+0000", 2023-09-30T09:48:06.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7929999999999999e-06 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972913+0000", 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2913e-05 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972924+0000", 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0859999999999999e-05 2023-09-30T09:48:06.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975474+0000", 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025492499999999999 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975483+0000", 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8919999999999995e-06 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.120688+0000", 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.145204678 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.120833+0000", 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000144955 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.121225+0000", 2023-09-30T09:48:06.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00039230599999999998 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.124776+0000", 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035502789999999999 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.126836+0000", 2023-09-30T09:48:06.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002060615 2023-09-30T09:48:06.274 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.274 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.274 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.274 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.129791+0000", 2023-09-30T09:48:06.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029550119999999999 2023-09-30T09:48:06.274 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.129855+0000", 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3948000000000005e-05 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.129971+0000", 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115957 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.275 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153604 8.13 8:cb19032d:::benchmark_data_smithi019_478679_object61156:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.972944+0000", 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 303.28535447600001, 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1685259029999999, 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:06.276 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153604 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972944+0000", 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972944+0000", 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972945+0000", 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7899999999999999e-07 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972947+0000", 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.537e-06 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972956+0000", 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6179999999999995e-06 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972962+0000", 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8050000000000003e-06 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975510+0000", 2023-09-30T09:48:06.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025486850000000002 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975517+0000", 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2069999999999998e-06 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.123223+0000", 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1477051220000001 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.123354+0000", 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013182600000000001 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.123631+0000", 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027705399999999999 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.126497+0000", 2023-09-30T09:48:06.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028658239999999999 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.128367+0000", 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018693570000000001 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141250+0000", 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012883321999999999 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.282 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141335+0000", 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5402999999999995e-05 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141470+0000", 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000134664 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.283 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153612 8.13 8:ce4cdea7:::benchmark_data_smithi019_478679_object61164:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.972969+0000", 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 303.28532975399997, 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.168764243, 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153612 2023-09-30T09:48:06.284 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972969+0000", 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972969+0000", 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972969+0000", 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7e-07 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972971+0000", 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.21e-06 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972978+0000", 2023-09-30T09:48:06.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5980000000000002e-06 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972983+0000", 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4360000000000001e-06 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975544+0000", 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002560756 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975552+0000", 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7160000000000003e-06 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.124813+0000", 2023-09-30T09:48:06.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.149260663 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.124944+0000", 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013185399999999999 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.125194+0000", 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024942299999999997 2023-09-30T09:48:06.289 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.126591+0000", 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0013973080000000001 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.129736+0000", 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003144441 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.290 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141566+0000", 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011830827 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141626+0000", 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.003e-05 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.291 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141733+0000", 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000106611 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153642 8.f 8:f10f80bc:::benchmark_data_smithi019_478679_object61194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:04.196235+0000", 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 302.06206339200003, 2023-09-30T09:48:06.292 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.064478372, 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153642 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.293 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196235+0000", 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196235+0000", 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.294 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196237+0000", 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.48e-06 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196242+0000", 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4079999999999997e-06 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196259+0000", 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.736e-05 2023-09-30T09:48:06.295 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196269+0000", 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4630000000000007e-06 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196353+0000", 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4209000000000006e-05 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.296 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196537+0000", 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018348999999999999 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3,4", 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196924+0000", 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00038737299999999999 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:06.297 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.198566+0000", 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0016421809999999999 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.198914+0000", 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00034747599999999999 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.200146+0000", 2023-09-30T09:48:06.298 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012326959999999999 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260516+0000", 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0603693430000001 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260584+0000", 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8002000000000004e-05 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.299 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260714+0000", 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012989100000000001 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153648 8.1f 8:fd9e162a:::benchmark_data_smithi019_478679_object61200:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:04.245434+0000", 2023-09-30T09:48:06.300 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 302.01286473099998, 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0155153640000001, 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153648 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.301 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245434+0000", 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245434+0000", 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.302 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245435+0000", 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.587e-06 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245440+0000", 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6800000000000001e-06 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.303 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245450+0000", 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5419999999999994e-06 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245455+0000", 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2909999999999998e-06 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245502+0000", 2023-09-30T09:48:06.304 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6783999999999998e-05 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245625+0000", 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012290099999999999 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3,4", 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245887+0000", 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000262174 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.305 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.246902+0000", 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001015143 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.247809+0000", 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00090754099999999999 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.248648+0000", 2023-09-30T09:48:06.306 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00083857300000000001 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260755+0000", 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0121074299999999 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260810+0000", 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4120999999999999e-05 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.307 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260949+0000", 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013959700000000001 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153659 8.1f 8:fcf7805c:::benchmark_data_smithi019_478679_object61211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:05.668494+0000", 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 300.589805033, 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.490733206, 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.308 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153659 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668494+0000", 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.309 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668494+0000", 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668495+0000", 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.827e-06 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.310 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668503+0000", 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8760000000000005e-06 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668525+0000", 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2130999999999999e-05 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668537+0000", 2023-09-30T09:48:06.311 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1717e-05 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668603+0000", 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5463000000000007e-05 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668720+0000", 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000117565 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.312 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3,4", 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668956+0000", 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000236214 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.670527+0000", 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015708300000000001 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.313 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.670834+0000", 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00030689600000000002 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.671351+0000", 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00051731800000000001 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.314 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.158999+0000", 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.487647202 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.159082+0000", 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3016999999999995e-05 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.159227+0000", 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014515 2023-09-30T09:48:06.315 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2291 9.f 9:f6cf5fc3:::benchmark_data_smithi019_560486_object2290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.043373+0000", 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 220.21492509699999, 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.042291273, 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:48:06.316 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2291 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043373+0000", 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043373+0000", 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043376+0000", 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2680000000000001e-06 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.318 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043413+0000", 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7765000000000002e-05 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043430+0000", 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6107999999999999e-05 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043443+0000", 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3502e-05 2023-09-30T09:48:06.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045256+0000", 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001812857 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045267+0000", 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1302e-05 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085561+0000", 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0402934189999999 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085665+0000", 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000104052 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.321 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2294 9.f 9:fece5195:::benchmark_data_smithi019_560486_object2293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.044894+0000", 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 220.213404768, 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.041032036, 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2294 2023-09-30T09:48:06.322 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.323 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.323 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.044894+0000", 2023-09-30T09:48:06.323 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.323 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.044894+0000", 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.044895+0000", 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0610000000000001e-06 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.044933+0000", 2023-09-30T09:48:06.324 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8374999999999998e-05 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045261+0000", 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000328241 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045274+0000", 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2401e-05 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045324+0000", 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0433000000000003e-05 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045333+0000", 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0960000000000007e-06 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085856+0000", 2023-09-30T09:48:06.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0405229039999999 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085926+0000", 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9524999999999996e-05 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2296 9.f 9:f85e51b4:::benchmark_data_smithi019_560486_object2295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:48:06.327 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.045296+0000", 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 220.21300262, 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0445070430000001, 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2296 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.328 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045296+0000", 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045296+0000", 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045297+0000", 2023-09-30T09:48:06.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8000000000000004e-07 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045332+0000", 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5549000000000001e-05 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045341+0000", 2023-09-30T09:48:06.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9090000000000006e-06 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045348+0000", 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8619999999999997e-06 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045476+0000", 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000127904 2023-09-30T09:48:06.331 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045487+0000", 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0564000000000001e-05 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085950+0000", 2023-09-30T09:48:06.332 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0404637800000001 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.086071+0000", 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012108 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.086237+0000", 2023-09-30T09:48:06.333 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016526400000000001 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087626+0000", 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001388937 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087772+0000", 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014617800000000001 2023-09-30T09:48:06.334 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.089501+0000", 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00172934 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.089571+0000", 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9715999999999995e-05 2023-09-30T09:48:06.335 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.089803+0000", 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00023207999999999999 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.336 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2303 9.f 9:f68e18ee:::benchmark_data_smithi019_560486_object2302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.045368+0000", 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 220.21293087800001, 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0475383970000001, 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2303 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.337 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.338 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045368+0000", 2023-09-30T09:48:06.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.338 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045368+0000", 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045368+0000", 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9800000000000003e-07 2023-09-30T09:48:06.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045403+0000", 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4819e-05 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045415+0000", 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1474999999999999e-05 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045423+0000", 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0399999999999993e-06 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045530+0000", 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000107414 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:06.341 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045541+0000", 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0934999999999999e-05 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.086823+0000", 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0412817050000001 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.086978+0000", 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015566199999999999 2023-09-30T09:48:06.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087167+0000", 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018856400000000001 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087731+0000", 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00056380500000000001 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.343 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.088006+0000", 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027526999999999999 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.092619+0000", 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0046127850000000003 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.092680+0000", 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0841000000000002e-05 2023-09-30T09:48:06.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.092906+0000", 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022638399999999999 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:39903 9.2 e499/483 9:495196ac:::benchmark_data_smithi019_560486_object39902:head v 499'2452, mlcod=499'2450)", 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:52.939778+0000", 2023-09-30T09:48:06.345 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 193.31852009799999, 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27977527899999999, 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939778+0000", 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939778+0000", 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939779+0000", 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8999999999999995e-07 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939800+0000", 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0455e-05 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939810+0000", 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0236e-05 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939817+0000", 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5590000000000002e-06 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.348 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939893+0000", 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6860000000000006e-05 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939912+0000", 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8742000000000001e-05 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.219523+0000", 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27961119400000001 2023-09-30T09:48:06.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.219554+0000", 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0443e-05 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:39905 9.3 e499/483 9:c278c8fc:::benchmark_data_smithi019_560486_object39904:head v 499'2617, mlcod=499'2614)", 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:52.940206+0000", 2023-09-30T09:48:06.350 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 193.31809226799999, 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27906742200000001, 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.940206+0000", 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.940206+0000", 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.940207+0000", 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3600000000000003e-07 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.940306+0000", 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9411000000000001e-05 2023-09-30T09:48:06.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.216799+0000", 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.276492403 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.216814+0000", 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4929999999999999e-05 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.216879+0000", 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5248999999999998e-05 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.216906+0000", 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7217e-05 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.219237+0000", 2023-09-30T09:48:06.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023303759999999999 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.219274+0000", 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7200000000000003e-05 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:157129 9.a e519/492 9:558aa271:::benchmark_data_smithi019_560486_object60384:head v 519'9922, mlcod=519'9920)", 2023-09-30T09:48:06.355 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:47:19.395799+0000", 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 46.862499659999997, 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15860339700000001, 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.395799+0000", 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.395799+0000", 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.395799+0000", 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7800000000000001e-07 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.395804+0000", 2023-09-30T09:48:06.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7119999999999999e-06 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.395813+0000", 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7900000000000005e-06 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.395818+0000", 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2000000000000002e-06 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.358 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.545899+0000", 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.150080928 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.545919+0000", 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9639999999999998e-05 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.554370+0000", 2023-09-30T09:48:06.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0084511129999999997 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:47:19.554402+0000", 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2435999999999999e-05 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184146 9.a e525/492 9:549708fe:::benchmark_data_smithi019_560486_object87401:head v 525'11572, mlcod=525'11568)", 2023-09-30T09:48:06.360 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.145546+0000", 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 6.1127521580000002, 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21242733599999999, 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145546+0000", 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145546+0000", 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145547+0000", 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4e-07 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145553+0000", 2023-09-30T09:48:06.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5319999999999997e-06 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145561+0000", 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2900000000000002e-06 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145569+0000", 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3110000000000006e-06 2023-09-30T09:48:06.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.148760+0000", 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031910350000000001 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.148782+0000", 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1480000000000001e-05 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.154088+0000", 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0053063429999999998 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357974+0000", 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20388550499999999 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184156 9.0 e525/483 9:0180b523:::benchmark_data_smithi019_560486_object87411:head v 525'11358, mlcod=525'11356)", 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.149851+0000", 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 6.108447215, 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20812422999999999, 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149851+0000", 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149851+0000", 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149852+0000", 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9000000000000005e-07 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149861+0000", 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2600000000000005e-06 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149873+0000", 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2109e-05 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149881+0000", 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2230000000000007e-06 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149954+0000", 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2893999999999997e-05 2023-09-30T09:48:06.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149973+0000", 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9381999999999998e-05 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.154102+0000", 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041292680000000002 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357976+0000", 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.203873104 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184153 9.2 e525/483 9:46c9a4a1:::benchmark_data_smithi019_560486_object87408:head v 525'11373, mlcod=525'11371)", 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.153325+0000", 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 6.1049739409999999, 2023-09-30T09:48:06.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.220288503, 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153325+0000", 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153325+0000", 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153326+0000", 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.116e-06 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153333+0000", 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0319999999999998e-06 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153344+0000", 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1558e-05 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153352+0000", 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6899999999999992e-06 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153409+0000", 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6789999999999997e-05 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153428+0000", 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9531e-05 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.373581+0000", 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22015299399999999 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.373613+0000", 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1792000000000003e-05 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184159 9.d e525/495 9:b38dcb7a:::benchmark_data_smithi019_560486_object87414:head v 525'11418, mlcod=525'11416)", 2023-09-30T09:48:06.376 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.153388+0000", 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 6.1049105160000003, 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21351836799999999, 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153388+0000", 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153388+0000", 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153389+0000", 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3399999999999999e-07 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153392+0000", 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7069999999999998e-06 2023-09-30T09:48:06.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153402+0000", 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4210000000000001e-06 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153407+0000", 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.95e-06 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357959+0000", 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20455210500000001 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357986+0000", 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7469000000000001e-05 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.366863+0000", 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088767259999999997 2023-09-30T09:48:06.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.366906+0000", 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3356000000000003e-05 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184160 9.a e525/492 9:532ad83b:::benchmark_data_smithi019_560486_object87415:head v 525'11573, mlcod=525'11569)", 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.153436+0000", 2023-09-30T09:48:06.381 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 6.1048621619999999, 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22014215200000001, 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153436+0000", 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.382 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153436+0000", 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153437+0000", 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4700000000000002e-07 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153440+0000", 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4309999999999998e-06 2023-09-30T09:48:06.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357934+0000", 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20449425500000001 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357950+0000", 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.579e-05 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.358005+0000", 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4580000000000003e-05 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.358025+0000", 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.037e-05 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.373535+0000", 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015509441000000001 2023-09-30T09:48:06.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.373579+0000", 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3937999999999998e-05 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184965 9.2 e525/483 9:4518caa7:::benchmark_data_smithi019_560486_object88220:head v 525'11417, mlcod=525'11414)", 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:01.369119+0000", 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 4.8891797759999998, 2023-09-30T09:48:06.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.17499641499999999, 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.369119+0000", 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.369119+0000", 2023-09-30T09:48:06.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.369120+0000", 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1699999999999997e-07 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.369126+0000", 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.054e-06 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.369136+0000", 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0431999999999999e-05 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.369143+0000", 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0979999999999996e-06 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.372053+0000", 2023-09-30T09:48:06.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029099379999999999 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.372074+0000", 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0771000000000002e-05 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.381202+0000", 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091284020000000007 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.544115+0000", 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.162913003 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184967 9.2 e525/483 9:4e99d555:::benchmark_data_smithi019_560486_object88222:head v 525'11418, mlcod=525'11414)", 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:01.374006+0000", 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 4.8842921590000001, 2023-09-30T09:48:06.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.174664821, 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.374006+0000", 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.374006+0000", 2023-09-30T09:48:06.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.374007+0000", 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1699999999999997e-07 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.374014+0000", 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0230000000000004e-06 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.374023+0000", 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9209999999999998e-06 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.374029+0000", 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4589999999999997e-06 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:06.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.377062+0000", 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00303367 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.377080+0000", 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7878000000000001e-05 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.548622+0000", 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.17154174599999999 2023-09-30T09:48:06.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:01.548671+0000", 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9206999999999998e-05 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:06.396 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:48:06.397 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:48:06.397 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:06.398 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:48:06.398 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:48:06.398 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:48:06.398 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:48:06.451 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:48:06.493 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:48:06.539 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:48:06.553 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:48:06.553 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:48:06.553 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:48:06.553 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:48:06.554 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:48:06.554 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:48:06.554 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:48:06.554 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:48:06.555 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:06.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:06.665 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:06.728 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:06.869 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:06.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:06.976 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:07.008 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:06 smithi167 ceph-mon[192880]: pgmap v1400: 119 pgs: 119 active+clean; 412 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-30T09:48:07.008 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2345663070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:07.030 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:07.123 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:07.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:07.236 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:07.307 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:06 smithi019 ceph-mon[216815]: pgmap v1400: 119 pgs: 119 active+clean; 412 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-30T09:48:07.307 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2345663070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:07.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:06 smithi019 ceph-mon[224043]: pgmap v1400: 119 pgs: 119 active+clean; 412 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-30T09:48:07.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2345663070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:07.310 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:07.455 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:48:07.500 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:07.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:07.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:07.604 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:07.629 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:07.673 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:07.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:07.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:07.740 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:48:07.790 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:07.859 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:07.946 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:07.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:07.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:08.016 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:48:08.056 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:08.110 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:08.196 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:08.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:08.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:08.301 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:08.381 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:08.481 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:08.545 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:08.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:08.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:08.652 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:08.724 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:08.848 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:08.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:08.969 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:09.015 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:09 smithi167 ceph-mon[192880]: pgmap v1401: 119 pgs: 119 active+clean; 412 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-09-30T09:48:09.036 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:09.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:09 smithi019 ceph-mon[216815]: pgmap v1401: 119 pgs: 119 active+clean; 412 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-09-30T09:48:09.292 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:09 smithi019 ceph-mon[224043]: pgmap v1401: 119 pgs: 119 active+clean; 412 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-09-30T09:48:09.340 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:09.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:09.448 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:09.504 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:09.768 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:09.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:09.877 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:09.941 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:10.053 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:10.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:10.168 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:48:10.169 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:10.220 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:10.321 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:10.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:10.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:10.425 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:10.435 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:48:10.435 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:48:10.512 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:10.741 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:10.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:10.847 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:10.899 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:10.906 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:10.953 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:10.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:10.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:11.057 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:11.151 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:11.276 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:11.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:11.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:11.293 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:11 smithi019 ceph-mon[216815]: pgmap v1402: 119 pgs: 119 active+clean; 188 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-30T09:48:11.293 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:11 smithi019 ceph-mon[224043]: pgmap v1402: 119 pgs: 119 active+clean; 188 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-30T09:48:11.391 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:11.395 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:11 smithi167 ceph-mon[192880]: pgmap v1402: 119 pgs: 119 active+clean; 188 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-30T09:48:11.424 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:48:11.475 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:11.569 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:11.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:11.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:11.681 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:11.749 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:11.812 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:11.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:11.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:11.887 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:48:11.888 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:48:11.888 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-30T09:48:11.889 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:48:11.915 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:11.943 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:11.954 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:48:11.987 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:11.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:11.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:12.036 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:48:12.093 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:12.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/527721228' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:12.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:12 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:12.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/527721228' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:12.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:12 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:12.185 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:12.206 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/527721228' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:12.207 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:12 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:12.646 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:12.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:12.762 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:12.791 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:13.184 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:13.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:13.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:13.296 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:13.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[216815]: pgmap v1403: 119 pgs: 119 active+clean; 188 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-30T09:48:13.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:13.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:48:13.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[216815]: failed to encode map e527 with expected crc 2023-09-30T09:48:13.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[216815]: failed to encode map e527 with expected crc 2023-09-30T09:48:13.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[216815]: osdmap e527: 8 total, 8 up, 8 in 2023-09-30T09:48:13.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[224043]: pgmap v1403: 119 pgs: 119 active+clean; 188 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-30T09:48:13.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:13.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:48:13.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[224043]: failed to encode map e527 with expected crc 2023-09-30T09:48:13.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[224043]: failed to encode map e527 with expected crc 2023-09-30T09:48:13.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:13 smithi019 ceph-mon[224043]: osdmap e527: 8 total, 8 up, 8 in 2023-09-30T09:48:13.354 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:13 smithi167 ceph-mon[192880]: pgmap v1403: 119 pgs: 119 active+clean; 188 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-30T09:48:13.354 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:13 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:13.354 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:13 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:48:13.355 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:13 smithi167 ceph-mon[192880]: failed to encode map e527 with expected crc 2023-09-30T09:48:13.355 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:13 smithi167 ceph-mon[192880]: failed to encode map e527 with expected crc 2023-09-30T09:48:13.355 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:13 smithi167 ceph-mon[192880]: osdmap e527: 8 total, 8 up, 8 in 2023-09-30T09:48:13.358 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:13.459 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:13.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:13.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:13.504 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Removed 96742 objects 2023-09-30T09:48:13.504 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Clean up completed and total clean up time :139.564 2023-09-30T09:48:13.574 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:13.602 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:13.651 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:13.734 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:48:13.746 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:13.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:13.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:13.753 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:48:13.772 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:48:13.792 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:48:13.812 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:48:13.831 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:48:13.851 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:13.853 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:48:13.877 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:48:13.878 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:48:13.878 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:48:13.878 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:48:13.878 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:48:13.879 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:48:13.879 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:48:13.879 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:48:13.879 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:48:13.879 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:13.924 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:13.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:13.972 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:48:13.972 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:48:13.972 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:48:13.973 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:48:13.973 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:48:14.028 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:14.058 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:14 smithi019 ceph-mon[216815]: failed to encode map e527 with expected crc 2023-09-30T09:48:14.245 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/880198556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:14.246 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:48:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:48:13] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.43.0" 2023-09-30T09:48:14.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:14 smithi019 ceph-mon[224043]: failed to encode map e527 with expected crc 2023-09-30T09:48:14.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/880198556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:14.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:14.247 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:14.247 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:14.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:14.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:14 smithi167 ceph-mon[192880]: failed to encode map e527 with expected crc 2023-09-30T09:48:14.286 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/880198556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:14.349 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:14.391 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:14.741 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:48:14.840 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:14.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:14.943 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:14.971 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:15.017 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:15.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:15.093 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:48:15.124 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:15.152 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:15.284 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:15 smithi167 ceph-mon[192880]: pgmap v1405: 119 pgs: 119 active+clean; 162 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-30T09:48:15.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:15 smithi019 ceph-mon[216815]: pgmap v1405: 119 pgs: 119 active+clean; 162 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-30T09:48:15.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:15 smithi019 ceph-mon[224043]: pgmap v1405: 119 pgs: 119 active+clean; 162 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-30T09:48:15.374 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:48:15.483 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:15.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:15.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:15.587 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:15.614 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:15.709 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:15.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:15.813 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:15.841 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:15.883 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:15.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:15.987 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:16.004 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:16.016 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:16.173 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:16.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:16.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:16.278 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:16.304 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:48:16.308 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:48:16.313 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:16.463 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:16.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:16.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:16.568 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:16.598 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:16.643 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:16.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:16.729 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:48:16.729 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:48:16.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:48:16.729 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:48:16.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.729 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2291 9.f 9.c3faf36f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:16.729 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 231.67500414400001, 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99786489, 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2291 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051293+0000", 2023-09-30T09:48:16.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051362+0000", 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9276999999999994e-05 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052566+0000", 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012036099999999999 2023-09-30T09:48:16.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052575+0000", 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3910000000000004e-06 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061665+0000", 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0090901720000000005 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061671+0000", 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3539999999999996e-06 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049150+0000", 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98747858099999997 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049159+0000", 2023-09-30T09:48:16.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9099999999999994e-06 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2293 9.a 9.c3c9b4aa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 231.67370402500001, 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98499129799999996, 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:48:16.735 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2293 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052593+0000", 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052645+0000", 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1387000000000001e-05 2023-09-30T09:48:16.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052655+0000", 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8919999999999995e-06 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052660+0000", 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4809999999999997e-06 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.059982+0000", 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0073216380000000001 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.059988+0000", 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7399999999999998e-06 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037576+0000", 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97758752599999998 2023-09-30T09:48:16.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037585+0000", 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4930000000000004e-06 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2294 9.f 9.a98a737f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:48:16.740 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 231.673595304, 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99648747699999995, 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2294 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:48:16.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052702+0000", 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052798+0000", 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5100999999999996e-05 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052809+0000", 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1833e-05 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052815+0000", 2023-09-30T09:48:16.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.356e-06 2023-09-30T09:48:16.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.744 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.744 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061690+0000", 2023-09-30T09:48:16.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088752989999999997 2023-09-30T09:48:16.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061694+0000", 2023-09-30T09:48:16.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0940000000000001e-06 2023-09-30T09:48:16.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049184+0000", 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98748982500000004 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049190+0000", 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4779999999999996e-06 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.746 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2296 9.f 9.2d8a7a1f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 231.673474658, 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99639533999999996, 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.747 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2296 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052823+0000", 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052859+0000", 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6331999999999997e-05 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052867+0000", 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8970000000000008e-06 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052873+0000", 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3149999999999999e-06 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061712+0000", 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.008839431 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061716+0000", 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7979999999999999e-06 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049212+0000", 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.987496598 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049219+0000", 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4300000000000003e-06 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2298 9.7 9.88338c87 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:48:16.752 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 231.67342048, 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99525176199999998, 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2298 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.753 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.754 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052877+0000", 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052906+0000", 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8574000000000001e-05 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052912+0000", 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5150000000000003e-06 2023-09-30T09:48:16.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.756 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.756 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052917+0000", 2023-09-30T09:48:16.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1230000000000002e-06 2023-09-30T09:48:16.756 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.756 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.756 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053439+0000", 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052202900000000003 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053447+0000", 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1620000000000008e-06 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.048119+0000", 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99467239399999996 2023-09-30T09:48:16.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.048130+0000", 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0394000000000001e-05 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.758 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2299 9.a 9.a4908ffa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:16.759 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:48:16.759 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 231.67337713399999, 2023-09-30T09:48:16.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.984693029, 2023-09-30T09:48:16.759 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.760 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:48:16.760 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.760 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.760 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.760 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2299 2023-09-30T09:48:16.760 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.760 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052947+0000", 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6486999999999999e-05 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052954+0000", 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6640000000000001e-06 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052958+0000", 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.617e-06 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.060007+0000", 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0070489289999999998 2023-09-30T09:48:16.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.060012+0000", 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7790000000000003e-06 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037607+0000", 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.977594764 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037614+0000", 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2309999999999999e-06 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2303 9.f 9.7718716f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 231.662843713, 2023-09-30T09:48:16.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98609501700000002, 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2303 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063453+0000", 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063499+0000", 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5393999999999999e-05 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063510+0000", 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1914e-05 2023-09-30T09:48:16.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063518+0000", 2023-09-30T09:48:16.769 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9719999999999992e-06 2023-09-30T09:48:16.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064775+0000", 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001256987 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064783+0000", 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.267e-06 2023-09-30T09:48:16.770 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049541+0000", 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98475831199999997 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049550+0000", 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5299999999999996e-06 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.771 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5456 9.f 9:f741c261:::benchmark_data_smithi019_560486_object5455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.04831428400001, 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99827741800000003, 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.772 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.773 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5456 2023-09-30T09:48:16.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.773 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:48:16.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677983+0000", 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678041+0000", 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7439999999999999e-05 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678050+0000", 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2380000000000003e-06 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678055+0000", 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2480000000000004e-06 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678212+0000", 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015715 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678221+0000", 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2140000000000002e-06 2023-09-30T09:48:16.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666463+0000", 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98824183200000004 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666571+0000", 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000107722 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666679+0000", 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108446 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669881+0000", 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032016509999999998 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676162+0000", 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062807169999999999 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676201+0000", 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9162000000000002e-05 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676235+0000", 2023-09-30T09:48:16.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4459000000000003e-05 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676261+0000", 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5897999999999998e-05 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5488 9.f 9:f5c1ce8d:::benchmark_data_smithi019_560486_object5487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.038347472, 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98845970699999997, 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5488 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.781 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:48:16.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687950+0000", 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687988+0000", 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8231e-05 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688001+0000", 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2577e-05 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688007+0000", 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3740000000000003e-06 2023-09-30T09:48:16.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688066+0000", 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8805999999999999e-05 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688075+0000", 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7499999999999998e-06 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667057+0000", 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97898119299999997 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667125+0000", 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7909000000000003e-05 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667219+0000", 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4759000000000005e-05 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669890+0000", 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026711130000000001 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676296+0000", 2023-09-30T09:48:16.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064055370000000002 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676351+0000", 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5433999999999998e-05 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676386+0000", 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4400000000000003e-05 2023-09-30T09:48:16.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676410+0000", 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4664e-05 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5500 9.f 9:fb77b28e:::benchmark_data_smithi019_560486_object5499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:48:16.789 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.035137384, 2023-09-30T09:48:16.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98537109199999995, 2023-09-30T09:48:16.790 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.790 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.790 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.790 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.790 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.790 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5500 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691160+0000", 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691215+0000", 2023-09-30T09:48:16.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5435e-05 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691223+0000", 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9340000000000006e-06 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691227+0000", 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7459999999999998e-06 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691295+0000", 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8145999999999994e-05 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691307+0000", 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1516e-05 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667479+0000", 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97617223600000003 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667613+0000", 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013427400000000001 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667811+0000", 2023-09-30T09:48:16.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019798 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669899+0000", 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002087977 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676442+0000", 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0065432470000000003 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676474+0000", 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1275000000000001e-05 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676512+0000", 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7997000000000002e-05 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676532+0000", 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0285999999999999e-05 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5531 9.f 9:fcece89e:::benchmark_data_smithi019_560486_object5530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:48:16.798 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.025646575, 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97602003800000003, 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5531 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700650+0000", 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700746+0000", 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5857000000000001e-05 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700758+0000", 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1983e-05 2023-09-30T09:48:16.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700765+0000", 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8279999999999997e-06 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700839+0000", 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4226000000000004e-05 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700857+0000", 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7193e-05 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668290+0000", 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96743334299999995 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668421+0000", 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013117099999999999 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668566+0000", 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014448900000000001 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671187+0000", 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002621529 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676578+0000", 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005391051 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676610+0000", 2023-09-30T09:48:16.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2039999999999998e-05 2023-09-30T09:48:16.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676647+0000", 2023-09-30T09:48:16.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7292000000000001e-05 2023-09-30T09:48:16.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676672+0000", 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4232000000000001e-05 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5545 9.f 9:fa26aa6f:::benchmark_data_smithi019_560486_object5544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:48:16.807 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.02069973100001, 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97120855500000003, 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5545 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705597+0000", 2023-09-30T09:48:16.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705693+0000", 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5401000000000004e-05 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705706+0000", 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2665e-05 2023-09-30T09:48:16.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705713+0000", 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9940000000000003e-06 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705830+0000", 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011760900000000001 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705836+0000", 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7610000000000004e-06 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668931+0000", 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96309502499999999 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669015+0000", 2023-09-30T09:48:16.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3947000000000001e-05 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669120+0000", 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010467900000000001 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671197+0000", 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020777399999999998 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676699+0000", 2023-09-30T09:48:16.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055018530000000001 2023-09-30T09:48:16.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676739+0000", 2023-09-30T09:48:16.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9949e-05 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676786+0000", 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6755999999999998e-05 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676807+0000", 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1189e-05 2023-09-30T09:48:16.815 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.816 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.816 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.816 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.816 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5551 9.f 9:f9ae74d4:::benchmark_data_smithi019_560486_object5550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.816 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:48:16.816 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.019784671, 2023-09-30T09:48:16.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97040703399999995, 2023-09-30T09:48:16.817 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.817 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.817 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.817 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.817 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5551 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.819 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:48:16.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.819 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.819 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706513+0000", 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706568+0000", 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5432000000000001e-05 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706580+0000", 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1494999999999999e-05 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706586+0000", 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7530000000000004e-06 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706645+0000", 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8591000000000002e-05 2023-09-30T09:48:16.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706653+0000", 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5250000000000005e-06 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669422+0000", 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96276874499999998 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669547+0000", 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012500799999999999 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669633+0000", 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5400000000000002e-05 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672244+0000", 2023-09-30T09:48:16.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026115729999999998 2023-09-30T09:48:16.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.824 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.824 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676836+0000", 2023-09-30T09:48:16.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045915579999999999 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676856+0000", 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.02e-05 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676898+0000", 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1789000000000003e-05 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676921+0000", 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2900000000000001e-05 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5571 9.f 9:f04c103d:::benchmark_data_smithi019_560486_object5570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.01362949899999, 2023-09-30T09:48:16.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96437061999999996, 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5571 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:48:16.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712668+0000", 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713054+0000", 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00038651499999999998 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713063+0000", 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6440000000000006e-06 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713068+0000", 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2120000000000002e-06 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713200+0000", 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000131977 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713210+0000", 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4250000000000004e-06 2023-09-30T09:48:16.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669945+0000", 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95673509599999995 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670055+0000", 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000110145 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670148+0000", 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3212000000000001e-05 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672252+0000", 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021043170000000001 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676960+0000", 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047072759999999998 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676992+0000", 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2808000000000002e-05 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677019+0000", 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6267000000000001e-05 2023-09-30T09:48:16.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677039+0000", 2023-09-30T09:48:16.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0647000000000001e-05 2023-09-30T09:48:16.834 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.834 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5581 9.f 9:fa532eaf:::benchmark_data_smithi019_560486_object5580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.01056050899999, 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96139814300000004, 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.835 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5581 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715737+0000", 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715765+0000", 2023-09-30T09:48:16.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8323e-05 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715780+0000", 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5475999999999999e-05 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715787+0000", 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9149999999999999e-06 2023-09-30T09:48:16.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715853+0000", 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5413000000000005e-05 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715862+0000", 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.594e-06 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670527+0000", 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95466473799999996 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670657+0000", 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012968699999999999 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670839+0000", 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000182461 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673403+0000", 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002564036 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677071+0000", 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036674810000000002 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677096+0000", 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5429999999999999e-05 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677119+0000", 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3090000000000001e-05 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677136+0000", 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6702e-05 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5591 9.f 9:fb1424fd:::benchmark_data_smithi019_560486_object5590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.007328462, 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95826697900000002, 2023-09-30T09:48:16.843 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5591 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:48:16.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:48:16.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718969+0000", 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719014+0000", 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4433e-05 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719031+0000", 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7078000000000001e-05 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719036+0000", 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2240000000000003e-06 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719098+0000", 2023-09-30T09:48:16.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2292999999999995e-05 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719107+0000", 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3319999999999992e-06 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671243+0000", 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95213663199999998 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671330+0000", 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6706999999999995e-05 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671432+0000", 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010215 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673409+0000", 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019769060000000001 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677162+0000", 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037529379999999999 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677190+0000", 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7702e-05 2023-09-30T09:48:16.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677216+0000", 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6894e-05 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677237+0000", 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0253000000000001e-05 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.851 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5593 9.f 9:f74f8c23:::benchmark_data_smithi019_560486_object5592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.006959967, 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95807467700000004, 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.852 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5593 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:48:16.853 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719338+0000", 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719402+0000", 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4404000000000002e-05 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.854 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719414+0000", 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1895999999999999e-05 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719420+0000", 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.365e-06 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.855 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719641+0000", 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000221122 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719653+0000", 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1884e-05 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671683+0000", 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95203014699999999 2023-09-30T09:48:16.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671776+0000", 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2331999999999996e-05 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671902+0000", 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000126666 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.674505+0000", 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026023959999999999 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677294+0000", 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027896100000000001 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677329+0000", 2023-09-30T09:48:16.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4668000000000001e-05 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677380+0000", 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0841999999999999e-05 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677413+0000", 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3028999999999998e-05 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.859 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5604 9.f 9:f2112614:::benchmark_data_smithi019_560486_object5603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 227.00206723599999, 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95332888500000001, 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.860 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5604 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724231+0000", 2023-09-30T09:48:16.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724230+0000", 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724380+0000", 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015008099999999999 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724392+0000", 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1518999999999999e-05 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724398+0000", 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5540000000000003e-06 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724454+0000", 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5358000000000001e-05 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.724465+0000", 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1226000000000001e-05 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672293+0000", 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94782834900000001 2023-09-30T09:48:16.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672431+0000", 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013751799999999999 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672607+0000", 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000176517 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.674510+0000", 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019026710000000001 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677453+0000", 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029434510000000001 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677481+0000", 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7932e-05 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677526+0000", 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4234999999999999e-05 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677560+0000", 2023-09-30T09:48:16.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4345999999999999e-05 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5614 9.f 9:fca5ec44:::benchmark_data_smithi019_560486_object5613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 226.99388394100001, 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.945425556, 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.868 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5614 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732414+0000", 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732413+0000", 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.870 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732525+0000", 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112094 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732544+0000", 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9120999999999999e-05 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732551+0000", 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9e-06 2023-09-30T09:48:16.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732616+0000", 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4272999999999999e-05 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.732625+0000", 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.166e-06 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672981+0000", 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94035580399999996 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673059+0000", 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8621999999999994e-05 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673154+0000", 2023-09-30T09:48:16.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4519999999999996e-05 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.675823+0000", 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002669205 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677652+0000", 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001828795 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677707+0000", 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5056000000000003e-05 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677772+0000", 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4801999999999994e-05 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677840+0000", 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8238999999999995e-05 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5618 9.f 9:fc43f802:::benchmark_data_smithi019_560486_object5617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 226.99308844399999, 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94477232899999997, 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:48:16.876 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5618 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733210+0000", 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733209+0000", 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:48:16.878 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733267+0000", 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7673999999999997e-05 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733281+0000", 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4701999999999999e-05 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733287+0000", 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9839999999999998e-06 2023-09-30T09:48:16.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733333+0000", 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5609000000000003e-05 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.733339+0000", 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7960000000000001e-06 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673445+0000", 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.94010585599999996 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673574+0000", 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129469 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673744+0000", 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017018 2023-09-30T09:48:16.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.675832+0000", 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002087696 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677879+0000", 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020473380000000001 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677911+0000", 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1739000000000002e-05 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677950+0000", 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8690999999999999e-05 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:48:16.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677982+0000", 2023-09-30T09:48:16.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2471999999999997e-05 2023-09-30T09:48:16.884 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.884 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.884 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.884 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:48:16.884 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:48:16.884 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:48:16.884 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:16.913 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:16.957 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:16.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:17.060 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:17.086 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:17.230 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:48:17.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:17 smithi019 ceph-mon[216815]: pgmap v1406: 119 pgs: 119 active+clean; 4.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-30T09:48:17.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:17 smithi019 ceph-mon[216815]: from='client.36516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:48:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:17 smithi019 ceph-mon[224043]: pgmap v1406: 119 pgs: 119 active+clean; 4.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-30T09:48:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:17 smithi019 ceph-mon[224043]: from='client.36516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:48:17.344 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:17.346 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:17 smithi167 ceph-mon[192880]: pgmap v1406: 119 pgs: 119 active+clean; 4.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-30T09:48:17.346 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:17 smithi167 ceph-mon[192880]: from='client.36516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:48:17.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:17.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:17.408 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:48:17.448 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:17.474 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:17.516 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:17.520 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1406,"stamp":"2023-09-30T09:48:15.898839+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790183,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5943,"num_read_kb":12147,"num_write":194393,"num_write_kb":6204214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6394,"num_bytes_recovered":368334450,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120730,"ondisk_log_size":120730,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4907228,"kb_used_data":171216,"kb_used_omap":30,"kb_used_meta":4735841,"kb_avail":745053988,"statfs":{"total":767960285184,"available":762935283712,"internally_reserved":0,"allocated":175325184,"data_stored":156166792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4849501538},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-426704920,"num_objects":-6512,"num_object_clones":0,"num_object_copies":-19536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6512,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007081"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":950,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990991+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:48:03.990991+0000","last_peered":"2023-09-30T09:48:03.990991+0000","last_clean":"2023-09-30T09:48:03.990991+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:48:03.990991+0000","last_undegraded":"2023-09-30T09:48:03.990991+0000","last_fullsized":"2023-09-30T09:48:03.990991+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":545,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990504+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:48:03.990504+0000","last_peered":"2023-09-30T09:48:03.990504+0000","last_clean":"2023-09-30T09:48:03.990504+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:48:03.990504+0000","last_undegraded":"2023-09-30T09:48:03.990504+0000","last_fullsized":"2023-09-30T09:48:03.990504+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":442,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.000918+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:48:04.000918+0000","last_peered":"2023-09-30T09:48:04.000918+0000","last_clean":"2023-09-30T09:48:04.000918+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:48:04.000918+0000","last_undegraded":"2023-09-30T09:48:04.000918+0000","last_fullsized":"2023-09-30T09:48:04.000918+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":952,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.995563+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:48:03.995563+0000","last_peered":"2023-09-30T09:48:03.995563+0000","last_clean":"2023-09-30T09:48:03.995563+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:48:03.995563+0000","last_undegraded":"2023-09-30T09:48:03.995563+0000","last_fullsized":"2023-09-30T09:48:03.995563+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":92,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001535+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:48:04.001535+0000","last_peered":"2023-09-30T09:48:04.001535+0000","last_clean":"2023-09-30T09:48:04.001535+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:48:04.001535+0000","last_undegraded":"2023-09-30T09:48:04.001535+0000","last_fullsized":"2023-09-30T09:48:04.001535+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":133,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019915+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:48:04.019915+0000","last_peered":"2023-09-30T09:48:04.019915+0000","last_clean":"2023-09-30T09:48:04.019915+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:48:04.019915+0000","last_undegraded":"2023-09-30T09:48:04.019915+0000","last_fullsized":"2023-09-30T09:48:04.019915+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":760,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990586+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:48:03.990586+0000","last_peered":"2023-09-30T09:48:03.990586+0000","last_clean":"2023-09-30T09:48:03.990586+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:48:03.990586+0000","last_undegraded":"2023-09-30T09:48:03.990586+0000","last_fullsized":"2023-09-30T09:48:03.990586+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":33,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.018933+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:48:04.018933+0000","last_peered":"2023-09-30T09:48:04.018933+0000","last_clean":"2023-09-30T09:48:04.018933+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:48:04.018933+0000","last_undegraded":"2023-09-30T09:48:04.018933+0000","last_fullsized":"2023-09-30T09:48:04.018933+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006800+0000","last_change":"2023-09-30T09:44:27.086035+0000","last_active":"2023-09-30T09:48:04.006800+0000","last_peered":"2023-09-30T09:48:04.006800+0000","last_clean":"2023-09-30T09:48:04.006800+0000","last_became_active":"2023-09-30T09:44:27.084499+0000","last_became_peered":"2023-09-30T09:44:27.084499+0000","last_unstale":"2023-09-30T09:48:04.006800+0000","last_undegraded":"2023-09-30T09:48:04.006800+0000","last_fullsized":"2023-09-30T09:48:04.006800+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:04:22.977794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":928,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001796+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:48:04.001796+0000","last_peered":"2023-09-30T09:48:04.001796+0000","last_clean":"2023-09-30T09:48:04.001796+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:48:04.001796+0000","last_undegraded":"2023-09-30T09:48:04.001796+0000","last_fullsized":"2023-09-30T09:48:04.001796+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":148,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.020439+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:48:04.020439+0000","last_peered":"2023-09-30T09:48:04.020439+0000","last_clean":"2023-09-30T09:48:04.020439+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:48:04.020439+0000","last_undegraded":"2023-09-30T09:48:04.020439+0000","last_fullsized":"2023-09-30T09:48:04.020439+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":108,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.996127+0000","last_change":"2023-09-30T09:44:27.071306+0000","last_active":"2023-09-30T09:48:03.996127+0000","last_peered":"2023-09-30T09:48:03.996127+0000","last_clean":"2023-09-30T09:48:03.996127+0000","last_became_active":"2023-09-30T09:44:27.070948+0000","last_became_peered":"2023-09-30T09:44:27.070948+0000","last_unstale":"2023-09-30T09:48:03.996127+0000","last_undegraded":"2023-09-30T09:48:03.996127+0000","last_fullsized":"2023-09-30T09:48:03.996127+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:07:48.649442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":717,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.014272+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:48:04.014272+0000","last_peered":"2023-09-30T09:48:04.014272+0000","last_clean":"2023-09-30T09:48:04.014272+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:48:04.014272+0000","last_undegraded":"2023-09-30T09:48:04.014272+0000","last_fullsized":"2023-09-30T09:48:04.014272+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":347,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991434+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:48:03.991434+0000","last_peered":"2023-09-30T09:48:03.991434+0000","last_clean":"2023-09-30T09:48:03.991434+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:48:03.991434+0000","last_undegraded":"2023-09-30T09:48:03.991434+0000","last_fullsized":"2023-09-30T09:48:03.991434+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":483,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001260+0000","last_change":"2023-09-30T09:43:45.120358+0000","last_active":"2023-09-30T09:48:04.001260+0000","last_peered":"2023-09-30T09:48:04.001260+0000","last_clean":"2023-09-30T09:48:04.001260+0000","last_became_active":"2023-09-30T09:43:45.120033+0000","last_became_peered":"2023-09-30T09:43:45.120033+0000","last_unstale":"2023-09-30T09:48:04.001260+0000","last_undegraded":"2023-09-30T09:48:04.001260+0000","last_fullsized":"2023-09-30T09:48:04.001260+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:06:02.027700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":656,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991175+0000","last_change":"2023-09-30T09:44:27.079071+0000","last_active":"2023-09-30T09:48:03.991175+0000","last_peered":"2023-09-30T09:48:03.991175+0000","last_clean":"2023-09-30T09:48:03.991175+0000","last_became_active":"2023-09-30T09:44:27.078369+0000","last_became_peered":"2023-09-30T09:44:27.078369+0000","last_unstale":"2023-09-30T09:48:03.991175+0000","last_undegraded":"2023-09-30T09:48:03.991175+0000","last_fullsized":"2023-09-30T09:48:03.991175+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:37.642304+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":241,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006412+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:48:04.006412+0000","last_peered":"2023-09-30T09:48:04.006412+0000","last_clean":"2023-09-30T09:48:04.006412+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:48:04.006412+0000","last_undegraded":"2023-09-30T09:48:04.006412+0000","last_fullsized":"2023-09-30T09:48:04.006412+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006157+0000","last_change":"2023-09-30T09:43:02.983243+0000","last_active":"2023-09-30T09:48:04.006157+0000","last_peered":"2023-09-30T09:48:04.006157+0000","last_clean":"2023-09-30T09:48:04.006157+0000","last_became_active":"2023-09-30T09:43:02.982865+0000","last_became_peered":"2023-09-30T09:43:02.982865+0000","last_unstale":"2023-09-30T09:48:04.006157+0000","last_undegraded":"2023-09-30T09:48:04.006157+0000","last_fullsized":"2023-09-30T09:48:04.006157+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:01.287893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":583,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019994+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:48:04.019994+0000","last_peered":"2023-09-30T09:48:04.019994+0000","last_clean":"2023-09-30T09:48:04.019994+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:48:04.019994+0000","last_undegraded":"2023-09-30T09:48:04.019994+0000","last_fullsized":"2023-09-30T09:48:04.019994+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":571,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991021+0000","last_change":"2023-09-30T09:43:03.019820+0000","last_active":"2023-09-30T09:48:03.991021+0000","last_peered":"2023-09-30T09:48:03.991021+0000","last_clean":"2023-09-30T09:48:03.991021+0000","last_became_active":"2023-09-30T09:43:02.999839+0000","last_became_peered":"2023-09-30T09:43:02.999839+0000","last_unstale":"2023-09-30T09:48:03.991021+0000","last_undegraded":"2023-09-30T09:48:03.991021+0000","last_fullsized":"2023-09-30T09:48:03.991021+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:37:21.808790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18888,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":549,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990936+0000","last_change":"2023-09-30T09:43:02.997129+0000","last_active":"2023-09-30T09:48:03.990936+0000","last_peered":"2023-09-30T09:48:03.990936+0000","last_clean":"2023-09-30T09:48:03.990936+0000","last_became_active":"2023-09-30T09:43:02.996839+0000","last_became_peered":"2023-09-30T09:43:02.996839+0000","last_unstale":"2023-09-30T09:48:03.990936+0000","last_undegraded":"2023-09-30T09:48:03.990936+0000","last_fullsized":"2023-09-30T09:48:03.990936+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:52:26.117515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":781,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.020710+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:48:04.020710+0000","last_peered":"2023-09-30T09:48:04.020710+0000","last_clean":"2023-09-30T09:48:04.020710+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:48:04.020710+0000","last_undegraded":"2023-09-30T09:48:04.020710+0000","last_fullsized":"2023-09-30T09:48:04.020710+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":177,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.004364+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:48:04.004364+0000","last_peered":"2023-09-30T09:48:04.004364+0000","last_clean":"2023-09-30T09:48:04.004364+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:48:04.004364+0000","last_undegraded":"2023-09-30T09:48:04.004364+0000","last_fullsized":"2023-09-30T09:48:04.004364+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":504,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.004509+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:48:04.004509+0000","last_peered":"2023-09-30T09:48:04.004509+0000","last_clean":"2023-09-30T09:48:04.004509+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:48:04.004509+0000","last_undegraded":"2023-09-30T09:48:04.004509+0000","last_fullsized":"2023-09-30T09:48:04.004509+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":828,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990775+0000","last_change":"2023-09-30T09:44:27.078363+0000","last_active":"2023-09-30T09:48:03.990775+0000","last_peered":"2023-09-30T09:48:03.990775+0000","last_clean":"2023-09-30T09:48:03.990775+0000","last_became_active":"2023-09-30T09:44:27.078021+0000","last_became_peered":"2023-09-30T09:44:27.078021+0000","last_unstale":"2023-09-30T09:48:03.990775+0000","last_undegraded":"2023-09-30T09:48:03.990775+0000","last_fullsized":"2023-09-30T09:48:03.990775+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:50:46.999629+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":107,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019324+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:48:04.019324+0000","last_peered":"2023-09-30T09:48:04.019324+0000","last_clean":"2023-09-30T09:48:04.019324+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:48:04.019324+0000","last_undegraded":"2023-09-30T09:48:04.019324+0000","last_fullsized":"2023-09-30T09:48:04.019324+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":202,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991391+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:48:03.991391+0000","last_peered":"2023-09-30T09:48:03.991391+0000","last_clean":"2023-09-30T09:48:03.991391+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:48:03.991391+0000","last_undegraded":"2023-09-30T09:48:03.991391+0000","last_fullsized":"2023-09-30T09:48:03.991391+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":929,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.998324+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:48:03.998324+0000","last_peered":"2023-09-30T09:48:03.998324+0000","last_clean":"2023-09-30T09:48:03.998324+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:48:03.998324+0000","last_undegraded":"2023-09-30T09:48:03.998324+0000","last_fullsized":"2023-09-30T09:48:03.998324+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":211,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001499+0000","last_change":"2023-09-30T09:43:02.991205+0000","last_active":"2023-09-30T09:48:04.001499+0000","last_peered":"2023-09-30T09:48:04.001499+0000","last_clean":"2023-09-30T09:48:04.001499+0000","last_became_active":"2023-09-30T09:43:02.990922+0000","last_became_peered":"2023-09-30T09:43:02.990922+0000","last_unstale":"2023-09-30T09:48:04.001499+0000","last_undegraded":"2023-09-30T09:48:04.001499+0000","last_fullsized":"2023-09-30T09:48:04.001499+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:32:32.795629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":107,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019696+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:48:04.019696+0000","last_peered":"2023-09-30T09:48:04.019696+0000","last_clean":"2023-09-30T09:48:04.019696+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:48:04.019696+0000","last_undegraded":"2023-09-30T09:48:04.019696+0000","last_fullsized":"2023-09-30T09:48:04.019696+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":937,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991483+0000","last_change":"2023-09-30T09:44:39.746223+0000","last_active":"2023-09-30T09:48:03.991483+0000","last_peered":"2023-09-30T09:48:03.991483+0000","last_clean":"2023-09-30T09:48:03.991483+0000","last_became_active":"2023-09-30T09:44:27.068172+0000","last_became_peered":"2023-09-30T09:44:27.068172+0000","last_unstale":"2023-09-30T09:48:03.991483+0000","last_undegraded":"2023-09-30T09:48:03.991483+0000","last_fullsized":"2023-09-30T09:48:03.991483+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:56:37.299956+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":123,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.009959+0000","last_change":"2023-09-30T09:45:25.704512+0000","last_active":"2023-09-30T09:48:04.009959+0000","last_peered":"2023-09-30T09:48:04.009959+0000","last_clean":"2023-09-30T09:48:04.009959+0000","last_became_active":"2023-09-30T09:45:25.699018+0000","last_became_peered":"2023-09-30T09:45:25.699018+0000","last_unstale":"2023-09-30T09:48:04.009959+0000","last_undegraded":"2023-09-30T09:48:04.009959+0000","last_fullsized":"2023-09-30T09:48:04.009959+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:00:52.799203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.996510+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:48:03.996510+0000","last_peered":"2023-09-30T09:48:03.996510+0000","last_clean":"2023-09-30T09:48:03.996510+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:48:03.996510+0000","last_undegraded":"2023-09-30T09:48:03.996510+0000","last_fullsized":"2023-09-30T09:48:03.996510+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"527'11974","reported_seq":12291,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.490302+0000","last_change":"2023-09-30T09:44:31.674615+0000","last_active":"2023-09-30T09:48:13.490302+0000","last_peered":"2023-09-30T09:48:13.490302+0000","last_clean":"2023-09-30T09:48:13.490302+0000","last_became_active":"2023-09-30T09:44:31.672908+0000","last_became_peered":"2023-09-30T09:44:31.672908+0000","last_unstale":"2023-09-30T09:48:13.490302+0000","last_undegraded":"2023-09-30T09:48:13.490302+0000","last_fullsized":"2023-09-30T09:48:13.490302+0000","mapping_epoch":495,"log_start":"504'4400","ondisk_log_start":"504'4400","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:35:26.487710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11974,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":903,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006307+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:48:04.006307+0000","last_peered":"2023-09-30T09:48:04.006307+0000","last_clean":"2023-09-30T09:48:04.006307+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:48:04.006307+0000","last_undegraded":"2023-09-30T09:48:04.006307+0000","last_fullsized":"2023-09-30T09:48:04.006307+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":203,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001915+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:48:04.001915+0000","last_peered":"2023-09-30T09:48:04.001915+0000","last_clean":"2023-09-30T09:48:04.001915+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:48:04.001915+0000","last_undegraded":"2023-09-30T09:48:04.001915+0000","last_fullsized":"2023-09-30T09:48:04.001915+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":148,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990692+0000","last_change":"2023-09-30T09:44:27.070829+0000","last_active":"2023-09-30T09:48:03.990692+0000","last_peered":"2023-09-30T09:48:03.990692+0000","last_clean":"2023-09-30T09:48:03.990692+0000","last_became_active":"2023-09-30T09:44:27.070528+0000","last_became_peered":"2023-09-30T09:44:27.070528+0000","last_unstale":"2023-09-30T09:48:03.990692+0000","last_undegraded":"2023-09-30T09:48:03.990692+0000","last_fullsized":"2023-09-30T09:48:03.990692+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:17.685961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"527'11944","reported_seq":11996,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.493357+0000","last_change":"2023-09-30T09:44:17.060472+0000","last_active":"2023-09-30T09:48:13.493357+0000","last_peered":"2023-09-30T09:48:13.493357+0000","last_clean":"2023-09-30T09:48:13.493357+0000","last_became_active":"2023-09-30T09:44:17.059630+0000","last_became_peered":"2023-09-30T09:44:17.059630+0000","last_unstale":"2023-09-30T09:48:13.493357+0000","last_undegraded":"2023-09-30T09:48:13.493357+0000","last_fullsized":"2023-09-30T09:48:13.493357+0000","mapping_epoch":483,"log_start":"512'5500","ondisk_log_start":"512'5500","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:03:12.093622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11944,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":921,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.996163+0000","last_change":"2023-09-30T09:44:15.182210+0000","last_active":"2023-09-30T09:48:03.996163+0000","last_peered":"2023-09-30T09:48:03.996163+0000","last_clean":"2023-09-30T09:48:03.996163+0000","last_became_active":"2023-09-30T09:43:13.258665+0000","last_became_peered":"2023-09-30T09:43:13.258665+0000","last_unstale":"2023-09-30T09:48:03.996163+0000","last_undegraded":"2023-09-30T09:48:03.996163+0000","last_fullsized":"2023-09-30T09:48:03.996163+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:44.574219+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":173,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006439+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:48:04.006439+0000","last_peered":"2023-09-30T09:48:04.006439+0000","last_clean":"2023-09-30T09:48:04.006439+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:48:04.006439+0000","last_undegraded":"2023-09-30T09:48:04.006439+0000","last_fullsized":"2023-09-30T09:48:04.006439+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":578,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990524+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:48:03.990524+0000","last_peered":"2023-09-30T09:48:03.990524+0000","last_clean":"2023-09-30T09:48:03.990524+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:48:03.990524+0000","last_undegraded":"2023-09-30T09:48:03.990524+0000","last_fullsized":"2023-09-30T09:48:03.990524+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"527'12182","reported_seq":12459,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.483222+0000","last_change":"2023-09-30T09:44:29.670663+0000","last_active":"2023-09-30T09:48:13.483222+0000","last_peered":"2023-09-30T09:48:13.483222+0000","last_clean":"2023-09-30T09:48:13.483222+0000","last_became_active":"2023-09-30T09:44:29.663729+0000","last_became_peered":"2023-09-30T09:44:29.663729+0000","last_unstale":"2023-09-30T09:48:13.483222+0000","last_undegraded":"2023-09-30T09:48:13.483222+0000","last_fullsized":"2023-09-30T09:48:13.483222+0000","mapping_epoch":493,"log_start":"512'5500","ondisk_log_start":"512'5500","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6682,"log_dups_size":0,"ondisk_log_size":6682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:49:38.314280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12182,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":697,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.007006+0000","last_change":"2023-09-30T09:44:27.085945+0000","last_active":"2023-09-30T09:48:04.007006+0000","last_peered":"2023-09-30T09:48:04.007006+0000","last_clean":"2023-09-30T09:48:04.007006+0000","last_became_active":"2023-09-30T09:44:27.084994+0000","last_became_peered":"2023-09-30T09:44:27.084994+0000","last_unstale":"2023-09-30T09:48:04.007006+0000","last_undegraded":"2023-09-30T09:48:04.007006+0000","last_fullsized":"2023-09-30T09:48:04.007006+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:11.677803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:11.677803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:07:43.101089+0000","scrub_duration":0.0018340839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":240,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001548+0000","last_change":"2023-09-30T09:44:27.069710+0000","last_active":"2023-09-30T09:48:04.001548+0000","last_peered":"2023-09-30T09:48:04.001548+0000","last_clean":"2023-09-30T09:48:04.001548+0000","last_became_active":"2023-09-30T09:44:27.069188+0000","last_became_peered":"2023-09-30T09:44:27.069188+0000","last_unstale":"2023-09-30T09:48:04.001548+0000","last_undegraded":"2023-09-30T09:48:04.001548+0000","last_fullsized":"2023-09-30T09:48:04.001548+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:57.540514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":96,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006233+0000","last_change":"2023-09-30T09:44:27.071819+0000","last_active":"2023-09-30T09:48:04.006233+0000","last_peered":"2023-09-30T09:48:04.006233+0000","last_clean":"2023-09-30T09:48:04.006233+0000","last_became_active":"2023-09-30T09:44:27.070978+0000","last_became_peered":"2023-09-30T09:44:27.070978+0000","last_unstale":"2023-09-30T09:48:04.006233+0000","last_undegraded":"2023-09-30T09:48:04.006233+0000","last_fullsized":"2023-09-30T09:48:04.006233+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:21.015268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"527'11896","reported_seq":12264,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.401789+0000","last_change":"2023-09-30T09:44:31.680840+0000","last_active":"2023-09-30T09:48:13.401789+0000","last_peered":"2023-09-30T09:48:13.401789+0000","last_clean":"2023-09-30T09:48:13.401789+0000","last_became_active":"2023-09-30T09:44:31.680056+0000","last_became_peered":"2023-09-30T09:44:31.680056+0000","last_unstale":"2023-09-30T09:48:13.401789+0000","last_undegraded":"2023-09-30T09:48:13.401789+0000","last_fullsized":"2023-09-30T09:48:13.401789+0000","mapping_epoch":495,"log_start":"497'1800","ondisk_log_start":"497'1800","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10096,"log_dups_size":0,"ondisk_log_size":10096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:29.227038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11896,"num_write_kb":380672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":762,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001079+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:48:04.001079+0000","last_peered":"2023-09-30T09:48:04.001079+0000","last_clean":"2023-09-30T09:48:04.001079+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:48:04.001079+0000","last_undegraded":"2023-09-30T09:48:04.001079+0000","last_fullsized":"2023-09-30T09:48:04.001079+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":462,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991107+0000","last_change":"2023-09-30T09:44:27.062733+0000","last_active":"2023-09-30T09:48:03.991107+0000","last_peered":"2023-09-30T09:48:03.991107+0000","last_clean":"2023-09-30T09:48:03.991107+0000","last_became_active":"2023-09-30T09:44:27.060745+0000","last_became_peered":"2023-09-30T09:44:27.060745+0000","last_unstale":"2023-09-30T09:48:03.991107+0000","last_undegraded":"2023-09-30T09:48:03.991107+0000","last_fullsized":"2023-09-30T09:48:03.991107+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:21:44.348802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019687+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:48:04.019687+0000","last_peered":"2023-09-30T09:48:04.019687+0000","last_clean":"2023-09-30T09:48:04.019687+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:48:04.019687+0000","last_undegraded":"2023-09-30T09:48:04.019687+0000","last_fullsized":"2023-09-30T09:48:04.019687+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"527'12132","reported_seq":12184,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.490330+0000","last_change":"2023-09-30T09:44:17.060356+0000","last_active":"2023-09-30T09:48:13.490330+0000","last_peered":"2023-09-30T09:48:13.490330+0000","last_clean":"2023-09-30T09:48:13.490330+0000","last_became_active":"2023-09-30T09:44:17.059974+0000","last_became_peered":"2023-09-30T09:44:17.059974+0000","last_unstale":"2023-09-30T09:48:13.490330+0000","last_undegraded":"2023-09-30T09:48:13.490330+0000","last_fullsized":"2023-09-30T09:48:13.490330+0000","mapping_epoch":483,"log_start":"510'5100","ondisk_log_start":"510'5100","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:58.519932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12132,"num_write_kb":388224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1009,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001417+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:48:04.001417+0000","last_peered":"2023-09-30T09:48:04.001417+0000","last_clean":"2023-09-30T09:48:04.001417+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:48:04.001417+0000","last_undegraded":"2023-09-30T09:48:04.001417+0000","last_fullsized":"2023-09-30T09:48:04.001417+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":86,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019573+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:48:04.019573+0000","last_peered":"2023-09-30T09:48:04.019573+0000","last_clean":"2023-09-30T09:48:04.019573+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:48:04.019573+0000","last_undegraded":"2023-09-30T09:48:04.019573+0000","last_fullsized":"2023-09-30T09:48:04.019573+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":525,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991097+0000","last_change":"2023-09-30T09:43:02.997165+0000","last_active":"2023-09-30T09:48:03.991097+0000","last_peered":"2023-09-30T09:48:03.991097+0000","last_clean":"2023-09-30T09:48:03.991097+0000","last_became_active":"2023-09-30T09:43:02.996821+0000","last_became_peered":"2023-09-30T09:43:02.996821+0000","last_unstale":"2023-09-30T09:48:03.991097+0000","last_undegraded":"2023-09-30T09:48:03.991097+0000","last_fullsized":"2023-09-30T09:48:03.991097+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:04.954889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"527'11954","reported_seq":12006,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.488583+0000","last_change":"2023-09-30T09:44:17.000904+0000","last_active":"2023-09-30T09:48:13.488583+0000","last_peered":"2023-09-30T09:48:13.488583+0000","last_clean":"2023-09-30T09:48:13.488583+0000","last_became_active":"2023-09-30T09:44:17.000630+0000","last_became_peered":"2023-09-30T09:44:17.000630+0000","last_unstale":"2023-09-30T09:48:13.488583+0000","last_undegraded":"2023-09-30T09:48:13.488583+0000","last_fullsized":"2023-09-30T09:48:13.488583+0000","mapping_epoch":483,"log_start":"510'4900","ondisk_log_start":"510'4900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:07:36.765501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11954,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":1008,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.014553+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:48:04.014553+0000","last_peered":"2023-09-30T09:48:04.014553+0000","last_clean":"2023-09-30T09:48:04.014553+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:48:04.014553+0000","last_undegraded":"2023-09-30T09:48:04.014553+0000","last_fullsized":"2023-09-30T09:48:04.014553+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":82,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006080+0000","last_change":"2023-09-30T09:43:02.982316+0000","last_active":"2023-09-30T09:48:04.006080+0000","last_peered":"2023-09-30T09:48:04.006080+0000","last_clean":"2023-09-30T09:48:04.006080+0000","last_became_active":"2023-09-30T09:43:02.982020+0000","last_became_peered":"2023-09-30T09:43:02.982020+0000","last_unstale":"2023-09-30T09:48:04.006080+0000","last_undegraded":"2023-09-30T09:48:04.006080+0000","last_fullsized":"2023-09-30T09:48:04.006080+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:34.719162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":221,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990526+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:48:03.990526+0000","last_peered":"2023-09-30T09:48:03.990526+0000","last_clean":"2023-09-30T09:48:03.990526+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:48:03.990526+0000","last_undegraded":"2023-09-30T09:48:03.990526+0000","last_fullsized":"2023-09-30T09:48:03.990526+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"527'12270","reported_seq":12322,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.483740+0000","last_change":"2023-09-30T09:44:17.059955+0000","last_active":"2023-09-30T09:48:13.483740+0000","last_peered":"2023-09-30T09:48:13.483740+0000","last_clean":"2023-09-30T09:48:13.483740+0000","last_became_active":"2023-09-30T09:44:17.059587+0000","last_became_peered":"2023-09-30T09:44:17.059587+0000","last_unstale":"2023-09-30T09:48:13.483740+0000","last_undegraded":"2023-09-30T09:48:13.483740+0000","last_fullsized":"2023-09-30T09:48:13.483740+0000","mapping_epoch":483,"log_start":"513'6500","ondisk_log_start":"513'6500","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:49.398874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12270,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":866,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019744+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:48:04.019744+0000","last_peered":"2023-09-30T09:48:04.019744+0000","last_clean":"2023-09-30T09:48:04.019744+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:48:04.019744+0000","last_undegraded":"2023-09-30T09:48:04.019744+0000","last_fullsized":"2023-09-30T09:48:04.019744+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":532,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990480+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:48:03.990480+0000","last_peered":"2023-09-30T09:48:03.990480+0000","last_clean":"2023-09-30T09:48:03.990480+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:48:03.990480+0000","last_undegraded":"2023-09-30T09:48:03.990480+0000","last_fullsized":"2023-09-30T09:48:03.990480+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":539,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001195+0000","last_change":"2023-09-30T09:43:02.992819+0000","last_active":"2023-09-30T09:48:04.001195+0000","last_peered":"2023-09-30T09:48:04.001195+0000","last_clean":"2023-09-30T09:48:04.001195+0000","last_became_active":"2023-09-30T09:43:02.992470+0000","last_became_peered":"2023-09-30T09:43:02.992470+0000","last_unstale":"2023-09-30T09:48:04.001195+0000","last_undegraded":"2023-09-30T09:48:04.001195+0000","last_fullsized":"2023-09-30T09:48:04.001195+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:50:11.495240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"527'11936","reported_seq":11992,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.451788+0000","last_change":"2023-09-30T09:44:17.060080+0000","last_active":"2023-09-30T09:48:13.451788+0000","last_peered":"2023-09-30T09:48:13.451788+0000","last_clean":"2023-09-30T09:48:13.451788+0000","last_became_active":"2023-09-30T09:44:17.059766+0000","last_became_peered":"2023-09-30T09:44:17.059766+0000","last_unstale":"2023-09-30T09:48:13.451788+0000","last_undegraded":"2023-09-30T09:48:13.451788+0000","last_fullsized":"2023-09-30T09:48:13.451788+0000","mapping_epoch":483,"log_start":"497'1900","ondisk_log_start":"497'1900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":10036,"log_dups_size":0,"ondisk_log_size":10036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:11.681255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11936,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":730,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.013887+0000","last_change":"2023-09-30T09:44:27.069348+0000","last_active":"2023-09-30T09:48:04.013887+0000","last_peered":"2023-09-30T09:48:04.013887+0000","last_clean":"2023-09-30T09:48:04.013887+0000","last_became_active":"2023-09-30T09:44:27.069028+0000","last_became_peered":"2023-09-30T09:44:27.069028+0000","last_unstale":"2023-09-30T09:48:04.013887+0000","last_undegraded":"2023-09-30T09:48:04.013887+0000","last_fullsized":"2023-09-30T09:48:04.013887+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:15.492081+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.996119+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:48:03.996119+0000","last_peered":"2023-09-30T09:48:03.996119+0000","last_clean":"2023-09-30T09:48:03.996119+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:48:03.996119+0000","last_undegraded":"2023-09-30T09:48:03.996119+0000","last_fullsized":"2023-09-30T09:48:03.996119+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":212,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.000881+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:48:04.000881+0000","last_peered":"2023-09-30T09:48:04.000881+0000","last_clean":"2023-09-30T09:48:04.000881+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:48:04.000881+0000","last_undegraded":"2023-09-30T09:48:04.000881+0000","last_fullsized":"2023-09-30T09:48:04.000881+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"527'12104","reported_seq":12415,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.501946+0000","last_change":"2023-09-30T09:44:30.675992+0000","last_active":"2023-09-30T09:48:13.501946+0000","last_peered":"2023-09-30T09:48:13.501946+0000","last_clean":"2023-09-30T09:48:13.501946+0000","last_became_active":"2023-09-30T09:44:30.665849+0000","last_became_peered":"2023-09-30T09:44:30.665849+0000","last_unstale":"2023-09-30T09:48:13.501946+0000","last_undegraded":"2023-09-30T09:48:13.501946+0000","last_fullsized":"2023-09-30T09:48:13.501946+0000","mapping_epoch":494,"log_start":"512'5400","ondisk_log_start":"512'5400","created":483,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:48.025131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12104,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":769,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.005897+0000","last_change":"2023-09-30T09:43:09.698864+0000","last_active":"2023-09-30T09:48:04.005897+0000","last_peered":"2023-09-30T09:48:04.005897+0000","last_clean":"2023-09-30T09:48:04.005897+0000","last_became_active":"2023-09-30T09:43:09.698428+0000","last_became_peered":"2023-09-30T09:43:09.698428+0000","last_unstale":"2023-09-30T09:48:04.005897+0000","last_undegraded":"2023-09-30T09:48:04.005897+0000","last_fullsized":"2023-09-30T09:48:04.005897+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:29:30.978674+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":758,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019514+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:48:04.019514+0000","last_peered":"2023-09-30T09:48:04.019514+0000","last_clean":"2023-09-30T09:48:04.019514+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:48:04.019514+0000","last_undegraded":"2023-09-30T09:48:04.019514+0000","last_fullsized":"2023-09-30T09:48:04.019514+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":455,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991537+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:48:03.991537+0000","last_peered":"2023-09-30T09:48:03.991537+0000","last_clean":"2023-09-30T09:48:03.991537+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:48:03.991537+0000","last_undegraded":"2023-09-30T09:48:03.991537+0000","last_fullsized":"2023-09-30T09:48:03.991537+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":144,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991224+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:48:03.991224+0000","last_peered":"2023-09-30T09:48:03.991224+0000","last_clean":"2023-09-30T09:48:03.991224+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:48:03.991224+0000","last_undegraded":"2023-09-30T09:48:03.991224+0000","last_fullsized":"2023-09-30T09:48:03.991224+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"527'12128","reported_seq":12485,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.471060+0000","last_change":"2023-09-30T09:44:29.664524+0000","last_active":"2023-09-30T09:48:13.471060+0000","last_peered":"2023-09-30T09:48:13.471060+0000","last_clean":"2023-09-30T09:48:13.471060+0000","last_became_active":"2023-09-30T09:44:29.663303+0000","last_became_peered":"2023-09-30T09:44:29.663303+0000","last_unstale":"2023-09-30T09:48:13.471060+0000","last_undegraded":"2023-09-30T09:48:13.471060+0000","last_fullsized":"2023-09-30T09:48:13.471060+0000","mapping_epoch":493,"log_start":"504'4600","ondisk_log_start":"504'4600","created":483,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7528,"log_dups_size":0,"ondisk_log_size":7528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:24:56.080075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12128,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":968,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990859+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:48:03.990859+0000","last_peered":"2023-09-30T09:48:03.990859+0000","last_clean":"2023-09-30T09:48:03.990859+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:48:03.990859+0000","last_undegraded":"2023-09-30T09:48:03.990859+0000","last_fullsized":"2023-09-30T09:48:03.990859+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":474,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006314+0000","last_change":"2023-09-30T09:44:27.071942+0000","last_active":"2023-09-30T09:48:04.006314+0000","last_peered":"2023-09-30T09:48:04.006314+0000","last_clean":"2023-09-30T09:48:04.006314+0000","last_became_active":"2023-09-30T09:44:27.071351+0000","last_became_peered":"2023-09-30T09:44:27.071351+0000","last_unstale":"2023-09-30T09:48:04.006314+0000","last_undegraded":"2023-09-30T09:48:04.006314+0000","last_fullsized":"2023-09-30T09:48:04.006314+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:33.393664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":546,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991030+0000","last_change":"2023-09-30T09:43:02.982556+0000","last_active":"2023-09-30T09:48:03.991030+0000","last_peered":"2023-09-30T09:48:03.991030+0000","last_clean":"2023-09-30T09:48:03.991030+0000","last_became_active":"2023-09-30T09:43:02.982203+0000","last_became_peered":"2023-09-30T09:43:02.982203+0000","last_unstale":"2023-09-30T09:48:03.991030+0000","last_undegraded":"2023-09-30T09:48:03.991030+0000","last_fullsized":"2023-09-30T09:48:03.991030+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:11:51.599950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":287,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.014344+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:48:04.014344+0000","last_peered":"2023-09-30T09:48:04.014344+0000","last_clean":"2023-09-30T09:48:04.014344+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:48:04.014344+0000","last_undegraded":"2023-09-30T09:48:04.014344+0000","last_fullsized":"2023-09-30T09:48:04.014344+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"527'12172","reported_seq":12529,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.489871+0000","last_change":"2023-09-30T09:44:28.792471+0000","last_active":"2023-09-30T09:48:13.489871+0000","last_peered":"2023-09-30T09:48:13.489871+0000","last_clean":"2023-09-30T09:48:13.489871+0000","last_became_active":"2023-09-30T09:44:28.783982+0000","last_became_peered":"2023-09-30T09:44:28.783982+0000","last_unstale":"2023-09-30T09:48:13.489871+0000","last_undegraded":"2023-09-30T09:48:13.489871+0000","last_fullsized":"2023-09-30T09:48:13.489871+0000","mapping_epoch":492,"log_start":"512'5500","ondisk_log_start":"512'5500","created":483,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:47.750775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12172,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":946,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001267+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:48:04.001267+0000","last_peered":"2023-09-30T09:48:04.001267+0000","last_clean":"2023-09-30T09:48:04.001267+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:48:04.001267+0000","last_undegraded":"2023-09-30T09:48:04.001267+0000","last_fullsized":"2023-09-30T09:48:04.001267+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":796,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.013981+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:48:04.013981+0000","last_peered":"2023-09-30T09:48:04.013981+0000","last_clean":"2023-09-30T09:48:04.013981+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:48:04.013981+0000","last_undegraded":"2023-09-30T09:48:04.013981+0000","last_fullsized":"2023-09-30T09:48:04.013981+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":290,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.014388+0000","last_change":"2023-09-30T09:43:02.977973+0000","last_active":"2023-09-30T09:48:04.014388+0000","last_peered":"2023-09-30T09:48:04.014388+0000","last_clean":"2023-09-30T09:48:04.014388+0000","last_became_active":"2023-09-30T09:43:02.977581+0000","last_became_peered":"2023-09-30T09:43:02.977581+0000","last_unstale":"2023-09-30T09:48:04.014388+0000","last_undegraded":"2023-09-30T09:48:04.014388+0000","last_fullsized":"2023-09-30T09:48:04.014388+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:41.283880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":376,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991417+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:48:03.991417+0000","last_peered":"2023-09-30T09:48:03.991417+0000","last_clean":"2023-09-30T09:48:03.991417+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:48:03.991417+0000","last_undegraded":"2023-09-30T09:48:03.991417+0000","last_fullsized":"2023-09-30T09:48:03.991417+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"527'12018","reported_seq":12335,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.422405+0000","last_change":"2023-09-30T09:44:31.674487+0000","last_active":"2023-09-30T09:48:13.422405+0000","last_peered":"2023-09-30T09:48:13.422405+0000","last_clean":"2023-09-30T09:48:13.422405+0000","last_became_active":"2023-09-30T09:44:31.672940+0000","last_became_peered":"2023-09-30T09:44:31.672940+0000","last_unstale":"2023-09-30T09:48:13.422405+0000","last_undegraded":"2023-09-30T09:48:13.422405+0000","last_fullsized":"2023-09-30T09:48:13.422405+0000","mapping_epoch":495,"log_start":"504'4500","ondisk_log_start":"504'4500","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:23:49.395047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12018,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1088,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.995715+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:48:03.995715+0000","last_peered":"2023-09-30T09:48:03.995715+0000","last_clean":"2023-09-30T09:48:03.995715+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:48:03.995715+0000","last_undegraded":"2023-09-30T09:48:03.995715+0000","last_fullsized":"2023-09-30T09:48:03.995715+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":889,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001763+0000","last_change":"2023-09-30T09:43:05.979385+0000","last_active":"2023-09-30T09:48:04.001763+0000","last_peered":"2023-09-30T09:48:04.001763+0000","last_clean":"2023-09-30T09:48:04.001763+0000","last_became_active":"2023-09-30T09:43:05.978961+0000","last_became_peered":"2023-09-30T09:43:05.978961+0000","last_unstale":"2023-09-30T09:48:04.001763+0000","last_undegraded":"2023-09-30T09:48:04.001763+0000","last_fullsized":"2023-09-30T09:48:04.001763+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:17:30.498636+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":103,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991278+0000","last_change":"2023-09-30T09:44:27.084414+0000","last_active":"2023-09-30T09:48:03.991278+0000","last_peered":"2023-09-30T09:48:03.991278+0000","last_clean":"2023-09-30T09:48:03.991278+0000","last_became_active":"2023-09-30T09:44:27.081215+0000","last_became_peered":"2023-09-30T09:44:27.081215+0000","last_unstale":"2023-09-30T09:48:03.991278+0000","last_undegraded":"2023-09-30T09:48:03.991278+0000","last_fullsized":"2023-09-30T09:48:03.991278+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:29:26.332125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":82,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006288+0000","last_change":"2023-09-30T09:43:02.997463+0000","last_active":"2023-09-30T09:48:04.006288+0000","last_peered":"2023-09-30T09:48:04.006288+0000","last_clean":"2023-09-30T09:48:04.006288+0000","last_became_active":"2023-09-30T09:43:02.996657+0000","last_became_peered":"2023-09-30T09:43:02.996657+0000","last_unstale":"2023-09-30T09:48:04.006288+0000","last_undegraded":"2023-09-30T09:48:04.006288+0000","last_fullsized":"2023-09-30T09:48:04.006288+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:32:25.456368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"527'12326","reported_seq":22362,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.439352+0000","last_change":"2023-09-30T09:47:51.102725+0000","last_active":"2023-09-30T09:48:13.439352+0000","last_peered":"2023-09-30T09:48:13.439352+0000","last_clean":"2023-09-30T09:48:13.439352+0000","last_became_active":"2023-09-30T09:47:50.995967+0000","last_became_peered":"2023-09-30T09:47:50.995967+0000","last_unstale":"2023-09-30T09:48:13.439352+0000","last_undegraded":"2023-09-30T09:48:13.439352+0000","last_fullsized":"2023-09-30T09:48:13.439352+0000","mapping_epoch":523,"log_start":"504'4778","ondisk_log_start":"504'4778","created":483,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":7548,"log_dups_size":0,"ondisk_log_size":7548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:15.442443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12326,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4971,"num_bytes_recovered":280821760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":788,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.007121+0000","last_change":"2023-09-30T09:43:05.973209+0000","last_active":"2023-09-30T09:48:04.007121+0000","last_peered":"2023-09-30T09:48:04.007121+0000","last_clean":"2023-09-30T09:48:04.007121+0000","last_became_active":"2023-09-30T09:43:05.972861+0000","last_became_peered":"2023-09-30T09:43:05.972861+0000","last_unstale":"2023-09-30T09:48:04.007121+0000","last_undegraded":"2023-09-30T09:48:04.007121+0000","last_fullsized":"2023-09-30T09:48:04.007121+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.041353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.041353+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:20:45.301935+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":935,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001692+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:48:04.001692+0000","last_peered":"2023-09-30T09:48:04.001692+0000","last_clean":"2023-09-30T09:48:04.001692+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:48:04.001692+0000","last_undegraded":"2023-09-30T09:48:04.001692+0000","last_fullsized":"2023-09-30T09:48:04.001692+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":140,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.020421+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:48:04.020421+0000","last_peered":"2023-09-30T09:48:04.020421+0000","last_clean":"2023-09-30T09:48:04.020421+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:48:04.020421+0000","last_undegraded":"2023-09-30T09:48:04.020421+0000","last_fullsized":"2023-09-30T09:48:04.020421+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":33,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019617+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:48:04.019617+0000","last_peered":"2023-09-30T09:48:04.019617+0000","last_clean":"2023-09-30T09:48:04.019617+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:48:04.019617+0000","last_undegraded":"2023-09-30T09:48:04.019617+0000","last_fullsized":"2023-09-30T09:48:04.019617+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"527'12112","reported_seq":12164,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.464237+0000","last_change":"2023-09-30T09:44:17.002803+0000","last_active":"2023-09-30T09:48:13.464237+0000","last_peered":"2023-09-30T09:48:13.464237+0000","last_clean":"2023-09-30T09:48:13.464237+0000","last_became_active":"2023-09-30T09:44:17.002567+0000","last_became_peered":"2023-09-30T09:44:17.002567+0000","last_unstale":"2023-09-30T09:48:13.464237+0000","last_undegraded":"2023-09-30T09:48:13.464237+0000","last_fullsized":"2023-09-30T09:48:13.464237+0000","mapping_epoch":483,"log_start":"513'6300","ondisk_log_start":"513'6300","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5812,"log_dups_size":0,"ondisk_log_size":5812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:36:07.624664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12112,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":779,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006615+0000","last_change":"2023-09-30T09:43:05.976808+0000","last_active":"2023-09-30T09:48:04.006615+0000","last_peered":"2023-09-30T09:48:04.006615+0000","last_clean":"2023-09-30T09:48:04.006615+0000","last_became_active":"2023-09-30T09:43:05.976589+0000","last_became_peered":"2023-09-30T09:43:05.976589+0000","last_unstale":"2023-09-30T09:48:04.006615+0000","last_undegraded":"2023-09-30T09:48:04.006615+0000","last_fullsized":"2023-09-30T09:48:04.006615+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:04:26.740162+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1075,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006498+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:48:04.006498+0000","last_peered":"2023-09-30T09:48:04.006498+0000","last_clean":"2023-09-30T09:48:04.006498+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:48:04.006498+0000","last_undegraded":"2023-09-30T09:48:04.006498+0000","last_fullsized":"2023-09-30T09:48:04.006498+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":271,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.014208+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:48:04.014208+0000","last_peered":"2023-09-30T09:48:04.014208+0000","last_clean":"2023-09-30T09:48:04.014208+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:48:04.014208+0000","last_undegraded":"2023-09-30T09:48:04.014208+0000","last_fullsized":"2023-09-30T09:48:04.014208+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":140,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019836+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:48:04.019836+0000","last_peered":"2023-09-30T09:48:04.019836+0000","last_clean":"2023-09-30T09:48:04.019836+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:48:04.019836+0000","last_undegraded":"2023-09-30T09:48:04.019836+0000","last_fullsized":"2023-09-30T09:48:04.019836+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"527'12010","reported_seq":12397,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.408986+0000","last_change":"2023-09-30T09:44:31.683115+0000","last_active":"2023-09-30T09:48:13.408986+0000","last_peered":"2023-09-30T09:48:13.408986+0000","last_clean":"2023-09-30T09:48:13.408986+0000","last_became_active":"2023-09-30T09:44:31.679618+0000","last_became_peered":"2023-09-30T09:44:31.679618+0000","last_unstale":"2023-09-30T09:48:13.408986+0000","last_undegraded":"2023-09-30T09:48:13.408986+0000","last_fullsized":"2023-09-30T09:48:13.408986+0000","mapping_epoch":495,"log_start":"513'6200","ondisk_log_start":"513'6200","created":483,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":5810,"log_dups_size":0,"ondisk_log_size":5810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:13:15.762082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12010,"num_write_kb":384257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1046,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.000670+0000","last_change":"2023-09-30T09:43:05.974917+0000","last_active":"2023-09-30T09:48:04.000670+0000","last_peered":"2023-09-30T09:48:04.000670+0000","last_clean":"2023-09-30T09:48:04.000670+0000","last_became_active":"2023-09-30T09:43:05.974593+0000","last_became_peered":"2023-09-30T09:43:05.974593+0000","last_unstale":"2023-09-30T09:48:04.000670+0000","last_undegraded":"2023-09-30T09:48:04.000670+0000","last_fullsized":"2023-09-30T09:48:04.000670+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:32.950674+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":558,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001755+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:48:04.001755+0000","last_peered":"2023-09-30T09:48:04.001755+0000","last_clean":"2023-09-30T09:48:04.001755+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:48:04.001755+0000","last_undegraded":"2023-09-30T09:48:04.001755+0000","last_fullsized":"2023-09-30T09:48:04.001755+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1245,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006564+0000","last_change":"2023-09-30T09:43:13.093099+0000","last_active":"2023-09-30T09:48:04.006564+0000","last_peered":"2023-09-30T09:48:04.006564+0000","last_clean":"2023-09-30T09:48:04.006564+0000","last_became_active":"2023-09-30T09:43:13.092545+0000","last_became_peered":"2023-09-30T09:43:13.092545+0000","last_unstale":"2023-09-30T09:48:04.006564+0000","last_undegraded":"2023-09-30T09:48:04.006564+0000","last_fullsized":"2023-09-30T09:48:04.006564+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:09:05.634502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":562,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.001606+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:48:04.001606+0000","last_peered":"2023-09-30T09:48:04.001606+0000","last_clean":"2023-09-30T09:48:04.001606+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:48:04.001606+0000","last_undegraded":"2023-09-30T09:48:04.001606+0000","last_fullsized":"2023-09-30T09:48:04.001606+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"527'12328","reported_seq":12380,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-30T09:48:13.493230+0000","last_change":"2023-09-30T09:44:17.060337+0000","last_active":"2023-09-30T09:48:13.493230+0000","last_peered":"2023-09-30T09:48:13.493230+0000","last_clean":"2023-09-30T09:48:13.493230+0000","last_became_active":"2023-09-30T09:44:17.060062+0000","last_became_peered":"2023-09-30T09:44:17.060062+0000","last_unstale":"2023-09-30T09:48:13.493230+0000","last_undegraded":"2023-09-30T09:48:13.493230+0000","last_fullsized":"2023-09-30T09:48:13.493230+0000","mapping_epoch":483,"log_start":"512'5900","ondisk_log_start":"512'5900","created":483,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:44:15.980363+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.980363+0000","objects_scrubbed":0,"log_size":6428,"log_dups_size":0,"ondisk_log_size":6428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:33:24.620824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12328,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":910,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.995995+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:48:03.995995+0000","last_peered":"2023-09-30T09:48:03.995995+0000","last_clean":"2023-09-30T09:48:03.995995+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:48:03.995995+0000","last_undegraded":"2023-09-30T09:48:03.995995+0000","last_fullsized":"2023-09-30T09:48:03.995995+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":97,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006932+0000","last_change":"2023-09-30T09:43:04.210910+0000","last_active":"2023-09-30T09:48:04.006932+0000","last_peered":"2023-09-30T09:48:04.006932+0000","last_clean":"2023-09-30T09:48:04.006932+0000","last_became_active":"2023-09-30T09:43:02.996240+0000","last_became_peered":"2023-09-30T09:43:02.996240+0000","last_unstale":"2023-09-30T09:48:04.006932+0000","last_undegraded":"2023-09-30T09:48:04.006932+0000","last_fullsized":"2023-09-30T09:48:04.006932+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:53:52.821557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":103,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019841+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:48:04.019841+0000","last_peered":"2023-09-30T09:48:04.019841+0000","last_clean":"2023-09-30T09:48:04.019841+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:48:04.019841+0000","last_undegraded":"2023-09-30T09:48:04.019841+0000","last_fullsized":"2023-09-30T09:48:04.019841+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":528,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990446+0000","last_change":"2023-09-30T09:44:27.084140+0000","last_active":"2023-09-30T09:48:03.990446+0000","last_peered":"2023-09-30T09:48:03.990446+0000","last_clean":"2023-09-30T09:48:03.990446+0000","last_became_active":"2023-09-30T09:44:27.083813+0000","last_became_peered":"2023-09-30T09:44:27.083813+0000","last_unstale":"2023-09-30T09:48:03.990446+0000","last_undegraded":"2023-09-30T09:48:03.990446+0000","last_fullsized":"2023-09-30T09:48:03.990446+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:04.277758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":847,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.020274+0000","last_change":"2023-09-30T09:45:28.768689+0000","last_active":"2023-09-30T09:48:04.020274+0000","last_peered":"2023-09-30T09:48:04.020274+0000","last_clean":"2023-09-30T09:48:04.020274+0000","last_became_active":"2023-09-30T09:45:28.768243+0000","last_became_peered":"2023-09-30T09:45:28.768243+0000","last_unstale":"2023-09-30T09:48:04.020274+0000","last_undegraded":"2023-09-30T09:48:04.020274+0000","last_fullsized":"2023-09-30T09:48:04.020274+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:46:42.284102+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":33,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.019785+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:48:04.019785+0000","last_peered":"2023-09-30T09:48:04.019785+0000","last_clean":"2023-09-30T09:48:04.019785+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:48:04.019785+0000","last_undegraded":"2023-09-30T09:48:04.019785+0000","last_fullsized":"2023-09-30T09:48:04.019785+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":233,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.014138+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:48:04.014138+0000","last_peered":"2023-09-30T09:48:04.014138+0000","last_clean":"2023-09-30T09:48:04.014138+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:48:04.014138+0000","last_undegraded":"2023-09-30T09:48:04.014138+0000","last_fullsized":"2023-09-30T09:48:04.014138+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":641,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006678+0000","last_change":"2023-09-30T09:44:14.674504+0000","last_active":"2023-09-30T09:48:04.006678+0000","last_peered":"2023-09-30T09:48:04.006678+0000","last_clean":"2023-09-30T09:48:04.006678+0000","last_became_active":"2023-09-30T09:43:45.123475+0000","last_became_peered":"2023-09-30T09:43:45.123475+0000","last_unstale":"2023-09-30T09:48:04.006678+0000","last_undegraded":"2023-09-30T09:48:04.006678+0000","last_fullsized":"2023-09-30T09:48:04.006678+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:15:30.275762+0000","scrub_duration":0.001521318,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":455,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991029+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:48:03.991029+0000","last_peered":"2023-09-30T09:48:03.991029+0000","last_clean":"2023-09-30T09:48:03.991029+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:48:03.991029+0000","last_undegraded":"2023-09-30T09:48:03.991029+0000","last_fullsized":"2023-09-30T09:48:03.991029+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":241,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990610+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:48:03.990610+0000","last_peered":"2023-09-30T09:48:03.990610+0000","last_clean":"2023-09-30T09:48:03.990610+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:48:03.990610+0000","last_undegraded":"2023-09-30T09:48:03.990610+0000","last_fullsized":"2023-09-30T09:48:03.990610+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":397,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991739+0000","last_change":"2023-09-30T09:44:27.069906+0000","last_active":"2023-09-30T09:48:03.991739+0000","last_peered":"2023-09-30T09:48:03.991739+0000","last_clean":"2023-09-30T09:48:03.991739+0000","last_became_active":"2023-09-30T09:44:27.068257+0000","last_became_peered":"2023-09-30T09:44:27.068257+0000","last_unstale":"2023-09-30T09:48:03.991739+0000","last_undegraded":"2023-09-30T09:48:03.991739+0000","last_fullsized":"2023-09-30T09:48:03.991739+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:00:30.119339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":261,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.991224+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:48:03.991224+0000","last_peered":"2023-09-30T09:48:03.991224+0000","last_clean":"2023-09-30T09:48:03.991224+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:48:03.991224+0000","last_undegraded":"2023-09-30T09:48:03.991224+0000","last_fullsized":"2023-09-30T09:48:03.991224+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":854,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.004424+0000","last_change":"2023-09-30T09:44:15.416601+0000","last_active":"2023-09-30T09:48:04.004424+0000","last_peered":"2023-09-30T09:48:04.004424+0000","last_clean":"2023-09-30T09:48:04.004424+0000","last_became_active":"2023-09-30T09:43:05.980413+0000","last_became_peered":"2023-09-30T09:43:05.980413+0000","last_unstale":"2023-09-30T09:48:04.004424+0000","last_undegraded":"2023-09-30T09:48:04.004424+0000","last_fullsized":"2023-09-30T09:48:04.004424+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:33.943313+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":84,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.006515+0000","last_change":"2023-09-30T09:43:45.120923+0000","last_active":"2023-09-30T09:48:04.006515+0000","last_peered":"2023-09-30T09:48:04.006515+0000","last_clean":"2023-09-30T09:48:04.006515+0000","last_became_active":"2023-09-30T09:43:45.120651+0000","last_became_peered":"2023-09-30T09:43:45.120651+0000","last_unstale":"2023-09-30T09:48:04.006515+0000","last_undegraded":"2023-09-30T09:48:04.006515+0000","last_fullsized":"2023-09-30T09:48:04.006515+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:37.307199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":478,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.013938+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:48:04.013938+0000","last_peered":"2023-09-30T09:48:04.013938+0000","last_clean":"2023-09-30T09:48:04.013938+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:48:04.013938+0000","last_undegraded":"2023-09-30T09:48:04.013938+0000","last_fullsized":"2023-09-30T09:48:04.013938+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":897,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.000553+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:48:04.000553+0000","last_peered":"2023-09-30T09:48:04.000553+0000","last_clean":"2023-09-30T09:48:04.000553+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:48:04.000553+0000","last_undegraded":"2023-09-30T09:48:04.000553+0000","last_fullsized":"2023-09-30T09:48:04.000553+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":39,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:04.020094+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:48:04.020094+0000","last_peered":"2023-09-30T09:48:04.020094+0000","last_clean":"2023-09-30T09:48:04.020094+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:48:04.020094+0000","last_undegraded":"2023-09-30T09:48:04.020094+0000","last_fullsized":"2023-09-30T09:48:04.020094+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":542,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-30T09:48:03.990327+0000","last_change":"2023-09-30T09:43:02.982448+0000","last_active":"2023-09-30T09:48:03.990327+0000","last_peered":"2023-09-30T09:48:03.990327+0000","last_clean":"2023-09-30T09:48:03.990327+0000","last_became_active":"2023-09-30T09:43:02.974640+0000","last_became_peered":"2023-09-30T09:43:02.974640+0000","last_unstale":"2023-09-30T09:48:03.990327+0000","last_undegraded":"2023-09-30T09:48:03.990327+0000","last_fullsized":"2023-09-30T09:48:03.990327+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:13.017080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":22130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193486,"num_write_kb":6191489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6114,"num_bytes_recovered":355729408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65081344,"data_stored":65077272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114708,"ondisk_log_size":114708,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234438,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":575196,"kb_used_data":12308,"kb_used_omap":0,"kb_used_meta":562880,"kb_avail":93169956,"statfs":{"total":95995035648,"available":95406034944,"internally_reserved":0,"allocated":12603392,"data_stored":10851108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":576389120},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:48:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0419999999999998,"5min":7.1319999999999997,"15min":5.226},"min":{"1min":0.29999999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.7629999999999999,"5min":124.74299999999999,"15min":215.416},"last":0.28299999999999997},{"interface":"front","average":{"1min":2.9470000000000001,"5min":7.0510000000000002,"15min":5.1500000000000004},"min":{"1min":0.34999999999999998,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":10.130000000000001,"5min":124.69499999999999,"15min":215.041},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:48:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":7.8150000000000004,"15min":4.133},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25},"max":{"1min":10.749000000000001,"5min":157.578,"15min":157.578},"last":0.32300000000000001},{"interface":"front","average":{"1min":3.4239999999999999,"5min":7.8079999999999998,"15min":4.2489999999999997},"min":{"1min":0.38600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":12.013999999999999,"5min":157.589,"15min":157.589},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":3.9289999999999998,"5min":3.1749999999999998,"15min":3.762},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.24299999999999999},"max":{"1min":10.173999999999999,"5min":104.88500000000001,"15min":117.14},"last":0.38700000000000001},{"interface":"front","average":{"1min":3.5270000000000001,"5min":3.1680000000000001,"15min":3.7149999999999999},"min":{"1min":0.436,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":16.135000000000002,"5min":104.867,"15min":117.044},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":7.8529999999999998,"5min":8.1449999999999996,"15min":8.6750000000000007},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":97.037000000000006,"5min":154.06700000000001,"15min":277.57799999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":8.9039999999999999,"5min":8.4440000000000008,"15min":8.9420000000000002},"min":{"1min":0.42999999999999999,"5min":0.36499999999999999,"15min":0.32200000000000001},"max":{"1min":95.135000000000005,"5min":154.24799999999999,"15min":278.69299999999998},"last":0.51100000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.3779999999999999,"15min":1.6839999999999999},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":7.6260000000000003,"5min":25.192,"15min":175.72},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.2290000000000001,"15min":1.5389999999999999},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.252},"max":{"1min":3.3100000000000001,"5min":24.521000000000001,"15min":179.048},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:47:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":7.6619999999999999,"15min":10.548999999999999},"min":{"1min":0.32700000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.266,"5min":197.75700000000001,"15min":197.75700000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.5780000000000001,"5min":7.79,"15min":10.481},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.7089999999999996,"5min":197.63300000000001,"15min":197.63300000000001},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.046,"15min":2.2589999999999999},"min":{"1min":0.41499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5830000000000002,"5min":7.5830000000000002,"15min":206.244},"last":0.441},{"interface":"front","average":{"1min":1.5009999999999999,"5min":1.05,"15min":2.2189999999999999},"min":{"1min":0.46700000000000003,"5min":0.31900000000000001,"15min":0.29199999999999998},"max":{"1min":3.9420000000000002,"5min":5.8520000000000003,"15min":202.78800000000001},"last":0.52900000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496076,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":529732,"kb_used_data":6076,"kb_used_omap":0,"kb_used_meta":523648,"kb_avail":93215420,"statfs":{"total":95995035648,"available":95452590080,"internally_reserved":0,"allocated":6221824,"data_stored":4492833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":536215552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:48:14 2023","interfaces":[{"interface":"back","average":{"1min":4.3129999999999997,"5min":4.351,"15min":5.5750000000000002},"min":{"1min":0.26300000000000001,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":40.536000000000001,"5min":95.043999999999997,"15min":187.19800000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":4.7699999999999996,"5min":4.4409999999999998,"15min":5.5789999999999997},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":40.317999999999998,"5min":94.855999999999995,"15min":187.46700000000001},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":1.8300000000000001,"15min":2.0139999999999998},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":13.494999999999999,"5min":32.814,"15min":69.569999999999993},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.6480000000000001,"5min":1.9710000000000001,"15min":2.2120000000000002},"min":{"1min":0.34999999999999998,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":8.5869999999999997,"5min":32.856000000000002,"15min":70.471999999999994},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":3.859,"15min":4.069},"min":{"1min":0.28899999999999998,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":11.432,"5min":207.02500000000001,"15min":207.02500000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.234,"5min":3.758,"15min":4.1609999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":11.585000000000001,"5min":207.114,"15min":207.114},"last":0.376}]},{"osd":3,"last update":"Sat Sep 30 09:48:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":3.577,"15min":3.8999999999999999},"min":{"1min":0.35299999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.5039999999999996,"5min":107.11799999999999,"15min":212.44800000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.2010000000000001,"5min":3.7309999999999999,"15min":4.1710000000000003},"min":{"1min":0.33800000000000002,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":6.5640000000000001,"5min":106.982,"15min":210.84299999999999},"last":0.439}]},{"osd":4,"last update":"Sat Sep 30 09:48:14 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":1.875,"15min":1.3640000000000001},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":11.499000000000001,"5min":63.798000000000002,"15min":63.798000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.768,"15min":1.258},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":11.064,"5min":63.133000000000003,"15min":63.133000000000003},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":1.825,"15min":1.974},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.367000000000001,"5min":14.974,"15min":14.974},"last":0.308},{"interface":"front","average":{"1min":2.1739999999999999,"5min":2.1880000000000002,"15min":2.4809999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.669,"5min":16.443000000000001,"15min":16.443000000000001},"last":0.41799999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.2090000000000001,"15min":1.633},"min":{"1min":0.378,"5min":0.28599999999999998,"15min":0.26500000000000001},"max":{"1min":7.1639999999999997,"5min":11.276999999999999,"15min":138.191},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.3519999999999999,"5min":1.27,"15min":1.528},"min":{"1min":0.36899999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.521000000000001,"5min":11.521000000000001,"15min":140.33500000000001},"last":0.67300000000000004}]}]},{"osd":1,"up_from":301,"seq":1292785156273,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":738684,"kb_used_data":71092,"kb_used_omap":9,"kb_used_meta":667574,"kb_avail":93006468,"statfs":{"total":95995035648,"available":95238623232,"internally_reserved":0,"allocated":72798208,"data_stored":69810141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":683595794},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":5.2460000000000004,"5min":4.9950000000000001,"15min":3.9119999999999999},"min":{"1min":0.30099999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":48.680999999999997,"5min":142.08500000000001,"15min":142.08500000000001},"last":5.2670000000000003},{"interface":"front","average":{"1min":5.8159999999999998,"5min":5.0279999999999996,"15min":3.8100000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":48.715000000000003,"5min":142.137,"15min":142.137},"last":5.2910000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":7.1529999999999996,"5min":3.573,"15min":2.8639999999999999},"min":{"1min":0.38800000000000001,"5min":0.184,"15min":0.184},"max":{"1min":64.759,"5min":64.759,"15min":64.759},"last":4.1639999999999997},{"interface":"front","average":{"1min":6.633,"5min":3.6659999999999999,"15min":3.0619999999999998},"min":{"1min":0.29199999999999998,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":62.954000000000001,"5min":68.373999999999995,"15min":68.373999999999995},"last":5.3259999999999996}]},{"osd":3,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":8.0359999999999996,"5min":7.2190000000000003,"15min":5.9960000000000004},"min":{"1min":0.254,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":48.732999999999997,"5min":199.41,"15min":199.41},"last":5.5990000000000002},{"interface":"front","average":{"1min":5.6820000000000004,"5min":6.7380000000000004,"15min":5.851},"min":{"1min":0.375,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":48.576999999999998,"5min":199.58099999999999,"15min":199.58099999999999},"last":4.2439999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":5.306,"5min":2.0190000000000001,"15min":2.3519999999999999},"min":{"1min":0.38900000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":48.604999999999997,"5min":48.604999999999997,"15min":142.77099999999999},"last":5.173},{"interface":"front","average":{"1min":2.919,"5min":1.9550000000000001,"15min":2.5219999999999998},"min":{"1min":0.39300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.5990000000000002,"5min":8.6319999999999997,"15min":144.898},"last":8.5690000000000008}]},{"osd":5,"last update":"Sat Sep 30 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":2.206,"15min":2.024},"min":{"1min":0.41099999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":8.6790000000000003,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":5.6429999999999998},{"interface":"front","average":{"1min":2.198,"5min":1.54,"15min":1.3280000000000001},"min":{"1min":0.39100000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.891,"5min":15.250999999999999,"15min":15.250999999999999},"last":5.2089999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":2.319,"5min":1.871,"15min":4.5830000000000002},"min":{"1min":0.442,"5min":0.316,"15min":0.29899999999999999},"max":{"1min":7.2729999999999997,"5min":30.986999999999998,"15min":209.221},"last":5.2359999999999998},{"interface":"front","average":{"1min":2.2589999999999999,"5min":1.7829999999999999,"15min":4.5119999999999996},"min":{"1min":0.45400000000000001,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":10.266999999999999,"5min":28.710999999999999,"15min":208.43600000000001},"last":5.1210000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.579,"15min":3.2440000000000002},"min":{"1min":0.40400000000000003,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":5.8600000000000003,"5min":9.3879999999999999,"15min":165.59399999999999},"last":4.1239999999999997},{"interface":"front","average":{"1min":2.238,"5min":1.6140000000000001,"15min":3.2970000000000002},"min":{"1min":0.42399999999999999,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":5.8029999999999999,"5min":7.5460000000000003,"15min":165.833},"last":4.1970000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926172,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670536,"kb_used_data":11836,"kb_used_omap":5,"kb_used_meta":658682,"kb_avail":93074616,"statfs":{"total":95995035648,"available":95308406784,"internally_reserved":0,"allocated":12120064,"data_stored":8902316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":674490621},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":1.984,"15min":2.085},"min":{"1min":0.22700000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":13.308999999999999,"5min":57.482999999999997,"15min":57.482999999999997},"last":0.248},{"interface":"front","average":{"1min":3.9420000000000002,"5min":2.9260000000000002,"15min":3.6549999999999998},"min":{"1min":0.26000000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.554,"5min":57.140999999999998,"15min":138.92400000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":6.0810000000000004,"5min":4.0350000000000001,"15min":3.6200000000000001},"min":{"1min":0.376,"5min":0.23799999999999999,"15min":0.20799999999999999},"max":{"1min":19.504000000000001,"5min":55.137,"15min":127.73399999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":5.7590000000000003,"5min":4.3019999999999996,"15min":3.8860000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":19.042999999999999,"5min":56.963999999999999,"15min":127.69},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":5.5099999999999998,"5min":5.9870000000000001,"15min":5.4249999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":16.594000000000001,"5min":125.562,"15min":218.06999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":3.0680000000000001,"5min":4.984,"15min":4.2599999999999998},"min":{"1min":0.39600000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":13.361000000000001,"5min":117.295,"15min":218.63200000000001},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.7989999999999999,"15min":2.012},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.8710000000000004,"5min":72.403999999999996,"15min":72.403999999999996},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1960000000000002,"15min":2.8260000000000001},"min":{"1min":0.38600000000000001,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":9.4870000000000001,"5min":72.162999999999997,"15min":136.262},"last":0.64100000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:47:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2570000000000001,"5min":2.0950000000000002,"15min":1.413},"min":{"1min":0.35299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":13.162000000000001,"5min":13.162000000000001,"15min":13.162000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.2150000000000001,"15min":1.1040000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":13.422000000000001,"5min":13.422000000000001,"15min":13.422000000000001},"last":0.625}]},{"osd":6,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.958,"15min":3.508},"min":{"1min":0.38200000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":20.706,"5min":57.384,"15min":214.785},"last":0.57799999999999996},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.3700000000000001,"15min":3.8370000000000002},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.684000000000001,"5min":57.109000000000002,"15min":214.93899999999999},"last":0.61099999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.8120000000000001,"15min":2.0720000000000001},"min":{"1min":0.42499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.6549999999999994,"5min":57.776000000000003,"15min":97.382000000000005},"last":19.103000000000002},{"interface":"front","average":{"1min":2.6349999999999998,"5min":2.363,"15min":2.3940000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":9.7149999999999999,"5min":57.179000000000002,"15min":97.015000000000001},"last":19.187000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599113,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":828452,"kb_used_data":30360,"kb_used_omap":5,"kb_used_meta":798074,"kb_avail":92916700,"statfs":{"total":95995035648,"available":95146700800,"internally_reserved":0,"allocated":31088640,"data_stored":28132483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":817228429},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":11.547000000000001,"5min":7.6280000000000001,"15min":4.0439999999999996},"min":{"1min":0.189,"5min":0.189,"15min":0.187},"max":{"1min":202.40899999999999,"5min":202.40899999999999,"15min":202.40899999999999},"last":0.23999999999999999},{"interface":"front","average":{"1min":11.834,"5min":7.7530000000000001,"15min":4.1269999999999998},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":202.38800000000001,"5min":202.38800000000001,"15min":202.38800000000001},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":12.548999999999999,"5min":7.056,"15min":4.391},"min":{"1min":0.30299999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":202.369,"5min":202.369,"15min":202.369},"last":99.638999999999996},{"interface":"front","average":{"1min":12.036,"5min":7.2880000000000003,"15min":4.2309999999999999},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":202.54499999999999,"5min":202.54499999999999,"15min":202.54499999999999},"last":102.131}]},{"osd":3,"last update":"Sat Sep 30 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":12.154,"5min":7.2050000000000001,"15min":5.8620000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":202.59,"5min":202.59,"15min":202.59},"last":0.31},{"interface":"front","average":{"1min":12.242000000000001,"5min":7.29,"15min":5.7210000000000001},"min":{"1min":0.29699999999999999,"5min":0.224,"15min":0.224},"max":{"1min":202.34200000000001,"5min":202.34200000000001,"15min":202.34200000000001},"last":0.36599999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":22.321000000000002,"5min":6.1150000000000002,"15min":3.3149999999999999},"min":{"1min":0.39900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":247.655,"5min":247.655,"15min":247.655},"last":0.45200000000000001},{"interface":"front","average":{"1min":22.981000000000002,"5min":6.327,"15min":3.3330000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":248.012,"5min":248.012,"15min":248.012},"last":0.497}]},{"osd":5,"last update":"Sat Sep 30 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":11.856,"5min":4.0570000000000004,"15min":2.069},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":243.86500000000001,"5min":243.86500000000001,"15min":243.86500000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":12.890000000000001,"5min":4.3949999999999996,"15min":2.3420000000000001},"min":{"1min":0.44,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":244.02199999999999,"5min":244.02199999999999,"15min":244.02199999999999},"last":0.60999999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":11.365,"5min":3.5649999999999999,"15min":3.5110000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":202.70599999999999,"5min":202.70599999999999,"15min":202.70599999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":11.484,"5min":3.8839999999999999,"15min":3.9399999999999999},"min":{"1min":0.438,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":202.916,"5min":202.916,"15min":202.916},"last":0.56899999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":29.030000000000001,"5min":7.7949999999999999,"15min":4.2309999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":244.911,"5min":244.911,"15min":244.911},"last":0.55500000000000005},{"interface":"front","average":{"1min":28.515000000000001,"5min":7.8369999999999997,"15min":4.5640000000000001},"min":{"1min":0.42399999999999999,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":244.66,"5min":244.66,"15min":244.66},"last":0.51500000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435586,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":890312,"kb_used_data":22076,"kb_used_omap":2,"kb_used_meta":868157,"kb_avail":92854840,"statfs":{"total":95995035648,"available":95083356160,"internally_reserved":0,"allocated":22605824,"data_stored":19556001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":888993127},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":5.0759999999999996,"5min":5.2880000000000003,"15min":4.9829999999999997},"min":{"1min":0.22700000000000001,"5min":0.183,"15min":0.183},"max":{"1min":45.957999999999998,"5min":144.72,"15min":205.73099999999999},"last":9.8390000000000004},{"interface":"front","average":{"1min":4.8479999999999999,"5min":5.2919999999999998,"15min":4.9100000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":44.298999999999999,"5min":142.36600000000001,"15min":205.495},"last":16.347999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":4.0839999999999996,"15min":6.2290000000000001},"min":{"1min":0.27600000000000002,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":14.055,"5min":142.434,"15min":273.99599999999998},"last":15.784000000000001},{"interface":"front","average":{"1min":3.4329999999999998,"5min":4.3170000000000002,"15min":6.0910000000000002},"min":{"1min":0.38300000000000001,"5min":0.28199999999999997,"15min":0.20699999999999999},"max":{"1min":12.470000000000001,"5min":142.44900000000001,"15min":274.21600000000001},"last":0.311}]},{"osd":2,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":3.8239999999999998,"5min":5.3940000000000001,"15min":5.7999999999999998},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":12.561999999999999,"5min":146.92599999999999,"15min":206.69},"last":0.746},{"interface":"front","average":{"1min":3.5979999999999999,"5min":4.9619999999999997,"15min":5.5789999999999997},"min":{"1min":0.38900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.834,"5min":145.75299999999999,"15min":209.22800000000001},"last":9.7759999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.3180000000000001,"15min":2.0920000000000001},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.2919999999999998,"5min":81.677000000000007,"15min":81.677000000000007},"last":9.8719999999999999},{"interface":"front","average":{"1min":3.0499999999999998,"5min":2.6360000000000001,"15min":2.04},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":18.943000000000001,"5min":80.623000000000005,"15min":80.623000000000005},"last":15.750999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:47:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5609999999999999,"5min":1.9390000000000001,"15min":1.6639999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.194000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":2.5430000000000001,"5min":1.536,"15min":1.1459999999999999},"min":{"1min":0.33700000000000002,"5min":0.311,"15min":0.311},"max":{"1min":8.0399999999999991,"5min":8.0399999999999991,"15min":8.0399999999999991},"last":15.715}]},{"osd":6,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":1.498,"15min":1.9790000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33200000000000002},"max":{"1min":10.837999999999999,"5min":10.837999999999999,"15min":60.698},"last":9.9009999999999998},{"interface":"front","average":{"1min":2.4369999999999998,"5min":1.659,"15min":1.9650000000000001},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.33400000000000002},"max":{"1min":8.2050000000000001,"5min":10.356999999999999,"15min":12.121},"last":15.666}]},{"osd":7,"last update":"Sat Sep 30 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":3.488,"15min":2.7120000000000002},"min":{"1min":0.42499999999999999,"5min":0.36399999999999999,"15min":0.35299999999999998},"max":{"1min":11.753,"5min":213.548,"15min":213.548},"last":5.7409999999999997},{"interface":"front","average":{"1min":3.2559999999999998,"5min":3.3290000000000002,"15min":2.605},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.888999999999999,"5min":213.583,"15min":213.583},"last":9.9290000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019349,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":502152,"kb_used_data":11456,"kb_used_omap":0,"kb_used_meta":490688,"kb_avail":93243000,"statfs":{"total":95995035648,"available":95480832000,"internally_reserved":0,"allocated":11730944,"data_stored":9997972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":502464512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:48:00 2023","interfaces":[{"interface":"back","average":{"1min":4.4560000000000004,"5min":2.734,"15min":2.8119999999999998},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":53.241999999999997,"5min":53.241999999999997,"15min":105.179},"last":107.43000000000001},{"interface":"front","average":{"1min":5.5339999999999998,"5min":2.9620000000000002,"15min":3.0659999999999998},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":53.340000000000003,"5min":53.340000000000003,"15min":104.70999999999999},"last":107.373}]},{"osd":1,"last update":"Sat Sep 30 09:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4660000000000002,"5min":2.8079999999999998,"15min":3.4009999999999998},"min":{"1min":0.28000000000000003,"5min":0.25,"15min":0.25},"max":{"1min":7.7619999999999996,"5min":80.712000000000003,"15min":160.69999999999999},"last":108.551},{"interface":"front","average":{"1min":2.7829999999999999,"5min":3.2559999999999998,"15min":3.4420000000000002},"min":{"1min":0.41399999999999998,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":8.0150000000000006,"5min":80.810000000000002,"15min":160.50700000000001},"last":108.884}]},{"osd":2,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":3.8900000000000001,"15min":3.5699999999999998},"min":{"1min":0.40999999999999998,"5min":0.33600000000000002,"15min":0.27300000000000002},"max":{"1min":7.415,"5min":117.858,"15min":142.88999999999999},"last":0.501},{"interface":"front","average":{"1min":2.7989999999999999,"5min":4.1420000000000003,"15min":3.6429999999999998},"min":{"1min":0.42899999999999999,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":10.252000000000001,"5min":117.871,"15min":142.90700000000001},"last":0.435}]},{"osd":3,"last update":"Sat Sep 30 09:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7200000000000002,"5min":2.3599999999999999,"15min":2.2669999999999999},"min":{"1min":0.503,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":7.6159999999999997,"5min":29.41,"15min":52.762},"last":105.142},{"interface":"front","average":{"1min":2.157,"5min":2.2200000000000002,"15min":2.2469999999999999},"min":{"1min":0.41299999999999998,"5min":0.36699999999999999,"15min":0.29299999999999998},"max":{"1min":6.6210000000000004,"5min":29.401,"15min":52.790999999999997},"last":105.157}]},{"osd":5,"last update":"Sat Sep 30 09:47:26 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":7.0880000000000001,"15min":9.5180000000000007},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.1470000000000002,"5min":140.11099999999999,"15min":140.11099999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.2889999999999999,"5min":7.0179999999999998,"15min":9.516},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.8789999999999996,"5min":140.065,"15min":140.065},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:47:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":0.84799999999999998,"15min":2.1850000000000001},"min":{"1min":0.34499999999999997,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":9.1470000000000002,"5min":9.1470000000000002,"15min":202.72499999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.026,"5min":0.78300000000000003,"15min":2.1720000000000002},"min":{"1min":0.311,"5min":0.253,"15min":0.249},"max":{"1min":5.2539999999999996,"5min":7.173,"15min":202.42400000000001},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.6440000000000001,"15min":2.395},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.4740000000000002,"5min":161.72999999999999,"15min":161.72999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.619,"5min":2.7000000000000002,"15min":2.298},"min":{"1min":0.41799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.5919999999999996,"5min":162.60900000000001,"15min":162.60900000000001},"last":0.56799999999999995}]}]},{"osd":5,"up_from":489,"seq":2100239007791,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":172164,"kb_used_data":6012,"kb_used_omap":6,"kb_used_meta":166137,"kb_avail":93572988,"statfs":{"total":95995035648,"available":95818739712,"internally_reserved":0,"allocated":6156288,"data_stored":4423938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":170124383},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":6.7590000000000003,"5min":17.806999999999999,"15min":23.475000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":89.805999999999997,"5min":281.10599999999999,"15min":281.10599999999999},"last":2.2349999999999999},{"interface":"front","average":{"1min":6.4459999999999997,"5min":18.018000000000001,"15min":23.738},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":88.313999999999993,"5min":282.07400000000001,"15min":282.07400000000001},"last":2.052}]},{"osd":1,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":6.7160000000000002,"5min":11.051,"15min":13.630000000000001},"min":{"1min":0.31900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":76.828999999999994,"5min":273.48200000000003,"15min":273.48200000000003},"last":0.83399999999999996},{"interface":"front","average":{"1min":5.3710000000000004,"5min":10.403,"15min":12.871},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":76.805000000000007,"5min":273.50200000000001,"15min":273.50200000000001},"last":0.74199999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":7.258,"5min":4.3230000000000004,"15min":2.0680000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":95.641000000000005,"5min":163.23099999999999,"15min":163.23099999999999},"last":2.0150000000000001},{"interface":"front","average":{"1min":6.7030000000000003,"5min":4.6449999999999996,"15min":2.5750000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":95.484999999999999,"5min":162.71100000000001,"15min":162.71100000000001},"last":0.86399999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":7.5179999999999998,"5min":10.975,"15min":13.491},"min":{"1min":0.45900000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":79.421999999999997,"5min":274.01799999999997,"15min":274.01799999999997},"last":0.88700000000000001},{"interface":"front","average":{"1min":7.2709999999999999,"5min":3.1469999999999998,"15min":2.2029999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":79.459000000000003,"5min":79.459000000000003,"15min":79.459000000000003},"last":0.78100000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1,"15min":0.96799999999999997},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.46,"5min":5.46,"15min":5.46},"last":2.077},{"interface":"front","average":{"1min":1.554,"5min":1.25,"15min":1.1799999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.8239999999999998,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":1.823}]},{"osd":6,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.516,"15min":1.6020000000000001},"min":{"1min":0.33000000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7569999999999997,"5min":14.371,"15min":14.371},"last":6.5789999999999997},{"interface":"front","average":{"1min":1.778,"5min":1.403,"15min":1.3939999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.8979999999999997,"5min":14.454000000000001,"15min":14.454000000000001},"last":2.101}]},{"osd":7,"last update":"Sat Sep 30 09:47:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.397,"15min":1.3280000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.7560000000000002,"5min":9.7560000000000002,"15min":9.7560000000000002},"last":1.556},{"interface":"front","average":{"1min":1.744,"5min":1.069,"15min":0.96699999999999997},"min":{"1min":0.34899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.8529999999999998,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":2.1259999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":39063552,"data_stored":39059480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":23527424,"data_stored":23527424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2490368,"data_stored":2490368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:48:17.530 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:48:17.530 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 71 2023-09-30T09:48:17.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:17.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:17.633 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:17.659 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:17.701 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:17.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:17.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:17.750 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:17.809 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:17.834 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:17.876 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:17.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:17.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:17.981 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:18.014 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:18.127 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:18.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:18.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:18.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:18.146 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:18.232 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:18.259 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:18.301 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:18.305 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:18.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:18.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:18.407 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:18.438 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:18.585 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:18.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:18.690 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:18.717 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:18.759 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:18.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:18.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:18.867 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:18.893 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:18.935 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:18.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:19.042 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:19.065 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:19.067 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:19.069 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:19.073 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:48:19.111 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:19.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:19.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:19.216 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:19.242 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:19.283 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:19.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:19 smithi019 ceph-mon[216815]: pgmap v1407: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-09-30T09:48:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2268956807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:19 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:19 smithi019 ceph-mon[224043]: pgmap v1407: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-09-30T09:48:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2268956807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:19 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:19.391 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:19.420 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:19 smithi167 ceph-mon[192880]: pgmap v1407: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-09-30T09:48:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2268956807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:19 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:19.749 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:19.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:19.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:19.853 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:19.886 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:20.035 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:20.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:20.139 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:20.169 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:20.262 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:20.262 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:20.262 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[216815]: failed to encode map e528 with expected crc 2023-09-30T09:48:20.263 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[216815]: failed to encode map e528 with expected crc 2023-09-30T09:48:20.263 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[216815]: osdmap e528: 8 total, 8 up, 8 in 2023-09-30T09:48:20.263 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:20.263 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:20.263 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[224043]: failed to encode map e528 with expected crc 2023-09-30T09:48:20.263 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[224043]: failed to encode map e528 with expected crc 2023-09-30T09:48:20.263 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:20 smithi019 ceph-mon[224043]: osdmap e528: 8 total, 8 up, 8 in 2023-09-30T09:48:20.496 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:48:20.519 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:48:20.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:20 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:20 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:20 smithi167 ceph-mon[192880]: failed to encode map e528 with expected crc 2023-09-30T09:48:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:20 smithi167 ceph-mon[192880]: failed to encode map e528 with expected crc 2023-09-30T09:48:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:20 smithi167 ceph-mon[192880]: osdmap e528: 8 total, 8 up, 8 in 2023-09-30T09:48:20.539 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:48:20.559 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:48:20.579 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:48:20.599 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:48:20.618 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:48:20.638 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:48:20.639 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:48:20.639 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:48:20.639 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:48:20.639 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:48:20.639 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:48:20.640 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:48:20.640 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:48:20.640 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:48:21.321 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:21.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:21 smithi019 ceph-mon[224043]: pgmap v1409: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-09-30T09:48:21.323 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:21 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1965409734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:21.323 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:21 smithi019 ceph-mon[216815]: pgmap v1409: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-09-30T09:48:21.324 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:21 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1965409734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:21.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:21.425 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:21.455 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:21 smithi167 ceph-mon[192880]: pgmap v1409: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-09-30T09:48:21.455 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:21 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1965409734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:21.458 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:21.508 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:21.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:21.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:21.616 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:21.647 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:21.893 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:21.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:21.998 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:22.030 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:22.082 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:22.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:22.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:22.185 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:22.214 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:22.258 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:22.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:22.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:22.313 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:48:22.363 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:22.392 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:22.574 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:22.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:22.679 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:22.707 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:22.751 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:22.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:22.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:22.858 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:22.894 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:22.941 INFO:teuthology.orchestra.run.smithi019.stderr:Error EINVAL: specified pg_num 71 < pg_num_max 32 2023-09-30T09:48:22.949 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:22.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:22.983 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:48:22.983 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:48:23.054 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:23.081 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:23.122 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:23.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:23.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:23.198 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:23 smithi019 ceph-mon[216815]: pgmap v1410: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 297 op/s 2023-09-30T09:48:23.198 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1426083845' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-30T09:48:23.198 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:23 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-30T09:48:23.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:23 smithi019 ceph-mon[224043]: pgmap v1410: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 297 op/s 2023-09-30T09:48:23.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1426083845' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-30T09:48:23.199 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:23 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-30T09:48:23.228 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:23.257 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:23 smithi167 ceph-mon[192880]: pgmap v1410: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 297 op/s 2023-09-30T09:48:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1426083845' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-30T09:48:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:23 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-30T09:48:24.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:48:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:48:23] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2023-09-30T09:48:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:48:24.121 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-30T09:48:24.122 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-30T09:48:24.134 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:24.158 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:24.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:24.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:24.164 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:48:24.265 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:24.298 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:24.317 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:48:24.347 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:24.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:24.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:24.454 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:24.493 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:24.853 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:24.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:24.959 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:24.988 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:25.032 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:25.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:25.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:25.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:24 smithi019 ceph-mon[216815]: pgmap v1411: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 252 op/s 2023-09-30T09:48:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:24 smithi019 ceph-mon[224043]: pgmap v1411: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 252 op/s 2023-09-30T09:48:25.136 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:25.165 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:24 smithi167 ceph-mon[192880]: pgmap v1411: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 252 op/s 2023-09-30T09:48:25.168 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:25.217 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:25.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:25.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:25.324 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:25.353 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:25.387 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:25.396 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:25.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:25.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:25.436 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:25.500 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:25.503 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:25.530 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:25.571 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:25.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:25.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:25.679 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:25.712 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:25.820 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:25.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:25.923 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:25.954 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:26.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:25 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:48:26.008 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:25 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:48:26.101 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:26.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:26.207 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:26.237 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:26.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:25 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:48:27.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[216815]: pgmap v1412: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1426057611' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:48:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[224043]: pgmap v1412: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:27.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1426057611' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:48:27.164 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:48:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:27 smithi167 ceph-mon[192880]: pgmap v1412: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:27 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:27.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1426057611' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:48:27.645 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:27.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:27.749 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:27.858 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:28.104 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:28.108 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:28 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:28.108 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:28 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1426057611' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:48:28.108 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:28 smithi167 ceph-mon[192880]: failed to encode map e529 with expected crc 2023-09-30T09:48:28.109 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:28 smithi167 ceph-mon[192880]: osdmap e529: 8 total, 8 up, 8 in 2023-09-30T09:48:28.109 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:28 smithi167 ceph-mon[192880]: failed to encode map e529 with expected crc 2023-09-30T09:48:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:28.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:28.210 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:28.238 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:28.238 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1426057611' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:48:28.239 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[216815]: failed to encode map e529 with expected crc 2023-09-30T09:48:28.239 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[216815]: osdmap e529: 8 total, 8 up, 8 in 2023-09-30T09:48:28.239 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[216815]: failed to encode map e529 with expected crc 2023-09-30T09:48:28.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:28.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1426057611' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:48:28.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[224043]: failed to encode map e529 with expected crc 2023-09-30T09:48:28.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[224043]: osdmap e529: 8 total, 8 up, 8 in 2023-09-30T09:48:28.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:28 smithi019 ceph-mon[224043]: failed to encode map e529 with expected crc 2023-09-30T09:48:28.241 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:28.571 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:28.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:28.674 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:28.701 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:28.745 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:28.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:28.851 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:28.878 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:28.975 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:28.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:28.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:29.079 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:29.104 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:29.122 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:48:29.122 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:48:29.123 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [1, 4, 5, 2, 7, 0, 3, 6] 2023-09-30T09:48:29.123 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-30T09:48:29.248 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:29.254 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:29 smithi167 ceph-mon[192880]: failed to encode map e529 with expected crc 2023-09-30T09:48:29.254 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:29 smithi167 ceph-mon[192880]: pgmap v1414: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:29.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:29.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:29 smithi019 ceph-mon[216815]: failed to encode map e529 with expected crc 2023-09-30T09:48:29.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:29 smithi019 ceph-mon[216815]: pgmap v1414: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:29 smithi019 ceph-mon[224043]: failed to encode map e529 with expected crc 2023-09-30T09:48:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:29 smithi019 ceph-mon[224043]: pgmap v1414: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:29.356 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:29.381 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:29.425 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:29.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:29.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:29.534 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:29.552 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:48:29.552 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:48:29.561 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:29.608 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:29.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:29.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:29.716 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:29.758 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:29.887 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:29.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:29.991 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:30.018 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:30.368 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_4' removed 2023-09-30T09:48:30.683 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:30 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.683 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:30 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.683 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/215054293' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:48:30.683 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:30 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:48:30.683 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:30 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.683 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:30 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/215054293' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:48:30.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:48:30.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.732 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.732 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.733 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/215054293' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:48:30.733 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:48:30.733 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:30.733 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:30 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:31.078 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:48:31.447 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:31.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[216815]: pgmap v1415: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:31.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:48:31.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[216815]: failed to encode map e530 with expected crc 2023-09-30T09:48:31.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[216815]: osdmap e530: 8 total, 8 up, 8 in 2023-09-30T09:48:31.449 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[216815]: failed to encode map e530 with expected crc 2023-09-30T09:48:31.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:31.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:31.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[224043]: pgmap v1415: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:31.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:48:31.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[224043]: failed to encode map e530 with expected crc 2023-09-30T09:48:31.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[224043]: osdmap e530: 8 total, 8 up, 8 in 2023-09-30T09:48:31.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:31 smithi019 ceph-mon[224043]: failed to encode map e530 with expected crc 2023-09-30T09:48:31.503 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:31.507 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:48:31.528 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:48:31.541 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:48:31.552 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:31.566 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:48:31.582 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:31.589 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:48:31.616 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:48:31.636 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:48:31.655 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:48:31.659 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:31 smithi167 ceph-mon[192880]: pgmap v1415: 119 pgs: 119 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:31.660 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:48:31.660 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:31 smithi167 ceph-mon[192880]: failed to encode map e530 with expected crc 2023-09-30T09:48:31.660 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:31 smithi167 ceph-mon[192880]: osdmap e530: 8 total, 8 up, 8 in 2023-09-30T09:48:31.660 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:31 smithi167 ceph-mon[192880]: failed to encode map e530 with expected crc 2023-09-30T09:48:31.676 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:31.681 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:48:31.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:31.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:31.684 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:48:31.684 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:48:31.684 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:48:31.684 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:48:31.684 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:48:31.684 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:48:31.684 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:48:31.684 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:48:31.754 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-30T09:48:31.755 INFO:tasks.radosbench:Beginning radosbench... 2023-09-30T09:48:31.755 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-30T09:48:31.756 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-30T09:48:31.783 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:31.814 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:31.911 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:31.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:32.015 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:32.056 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:32.663 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:32.791 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:32 smithi167 ceph-mon[192880]: failed to encode map e530 with expected crc 2023-09-30T09:48:32.792 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3481620646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:32.792 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:32 smithi019 ceph-mon[216815]: failed to encode map e530 with expected crc 2023-09-30T09:48:32.792 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3481620646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:32.964 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:32 smithi019 ceph-mon[224043]: failed to encode map e530 with expected crc 2023-09-30T09:48:32.964 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3481620646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:32.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:32.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:33.066 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:33.123 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:33.191 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:33.605 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:33 smithi019 ceph-mon[224043]: pgmap v1417: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:33.605 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:33.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:33 smithi019 ceph-mon[216815]: pgmap v1417: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:33.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:33.630 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:33.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:33.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:33.735 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:33.766 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:33 smithi167 ceph-mon[192880]: pgmap v1417: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:33.766 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:33.769 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:33.822 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:33.866 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:33.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:33.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:33.948 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:48:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:48:33] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2023-09-30T09:48:33.976 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:34.004 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:34.048 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:34.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:34.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:34.153 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:34.192 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:34.378 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:34.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:34 smithi019 conmon[216791]: 2023-09-30T09:48:34.439+0000 7faf52971700 -1 mon.a@0(leader).osd e530 definitely_dead 0 2023-09-30T09:48:34.597 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:34.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:34.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:34.702 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:34.737 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:34.786 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:34.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:34.891 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:34.892 INFO:teuthology.orchestra.run.smithi019.stderr:marked out osd.6. 2023-09-30T09:48:34.926 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:34 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:34 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:34 smithi167 ceph-mon[192880]: pgmap v1418: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1507037752' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-30T09:48:35.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:34 smithi167 ceph-mon[192880]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-30T09:48:35.125 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:48:35.152 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:35.152 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:35.152 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[224043]: pgmap v1418: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:35.152 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1507037752' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-30T09:48:35.152 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[224043]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-30T09:48:35.152 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:35.152 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:35.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[216815]: pgmap v1418: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-30T09:48:35.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1507037752' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-30T09:48:35.153 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:34 smithi019 ceph-mon[216815]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-30T09:48:35.574 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:35.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:35.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:35.678 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:35.710 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:35.760 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:35.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:35.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:35.865 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:35.905 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:36.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1507037752' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-30T09:48:36.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:35 smithi167 ceph-mon[192880]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:35 smithi167 ceph-mon[192880]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:35 smithi167 ceph-mon[192880]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:35 smithi167 ceph-mon[192880]: osdmap e531: 8 total, 8 up, 7 in 2023-09-30T09:48:36.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:35 smithi167 ceph-mon[192880]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:35 smithi167 ceph-mon[192880]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.199 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1507037752' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-30T09:48:36.199 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[216815]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[216815]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[216815]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[216815]: osdmap e531: 8 total, 8 up, 7 in 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[216815]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[216815]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1507037752' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[224043]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[224043]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[224043]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.200 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[224043]: osdmap e531: 8 total, 8 up, 7 in 2023-09-30T09:48:36.201 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[224043]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.201 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:35 smithi019 ceph-mon[224043]: failed to encode map e531 with expected crc 2023-09-30T09:48:36.861 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:36.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:36.965 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:36.991 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:37.036 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:37.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:37.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:37.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: osdmap e532: 8 total, 8 up, 7 in 2023-09-30T09:48:37.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: pgmap v1421: 103 pgs: 103 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:37.120 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[216815]: osdmap e533: 8 total, 8 up, 7 in 2023-09-30T09:48:37.120 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.120 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.120 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.120 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: osdmap e532: 8 total, 8 up, 7 in 2023-09-30T09:48:37.120 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: pgmap v1421: 103 pgs: 103 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:37.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:36 smithi019 ceph-mon[224043]: osdmap e533: 8 total, 8 up, 7 in 2023-09-30T09:48:37.143 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:37.177 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: osdmap e532: 8 total, 8 up, 7 in 2023-09-30T09:48:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: failed to encode map e532 with expected crc 2023-09-30T09:48:37.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: pgmap v1421: 103 pgs: 103 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:37.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:36 smithi167 ceph-mon[192880]: osdmap e533: 8 total, 8 up, 7 in 2023-09-30T09:48:38.088 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:54238 8.c 8:366ec1be:::benchmark_data_smithi019_478679_object54237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 592.84065062800005, 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0706041239999999, 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.089 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 54238 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639824+0000", 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4300000000000001e-07 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639843+0000", 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8371000000000001e-05 2023-09-30T09:48:38.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639856+0000", 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3366999999999999e-05 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639862+0000", 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.161e-06 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639930+0000", 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7676999999999996e-05 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:44.639945+0000", 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5138e-05 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:45.597093+0000", 2023-09-30T09:48:38.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95714748900000002 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:45.597106+0000", 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3617e-05 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676568+0000", 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0794613529999999 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.094 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676752+0000", 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018476000000000001 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.676942+0000", 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018961800000000001 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708135+0000", 2023-09-30T09:48:38.095 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031193262999999999 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708162+0000", 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6557999999999999e-05 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708218+0000", 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5918000000000002e-05 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.096 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.708303+0000", 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5107999999999999e-05 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:38:46.710428+0000", 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021247829999999999 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76883 8.16 8:6fedded8:::benchmark_data_smithi019_478679_object76882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e379)", 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 562.839378592, 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.937896308, 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76883 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.098 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641096+0000", 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9400000000000005e-07 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641122+0000", 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5469e-05 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641129+0000", 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7079999999999997e-06 2023-09-30T09:48:38.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.641135+0000", 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8420000000000001e-06 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.735104+0000", 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.093968634999999995 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.101 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:14.735114+0000", 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0591000000000001e-05 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.623444+0000", 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.888329232 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.623456+0000", 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2207e-05 2023-09-30T09:48:38.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.570797+0000", 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94734084100000004 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.570930+0000", 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013319100000000001 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:48:38.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.571052+0000", 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012170700000000001 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.574539+0000", 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034873180000000001 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576645+0000", 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021061999999999999 2023-09-30T09:48:38.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578718+0000", 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020728700000000001 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578793+0000", 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4670000000000002e-05 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578992+0000", 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019923299999999999 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76913 8.16 8:6fac3ea3:::benchmark_data_smithi019_478679_object76912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 561.847955082, 2023-09-30T09:48:38.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9468952980000001, 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76913 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632519+0000", 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632521+0000", 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.545e-06 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632547+0000", 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6449999999999999e-05 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632566+0000", 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8515000000000001e-05 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632576+0000", 2023-09-30T09:48:38.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0199e-05 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632655+0000", 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9171999999999993e-05 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.632669+0000", 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4055e-05 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624641+0000", 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99197187499999995 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624650+0000", 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9570000000000008e-06 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572117+0000", 2023-09-30T09:48:38.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94746728000000002 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572215+0000", 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8041000000000005e-05 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572312+0000", 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6371000000000003e-05 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576694+0000", 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043828060000000004 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576706+0000", 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1773999999999999e-05 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579137+0000", 2023-09-30T09:48:38.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024311419999999999 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579213+0000", 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5665000000000001e-05 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.579414+0000", 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020145100000000001 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.114 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:76925 8.16 8:6e562ee8:::benchmark_data_smithi019_478679_object76924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 561.82382757200003, 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9248296740000002, 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.115 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 76925 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656647+0000", 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656648+0000", 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.443e-06 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656673+0000", 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4562e-05 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.117 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656689+0000", 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5948999999999999e-05 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656697+0000", 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9789999999999993e-06 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656753+0000", 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6739e-05 2023-09-30T09:48:38.118 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:15.656760+0000", 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4710000000000004e-06 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624673+0000", 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96791319200000003 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:16.624681+0000", 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9060000000000002e-06 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.572984+0000", 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94830274000000003 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573098+0000", 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000114694 2023-09-30T09:48:38.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.573214+0000", 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115318 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.576712+0000", 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003498143 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.578659+0000", 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001947308 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581148+0000", 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002488873 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581230+0000", 2023-09-30T09:48:38.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2106000000000003e-05 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:39:17.581476+0000", 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024625100000000001 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:126957 8.11 8:8c2c7c44:::benchmark_data_smithi019_478679_object34509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:48:38.123 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 409.56582265200001, 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8430575170000001, 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 126957 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.124 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914653+0000", 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2550000000000001e-06 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914659+0000", 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5400000000000001e-06 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914676+0000", 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6543999999999999e-05 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914685+0000", 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5109999999999992e-06 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914770+0000", 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4579000000000005e-05 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914933+0000", 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016312100000000001 2023-09-30T09:48:38.127 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915080+0000", 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000146699 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.917700+0000", 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026206659999999998 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.919771+0000", 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002070985 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757615+0000", 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.837843213 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757648+0000", 2023-09-30T09:48:38.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3256000000000001e-05 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757709+0000", 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1148000000000005e-05 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136440 8.19 8:9e052cf0:::benchmark_data_smithi019_478679_object43992:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:48:38.130 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 376.05196950599998, 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8141369350000001, 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136440 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.131 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6399999999999998e-07 2023-09-30T09:48:38.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428507+0000", 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0959999999999999e-06 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428513+0000", 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1639999999999998e-06 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428519+0000", 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.541e-06 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429278+0000", 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00075965099999999997 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429285+0000", 2023-09-30T09:48:38.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.618e-06 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.195903+0000", 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7666181060000001 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.196239+0000", 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033550600000000001 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.196875+0000", 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000636519 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.199780+0000", 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002904883 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208312+0000", 2023-09-30T09:48:38.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0085319590000000004 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242090+0000", 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.033777612999999998 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242261+0000", 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000171507 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242642+0000", 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00038040799999999999 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136457 8.19 8:9a6b3ff1:::benchmark_data_smithi019_478679_object44009:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 376.051948622, 2023-09-30T09:48:38.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.814734099, 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136457 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.139 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9700000000000002e-07 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428528+0000", 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.905e-06 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428552+0000", 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.366e-05 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428558+0000", 2023-09-30T09:48:38.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1160000000000004e-06 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429324+0000", 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00076669300000000004 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429333+0000", 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1120000000000002e-06 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.199884+0000", 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7705505289999999 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.200283+0000", 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00039902299999999998 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.200894+0000", 2023-09-30T09:48:38.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00061094000000000003 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208335+0000", 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0074407869999999999 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208590+0000", 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025573000000000002 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.144 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242752+0000", 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.034161527999999997 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242903+0000", 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000150569 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243260+0000", 2023-09-30T09:48:38.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035711 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136497 8.19 8:9ab7e4b6:::benchmark_data_smithi019_478679_object44049:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 376.05190731599998, 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.815323341, 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.146 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136497 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3499999999999996e-07 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428570+0000", 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.548e-06 2023-09-30T09:48:38.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428577+0000", 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3159999999999999e-06 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428583+0000", 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.592e-06 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429357+0000", 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00077461499999999998 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429365+0000", 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4390000000000003e-06 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.202754+0000", 2023-09-30T09:48:38.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7733887419999999 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.203035+0000", 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00028132800000000001 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.203627+0000", 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00059232499999999999 2023-09-30T09:48:38.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208358+0000", 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004730845 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.211841+0000", 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034829769999999999 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243353+0000", 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031512327 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243486+0000", 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013215899999999999 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243890+0000", 2023-09-30T09:48:38.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00040459300000000002 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153596 8.6 8:63ca80c0:::benchmark_data_smithi019_478679_object61148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 335.47502355199998, 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2178367450000001, 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.154 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153596 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005452+0000", 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-06 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005458+0000", 2023-09-30T09:48:38.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0240000000000003e-06 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005476+0000", 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7638999999999998e-05 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005484+0000", 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6649999999999992e-06 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005563+0000", 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8406999999999997e-05 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005575+0000", 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2533e-05 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961360+0000", 2023-09-30T09:48:38.158 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.955784462 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961374+0000", 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4756999999999999e-05 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197062+0000", 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2356875270000001 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197198+0000", 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013569999999999999 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197626+0000", 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00042856900000000001 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.202382+0000", 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0047554479999999998 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.222890+0000", 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020507803000000002 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.222938+0000", 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8897999999999998e-05 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223015+0000", 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6330999999999998e-05 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223287+0000", 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027260099999999997 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.162 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153617 8.6 8:671a25b8:::benchmark_data_smithi019_478679_object61169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 335.40113226900002, 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1478323110000002, 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153617 2023-09-30T09:48:38.163 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079343+0000", 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0410000000000001e-06 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079347+0000", 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8829999999999999e-06 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079359+0000", 2023-09-30T09:48:38.165 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1719000000000001e-05 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079364+0000", 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8889999999999999e-06 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079445+0000", 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0644999999999998e-05 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.166 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079460+0000", 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5247e-05 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961403+0000", 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.88194313400000002 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.167 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961413+0000", 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9450000000000005e-06 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.199966+0000", 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2385522090000001 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.200059+0000", 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3688000000000002e-05 2023-09-30T09:48:38.168 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.200467+0000", 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000408093 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223502+0000", 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.023034289999999999 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.169 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223531+0000", 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9584e-05 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.226902+0000", 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033704429999999999 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.226956+0000", 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4129000000000003e-05 2023-09-30T09:48:38.170 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.227174+0000", 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021837199999999999 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2292 9.8 9:1a1fa048:::benchmark_data_smithi019_560486_object2291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:48:38.171 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 252.43849343100001, 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.046112114, 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2292 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.172 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041982+0000", 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.375e-06 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.173 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042038+0000", 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6124e-05 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042052+0000", 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3366e-05 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.174 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042061+0000", 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9039999999999998e-06 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.043012+0000", 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00095105899999999998 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.043037+0000", 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5621000000000001e-05 2023-09-30T09:48:38.175 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046269+0000", 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.003231505 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046280+0000", 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1532000000000001e-05 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.176 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080018+0000", 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0337373249999999 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080180+0000", 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000162268 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080377+0000", 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000197295 2023-09-30T09:48:38.177 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080973+0000", 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000596292 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081788+0000", 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00081490800000000004 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.178 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087843+0000", 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0060543369999999999 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087903+0000", 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9904000000000001e-05 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.088093+0000", 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000190299 2023-09-30T09:48:38.179 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2304 9.5 9:a61d12ad:::benchmark_data_smithi019_560486_object2303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 252.426117428, 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0366920450000001, 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.180 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2304 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.181 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054358+0000", 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.497e-06 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054413+0000", 2023-09-30T09:48:38.182 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.486e-05 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054425+0000", 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.221e-05 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054436+0000", 2023-09-30T09:48:38.183 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0822000000000001e-05 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.055599+0000", 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001163182 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.055607+0000", 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2219999999999996e-06 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.184 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.039370+0000", 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98376339800000001 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.039387+0000", 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6816999999999998e-05 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081036+0000", 2023-09-30T09:48:38.185 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0416488429999999 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081162+0000", 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012657 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081327+0000", 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000165043 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.186 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081830+0000", 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00050259000000000002 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082960+0000", 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001130147 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090727+0000", 2023-09-30T09:48:38.187 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077666380000000002 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090794+0000", 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7787999999999995e-05 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.091049+0000", 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025441800000000002 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.188 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2309 9.8 9:1c660290:::benchmark_data_smithi019_560486_object2308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 252.40694912199999, 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0171991079999998, 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:38.189 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2309 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.190 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073526+0000", 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-06 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073572+0000", 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5779999999999999e-05 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.191 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073585+0000", 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2870999999999999e-05 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073592+0000", 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5150000000000003e-06 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073661+0000", 2023-09-30T09:48:38.192 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9664000000000003e-05 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073675+0000", 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-05 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046310+0000", 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.972634836 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.193 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046319+0000", 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.329e-06 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081008+0000", 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.034688308 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081125+0000", 2023-09-30T09:48:38.194 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011733199999999999 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081251+0000", 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012646599999999999 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081666+0000", 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00041515399999999999 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.195 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082565+0000", 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00089839100000000001 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090410+0000", 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.007844976 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.196 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090481+0000", 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1354999999999997e-05 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090724+0000", 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024294 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.197 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2312 9.9 9:939ceb45:::benchmark_data_smithi019_560486_object2311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 252.40096139299999, 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0117532649999998, 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2312 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.198 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079514+0000", 2023-09-30T09:48:38.199 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1200000000000002e-07 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079550+0000", 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6514000000000003e-05 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079562+0000", 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2275e-05 2023-09-30T09:48:38.200 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079568+0000", 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0109999999999997e-06 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079619+0000", 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0250999999999997e-05 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.201 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079631+0000", 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2756e-05 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043823+0000", 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96419190499999996 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043838+0000", 2023-09-30T09:48:38.202 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4323e-05 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080176+0000", 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.036337939 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080316+0000", 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014040999999999999 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.203 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080506+0000", 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019016399999999999 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081229+0000", 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00072260300000000005 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082898+0000", 2023-09-30T09:48:38.204 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001669366 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090790+0000", 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0078920009999999992 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090859+0000", 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9210999999999999e-05 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.205 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.091266+0000", 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00040672400000000002 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPushReply(3.10 532/531 [PushReplyOp(3:0d82a743:::600.00000000:head)])", 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:35.903915+0000", 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 1.5765589630000001, 2023-09-30T09:48:38.206 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000299451, 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.903915+0000", 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.903915+0000", 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.207 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.903916+0000", 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3200000000000004e-07 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.903922+0000", 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5899999999999998e-06 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.208 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.903933+0000", 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1059e-05 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.903942+0000", 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.251e-06 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.903999+0000", 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7266999999999999e-05 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.209 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.904215+0000", 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021555200000000001 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPushReply(3.10 532/531 [PushReplyOp(3:09401a81:::601.00000000:head)])", 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:35.905171+0000", 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 1.5753031420000001, 2023-09-30T09:48:38.210 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035369900000000002, 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.905171+0000", 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.905171+0000", 2023-09-30T09:48:38.211 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.905172+0000", 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3300000000000002e-07 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.905177+0000", 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9180000000000002e-06 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.212 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.905205+0000", 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8501999999999999e-05 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.905214+0000", 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0280000000000006e-06 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.905288+0000", 2023-09-30T09:48:38.213 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4239999999999994e-05 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.905525+0000", 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00023637800000000001 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPushReply(3.10 532/531 [PushReplyOp(3:0bd6d154:::602.00000000:head)])", 2023-09-30T09:48:38.214 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:35.906492+0000", 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 1.5739818400000001, 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00036445899999999997, 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.906492+0000", 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.215 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.906492+0000", 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.906493+0000", 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.99e-07 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.906498+0000", 2023-09-30T09:48:38.216 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4439999999999999e-06 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.906512+0000", 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4116e-05 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.906521+0000", 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7229999999999993e-06 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.217 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.906581+0000", 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9694999999999997e-05 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.906857+0000", 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027588200000000003 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.218 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPushReply(3.10 532/531 [PushReplyOp(3:0d89b25e:::603.00000000:head)])", 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:35.909537+0000", 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 1.5709369949999998, 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00032412800000000002, 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.909537+0000", 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.219 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.909537+0000", 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.909538+0000", 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6399999999999998e-07 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.909542+0000", 2023-09-30T09:48:38.220 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0790000000000002e-06 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.909550+0000", 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8470000000000002e-06 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.909560+0000", 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9499999999999996e-06 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.221 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.909628+0000", 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8077999999999994e-05 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.909861+0000", 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00023280999999999999 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.222 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(2.2 532/531 [PushOp(2:5cea7035:::rbd_info:head, version: 72'2, data_included: [0~19], data_size: 19, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(2:5cea7035:::rbd_info:head@72'2, size: 19, copy_subset: [(0, 19)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 19, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:35.920288+0000", 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 1.5601862369999999, 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00053260199999999997, 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920288+0000", 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.223 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920288+0000", 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920289+0000", 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8999999999999996e-07 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.224 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920305+0000", 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6631000000000001e-05 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920329+0000", 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3926000000000001e-05 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920347+0000", 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8238999999999999e-05 2023-09-30T09:48:38.225 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920460+0000", 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000113037 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920475+0000", 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4669e-05 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:48:38.226 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:48:38.227 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:35.920821+0000", 2023-09-30T09:48:38.227 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00034550999999999999 2023-09-30T09:48:38.227 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.227 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.227 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.227 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:48:38.227 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:48:38.227 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:48:38.228 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:37 smithi167 ceph-mon[192880]: osdmap e534: 8 total, 8 up, 7 in 2023-09-30T09:48:38.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[216815]: osdmap e534: 8 total, 8 up, 7 in 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: failed to encode map e533 with expected crc 2023-09-30T09:48:38.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:37 smithi019 ceph-mon[224043]: osdmap e534: 8 total, 8 up, 7 in 2023-09-30T09:48:38.902 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:48:38.912 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.912 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.912 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.912 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.913 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: failed to encode map e534 with expected crc 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: pgmap v1424: 103 pgs: 103 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/338755244' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/338755244' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:38.914 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[224043]: osdmap e535: 8 total, 8 up, 7 in 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: failed to encode map e534 with expected crc 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: failed to encode map e534 with expected crc 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: failed to encode map e534 with expected crc 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: failed to encode map e534 with expected crc 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: failed to encode map e534 with expected crc 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: pgmap v1424: 103 pgs: 103 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/338755244' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:48:39.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:48:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/338755244' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:39.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:38 smithi167 ceph-mon[192880]: osdmap e535: 8 total, 8 up, 7 in 2023-09-30T09:48:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: failed to encode map e534 with expected crc 2023-09-30T09:48:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: pgmap v1424: 103 pgs: 103 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/338755244' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:48:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:48:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:48:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/338755244' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:48:39.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:38 smithi019 ceph-mon[216815]: osdmap e535: 8 total, 8 up, 7 in 2023-09-30T09:48:39.515 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:39.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:39.619 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:39.646 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:39.688 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:39.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:39.795 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:39.827 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:39.988 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_5' created 2023-09-30T09:48:40.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1316976297' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1316976297' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-30T09:48:40.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: failed to encode map e535 with expected crc 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1316976297' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-30T09:48:40.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-30T09:48:40.375 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:48:41.180 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:41.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: 1.3 scrub starts 2023-09-30T09:48:41.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: 1.3 scrub ok 2023-09-30T09:48:41.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: pgmap v1426: 103 pgs: 4 peering, 99 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 102 KiB/s, 16 objects/s recovering 2023-09-30T09:48:41.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-30T09:48:41.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: osdmap e536: 8 total, 8 up, 7 in 2023-09-30T09:48:41.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: 1.f scrub starts 2023-09-30T09:48:41.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[224043]: 1.f scrub ok 2023-09-30T09:48:41.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: 1.3 scrub starts 2023-09-30T09:48:41.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: 1.3 scrub ok 2023-09-30T09:48:41.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: pgmap v1426: 103 pgs: 4 peering, 99 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 102 KiB/s, 16 objects/s recovering 2023-09-30T09:48:41.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-30T09:48:41.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: osdmap e536: 8 total, 8 up, 7 in 2023-09-30T09:48:41.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: 1.f scrub starts 2023-09-30T09:48:41.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:40 smithi019 ceph-mon[216815]: 1.f scrub ok 2023-09-30T09:48:41.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:41.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:41.265 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3] out_osds: [6] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:48:41.265 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: 1.3 scrub starts 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: 1.3 scrub ok 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: pgmap v1426: 103 pgs: 4 peering, 99 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 102 KiB/s, 16 objects/s recovering 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: osdmap e536: 8 total, 8 up, 7 in 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: failed to encode map e536 with expected crc 2023-09-30T09:48:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: 1.f scrub starts 2023-09-30T09:48:41.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:40 smithi167 ceph-mon[192880]: 1.f scrub ok 2023-09-30T09:48:41.287 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:41.316 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:41.360 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:41.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:41.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:41.465 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:41.491 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:41.532 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:41.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:41.637 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:41.663 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:41.705 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:41.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:41.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:41.810 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:41.836 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:41.878 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:41.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:41.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:41.985 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:42.011 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:42.053 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:42.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:42.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[216815]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[216815]: osdmap e537: 8 total, 8 up, 7 in 2023-09-30T09:48:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[216815]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[216815]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:42.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[216815]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[224043]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[224043]: osdmap e537: 8 total, 8 up, 7 in 2023-09-30T09:48:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[224043]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[224043]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:42.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:41 smithi019 ceph-mon[224043]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.160 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:42.190 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:42.228 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-30T09:48:42.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:41 smithi167 ceph-mon[192880]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:41 smithi167 ceph-mon[192880]: osdmap e537: 8 total, 8 up, 7 in 2023-09-30T09:48:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:41 smithi167 ceph-mon[192880]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:41 smithi167 ceph-mon[192880]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:41 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:42.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:41 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:42.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:41 smithi167 ceph-mon[192880]: failed to encode map e537 with expected crc 2023-09-30T09:48:42.295 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:42.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:42.399 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:42.400 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:42.426 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:42.436 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:48:42.459 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:48:42.467 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:42.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:42.480 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:48:42.509 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:48:42.530 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:48:42.552 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:48:42.573 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:42.575 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:48:42.597 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:48:42.600 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:48:42.600 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:48:42.600 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:48:42.600 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:48:42.600 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:48:42.600 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:48:42.601 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:48:42.601 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:48:42.604 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:42.652 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:42.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:42.756 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:42.783 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:42.826 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:42.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:42.931 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:42.961 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:43.161 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: pgmap v1429: 119 pgs: 16 unknown, 4 peering, 99 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 103 KiB/s, 16 objects/s recovering 2023-09-30T09:48:43.161 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: failed to encode map e538 with expected crc 2023-09-30T09:48:43.161 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: failed to encode map e538 with expected crc 2023-09-30T09:48:43.161 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: osdmap e538: 8 total, 8 up, 7 in 2023-09-30T09:48:43.161 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:48:43.161 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3630310886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: pgmap v1429: 119 pgs: 16 unknown, 4 peering, 99 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 103 KiB/s, 16 objects/s recovering 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: failed to encode map e538 with expected crc 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: failed to encode map e538 with expected crc 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: osdmap e538: 8 total, 8 up, 7 in 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:48:43.162 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:48:43.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:48:43.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:43.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3630310886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:43.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: pgmap v1429: 119 pgs: 16 unknown, 4 peering, 99 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 103 KiB/s, 16 objects/s recovering 2023-09-30T09:48:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: failed to encode map e538 with expected crc 2023-09-30T09:48:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: failed to encode map e538 with expected crc 2023-09-30T09:48:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: osdmap e538: 8 total, 8 up, 7 in 2023-09-30T09:48:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-09-30T09:48:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:48:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:48:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:43.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3630310886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:43.829 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:48:43.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:48:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:48:43] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2023-09-30T09:48:44.161 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:44.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:44.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:44.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:43 smithi019 ceph-mon[224043]: failed to encode map e538 with expected crc 2023-09-30T09:48:44.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:43 smithi019 ceph-mon[216815]: failed to encode map e538 with expected crc 2023-09-30T09:48:44.265 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:44.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:43 smithi167 ceph-mon[192880]: failed to encode map e538 with expected crc 2023-09-30T09:48:44.297 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:44.303 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:44.339 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:44.346 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:44.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:44.453 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:44.485 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:44.591 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:44.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:44.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:44.694 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:44.722 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:44.973 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:44.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:44.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:45.077 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:45.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:44 smithi019 ceph-mon[216815]: pgmap v1431: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 4 objects/s recovering 2023-09-30T09:48:45.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:44 smithi019 ceph-mon[224043]: pgmap v1431: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 4 objects/s recovering 2023-09-30T09:48:45.117 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:45.159 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:48:45.159 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:48:45.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:44 smithi167 ceph-mon[192880]: pgmap v1431: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 4 objects/s recovering 2023-09-30T09:48:45.826 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:45.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:45.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:45.922 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:48:45.929 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:45.994 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:46.700 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:46.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:46.806 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:46.833 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:46.988 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:48:47.131 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:47.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:47.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:47.235 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:47.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:46 smithi019 ceph-mon[224043]: pgmap v1432: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:47.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:46 smithi019 ceph-mon[216815]: pgmap v1432: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:47.267 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:47.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:46 smithi167 ceph-mon[192880]: pgmap v1432: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:47.523 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:47.795 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:47.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:47.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:47.877 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:47.901 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:47.932 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:47.952 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2359117235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:47.952 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:47 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:47.967 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:48:47.983 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:47.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:47.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:48.089 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:48.117 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:48.162 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:48.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:48.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2359117235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:48.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:47 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:48.269 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2359117235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:48.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:47 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:48:48.297 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:48.340 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:48.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:48.447 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:48.478 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:49.008 INFO:teuthology.orchestra.run.smithi019.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-30T09:48:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: pgmap v1433: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:48:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: failed to encode map e539 with expected crc 2023-09-30T09:48:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: osdmap e539: 8 total, 8 up, 7 in 2023-09-30T09:48:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:49.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:49.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3870889743' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:48:49.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:48:49.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: pgmap v1433: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: failed to encode map e539 with expected crc 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: osdmap e539: 8 total, 8 up, 7 in 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3870889743' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: pgmap v1433: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:48:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:48:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: failed to encode map e539 with expected crc 2023-09-30T09:48:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: osdmap e539: 8 total, 8 up, 7 in 2023-09-30T09:48:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:48:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3870889743' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:48:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:48:49.579 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:49.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:49.685 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:49.717 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:49.766 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:49.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:49.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:49.873 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:49.901 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:49.945 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:49.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:49.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:50.052 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:50.079 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:50.121 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:50.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:50.190 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[216815]: failed to encode map e539 with expected crc 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[216815]: failed to encode map e540 with expected crc 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[216815]: osdmap e540: 8 total, 8 up, 7 in 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[216815]: failed to encode map e540 with expected crc 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[224043]: failed to encode map e539 with expected crc 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[224043]: failed to encode map e540 with expected crc 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[224043]: osdmap e540: 8 total, 8 up, 7 in 2023-09-30T09:48:50.191 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:50 smithi019 ceph-mon[224043]: failed to encode map e540 with expected crc 2023-09-30T09:48:50.228 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:48:50.254 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:50 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-30T09:48:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:50 smithi167 ceph-mon[192880]: failed to encode map e539 with expected crc 2023-09-30T09:48:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:50 smithi167 ceph-mon[192880]: failed to encode map e540 with expected crc 2023-09-30T09:48:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:50 smithi167 ceph-mon[192880]: osdmap e540: 8 total, 8 up, 7 in 2023-09-30T09:48:50.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:50 smithi167 ceph-mon[192880]: failed to encode map e540 with expected crc 2023-09-30T09:48:50.297 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:48:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:50.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:48:50.401 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:50.431 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:50.865 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:50.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:50.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:50.969 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:50.997 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:51.040 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:51.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:51.139 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:51 smithi019 ceph-mon[216815]: pgmap v1436: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:51.139 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:51 smithi019 ceph-mon[216815]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:48:51.139 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:51 smithi019 ceph-mon[224043]: pgmap v1436: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:51.140 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:51 smithi019 ceph-mon[224043]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:48:51.145 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:51.187 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:51.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:51 smithi167 ceph-mon[192880]: pgmap v1436: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:51.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:51 smithi167 ceph-mon[192880]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-09-30T09:48:51.338 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:51.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:51.402 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:48:51.421 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:48:51.440 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:48:51.552 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:51.554 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:48:51.554 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:48:51.554 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:48:51.554 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:48:51.555 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:48:51.579 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:51.622 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:51.651 DEBUG:teuthology.orchestra.run.smithi019:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-30T09:48:51.651 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-30T09:48:51.652 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:48:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:51.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:51.655 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-30T09:48:51.655 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-30T09:48:51.707 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:hints = 1 2023-09-30T09:48:51.707 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-30T09:48:51.707 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object prefix: benchmark_data_smithi019_617837 2023-09-30T09:48:51.707 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:48:51.708 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 0 0 0 0 0 0 - 0 2023-09-30T09:48:51.754 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:48:51.789 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:51.852 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:48:51.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:51.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:48:51.915 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:48:51.918 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:48:51.918 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:48:51.918 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:48:51.918 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:48:51.918 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:48:51.958 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:51.997 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:52.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2736391029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:52.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2736391029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:52.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2736391029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:48:52.707 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 1 16 2520 2504 156.467 156.5 0.00257377 0.00637419 2023-09-30T09:48:52.961 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:52.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:52.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:53.072 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:53.114 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:53.175 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:53.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:53.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:53 smithi167 ceph-mon[192880]: pgmap v1437: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:53.280 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:53.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:53 smithi019 ceph-mon[216815]: pgmap v1437: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:53.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:53 smithi019 ceph-mon[224043]: pgmap v1437: 119 pgs: 119 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-09-30T09:48:53.308 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:53.710 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 2 16 5346 5330 166.345 176.625 0.00304785 0.0060005 2023-09-30T09:48:53.863 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:53.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:53.969 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:53.994 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:48:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:48:53] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.43.0" 2023-09-30T09:48:54.013 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:48:54.058 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:54.118 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:54.144 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:54.248 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:54.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:54.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:54.358 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:54.419 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:54.545 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:54.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:54.650 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:54.709 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:54.710 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 3 16 7754 7738 161.063 150.5 0.00390153 0.00619389 2023-09-30T09:48:54.819 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:54.819 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:54 smithi019 ceph-mon[216815]: pgmap v1438: 119 pgs: 119 active+clean; 137 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 17 MiB/s wr, 264 op/s 2023-09-30T09:48:54.820 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:54.820 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:54 smithi019 ceph-mon[224043]: pgmap v1438: 119 pgs: 119 active+clean; 137 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 17 MiB/s wr, 264 op/s 2023-09-30T09:48:55.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:48:55.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:54 smithi167 ceph-mon[192880]: pgmap v1438: 119 pgs: 119 active+clean; 137 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 17 MiB/s wr, 264 op/s 2023-09-30T09:48:55.319 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:55.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:55.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:55.424 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:48:55.483 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:55.686 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:48:55.710 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 4 16 10472 10456 163.248 169.875 0.00891067 0.00611674 2023-09-30T09:48:56.251 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:48:56.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:56.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:48:56.357 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:56.413 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:56.710 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 5 16 13001 12985 162.209 158.062 0.00637676 0.00615823 2023-09-30T09:48:57.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:56 smithi167 ceph-mon[192880]: pgmap v1439: 119 pgs: 119 active+clean; 403 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s 2023-09-30T09:48:57.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:56 smithi019 ceph-mon[216815]: pgmap v1439: 119 pgs: 119 active+clean; 403 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s 2023-09-30T09:48:57.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:56 smithi019 ceph-mon[224043]: pgmap v1439: 119 pgs: 119 active+clean; 403 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s 2023-09-30T09:48:57.651 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:57.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:57.711 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 6 15 15770 15755 164.025 173.125 0.0042987 0.00609089 2023-09-30T09:48:57.758 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:57.806 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:58.257 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:48:58.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:58.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:48:58.369 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:48:58.450 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:58.569 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:58.596 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:48:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:58.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:48:58.663 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:48:58.705 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:48:58.710 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 7 16 17801 17785 158.718 126.875 0.00351194 0.00629389 2023-09-30T09:48:58.750 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:58.816 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:48:58.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:58.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:48:58.924 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:48:58.969 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:59.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:58 smithi167 ceph-mon[192880]: pgmap v1440: 119 pgs: 119 active+clean; 403 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s 2023-09-30T09:48:59.045 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:48:59.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:59.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:48:59.150 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:48:59.206 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:58 smithi019 ceph-mon[216815]: pgmap v1440: 119 pgs: 119 active+clean; 403 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s 2023-09-30T09:48:59.207 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:58 smithi019 ceph-mon[224043]: pgmap v1440: 119 pgs: 119 active+clean; 403 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s 2023-09-30T09:48:59.221 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:59.444 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:48:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:48:59.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:48:59.556 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:48:59.604 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:48:59.712 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 8 16 19619 19603 153.068 113.625 0.00474382 0.00652752 2023-09-30T09:48:59.933 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:48:59.944 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:48:59.990 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:49:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3177988762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:48:59 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:00.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3177988762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:00.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:48:59 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3177988762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:48:59 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:00.712 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 9 16 21769 21753 150.99 134.375 0.00908047 0.00661766 2023-09-30T09:49:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: from='client.46589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:49:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: pgmap v1441: 119 pgs: 119 active+clean; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-30T09:49:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: osdmap e541: 8 total, 8 up, 7 in 2023-09-30T09:49:01.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:00 smithi167 ceph-mon[192880]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: from='client.46589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:49:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: pgmap v1441: 119 pgs: 119 active+clean; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-30T09:49:01.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: osdmap e541: 8 total, 8 up, 7 in 2023-09-30T09:49:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[216815]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: from='client.46589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:49:01.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: pgmap v1441: 119 pgs: 119 active+clean; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-30T09:49:01.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:01.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:01.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: osdmap e541: 8 total, 8 up, 7 in 2023-09-30T09:49:01.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:00 smithi019 ceph-mon[224043]: failed to encode map e541 with expected crc 2023-09-30T09:49:01.712 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 10 16 24092 24076 150.4 145.188 0.0101768 0.00664308 2023-09-30T09:49:02.348 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:02.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:02.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:02.453 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:49:02.489 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:02.568 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:49:02.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:02.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:49:02.678 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:02.713 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 11 16 26132 26116 148.317 127.5 0.00800281 0.0067365 2023-09-30T09:49:02.742 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:02.997 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:02 smithi019 ceph-mon[216815]: pgmap v1443: 119 pgs: 119 active+clean; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-30T09:49:03.167 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:03.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:03.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:03.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:02 smithi019 ceph-mon[224043]: pgmap v1443: 119 pgs: 119 active+clean; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-30T09:49:03.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:02 smithi167 ceph-mon[192880]: pgmap v1443: 119 pgs: 119 active+clean; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-30T09:49:03.282 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:03.315 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:03.382 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:03.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:03.487 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:03.554 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:03.713 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 12 16 27907 27891 145.202 110.938 0.00203099 0.00685097 2023-09-30T09:49:04.058 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:49:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:49:03] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-09-30T09:49:04.058 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:04.058 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:04.058 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:04.058 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:04.477 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:04.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:04.584 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:49:04.658 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:04.713 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 13 16 29166 29150 140.086 78.6875 0.00204532 0.00712175 2023-09-30T09:49:05.043 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1440,"stamp":"2023-09-30T09:48:57.910706+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":422975399,"num_objects":6409,"num_object_clones":0,"num_object_copies":19227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6409,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":7288,"num_write_kb":421109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":348,"num_bytes_recovered":12937533,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12403,"ondisk_log_size":12403,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5304308,"kb_used_data":1093692,"kb_used_omap":30,"kb_used_meta":4210529,"kb_avail":650911756,"statfs":{"total":671965249536,"available":666533638144,"internally_reserved":0,"allocated":1119940608,"data_stored":1102261503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":4311582050},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,6,2,0,0,0,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":418185216,"num_objects":6381,"num_object_clones":0,"num_object_copies":19143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6381,"num_write_kb":408384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.950206"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":963,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964219+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:48:47.964219+0000","last_peered":"2023-09-30T09:48:47.964219+0000","last_clean":"2023-09-30T09:48:47.964219+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:48:47.964219+0000","last_undegraded":"2023-09-30T09:48:47.964219+0000","last_fullsized":"2023-09-30T09:48:47.964219+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":557,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950007+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:48:41.950007+0000","last_peered":"2023-09-30T09:48:41.950007+0000","last_clean":"2023-09-30T09:48:41.950007+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:48:41.950007+0000","last_undegraded":"2023-09-30T09:48:41.950007+0000","last_fullsized":"2023-09-30T09:48:41.950007+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":454,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950850+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:48:41.950850+0000","last_peered":"2023-09-30T09:48:41.950850+0000","last_clean":"2023-09-30T09:48:41.950850+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:48:41.950850+0000","last_undegraded":"2023-09-30T09:48:41.950850+0000","last_fullsized":"2023-09-30T09:48:41.950850+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":965,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964628+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:48:47.964628+0000","last_peered":"2023-09-30T09:48:47.964628+0000","last_clean":"2023-09-30T09:48:47.964628+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:48:47.964628+0000","last_undegraded":"2023-09-30T09:48:47.964628+0000","last_fullsized":"2023-09-30T09:48:47.964628+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":105,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.148240+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:48:48.148240+0000","last_peered":"2023-09-30T09:48:48.148240+0000","last_clean":"2023-09-30T09:48:48.148240+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:48:48.148240+0000","last_undegraded":"2023-09-30T09:48:48.148240+0000","last_fullsized":"2023-09-30T09:48:48.148240+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":147,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090675+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:48:48.090675+0000","last_peered":"2023-09-30T09:48:48.090675+0000","last_clean":"2023-09-30T09:48:48.090675+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:48:48.090675+0000","last_undegraded":"2023-09-30T09:48:48.090675+0000","last_fullsized":"2023-09-30T09:48:48.090675+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":772,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950098+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:48:41.950098+0000","last_peered":"2023-09-30T09:48:41.950098+0000","last_clean":"2023-09-30T09:48:41.950098+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:48:41.950098+0000","last_undegraded":"2023-09-30T09:48:41.950098+0000","last_fullsized":"2023-09-30T09:48:41.950098+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":46,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090035+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:48:48.090035+0000","last_peered":"2023-09-30T09:48:48.090035+0000","last_clean":"2023-09-30T09:48:48.090035+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:48:48.090035+0000","last_undegraded":"2023-09-30T09:48:48.090035+0000","last_fullsized":"2023-09-30T09:48:48.090035+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":19,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090208+0000","last_change":"2023-09-30T09:48:35.900471+0000","last_active":"2023-09-30T09:48:48.090208+0000","last_peered":"2023-09-30T09:48:48.090208+0000","last_clean":"2023-09-30T09:48:48.090208+0000","last_became_active":"2023-09-30T09:48:35.900122+0000","last_became_peered":"2023-09-30T09:48:35.900122+0000","last_unstale":"2023-09-30T09:48:48.090208+0000","last_undegraded":"2023-09-30T09:48:48.090208+0000","last_fullsized":"2023-09-30T09:48:48.090208+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:00:14.955216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":941,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636778+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:48:48.636778+0000","last_peered":"2023-09-30T09:48:48.636778+0000","last_clean":"2023-09-30T09:48:48.636778+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:48:48.636778+0000","last_undegraded":"2023-09-30T09:48:48.636778+0000","last_fullsized":"2023-09-30T09:48:48.636778+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":161,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090709+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:48:48.090709+0000","last_peered":"2023-09-30T09:48:48.090709+0000","last_clean":"2023-09-30T09:48:48.090709+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:48:48.090709+0000","last_undegraded":"2023-09-30T09:48:48.090709+0000","last_fullsized":"2023-09-30T09:48:48.090709+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":129,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.148116+0000","last_change":"2023-09-30T09:48:35.910904+0000","last_active":"2023-09-30T09:48:48.148116+0000","last_peered":"2023-09-30T09:48:48.148116+0000","last_clean":"2023-09-30T09:48:48.148116+0000","last_became_active":"2023-09-30T09:48:35.910543+0000","last_became_peered":"2023-09-30T09:48:35.910543+0000","last_unstale":"2023-09-30T09:48:48.148116+0000","last_undegraded":"2023-09-30T09:48:48.148116+0000","last_fullsized":"2023-09-30T09:48:48.148116+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:39:43.053423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":730,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.962470+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:48:47.962470+0000","last_peered":"2023-09-30T09:48:47.962470+0000","last_clean":"2023-09-30T09:48:47.962470+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:48:47.962470+0000","last_undegraded":"2023-09-30T09:48:47.962470+0000","last_fullsized":"2023-09-30T09:48:47.962470+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":360,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964560+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:48:47.964560+0000","last_peered":"2023-09-30T09:48:47.964560+0000","last_clean":"2023-09-30T09:48:47.964560+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:48:47.964560+0000","last_undegraded":"2023-09-30T09:48:47.964560+0000","last_fullsized":"2023-09-30T09:48:47.964560+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":506,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636558+0000","last_change":"2023-09-30T09:48:35.905232+0000","last_active":"2023-09-30T09:48:48.636558+0000","last_peered":"2023-09-30T09:48:48.636558+0000","last_clean":"2023-09-30T09:48:48.636558+0000","last_became_active":"2023-09-30T09:48:35.904896+0000","last_became_peered":"2023-09-30T09:48:35.904896+0000","last_unstale":"2023-09-30T09:48:48.636558+0000","last_undegraded":"2023-09-30T09:48:48.636558+0000","last_fullsized":"2023-09-30T09:48:48.636558+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:08:31.889458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":689,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964752+0000","last_change":"2023-09-30T09:48:38.908417+0000","last_active":"2023-09-30T09:48:47.964752+0000","last_peered":"2023-09-30T09:48:47.964752+0000","last_clean":"2023-09-30T09:48:47.964752+0000","last_became_active":"2023-09-30T09:48:38.907973+0000","last_became_peered":"2023-09-30T09:48:38.907973+0000","last_unstale":"2023-09-30T09:48:47.964752+0000","last_undegraded":"2023-09-30T09:48:47.964752+0000","last_fullsized":"2023-09-30T09:48:47.964752+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:43:42.475428+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":254,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636890+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:48:48.636890+0000","last_peered":"2023-09-30T09:48:48.636890+0000","last_clean":"2023-09-30T09:48:48.636890+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:48:48.636890+0000","last_undegraded":"2023-09-30T09:48:48.636890+0000","last_fullsized":"2023-09-30T09:48:48.636890+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":323,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.963395+0000","last_change":"2023-09-30T09:48:35.898670+0000","last_active":"2023-09-30T09:48:47.963395+0000","last_peered":"2023-09-30T09:48:47.963395+0000","last_clean":"2023-09-30T09:48:47.963395+0000","last_became_active":"2023-09-30T09:48:35.898345+0000","last_became_peered":"2023-09-30T09:48:35.898345+0000","last_unstale":"2023-09-30T09:48:47.963395+0000","last_undegraded":"2023-09-30T09:48:47.963395+0000","last_fullsized":"2023-09-30T09:48:47.963395+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:31:10.488807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":596,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090708+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:48:48.090708+0000","last_peered":"2023-09-30T09:48:48.090708+0000","last_clean":"2023-09-30T09:48:48.090708+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:48:48.090708+0000","last_undegraded":"2023-09-30T09:48:48.090708+0000","last_fullsized":"2023-09-30T09:48:48.090708+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":603,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950632+0000","last_change":"2023-09-30T09:48:35.923239+0000","last_active":"2023-09-30T09:48:41.950632+0000","last_peered":"2023-09-30T09:48:41.950632+0000","last_clean":"2023-09-30T09:48:41.950632+0000","last_became_active":"2023-09-30T09:48:35.914812+0000","last_became_peered":"2023-09-30T09:48:35.914812+0000","last_unstale":"2023-09-30T09:48:41.950632+0000","last_undegraded":"2023-09-30T09:48:41.950632+0000","last_fullsized":"2023-09-30T09:48:41.950632+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:58:54.128062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23610,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":571,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950277+0000","last_change":"2023-09-30T09:48:35.919149+0000","last_active":"2023-09-30T09:48:41.950277+0000","last_peered":"2023-09-30T09:48:41.950277+0000","last_clean":"2023-09-30T09:48:41.950277+0000","last_became_active":"2023-09-30T09:48:35.918791+0000","last_became_peered":"2023-09-30T09:48:35.918791+0000","last_unstale":"2023-09-30T09:48:41.950277+0000","last_undegraded":"2023-09-30T09:48:41.950277+0000","last_fullsized":"2023-09-30T09:48:41.950277+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:31.788244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":794,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090133+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:48:48.090133+0000","last_peered":"2023-09-30T09:48:48.090133+0000","last_clean":"2023-09-30T09:48:48.090133+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:48:48.090133+0000","last_undegraded":"2023-09-30T09:48:48.090133+0000","last_fullsized":"2023-09-30T09:48:48.090133+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":190,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636470+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:48:48.636470+0000","last_peered":"2023-09-30T09:48:48.636470+0000","last_clean":"2023-09-30T09:48:48.636470+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:48:48.636470+0000","last_undegraded":"2023-09-30T09:48:48.636470+0000","last_fullsized":"2023-09-30T09:48:48.636470+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":517,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636950+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:48:48.636950+0000","last_peered":"2023-09-30T09:48:48.636950+0000","last_clean":"2023-09-30T09:48:48.636950+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:48:48.636950+0000","last_undegraded":"2023-09-30T09:48:48.636950+0000","last_fullsized":"2023-09-30T09:48:48.636950+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":861,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964000+0000","last_change":"2023-09-30T09:48:38.909653+0000","last_active":"2023-09-30T09:48:47.964000+0000","last_peered":"2023-09-30T09:48:47.964000+0000","last_clean":"2023-09-30T09:48:47.964000+0000","last_became_active":"2023-09-30T09:48:38.909215+0000","last_became_peered":"2023-09-30T09:48:38.909215+0000","last_unstale":"2023-09-30T09:48:47.964000+0000","last_undegraded":"2023-09-30T09:48:47.964000+0000","last_fullsized":"2023-09-30T09:48:47.964000+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:14:44.615680+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":120,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090360+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:48:48.090360+0000","last_peered":"2023-09-30T09:48:48.090360+0000","last_clean":"2023-09-30T09:48:48.090360+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:48:48.090360+0000","last_undegraded":"2023-09-30T09:48:48.090360+0000","last_fullsized":"2023-09-30T09:48:48.090360+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":215,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964349+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:48:47.964349+0000","last_peered":"2023-09-30T09:48:47.964349+0000","last_clean":"2023-09-30T09:48:47.964349+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:48:47.964349+0000","last_undegraded":"2023-09-30T09:48:47.964349+0000","last_fullsized":"2023-09-30T09:48:47.964349+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":942,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.148630+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:48:48.148630+0000","last_peered":"2023-09-30T09:48:48.148630+0000","last_clean":"2023-09-30T09:48:48.148630+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:48:48.148630+0000","last_undegraded":"2023-09-30T09:48:48.148630+0000","last_fullsized":"2023-09-30T09:48:48.148630+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":234,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636567+0000","last_change":"2023-09-30T09:48:35.914208+0000","last_active":"2023-09-30T09:48:48.636567+0000","last_peered":"2023-09-30T09:48:48.636567+0000","last_clean":"2023-09-30T09:48:48.636567+0000","last_became_active":"2023-09-30T09:48:35.914018+0000","last_became_peered":"2023-09-30T09:48:35.914018+0000","last_unstale":"2023-09-30T09:48:48.636567+0000","last_undegraded":"2023-09-30T09:48:48.636567+0000","last_fullsized":"2023-09-30T09:48:48.636567+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:23:01.651345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":120,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090469+0000","last_change":"2023-09-30T09:42:48.643526+0000","last_active":"2023-09-30T09:48:48.090469+0000","last_peered":"2023-09-30T09:48:48.090469+0000","last_clean":"2023-09-30T09:48:48.090469+0000","last_became_active":"2023-09-30T09:42:48.643343+0000","last_became_peered":"2023-09-30T09:42:48.643343+0000","last_unstale":"2023-09-30T09:48:48.090469+0000","last_undegraded":"2023-09-30T09:48:48.090469+0000","last_fullsized":"2023-09-30T09:48:48.090469+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:09.505985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":970,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964690+0000","last_change":"2023-09-30T09:48:38.908387+0000","last_active":"2023-09-30T09:48:47.964690+0000","last_peered":"2023-09-30T09:48:47.964690+0000","last_clean":"2023-09-30T09:48:47.964690+0000","last_became_active":"2023-09-30T09:48:38.907935+0000","last_became_peered":"2023-09-30T09:48:38.907935+0000","last_unstale":"2023-09-30T09:48:47.964690+0000","last_undegraded":"2023-09-30T09:48:47.964690+0000","last_fullsized":"2023-09-30T09:48:47.964690+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:32:04.830035+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":113,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090469+0000","last_change":"2023-09-30T09:48:35.910222+0000","last_active":"2023-09-30T09:48:48.090469+0000","last_peered":"2023-09-30T09:48:48.090469+0000","last_clean":"2023-09-30T09:48:48.090469+0000","last_became_active":"2023-09-30T09:48:35.900106+0000","last_became_peered":"2023-09-30T09:48:35.900106+0000","last_unstale":"2023-09-30T09:48:48.090469+0000","last_undegraded":"2023-09-30T09:48:48.090469+0000","last_fullsized":"2023-09-30T09:48:48.090469+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:23:34.139486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":105,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.148730+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:48:48.148730+0000","last_peered":"2023-09-30T09:48:48.148730+0000","last_clean":"2023-09-30T09:48:48.148730+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:48:48.148730+0000","last_undegraded":"2023-09-30T09:48:48.148730+0000","last_fullsized":"2023-09-30T09:48:48.148730+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"540'438","reported_seq":453,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.396575+0000","last_change":"2023-09-30T09:48:40.945814+0000","last_active":"2023-09-30T09:48:54.396575+0000","last_peered":"2023-09-30T09:48:54.396575+0000","last_clean":"2023-09-30T09:48:54.396575+0000","last_became_active":"2023-09-30T09:48:40.945431+0000","last_became_peered":"2023-09-30T09:48:40.945431+0000","last_unstale":"2023-09-30T09:48:54.396575+0000","last_undegraded":"2023-09-30T09:48:54.396575+0000","last_fullsized":"2023-09-30T09:48:54.396575+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":438,"log_dups_size":0,"ondisk_log_size":438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:36:56.353301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28704768,"num_objects":438,"num_object_clones":0,"num_object_copies":1314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":438,"num_write_kb":28032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":916,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636782+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:48:48.636782+0000","last_peered":"2023-09-30T09:48:48.636782+0000","last_clean":"2023-09-30T09:48:48.636782+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:48:48.636782+0000","last_undegraded":"2023-09-30T09:48:48.636782+0000","last_fullsized":"2023-09-30T09:48:48.636782+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":216,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636906+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:48:48.636906+0000","last_peered":"2023-09-30T09:48:48.636906+0000","last_clean":"2023-09-30T09:48:48.636906+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:48:48.636906+0000","last_undegraded":"2023-09-30T09:48:48.636906+0000","last_fullsized":"2023-09-30T09:48:48.636906+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":171,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.963998+0000","last_change":"2023-09-30T09:48:35.909910+0000","last_active":"2023-09-30T09:48:47.963998+0000","last_peered":"2023-09-30T09:48:47.963998+0000","last_clean":"2023-09-30T09:48:47.963998+0000","last_became_active":"2023-09-30T09:48:35.909573+0000","last_became_peered":"2023-09-30T09:48:35.909573+0000","last_unstale":"2023-09-30T09:48:47.963998+0000","last_undegraded":"2023-09-30T09:48:47.963998+0000","last_fullsized":"2023-09-30T09:48:47.963998+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:54:49.867201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"540'439","reported_seq":451,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.344530+0000","last_change":"2023-09-30T09:48:40.945116+0000","last_active":"2023-09-30T09:48:54.344530+0000","last_peered":"2023-09-30T09:48:54.344530+0000","last_clean":"2023-09-30T09:48:54.344530+0000","last_became_active":"2023-09-30T09:48:40.944827+0000","last_became_peered":"2023-09-30T09:48:40.944827+0000","last_unstale":"2023-09-30T09:48:54.344530+0000","last_undegraded":"2023-09-30T09:48:54.344530+0000","last_fullsized":"2023-09-30T09:48:54.344530+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":439,"log_dups_size":0,"ondisk_log_size":439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:42:25.620829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28770304,"num_objects":439,"num_object_clones":0,"num_object_copies":1317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":439,"num_write_kb":28096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":952,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.148920+0000","last_change":"2023-09-30T09:48:38.911923+0000","last_active":"2023-09-30T09:48:48.148920+0000","last_peered":"2023-09-30T09:48:48.148920+0000","last_clean":"2023-09-30T09:48:48.148920+0000","last_became_active":"2023-09-30T09:48:38.911492+0000","last_became_peered":"2023-09-30T09:48:38.911492+0000","last_unstale":"2023-09-30T09:48:48.148920+0000","last_undegraded":"2023-09-30T09:48:48.148920+0000","last_fullsized":"2023-09-30T09:48:48.148920+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:45.980104+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":186,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636823+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:48:48.636823+0000","last_peered":"2023-09-30T09:48:48.636823+0000","last_clean":"2023-09-30T09:48:48.636823+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:48:48.636823+0000","last_undegraded":"2023-09-30T09:48:48.636823+0000","last_fullsized":"2023-09-30T09:48:48.636823+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":590,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950205+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:48:41.950205+0000","last_peered":"2023-09-30T09:48:41.950205+0000","last_clean":"2023-09-30T09:48:41.950205+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:48:41.950205+0000","last_undegraded":"2023-09-30T09:48:41.950205+0000","last_fullsized":"2023-09-30T09:48:41.950205+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"540'475","reported_seq":490,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.394987+0000","last_change":"2023-09-30T09:48:40.935965+0000","last_active":"2023-09-30T09:48:54.394987+0000","last_peered":"2023-09-30T09:48:54.394987+0000","last_clean":"2023-09-30T09:48:54.394987+0000","last_became_active":"2023-09-30T09:48:40.935590+0000","last_became_peered":"2023-09-30T09:48:40.935590+0000","last_unstale":"2023-09-30T09:48:54.394987+0000","last_undegraded":"2023-09-30T09:48:54.394987+0000","last_fullsized":"2023-09-30T09:48:54.394987+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":475,"log_dups_size":0,"ondisk_log_size":475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:22.798805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31129600,"num_objects":475,"num_object_clones":0,"num_object_copies":1425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":475,"num_write_kb":30400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":603,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964644+0000","last_change":"2023-09-30T09:48:40.131482+0000","last_active":"2023-09-30T09:48:47.964644+0000","last_peered":"2023-09-30T09:48:47.964644+0000","last_clean":"2023-09-30T09:48:47.964644+0000","last_became_active":"2023-09-30T09:48:38.908784+0000","last_became_peered":"2023-09-30T09:48:38.908784+0000","last_unstale":"2023-09-30T09:48:47.964644+0000","last_undegraded":"2023-09-30T09:48:47.964644+0000","last_fullsized":"2023-09-30T09:48:47.964644+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:40.131415+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:40.131415+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:44:13.991726+0000","scrub_duration":0.0018063230000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":267,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636598+0000","last_change":"2023-09-30T09:48:35.922401+0000","last_active":"2023-09-30T09:48:48.636598+0000","last_peered":"2023-09-30T09:48:48.636598+0000","last_clean":"2023-09-30T09:48:48.636598+0000","last_became_active":"2023-09-30T09:48:35.916160+0000","last_became_peered":"2023-09-30T09:48:35.916160+0000","last_unstale":"2023-09-30T09:48:48.636598+0000","last_undegraded":"2023-09-30T09:48:48.636598+0000","last_fullsized":"2023-09-30T09:48:48.636598+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:56:59.795372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":449,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636463+0000","last_change":"2023-09-30T09:48:35.905197+0000","last_active":"2023-09-30T09:48:48.636463+0000","last_peered":"2023-09-30T09:48:48.636463+0000","last_clean":"2023-09-30T09:48:48.636463+0000","last_became_active":"2023-09-30T09:48:35.904749+0000","last_became_peered":"2023-09-30T09:48:35.904749+0000","last_unstale":"2023-09-30T09:48:48.636463+0000","last_undegraded":"2023-09-30T09:48:48.636463+0000","last_fullsized":"2023-09-30T09:48:48.636463+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:37:03.669930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"540'387","reported_seq":400,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.066912+0000","last_change":"2023-09-30T09:48:40.937574+0000","last_active":"2023-09-30T09:48:54.066912+0000","last_peered":"2023-09-30T09:48:54.066912+0000","last_clean":"2023-09-30T09:48:54.066912+0000","last_became_active":"2023-09-30T09:48:40.937127+0000","last_became_peered":"2023-09-30T09:48:40.937127+0000","last_unstale":"2023-09-30T09:48:54.066912+0000","last_undegraded":"2023-09-30T09:48:54.066912+0000","last_fullsized":"2023-09-30T09:48:54.066912+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":387,"log_dups_size":0,"ondisk_log_size":387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:24.051843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25362432,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":387,"num_write_kb":24768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":774,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950658+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:48:41.950658+0000","last_peered":"2023-09-30T09:48:41.950658+0000","last_clean":"2023-09-30T09:48:41.950658+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:48:41.950658+0000","last_undegraded":"2023-09-30T09:48:41.950658+0000","last_fullsized":"2023-09-30T09:48:41.950658+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":485,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964330+0000","last_change":"2023-09-30T09:48:35.897452+0000","last_active":"2023-09-30T09:48:47.964330+0000","last_peered":"2023-09-30T09:48:47.964330+0000","last_clean":"2023-09-30T09:48:47.964330+0000","last_became_active":"2023-09-30T09:48:35.897028+0000","last_became_peered":"2023-09-30T09:48:35.897028+0000","last_unstale":"2023-09-30T09:48:47.964330+0000","last_undegraded":"2023-09-30T09:48:47.964330+0000","last_fullsized":"2023-09-30T09:48:47.964330+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:27:31.911975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090627+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:48:48.090627+0000","last_peered":"2023-09-30T09:48:48.090627+0000","last_clean":"2023-09-30T09:48:48.090627+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:48:48.090627+0000","last_undegraded":"2023-09-30T09:48:48.090627+0000","last_fullsized":"2023-09-30T09:48:48.090627+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"540'381","reported_seq":392,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:53.874500+0000","last_change":"2023-09-30T09:48:40.937693+0000","last_active":"2023-09-30T09:48:53.874500+0000","last_peered":"2023-09-30T09:48:53.874500+0000","last_clean":"2023-09-30T09:48:53.874500+0000","last_became_active":"2023-09-30T09:48:40.937364+0000","last_became_peered":"2023-09-30T09:48:40.937364+0000","last_unstale":"2023-09-30T09:48:53.874500+0000","last_undegraded":"2023-09-30T09:48:53.874500+0000","last_fullsized":"2023-09-30T09:48:53.874500+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":381,"log_dups_size":0,"ondisk_log_size":381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:58.223575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":381,"num_write_kb":24384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1022,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636457+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:48:48.636457+0000","last_peered":"2023-09-30T09:48:48.636457+0000","last_clean":"2023-09-30T09:48:48.636457+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:48:48.636457+0000","last_undegraded":"2023-09-30T09:48:48.636457+0000","last_fullsized":"2023-09-30T09:48:48.636457+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":99,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090429+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:48:48.090429+0000","last_peered":"2023-09-30T09:48:48.090429+0000","last_clean":"2023-09-30T09:48:48.090429+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:48:48.090429+0000","last_undegraded":"2023-09-30T09:48:48.090429+0000","last_fullsized":"2023-09-30T09:48:48.090429+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":547,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950631+0000","last_change":"2023-09-30T09:48:35.913174+0000","last_active":"2023-09-30T09:48:41.950631+0000","last_peered":"2023-09-30T09:48:41.950631+0000","last_clean":"2023-09-30T09:48:41.950631+0000","last_became_active":"2023-09-30T09:48:35.912907+0000","last_became_peered":"2023-09-30T09:48:35.912907+0000","last_unstale":"2023-09-30T09:48:41.950631+0000","last_undegraded":"2023-09-30T09:48:41.950631+0000","last_fullsized":"2023-09-30T09:48:41.950631+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:30:45.904549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"540'422","reported_seq":437,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.033698+0000","last_change":"2023-09-30T09:48:40.945351+0000","last_active":"2023-09-30T09:48:54.033698+0000","last_peered":"2023-09-30T09:48:54.033698+0000","last_clean":"2023-09-30T09:48:54.033698+0000","last_became_active":"2023-09-30T09:48:40.945009+0000","last_became_peered":"2023-09-30T09:48:40.945009+0000","last_unstale":"2023-09-30T09:48:54.033698+0000","last_undegraded":"2023-09-30T09:48:54.033698+0000","last_fullsized":"2023-09-30T09:48:54.033698+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":422,"log_dups_size":0,"ondisk_log_size":422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:06.181942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27656192,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":422,"num_write_kb":27008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":1022,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.962011+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:48:47.962011+0000","last_peered":"2023-09-30T09:48:47.962011+0000","last_clean":"2023-09-30T09:48:47.962011+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:48:47.962011+0000","last_undegraded":"2023-09-30T09:48:47.962011+0000","last_fullsized":"2023-09-30T09:48:47.962011+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":382,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.949745+0000","last_change":"2023-09-30T09:48:35.911039+0000","last_active":"2023-09-30T09:48:41.949745+0000","last_peered":"2023-09-30T09:48:41.949745+0000","last_clean":"2023-09-30T09:48:41.949745+0000","last_became_active":"2023-09-30T09:48:35.909494+0000","last_became_peered":"2023-09-30T09:48:35.909494+0000","last_unstale":"2023-09-30T09:48:41.949745+0000","last_undegraded":"2023-09-30T09:48:41.949745+0000","last_fullsized":"2023-09-30T09:48:41.949745+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:30:33.598204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":233,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950316+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:48:41.950316+0000","last_peered":"2023-09-30T09:48:41.950316+0000","last_clean":"2023-09-30T09:48:41.950316+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:48:41.950316+0000","last_undegraded":"2023-09-30T09:48:41.950316+0000","last_fullsized":"2023-09-30T09:48:41.950316+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"540'350","reported_seq":361,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:53.890972+0000","last_change":"2023-09-30T09:48:40.940900+0000","last_active":"2023-09-30T09:48:53.890972+0000","last_peered":"2023-09-30T09:48:53.890972+0000","last_clean":"2023-09-30T09:48:53.890972+0000","last_became_active":"2023-09-30T09:48:40.940627+0000","last_became_peered":"2023-09-30T09:48:40.940627+0000","last_unstale":"2023-09-30T09:48:53.890972+0000","last_undegraded":"2023-09-30T09:48:53.890972+0000","last_fullsized":"2023-09-30T09:48:53.890972+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":350,"log_dups_size":0,"ondisk_log_size":350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:21:21.782902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22937600,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":350,"num_write_kb":22400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":879,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090690+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:48:48.090690+0000","last_peered":"2023-09-30T09:48:48.090690+0000","last_clean":"2023-09-30T09:48:48.090690+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:48:48.090690+0000","last_undegraded":"2023-09-30T09:48:48.090690+0000","last_fullsized":"2023-09-30T09:48:48.090690+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":544,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950060+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:48:41.950060+0000","last_peered":"2023-09-30T09:48:41.950060+0000","last_clean":"2023-09-30T09:48:41.950060+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:48:41.950060+0000","last_undegraded":"2023-09-30T09:48:41.950060+0000","last_fullsized":"2023-09-30T09:48:41.950060+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":561,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950843+0000","last_change":"2023-09-30T09:48:35.912344+0000","last_active":"2023-09-30T09:48:41.950843+0000","last_peered":"2023-09-30T09:48:41.950843+0000","last_clean":"2023-09-30T09:48:41.950843+0000","last_became_active":"2023-09-30T09:48:35.912031+0000","last_became_peered":"2023-09-30T09:48:35.912031+0000","last_unstale":"2023-09-30T09:48:41.950843+0000","last_undegraded":"2023-09-30T09:48:41.950843+0000","last_fullsized":"2023-09-30T09:48:41.950843+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:30.085847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"540'368","reported_seq":379,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:53.878509+0000","last_change":"2023-09-30T09:48:40.940915+0000","last_active":"2023-09-30T09:48:53.878509+0000","last_peered":"2023-09-30T09:48:53.878509+0000","last_clean":"2023-09-30T09:48:53.878509+0000","last_became_active":"2023-09-30T09:48:40.940606+0000","last_became_peered":"2023-09-30T09:48:40.940606+0000","last_unstale":"2023-09-30T09:48:53.878509+0000","last_undegraded":"2023-09-30T09:48:53.878509+0000","last_fullsized":"2023-09-30T09:48:53.878509+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":368,"log_dups_size":0,"ondisk_log_size":368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:11.151884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24117248,"num_objects":368,"num_object_clones":0,"num_object_copies":1104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":368,"num_write_kb":23552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":758,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.963478+0000","last_change":"2023-09-30T09:48:38.911486+0000","last_active":"2023-09-30T09:48:47.963478+0000","last_peered":"2023-09-30T09:48:47.963478+0000","last_clean":"2023-09-30T09:48:47.963478+0000","last_became_active":"2023-09-30T09:48:38.910968+0000","last_became_peered":"2023-09-30T09:48:38.910968+0000","last_unstale":"2023-09-30T09:48:47.963478+0000","last_undegraded":"2023-09-30T09:48:47.963478+0000","last_fullsized":"2023-09-30T09:48:47.963478+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:04:09.132023+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":105,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.148052+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:48:48.148052+0000","last_peered":"2023-09-30T09:48:48.148052+0000","last_clean":"2023-09-30T09:48:48.148052+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:48:48.148052+0000","last_undegraded":"2023-09-30T09:48:48.148052+0000","last_fullsized":"2023-09-30T09:48:48.148052+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":224,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950532+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:48:41.950532+0000","last_peered":"2023-09-30T09:48:41.950532+0000","last_clean":"2023-09-30T09:48:41.950532+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:48:41.950532+0000","last_undegraded":"2023-09-30T09:48:41.950532+0000","last_fullsized":"2023-09-30T09:48:41.950532+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"540'352","reported_seq":364,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:53.954529+0000","last_change":"2023-09-30T09:48:40.942045+0000","last_active":"2023-09-30T09:48:53.954529+0000","last_peered":"2023-09-30T09:48:53.954529+0000","last_clean":"2023-09-30T09:48:53.954529+0000","last_became_active":"2023-09-30T09:48:40.941786+0000","last_became_peered":"2023-09-30T09:48:40.941786+0000","last_unstale":"2023-09-30T09:48:53.954529+0000","last_undegraded":"2023-09-30T09:48:53.954529+0000","last_fullsized":"2023-09-30T09:48:53.954529+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":352,"log_dups_size":0,"ondisk_log_size":352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:03:11.354244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23068672,"num_objects":352,"num_object_clones":0,"num_object_copies":1056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":352,"num_write_kb":22528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":780,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636723+0000","last_change":"2023-09-30T09:48:38.908129+0000","last_active":"2023-09-30T09:48:48.636723+0000","last_peered":"2023-09-30T09:48:48.636723+0000","last_clean":"2023-09-30T09:48:48.636723+0000","last_became_active":"2023-09-30T09:48:38.907679+0000","last_became_peered":"2023-09-30T09:48:38.907679+0000","last_unstale":"2023-09-30T09:48:48.636723+0000","last_undegraded":"2023-09-30T09:48:48.636723+0000","last_fullsized":"2023-09-30T09:48:48.636723+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:08:43.871037+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":771,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090294+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:48:48.090294+0000","last_peered":"2023-09-30T09:48:48.090294+0000","last_clean":"2023-09-30T09:48:48.090294+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:48:48.090294+0000","last_undegraded":"2023-09-30T09:48:48.090294+0000","last_fullsized":"2023-09-30T09:48:48.090294+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":468,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964426+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:48:47.964426+0000","last_peered":"2023-09-30T09:48:47.964426+0000","last_clean":"2023-09-30T09:48:47.964426+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:48:47.964426+0000","last_undegraded":"2023-09-30T09:48:47.964426+0000","last_fullsized":"2023-09-30T09:48:47.964426+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":157,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964406+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:48:47.964406+0000","last_peered":"2023-09-30T09:48:47.964406+0000","last_clean":"2023-09-30T09:48:47.964406+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:48:47.964406+0000","last_undegraded":"2023-09-30T09:48:47.964406+0000","last_fullsized":"2023-09-30T09:48:47.964406+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"540'448","reported_seq":463,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.387708+0000","last_change":"2023-09-30T09:48:40.945374+0000","last_active":"2023-09-30T09:48:54.387708+0000","last_peered":"2023-09-30T09:48:54.387708+0000","last_clean":"2023-09-30T09:48:54.387708+0000","last_became_active":"2023-09-30T09:48:40.945039+0000","last_became_peered":"2023-09-30T09:48:40.945039+0000","last_unstale":"2023-09-30T09:48:54.387708+0000","last_undegraded":"2023-09-30T09:48:54.387708+0000","last_fullsized":"2023-09-30T09:48:54.387708+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":448,"log_dups_size":0,"ondisk_log_size":448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:08.124785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29360128,"num_objects":448,"num_object_clones":0,"num_object_copies":1344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":448,"num_write_kb":28672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":981,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964217+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:48:47.964217+0000","last_peered":"2023-09-30T09:48:47.964217+0000","last_clean":"2023-09-30T09:48:47.964217+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:48:47.964217+0000","last_undegraded":"2023-09-30T09:48:47.964217+0000","last_fullsized":"2023-09-30T09:48:47.964217+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":32,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090461+0000","last_change":"2023-09-30T09:48:35.925478+0000","last_active":"2023-09-30T09:48:48.090461+0000","last_peered":"2023-09-30T09:48:48.090461+0000","last_clean":"2023-09-30T09:48:48.090461+0000","last_became_active":"2023-09-30T09:48:35.901041+0000","last_became_peered":"2023-09-30T09:48:35.901041+0000","last_unstale":"2023-09-30T09:48:48.090461+0000","last_undegraded":"2023-09-30T09:48:48.090461+0000","last_fullsized":"2023-09-30T09:48:48.090461+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:00:47.891642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":568,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950382+0000","last_change":"2023-09-30T09:48:35.913944+0000","last_active":"2023-09-30T09:48:41.950382+0000","last_peered":"2023-09-30T09:48:41.950382+0000","last_clean":"2023-09-30T09:48:41.950382+0000","last_became_active":"2023-09-30T09:48:35.913292+0000","last_became_peered":"2023-09-30T09:48:35.913292+0000","last_unstale":"2023-09-30T09:48:41.950382+0000","last_undegraded":"2023-09-30T09:48:41.950382+0000","last_fullsized":"2023-09-30T09:48:41.950382+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:48:38.875604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":300,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.962542+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:48:47.962542+0000","last_peered":"2023-09-30T09:48:47.962542+0000","last_clean":"2023-09-30T09:48:47.962542+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:48:47.962542+0000","last_undegraded":"2023-09-30T09:48:47.962542+0000","last_fullsized":"2023-09-30T09:48:47.962542+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"540'416","reported_seq":428,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.319930+0000","last_change":"2023-09-30T09:48:40.944781+0000","last_active":"2023-09-30T09:48:54.319930+0000","last_peered":"2023-09-30T09:48:54.319930+0000","last_clean":"2023-09-30T09:48:54.319930+0000","last_became_active":"2023-09-30T09:48:40.944445+0000","last_became_peered":"2023-09-30T09:48:40.944445+0000","last_unstale":"2023-09-30T09:48:54.319930+0000","last_undegraded":"2023-09-30T09:48:54.319930+0000","last_fullsized":"2023-09-30T09:48:54.319930+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":416,"log_dups_size":0,"ondisk_log_size":416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:26:02.476879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27262976,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":416,"num_write_kb":26624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":959,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636638+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:48:48.636638+0000","last_peered":"2023-09-30T09:48:48.636638+0000","last_clean":"2023-09-30T09:48:48.636638+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:48:48.636638+0000","last_undegraded":"2023-09-30T09:48:48.636638+0000","last_fullsized":"2023-09-30T09:48:48.636638+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":809,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.963150+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:48:47.963150+0000","last_peered":"2023-09-30T09:48:47.963150+0000","last_clean":"2023-09-30T09:48:47.963150+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:48:47.963150+0000","last_undegraded":"2023-09-30T09:48:47.963150+0000","last_fullsized":"2023-09-30T09:48:47.963150+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":311,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.961926+0000","last_change":"2023-09-30T09:48:35.898350+0000","last_active":"2023-09-30T09:48:47.961926+0000","last_peered":"2023-09-30T09:48:47.961926+0000","last_clean":"2023-09-30T09:48:47.961926+0000","last_became_active":"2023-09-30T09:48:35.897864+0000","last_became_peered":"2023-09-30T09:48:35.897864+0000","last_unstale":"2023-09-30T09:48:47.961926+0000","last_undegraded":"2023-09-30T09:48:47.961926+0000","last_fullsized":"2023-09-30T09:48:47.961926+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:53:06.481686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":389,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964909+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:48:47.964909+0000","last_peered":"2023-09-30T09:48:47.964909+0000","last_clean":"2023-09-30T09:48:47.964909+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:48:47.964909+0000","last_undegraded":"2023-09-30T09:48:47.964909+0000","last_fullsized":"2023-09-30T09:48:47.964909+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"540'462","reported_seq":477,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.396699+0000","last_change":"2023-09-30T09:48:40.935961+0000","last_active":"2023-09-30T09:48:54.396699+0000","last_peered":"2023-09-30T09:48:54.396699+0000","last_clean":"2023-09-30T09:48:54.396699+0000","last_became_active":"2023-09-30T09:48:40.935769+0000","last_became_peered":"2023-09-30T09:48:40.935769+0000","last_unstale":"2023-09-30T09:48:54.396699+0000","last_undegraded":"2023-09-30T09:48:54.396699+0000","last_fullsized":"2023-09-30T09:48:54.396699+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":462,"log_dups_size":0,"ondisk_log_size":462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:12:56.433092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30277632,"num_objects":462,"num_object_clones":0,"num_object_copies":1386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":462,"num_write_kb":29568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1101,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964718+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:48:47.964718+0000","last_peered":"2023-09-30T09:48:47.964718+0000","last_clean":"2023-09-30T09:48:47.964718+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:48:47.964718+0000","last_undegraded":"2023-09-30T09:48:47.964718+0000","last_fullsized":"2023-09-30T09:48:47.964718+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":922,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636429+0000","last_change":"2023-09-30T09:48:38.904904+0000","last_active":"2023-09-30T09:48:48.636429+0000","last_peered":"2023-09-30T09:48:48.636429+0000","last_clean":"2023-09-30T09:48:48.636429+0000","last_became_active":"2023-09-30T09:48:38.904561+0000","last_became_peered":"2023-09-30T09:48:38.904561+0000","last_unstale":"2023-09-30T09:48:48.636429+0000","last_undegraded":"2023-09-30T09:48:48.636429+0000","last_fullsized":"2023-09-30T09:48:48.636429+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:30:58.322569+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":126,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964261+0000","last_change":"2023-09-30T09:48:35.916466+0000","last_active":"2023-09-30T09:48:47.964261+0000","last_peered":"2023-09-30T09:48:47.964261+0000","last_clean":"2023-09-30T09:48:47.964261+0000","last_became_active":"2023-09-30T09:48:35.916096+0000","last_became_peered":"2023-09-30T09:48:35.916096+0000","last_unstale":"2023-09-30T09:48:47.964261+0000","last_undegraded":"2023-09-30T09:48:47.964261+0000","last_fullsized":"2023-09-30T09:48:47.964261+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:06:10.322694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":434,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950867+0000","last_change":"2023-09-30T09:48:35.915151+0000","last_active":"2023-09-30T09:48:41.950867+0000","last_peered":"2023-09-30T09:48:41.950867+0000","last_clean":"2023-09-30T09:48:41.950867+0000","last_became_active":"2023-09-30T09:48:35.914780+0000","last_became_peered":"2023-09-30T09:48:35.914780+0000","last_unstale":"2023-09-30T09:48:41.950867+0000","last_undegraded":"2023-09-30T09:48:41.950867+0000","last_fullsized":"2023-09-30T09:48:41.950867+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:54:50.177325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"540'423","reported_seq":438,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:54.388840+0000","last_change":"2023-09-30T09:48:40.941098+0000","last_active":"2023-09-30T09:48:54.388840+0000","last_peered":"2023-09-30T09:48:54.388840+0000","last_clean":"2023-09-30T09:48:54.388840+0000","last_became_active":"2023-09-30T09:48:40.940802+0000","last_became_peered":"2023-09-30T09:48:40.940802+0000","last_unstale":"2023-09-30T09:48:54.388840+0000","last_undegraded":"2023-09-30T09:48:54.388840+0000","last_fullsized":"2023-09-30T09:48:54.388840+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":423,"log_dups_size":0,"ondisk_log_size":423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:33:38.250788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27721728,"num_objects":423,"num_object_clones":0,"num_object_copies":1269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":423,"num_write_kb":27072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":716,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950076+0000","last_change":"2023-09-30T09:48:39.230305+0000","last_active":"2023-09-30T09:48:41.950076+0000","last_peered":"2023-09-30T09:48:41.950076+0000","last_clean":"2023-09-30T09:48:41.950076+0000","last_became_active":"2023-09-30T09:48:38.909167+0000","last_became_peered":"2023-09-30T09:48:38.909167+0000","last_unstale":"2023-09-30T09:48:41.950076+0000","last_undegraded":"2023-09-30T09:48:41.950076+0000","last_fullsized":"2023-09-30T09:48:41.950076+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:39.230227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.230227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:31:02.995548+0000","scrub_duration":0.0016926949999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":948,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636405+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:48:48.636405+0000","last_peered":"2023-09-30T09:48:48.636405+0000","last_clean":"2023-09-30T09:48:48.636405+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:48:48.636405+0000","last_undegraded":"2023-09-30T09:48:48.636405+0000","last_fullsized":"2023-09-30T09:48:48.636405+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":154,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:49.085319+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:48:49.085319+0000","last_peered":"2023-09-30T09:48:49.085319+0000","last_clean":"2023-09-30T09:48:49.085319+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:48:49.085319+0000","last_undegraded":"2023-09-30T09:48:49.085319+0000","last_fullsized":"2023-09-30T09:48:49.085319+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":46,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090395+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:48:48.090395+0000","last_peered":"2023-09-30T09:48:48.090395+0000","last_clean":"2023-09-30T09:48:48.090395+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:48:48.090395+0000","last_undegraded":"2023-09-30T09:48:48.090395+0000","last_fullsized":"2023-09-30T09:48:48.090395+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"540'374","reported_seq":385,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:53.877520+0000","last_change":"2023-09-30T09:48:40.934636+0000","last_active":"2023-09-30T09:48:53.877520+0000","last_peered":"2023-09-30T09:48:53.877520+0000","last_clean":"2023-09-30T09:48:53.877520+0000","last_became_active":"2023-09-30T09:48:40.934291+0000","last_became_peered":"2023-09-30T09:48:40.934291+0000","last_unstale":"2023-09-30T09:48:53.877520+0000","last_undegraded":"2023-09-30T09:48:53.877520+0000","last_fullsized":"2023-09-30T09:48:53.877520+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":374,"log_dups_size":0,"ondisk_log_size":374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:38:52.121514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24510464,"num_objects":374,"num_object_clones":0,"num_object_copies":1122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":374,"num_write_kb":23936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":728,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090621+0000","last_change":"2023-09-30T09:48:38.912639+0000","last_active":"2023-09-30T09:48:48.090621+0000","last_peered":"2023-09-30T09:48:48.090621+0000","last_clean":"2023-09-30T09:48:48.090621+0000","last_became_active":"2023-09-30T09:48:38.912216+0000","last_became_peered":"2023-09-30T09:48:38.912216+0000","last_unstale":"2023-09-30T09:48:48.090621+0000","last_undegraded":"2023-09-30T09:48:48.090621+0000","last_fullsized":"2023-09-30T09:48:48.090621+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:07.170115+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1088,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636883+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:48:48.636883+0000","last_peered":"2023-09-30T09:48:48.636883+0000","last_clean":"2023-09-30T09:48:48.636883+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:48:48.636883+0000","last_undegraded":"2023-09-30T09:48:48.636883+0000","last_fullsized":"2023-09-30T09:48:48.636883+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":284,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.962391+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:48:47.962391+0000","last_peered":"2023-09-30T09:48:47.962391+0000","last_clean":"2023-09-30T09:48:47.962391+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:48:47.962391+0000","last_undegraded":"2023-09-30T09:48:47.962391+0000","last_fullsized":"2023-09-30T09:48:47.962391+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090781+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:48:48.090781+0000","last_peered":"2023-09-30T09:48:48.090781+0000","last_clean":"2023-09-30T09:48:48.090781+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:48:48.090781+0000","last_undegraded":"2023-09-30T09:48:48.090781+0000","last_fullsized":"2023-09-30T09:48:48.090781+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"540'376","reported_seq":387,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:53.887220+0000","last_change":"2023-09-30T09:48:40.945395+0000","last_active":"2023-09-30T09:48:53.887220+0000","last_peered":"2023-09-30T09:48:53.887220+0000","last_clean":"2023-09-30T09:48:53.887220+0000","last_became_active":"2023-09-30T09:48:40.945068+0000","last_became_peered":"2023-09-30T09:48:40.945068+0000","last_unstale":"2023-09-30T09:48:53.887220+0000","last_undegraded":"2023-09-30T09:48:53.887220+0000","last_fullsized":"2023-09-30T09:48:53.887220+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":376,"log_dups_size":0,"ondisk_log_size":376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:05:00.962112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24641536,"num_objects":376,"num_object_clones":0,"num_object_copies":1128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":376,"num_write_kb":24064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1078,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950712+0000","last_change":"2023-09-30T09:48:38.908757+0000","last_active":"2023-09-30T09:48:41.950712+0000","last_peered":"2023-09-30T09:48:41.950712+0000","last_clean":"2023-09-30T09:48:41.950712+0000","last_became_active":"2023-09-30T09:48:38.908306+0000","last_became_peered":"2023-09-30T09:48:38.908306+0000","last_unstale":"2023-09-30T09:48:41.950712+0000","last_undegraded":"2023-09-30T09:48:41.950712+0000","last_fullsized":"2023-09-30T09:48:41.950712+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:47:04.504815+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":571,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636797+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:48:48.636797+0000","last_peered":"2023-09-30T09:48:48.636797+0000","last_clean":"2023-09-30T09:48:48.636797+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:48:48.636797+0000","last_undegraded":"2023-09-30T09:48:48.636797+0000","last_fullsized":"2023-09-30T09:48:48.636797+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1267,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950760+0000","last_change":"2023-09-30T09:48:38.911478+0000","last_active":"2023-09-30T09:48:41.950760+0000","last_peered":"2023-09-30T09:48:41.950760+0000","last_clean":"2023-09-30T09:48:41.950760+0000","last_became_active":"2023-09-30T09:48:38.910854+0000","last_became_peered":"2023-09-30T09:48:38.910854+0000","last_unstale":"2023-09-30T09:48:41.950760+0000","last_undegraded":"2023-09-30T09:48:41.950760+0000","last_fullsized":"2023-09-30T09:48:41.950760+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:35:42.683028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":575,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636319+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:48:48.636319+0000","last_peered":"2023-09-30T09:48:48.636319+0000","last_clean":"2023-09-30T09:48:48.636319+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:48:48.636319+0000","last_undegraded":"2023-09-30T09:48:48.636319+0000","last_fullsized":"2023-09-30T09:48:48.636319+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"540'270","reported_seq":285,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:53.423286+0000","last_change":"2023-09-30T09:48:40.934540+0000","last_active":"2023-09-30T09:48:53.423286+0000","last_peered":"2023-09-30T09:48:53.423286+0000","last_clean":"2023-09-30T09:48:53.423286+0000","last_became_active":"2023-09-30T09:48:40.934267+0000","last_became_peered":"2023-09-30T09:48:40.934267+0000","last_unstale":"2023-09-30T09:48:53.423286+0000","last_undegraded":"2023-09-30T09:48:53.423286+0000","last_fullsized":"2023-09-30T09:48:53.423286+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":270,"log_dups_size":0,"ondisk_log_size":270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:51:07.655923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17694720,"num_objects":270,"num_object_clones":0,"num_object_copies":810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":270,"num_write_kb":17280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":923,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.148869+0000","last_change":"2023-09-30T09:44:27.063145+0000","last_active":"2023-09-30T09:48:48.148869+0000","last_peered":"2023-09-30T09:48:48.148869+0000","last_clean":"2023-09-30T09:48:48.148869+0000","last_became_active":"2023-09-30T09:44:27.062778+0000","last_became_peered":"2023-09-30T09:44:27.062778+0000","last_unstale":"2023-09-30T09:48:48.148869+0000","last_undegraded":"2023-09-30T09:48:48.148869+0000","last_fullsized":"2023-09-30T09:48:48.148869+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:17:24.038390+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":827,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636879+0000","last_change":"2023-09-30T09:48:35.928561+0000","last_active":"2023-09-30T09:48:48.636879+0000","last_peered":"2023-09-30T09:48:48.636879+0000","last_clean":"2023-09-30T09:48:48.636879+0000","last_became_active":"2023-09-30T09:48:35.909706+0000","last_became_peered":"2023-09-30T09:48:35.909706+0000","last_unstale":"2023-09-30T09:48:48.636879+0000","last_undegraded":"2023-09-30T09:48:48.636879+0000","last_fullsized":"2023-09-30T09:48:48.636879+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:42:32.083217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":117,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-09-30T09:48:49.165900+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:48:49.165900+0000","last_peered":"2023-09-30T09:48:49.165900+0000","last_clean":"2023-09-30T09:48:49.165900+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:48:49.165900+0000","last_undegraded":"2023-09-30T09:48:49.165900+0000","last_fullsized":"2023-09-30T09:48:49.165900+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":550,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950136+0000","last_change":"2023-09-30T09:48:35.918447+0000","last_active":"2023-09-30T09:48:41.950136+0000","last_peered":"2023-09-30T09:48:41.950136+0000","last_clean":"2023-09-30T09:48:41.950136+0000","last_became_active":"2023-09-30T09:48:35.918153+0000","last_became_peered":"2023-09-30T09:48:35.918153+0000","last_unstale":"2023-09-30T09:48:41.950136+0000","last_undegraded":"2023-09-30T09:48:41.950136+0000","last_fullsized":"2023-09-30T09:48:41.950136+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:43:28.672561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":874,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090032+0000","last_change":"2023-09-30T09:48:38.905810+0000","last_active":"2023-09-30T09:48:48.090032+0000","last_peered":"2023-09-30T09:48:48.090032+0000","last_clean":"2023-09-30T09:48:48.090032+0000","last_became_active":"2023-09-30T09:48:38.905438+0000","last_became_peered":"2023-09-30T09:48:38.905438+0000","last_unstale":"2023-09-30T09:48:48.090032+0000","last_undegraded":"2023-09-30T09:48:48.090032+0000","last_fullsized":"2023-09-30T09:48:48.090032+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:38:39.516217+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":46,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090750+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:48:48.090750+0000","last_peered":"2023-09-30T09:48:48.090750+0000","last_clean":"2023-09-30T09:48:48.090750+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:48:48.090750+0000","last_undegraded":"2023-09-30T09:48:48.090750+0000","last_fullsized":"2023-09-30T09:48:48.090750+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":246,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.962308+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:48:47.962308+0000","last_peered":"2023-09-30T09:48:47.962308+0000","last_clean":"2023-09-30T09:48:47.962308+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:48:47.962308+0000","last_undegraded":"2023-09-30T09:48:47.962308+0000","last_fullsized":"2023-09-30T09:48:47.962308+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":686,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636838+0000","last_change":"2023-09-30T09:48:38.912556+0000","last_active":"2023-09-30T09:48:48.636838+0000","last_peered":"2023-09-30T09:48:48.636838+0000","last_clean":"2023-09-30T09:48:48.636838+0000","last_became_active":"2023-09-30T09:48:38.912185+0000","last_became_peered":"2023-09-30T09:48:38.912185+0000","last_unstale":"2023-09-30T09:48:48.636838+0000","last_undegraded":"2023-09-30T09:48:48.636838+0000","last_fullsized":"2023-09-30T09:48:48.636838+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:26:30.556292+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":468,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964318+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:48:47.964318+0000","last_peered":"2023-09-30T09:48:47.964318+0000","last_clean":"2023-09-30T09:48:47.964318+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:48:47.964318+0000","last_undegraded":"2023-09-30T09:48:47.964318+0000","last_fullsized":"2023-09-30T09:48:47.964318+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":253,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.949927+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:48:41.949927+0000","last_peered":"2023-09-30T09:48:41.949927+0000","last_clean":"2023-09-30T09:48:41.949927+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:48:41.949927+0000","last_undegraded":"2023-09-30T09:48:41.949927+0000","last_fullsized":"2023-09-30T09:48:41.949927+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":424,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964602+0000","last_change":"2023-09-30T09:48:35.921387+0000","last_active":"2023-09-30T09:48:47.964602+0000","last_peered":"2023-09-30T09:48:47.964602+0000","last_clean":"2023-09-30T09:48:47.964602+0000","last_became_active":"2023-09-30T09:48:35.911929+0000","last_became_peered":"2023-09-30T09:48:35.911929+0000","last_unstale":"2023-09-30T09:48:47.964602+0000","last_undegraded":"2023-09-30T09:48:47.964602+0000","last_fullsized":"2023-09-30T09:48:47.964602+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:38:53.806721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":273,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.949827+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:48:41.949827+0000","last_peered":"2023-09-30T09:48:41.949827+0000","last_clean":"2023-09-30T09:48:41.949827+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:48:41.949827+0000","last_undegraded":"2023-09-30T09:48:41.949827+0000","last_fullsized":"2023-09-30T09:48:41.949827+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":887,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.636871+0000","last_change":"2023-09-30T09:48:38.908089+0000","last_active":"2023-09-30T09:48:48.636871+0000","last_peered":"2023-09-30T09:48:48.636871+0000","last_clean":"2023-09-30T09:48:48.636871+0000","last_became_active":"2023-09-30T09:48:38.907639+0000","last_became_peered":"2023-09-30T09:48:38.907639+0000","last_unstale":"2023-09-30T09:48:48.636871+0000","last_undegraded":"2023-09-30T09:48:48.636871+0000","last_fullsized":"2023-09-30T09:48:48.636871+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.416538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.416538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:25:41.900786+0000","scrub_duration":0.0014466209999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":368,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.964927+0000","last_change":"2023-09-30T09:48:35.898301+0000","last_active":"2023-09-30T09:48:47.964927+0000","last_peered":"2023-09-30T09:48:47.964927+0000","last_clean":"2023-09-30T09:48:47.964927+0000","last_became_active":"2023-09-30T09:48:35.897906+0000","last_became_peered":"2023-09-30T09:48:35.897906+0000","last_unstale":"2023-09-30T09:48:47.964927+0000","last_undegraded":"2023-09-30T09:48:47.964927+0000","last_fullsized":"2023-09-30T09:48:47.964927+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:44:13.422293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":492,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:47.963009+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:48:47.963009+0000","last_peered":"2023-09-30T09:48:47.963009+0000","last_clean":"2023-09-30T09:48:47.963009+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:48:47.963009+0000","last_undegraded":"2023-09-30T09:48:47.963009+0000","last_fullsized":"2023-09-30T09:48:47.963009+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":909,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.950614+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:48:41.950614+0000","last_peered":"2023-09-30T09:48:41.950614+0000","last_clean":"2023-09-30T09:48:41.950614+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:48:41.950614+0000","last_undegraded":"2023-09-30T09:48:41.950614+0000","last_fullsized":"2023-09-30T09:48:41.950614+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":52,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-09-30T09:48:48.090324+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:48:48.090324+0000","last_peered":"2023-09-30T09:48:48.090324+0000","last_clean":"2023-09-30T09:48:48.090324+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:48:48.090324+0000","last_undegraded":"2023-09-30T09:48:48.090324+0000","last_fullsized":"2023-09-30T09:48:48.090324+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":564,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-30T09:48:41.949760+0000","last_change":"2023-09-30T09:48:35.910729+0000","last_active":"2023-09-30T09:48:41.949760+0000","last_peered":"2023-09-30T09:48:41.949760+0000","last_clean":"2023-09-30T09:48:41.949760+0000","last_became_active":"2023-09-30T09:48:35.910406+0000","last_became_peered":"2023-09-30T09:48:35.910406+0000","last_unstale":"2023-09-30T09:48:41.949760+0000","last_undegraded":"2023-09-30T09:48:41.949760+0000","last_fullsized":"2023-09-30T09:48:41.949760+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:32:23.277215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":418185216,"num_objects":6381,"num_object_clones":0,"num_object_copies":19143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6381,"num_write_kb":408384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1270874112,"data_stored":1270874112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6381,"ondisk_log_size":6381,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":26922,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":242,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234446,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":642124,"kb_used_data":87556,"kb_used_omap":0,"kb_used_meta":554560,"kb_avail":93103028,"statfs":{"total":95995035648,"available":95337500672,"internally_reserved":0,"allocated":89657344,"data_stored":87895441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":567869440},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:48:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0419999999999998,"5min":7.1319999999999997,"15min":5.226},"min":{"1min":0.29999999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.7629999999999999,"5min":124.74299999999999,"15min":215.416},"last":0.78800000000000003},{"interface":"front","average":{"1min":2.9470000000000001,"5min":7.0510000000000002,"15min":5.1500000000000004},"min":{"1min":0.34999999999999998,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":10.130000000000001,"5min":124.69499999999999,"15min":215.041},"last":0.57199999999999995}]},{"osd":1,"last update":"Sat Sep 30 09:48:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":7.8150000000000004,"15min":4.133},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25},"max":{"1min":10.749000000000001,"5min":157.578,"15min":157.578},"last":0.69399999999999995},{"interface":"front","average":{"1min":3.4239999999999999,"5min":7.8079999999999998,"15min":4.2489999999999997},"min":{"1min":0.38600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":12.013999999999999,"5min":157.589,"15min":157.589},"last":0.745}]},{"osd":2,"last update":"Sat Sep 30 09:48:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9810000000000001,"5min":2.2949999999999999,"15min":3.7930000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.568,"5min":19.399000000000001,"15min":117.14},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.9139999999999999,"5min":2.2770000000000001,"15min":3.7589999999999999},"min":{"1min":0.374,"5min":0.29599999999999999,"15min":0.28399999999999997},"max":{"1min":10.083,"5min":18.603000000000002,"15min":117.044},"last":0.59899999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":8.4610000000000003,"15min":8.3160000000000007},"min":{"1min":0.34399999999999997,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":7.3979999999999997,"5min":154.06700000000001,"15min":277.57799999999997},"last":0.57799999999999996},{"interface":"front","average":{"1min":3.4729999999999999,"5min":8.8829999999999991,"15min":8.6379999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32200000000000001},"max":{"1min":11.343999999999999,"5min":154.24799999999999,"15min":278.69299999999998},"last":0.73199999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:48:44 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.581,"15min":1.6579999999999999},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.23799999999999999},"max":{"1min":10.789,"5min":25.192,"15min":175.72},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.395,"5min":1.4059999999999999,"15min":1.5489999999999999},"min":{"1min":0.35299999999999998,"5min":0.28999999999999998,"15min":0.252},"max":{"1min":10.231,"5min":24.521000000000001,"15min":179.048},"last":0.438}]},{"osd":5,"last update":"Sat Sep 30 09:48:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":5.6280000000000001,"15min":9.8710000000000004},"min":{"1min":0.32700000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.054,"5min":197.75700000000001,"15min":197.75700000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.633,"5min":5.7510000000000003,"15min":9.8010000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":8.157,"5min":197.63300000000001,"15min":197.63300000000001},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:48:44 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.214,"15min":2.2170000000000001},"min":{"1min":0.376,"5min":0.30399999999999999,"15min":0.27100000000000002},"max":{"1min":4.9109999999999996,"5min":7.5830000000000002,"15min":206.244},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.1319999999999999,"15min":2.1709999999999998},"min":{"1min":0.35699999999999998,"5min":0.31900000000000001,"15min":0.29199999999999998},"max":{"1min":3.9279999999999999,"5min":5.8520000000000003,"15min":202.78800000000001},"last":0.49299999999999999}]}]},{"osd":6,"up_from":0,"seq":197568496084,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":301,"seq":1292785156282,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":775900,"kb_used_data":116628,"kb_used_omap":9,"kb_used_meta":659254,"kb_avail":92969252,"statfs":{"total":95995035648,"available":95200514048,"internally_reserved":0,"allocated":119427072,"data_stored":116394509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":675076114},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":5.2460000000000004,"5min":4.9950000000000001,"15min":3.9119999999999999},"min":{"1min":0.30099999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":48.680999999999997,"5min":142.08500000000001,"15min":142.08500000000001},"last":0.25700000000000001},{"interface":"front","average":{"1min":5.8159999999999998,"5min":5.0279999999999996,"15min":3.8100000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":48.715000000000003,"5min":142.137,"15min":142.137},"last":0.443}]},{"osd":2,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":7.1529999999999996,"5min":3.573,"15min":2.8639999999999999},"min":{"1min":0.38800000000000001,"5min":0.184,"15min":0.184},"max":{"1min":64.759,"5min":64.759,"15min":64.759},"last":0.28100000000000003},{"interface":"front","average":{"1min":6.633,"5min":3.6659999999999999,"15min":3.0619999999999998},"min":{"1min":0.29199999999999998,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":62.954000000000001,"5min":68.373999999999995,"15min":68.373999999999995},"last":0.38300000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":8.0359999999999996,"5min":7.2190000000000003,"15min":5.9960000000000004},"min":{"1min":0.254,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":48.732999999999997,"5min":199.41,"15min":199.41},"last":0.34300000000000003},{"interface":"front","average":{"1min":5.6820000000000004,"5min":6.7380000000000004,"15min":5.851},"min":{"1min":0.375,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":48.576999999999998,"5min":199.58099999999999,"15min":199.58099999999999},"last":0.45600000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":5.306,"5min":2.0190000000000001,"15min":2.3519999999999999},"min":{"1min":0.38900000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":48.604999999999997,"5min":48.604999999999997,"15min":142.77099999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.919,"5min":1.9550000000000001,"15min":2.5219999999999998},"min":{"1min":0.39300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.5990000000000002,"5min":8.6319999999999997,"15min":144.898},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 30 09:48:33 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.536,"15min":2.1339999999999999},"min":{"1min":0.40500000000000003,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":22.259,"5min":22.259,"15min":22.259},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.9410000000000001,"5min":1.6839999999999999,"15min":1.3759999999999999},"min":{"1min":0.49199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.1630000000000003,"5min":15.250999999999999,"15min":15.250999999999999},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":2.319,"5min":1.871,"15min":4.5830000000000002},"min":{"1min":0.442,"5min":0.316,"15min":0.29899999999999999},"max":{"1min":7.2729999999999997,"5min":30.986999999999998,"15min":209.221},"last":0.67000000000000004},{"interface":"front","average":{"1min":2.2589999999999999,"5min":1.7829999999999999,"15min":4.5119999999999996},"min":{"1min":0.45400000000000001,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":10.266999999999999,"5min":28.710999999999999,"15min":208.43600000000001},"last":0.628}]},{"osd":7,"last update":"Sat Sep 30 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.579,"15min":3.2440000000000002},"min":{"1min":0.40400000000000003,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":5.8600000000000003,"5min":9.3879999999999999,"15min":165.59399999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.238,"5min":1.6140000000000001,"15min":3.2970000000000002},"min":{"1min":0.42399999999999999,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":5.8029999999999999,"5min":7.5460000000000003,"15min":165.833},"last":0.60399999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926180,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925992,"kb_used_data":275616,"kb_used_omap":5,"kb_used_meta":650362,"kb_avail":92819160,"statfs":{"total":95995035648,"available":95046819840,"internally_reserved":0,"allocated":282230784,"data_stored":278957398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":665970941},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":1.984,"15min":2.085},"min":{"1min":0.22700000000000001,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":13.308999999999999,"5min":57.482999999999997,"15min":57.482999999999997},"last":1.6699999999999999},{"interface":"front","average":{"1min":3.9420000000000002,"5min":2.9260000000000002,"15min":3.6549999999999998},"min":{"1min":0.26000000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.554,"5min":57.140999999999998,"15min":138.92400000000001},"last":2.3900000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:48:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3740000000000001,"5min":3.0819999999999999,"15min":3.7069999999999999},"min":{"1min":0.31900000000000001,"5min":0.246,"15min":0.20799999999999999},"max":{"1min":23.547999999999998,"5min":31.521000000000001,"15min":127.73399999999999},"last":0.625},{"interface":"front","average":{"1min":2.323,"5min":3.363,"15min":3.9700000000000002},"min":{"1min":0.307,"5min":0.249,"15min":0.249},"max":{"1min":23.789000000000001,"5min":31.478999999999999,"15min":127.69},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":2.9289999999999998,"5min":6.3029999999999999,"15min":5.5490000000000004},"min":{"1min":0.28599999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":15.391999999999999,"5min":125.562,"15min":218.06999999999999},"last":1.9019999999999999},{"interface":"front","average":{"1min":1.54,"5min":5.0750000000000002,"15min":4.2869999999999999},"min":{"1min":0.33600000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.202,"5min":117.295,"15min":218.63200000000001},"last":3.5030000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":4.1779999999999999,"5min":2.6560000000000001,"15min":2.2469999999999999},"min":{"1min":0.38800000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":23.658000000000001,"5min":57.066000000000003,"15min":72.403999999999996},"last":0.746},{"interface":"front","average":{"1min":2.633,"5min":2.7559999999999998,"15min":2.9390000000000001},"min":{"1min":0.39100000000000001,"5min":0.30299999999999999,"15min":0.29399999999999998},"max":{"1min":11.617000000000001,"5min":55.216999999999999,"15min":136.262},"last":0.76500000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:48:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":2.218,"15min":1.454},"min":{"1min":0.38500000000000001,"5min":0.317,"15min":0.317},"max":{"1min":7.6360000000000001,"5min":13.162000000000001,"15min":13.162000000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.2390000000000001,"15min":1.1120000000000001},"min":{"1min":0.35599999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.1379999999999999,"5min":13.422000000000001,"15min":13.422000000000001},"last":3.4279999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":12.973000000000001,"5min":4.3390000000000004,"15min":3.9390000000000001},"min":{"1min":0.39900000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":240.946,"5min":240.946,"15min":240.946},"last":2.8769999999999998},{"interface":"front","average":{"1min":16.146999999999998,"5min":5.4029999999999996,"15min":4.4749999999999996},"min":{"1min":0.42099999999999999,"5min":0.38600000000000001,"15min":0.34999999999999998},"max":{"1min":240.977,"5min":240.977,"15min":240.977},"last":0.71199999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":9.5180000000000007,"5min":3.468,"15min":2.6379999999999999},"min":{"1min":0.44500000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":146.828,"5min":146.828,"15min":146.828},"last":3.468},{"interface":"front","average":{"1min":12.778,"5min":4.6840000000000002,"15min":3.1749999999999998},"min":{"1min":0.46300000000000002,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":146.876,"5min":146.876,"15min":146.876},"last":0.66800000000000004}]}]},{"osd":2,"up_from":104,"seq":446676599121,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1019848,"kb_used_data":195392,"kb_used_omap":5,"kb_used_meta":824442,"kb_avail":92725304,"statfs":{"total":95995035648,"available":94950711296,"internally_reserved":0,"allocated":200081408,"data_stored":197069499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":844229261},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":11.547000000000001,"5min":7.6280000000000001,"15min":4.0439999999999996},"min":{"1min":0.189,"5min":0.189,"15min":0.187},"max":{"1min":202.40899999999999,"5min":202.40899999999999,"15min":202.40899999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":11.834,"5min":7.7530000000000001,"15min":4.1269999999999998},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":202.38800000000001,"5min":202.38800000000001,"15min":202.38800000000001},"last":0.55600000000000005}]},{"osd":1,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":12.548999999999999,"5min":7.056,"15min":4.391},"min":{"1min":0.30299999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":202.369,"5min":202.369,"15min":202.369},"last":0.39000000000000001},{"interface":"front","average":{"1min":12.036,"5min":7.2880000000000003,"15min":4.2309999999999999},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":202.54499999999999,"5min":202.54499999999999,"15min":202.54499999999999},"last":0.501}]},{"osd":3,"last update":"Sat Sep 30 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":12.154,"5min":7.2050000000000001,"15min":5.8620000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":202.59,"5min":202.59,"15min":202.59},"last":0.67200000000000004},{"interface":"front","average":{"1min":12.242000000000001,"5min":7.29,"15min":5.7210000000000001},"min":{"1min":0.29699999999999999,"5min":0.224,"15min":0.224},"max":{"1min":202.34200000000001,"5min":202.34200000000001,"15min":202.34200000000001},"last":0.57199999999999995}]},{"osd":4,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":22.321000000000002,"5min":6.1150000000000002,"15min":3.3149999999999999},"min":{"1min":0.39900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":247.655,"5min":247.655,"15min":247.655},"last":0.60299999999999998},{"interface":"front","average":{"1min":22.981000000000002,"5min":6.327,"15min":3.3330000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":248.012,"5min":248.012,"15min":248.012},"last":0.621}]},{"osd":5,"last update":"Sat Sep 30 09:48:34 2023","interfaces":[{"interface":"back","average":{"1min":11.827,"5min":6.2069999999999999,"15min":2.786},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":202.65600000000001,"5min":243.86500000000001,"15min":243.86500000000001},"last":0.76800000000000002},{"interface":"front","average":{"1min":12.071999999999999,"5min":6.5460000000000003,"15min":3.0590000000000002},"min":{"1min":0.40400000000000003,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":202.86799999999999,"5min":244.02199999999999,"15min":244.02199999999999},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":11.365,"5min":3.5649999999999999,"15min":3.5110000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":202.70599999999999,"5min":202.70599999999999,"15min":202.70599999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":11.484,"5min":3.8839999999999999,"15min":3.9399999999999999},"min":{"1min":0.438,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":202.916,"5min":202.916,"15min":202.916},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":29.030000000000001,"5min":7.7949999999999999,"15min":4.2309999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":244.911,"5min":244.911,"15min":244.911},"last":0.65900000000000003},{"interface":"front","average":{"1min":28.515000000000001,"5min":7.8369999999999997,"15min":4.5640000000000001},"min":{"1min":0.42399999999999999,"5min":0.40100000000000002,"15min":0.40000000000000002},"max":{"1min":244.66,"5min":244.66,"15min":244.66},"last":0.63600000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435594,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":958952,"kb_used_data":86624,"kb_used_omap":2,"kb_used_meta":872317,"kb_avail":92786200,"statfs":{"total":95995035648,"available":95013068800,"internally_reserved":0,"allocated":88702976,"data_stored":85596336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":893252967},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7679999999999998,"5min":4.4100000000000001,"15min":4.7030000000000003},"min":{"1min":0.191,"5min":0.191,"15min":0.183},"max":{"1min":18.006,"5min":87.367999999999995,"15min":205.73099999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":4.5609999999999999,"5min":4.5700000000000003,"15min":4.6909999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":16.347999999999999,"5min":85.700000000000003,"15min":205.495},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 30 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":4.0839999999999996,"15min":6.2290000000000001},"min":{"1min":0.27600000000000002,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":14.055,"5min":142.434,"15min":273.99599999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.4329999999999998,"5min":4.3170000000000002,"15min":6.0910000000000002},"min":{"1min":0.38300000000000001,"5min":0.28199999999999997,"15min":0.20699999999999999},"max":{"1min":12.470000000000001,"5min":142.44900000000001,"15min":274.21600000000001},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":5.6589999999999998,"15min":5.907},"min":{"1min":0.26600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":14.9,"5min":146.92599999999999,"15min":206.69},"last":0.54600000000000004},{"interface":"front","average":{"1min":4.4240000000000004,"5min":5.5700000000000003,"15min":5.8129999999999997},"min":{"1min":0.27000000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.228999999999999,"5min":145.75299999999999,"15min":209.22800000000001},"last":0.36599999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":2.4510000000000001,"15min":2.2309999999999999},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.8719999999999999,"5min":81.677000000000007,"15min":81.677000000000007},"last":0.45100000000000001},{"interface":"front","average":{"1min":3.4100000000000001,"5min":2.851,"15min":2.2069999999999999},"min":{"1min":0.44500000000000001,"5min":0.33200000000000002,"15min":0.29499999999999998},"max":{"1min":15.750999999999999,"5min":80.623000000000005,"15min":80.623000000000005},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:48:33 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.976,"15min":1.6759999999999999},"min":{"1min":0.35099999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.4619999999999997,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":3.1099999999999999,"5min":1.968,"15min":1.29},"min":{"1min":0.442,"5min":0.311,"15min":0.311},"max":{"1min":15.715,"5min":15.715,"15min":15.715},"last":0.64300000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":1.831,"15min":2.0710000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.9009999999999998,"5min":10.837999999999999,"15min":60.698},"last":0.53400000000000003},{"interface":"front","average":{"1min":4.2400000000000002,"5min":2.3039999999999998,"15min":2.1619999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.33400000000000002},"max":{"1min":15.666,"5min":15.666,"15min":15.666},"last":0.60399999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":3.9009999999999998,"15min":2.7280000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.257999999999999,"5min":213.548,"15min":213.548},"last":0.72499999999999998},{"interface":"front","average":{"1min":3.492,"5min":3.7919999999999998,"15min":2.6760000000000002},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.9290000000000003,"5min":213.583,"15min":213.583},"last":0.502}]}]},{"osd":4,"up_from":30,"seq":128849019357,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":656892,"kb_used_data":170676,"kb_used_omap":0,"kb_used_meta":486208,"kb_avail":93088260,"statfs":{"total":95995035648,"available":95322378240,"internally_reserved":0,"allocated":174772224,"data_stored":173025561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":497876992},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,6,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:48:00 2023","interfaces":[{"interface":"back","average":{"1min":4.4560000000000004,"5min":2.734,"15min":2.8119999999999998},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":53.241999999999997,"5min":53.241999999999997,"15min":105.179},"last":1.0169999999999999},{"interface":"front","average":{"1min":5.5339999999999998,"5min":2.9620000000000002,"15min":3.0659999999999998},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":53.340000000000003,"5min":53.340000000000003,"15min":104.70999999999999},"last":2.1589999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4660000000000002,"5min":2.8079999999999998,"15min":3.4009999999999998},"min":{"1min":0.28000000000000003,"5min":0.25,"15min":0.25},"max":{"1min":7.7619999999999996,"5min":80.712000000000003,"15min":160.69999999999999},"last":0.88600000000000001},{"interface":"front","average":{"1min":2.7829999999999999,"5min":3.2559999999999998,"15min":3.4420000000000002},"min":{"1min":0.41399999999999998,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":8.0150000000000006,"5min":80.810000000000002,"15min":160.50700000000001},"last":1.224}]},{"osd":2,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":3.8900000000000001,"15min":3.5699999999999998},"min":{"1min":0.40999999999999998,"5min":0.33600000000000002,"15min":0.27300000000000002},"max":{"1min":7.415,"5min":117.858,"15min":142.88999999999999},"last":1.3859999999999999},{"interface":"front","average":{"1min":2.7989999999999999,"5min":4.1420000000000003,"15min":3.6429999999999998},"min":{"1min":0.42899999999999999,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":10.252000000000001,"5min":117.871,"15min":142.90700000000001},"last":1.3600000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7200000000000002,"5min":2.3599999999999999,"15min":2.2669999999999999},"min":{"1min":0.503,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":7.6159999999999997,"5min":29.41,"15min":52.762},"last":1.0489999999999999},{"interface":"front","average":{"1min":2.157,"5min":2.2200000000000002,"15min":2.2469999999999999},"min":{"1min":0.41299999999999998,"5min":0.36699999999999999,"15min":0.29299999999999998},"max":{"1min":6.6210000000000004,"5min":29.401,"15min":52.790999999999997},"last":0.97699999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:48:28 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":5.1680000000000001,"15min":8.8780000000000001},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.8940000000000001,"5min":140.11099999999999,"15min":140.11099999999999},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.85699999999999998,"5min":5.0359999999999996,"15min":8.8559999999999999},"min":{"1min":0.34300000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.8900000000000001,"5min":140.065,"15min":140.065},"last":0.40799999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:48:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":1.0720000000000001,"15min":2.2130000000000001},"min":{"1min":0.34000000000000002,"5min":0.26300000000000001,"15min":0.23799999999999999},"max":{"1min":7.4749999999999996,"5min":9.1470000000000002,"15min":202.72499999999999},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.028,"5min":0.90400000000000003,"15min":2.165},"min":{"1min":0.36399999999999999,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":5.875,"5min":7.173,"15min":202.42400000000001},"last":0.875}]},{"osd":7,"last update":"Sat Sep 30 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.6440000000000001,"15min":2.395},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.4740000000000002,"5min":161.72999999999999,"15min":161.72999999999999},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.619,"5min":2.7000000000000002,"15min":2.298},"min":{"1min":0.41799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.5919999999999996,"5min":162.60900000000001,"15min":162.60900000000001},"last":0.91200000000000003}]}]},{"osd":5,"up_from":489,"seq":2100239007799,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":324600,"kb_used_data":161200,"kb_used_omap":6,"kb_used_meta":163385,"kb_avail":93420552,"statfs":{"total":95995035648,"available":95662645248,"internally_reserved":0,"allocated":165068800,"data_stored":163322759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":167306335},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:48:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9220000000000002,"5min":13.33,"15min":21.983000000000001},"min":{"1min":0.248,"5min":0.22,"15min":0.22},"max":{"1min":49.408000000000001,"5min":281.10599999999999,"15min":281.10599999999999},"last":0.313},{"interface":"front","average":{"1min":3.7160000000000002,"5min":13.442,"15min":22.213000000000001},"min":{"1min":0.435,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":51.515000000000001,"5min":282.07400000000001,"15min":282.07400000000001},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:48:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":8.3740000000000006,"15min":12.738},"min":{"1min":0.371,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.4139999999999997,"5min":273.48200000000003,"15min":273.48200000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.518,"5min":7.8860000000000001,"15min":12.032},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":6.9329999999999998,"5min":273.50200000000001,"15min":273.50200000000001},"last":0.45000000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:48:31 2023","interfaces":[{"interface":"back","average":{"1min":10.776999999999999,"5min":6.29,"15min":2.7240000000000002},"min":{"1min":0.34699999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":194.14400000000001,"5min":194.14400000000001,"15min":194.14400000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":10.315,"5min":6.399,"15min":3.1600000000000001},"min":{"1min":0.42399999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":198.083,"5min":198.083,"15min":198.083},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:48:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4239999999999999,"5min":8.5099999999999998,"15min":12.669},"min":{"1min":0.378,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.468,"5min":274.01799999999997,"15min":274.01799999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.9340000000000002,"5min":3.3879999999999999,"15min":2.2839999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.564,"5min":79.459000000000003,"15min":79.459000000000003},"last":0.48699999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:48:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":1.1200000000000001,"15min":1.008},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.3360000000000003,"5min":7.3360000000000003,"15min":7.3360000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.411,"5min":1.3029999999999999,"15min":1.198},"min":{"1min":0.318,"5min":0.317,"15min":0.317},"max":{"1min":6.4509999999999996,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.624}]},{"osd":6,"last update":"Sat Sep 30 09:48:31 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.5860000000000001,"15min":1.625},"min":{"1min":0.307,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.7650000000000006,"5min":14.371,"15min":14.371},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.873,"5min":1.5,"15min":1.4259999999999999},"min":{"1min":0.39100000000000001,"5min":0.316,"15min":0.316},"max":{"1min":7.9000000000000004,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:48:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.4630000000000001,"15min":1.3500000000000001},"min":{"1min":0.34999999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.7050000000000001,"5min":9.7560000000000002,"15min":9.7560000000000002},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.222,"5min":1.3300000000000001,"15min":1.054},"min":{"1min":0.34799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":11.000999999999999,"5min":11.000999999999999,"15min":11.000999999999999},"last":1.022}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":291831808,"data_stored":291831808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":123994112,"data_stored":123994112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":208076800,"data_stored":208076800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":110690304,"data_stored":110690304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":243793920,"data_stored":243793920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":171966464,"data_stored":171966464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":120520704,"data_stored":120520704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:49:05.052 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-30T09:49:05.052 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 81 2023-09-30T09:49:05.208 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:49:05.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:05.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:49:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:05 smithi167 ceph-mon[192880]: pgmap v1444: 119 pgs: 119 active+clean; 1.4 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-09-30T09:49:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]: dispatch 2023-09-30T09:49:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-30T09:49:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [7, 1]}]: dispatch 2023-09-30T09:49:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]: dispatch 2023-09-30T09:49:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 2]}]: dispatch 2023-09-30T09:49:05.317 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[224043]: pgmap v1444: 119 pgs: 119 active+clean; 1.4 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-09-30T09:49:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]: dispatch 2023-09-30T09:49:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-30T09:49:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [7, 1]}]: dispatch 2023-09-30T09:49:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]: dispatch 2023-09-30T09:49:05.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 2]}]: dispatch 2023-09-30T09:49:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[216815]: pgmap v1444: 119 pgs: 119 active+clean; 1.4 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-09-30T09:49:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]: dispatch 2023-09-30T09:49:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-30T09:49:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [7, 1]}]: dispatch 2023-09-30T09:49:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]: dispatch 2023-09-30T09:49:05.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 2]}]: dispatch 2023-09-30T09:49:05.415 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:05.649 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:05.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:05.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:05.713 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 14 16 30649 30633 136.701 92.6875 0.0092465 0.00730932 2023-09-30T09:49:05.756 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:49:05.771 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:106329 8.11 8:8971a34b:::benchmark_data_smithi019_478679_object13881:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:40:21.854198+0000", 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 523.908645224, 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1407501839999998, 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.772 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 106329 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.854198+0000", 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.773 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.854198+0000", 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.854200+0000", 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1789999999999998e-06 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.854209+0000", 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2499999999999995e-06 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.854224+0000", 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4630999999999999e-05 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.854234+0000", 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0118999999999999e-05 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.854317+0000", 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3585999999999996e-05 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.854331+0000", 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3568e-05 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:22.778321+0000", 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.92398959300000005 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:22.778333+0000", 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2196e-05 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.978311+0000", 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.199978429 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.978510+0000", 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019881400000000001 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.978915+0000", 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00040475800000000002 2023-09-30T09:49:05.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.982479+0000", 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035644359999999998 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.984865+0000", 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023852869999999998 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.993355+0000", 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0084905919999999999 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.993435+0000", 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9876000000000006e-05 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.994948+0000", 2023-09-30T09:49:05.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015128699999999999 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:106338 8.1 8:804ad605:::benchmark_data_smithi019_478679_object13890:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:40:21.868718+0000", 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 523.89412462799999, 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.113921999, 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.781 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 106338 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.868718+0000", 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.868718+0000", 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.868720+0000", 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.39e-06 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.868726+0000", 2023-09-30T09:49:05.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0209999999999996e-06 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.868743+0000", 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7813999999999999e-05 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.868753+0000", 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3640000000000005e-06 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.868831+0000", 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8285e-05 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:21.868844+0000", 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2957e-05 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:22.778292+0000", 2023-09-30T09:49:05.785 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90944813700000005 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:22.778302+0000", 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3260000000000002e-06 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.959211+0000", 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1809097529999999 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.959411+0000", 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000199209 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.959789+0000", 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00037824100000000003 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.962419+0000", 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026302080000000002 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.964369+0000", 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001949922 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.980852+0000", 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016483297000000001 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.980921+0000", 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8849000000000004e-05 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:40:23.982640+0000", 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0017192259999999999 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:126867 8.9 8:95ef1fdb:::benchmark_data_smithi019_478679_object34419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e429)", 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:46.708483+0000", 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 439.05436017900001, 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0489419290000002, 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 126867 2023-09-30T09:49:05.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:46.708483+0000", 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:46.708483+0000", 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:46.708485+0000", 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2759999999999999e-06 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:46.708495+0000", 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5999999999999996e-06 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:46.708526+0000", 2023-09-30T09:49:05.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1300000000000002e-05 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:46.708549+0000", 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2713000000000001e-05 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:46.708626+0000", 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7422999999999994e-05 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:46.708633+0000", 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6780000000000003e-06 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.182882+0000", 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4742497729999999 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.185553+0000", 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026705930000000002 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.189704+0000", 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041506879999999996 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.746786+0000", 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.55708238600000004 2023-09-30T09:49:05.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.749403+0000", 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002617093 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.750547+0000", 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011433540000000001 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.796 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.752706+0000", 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021591219999999999 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757425+0000", 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0047189299999999997 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:126957 8.11 e430/414 8:8c2c7c44:::benchmark_data_smithi019_478679_object34509:head v 430'6299, mlcod=407'5669)", 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:47.915204+0000", 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 437.847639141, 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.842278077, 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915204+0000", 2023-09-30T09:49:05.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915204+0000", 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915205+0000", 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8299999999999996e-07 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915212+0000", 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2420000000000003e-06 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915230+0000", 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8117999999999998e-05 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915237+0000", 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2880000000000004e-06 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.749662+0000", 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.834425247 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.749681+0000", 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8725000000000001e-05 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757449+0000", 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077676450000000001 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757482+0000", 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3028999999999998e-05 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:126964 8.1 e430/414 8:80550835:::benchmark_data_smithi019_478679_object34516:head v 430'6254, mlcod=407'5601)", 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:47.965225+0000", 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 437.79761759600001, 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7922756130000002, 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.803 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.965225+0000", 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.965225+0000", 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.965226+0000", 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8899999999999998e-07 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.965236+0000", 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8970000000000003e-06 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.965247+0000", 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.132e-05 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.965255+0000", 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.131e-06 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.806 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.751195+0000", 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7859402960000001 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.751215+0000", 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9925999999999999e-05 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.807 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757484+0000", 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062690929999999999 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757501+0000", 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7061000000000001e-05 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.808 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:126967 8.1 e430/414 8:87935f78:::benchmark_data_smithi019_478679_object34519:head v 430'6255, mlcod=407'5601)", 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:47.973009+0000", 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 437.78983339600001, 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7845078939999999, 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.973009+0000", 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.973009+0000", 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.973010+0000", 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.47e-07 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.973018+0000", 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6040000000000004e-06 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.973028+0000", 2023-09-30T09:49:05.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.039e-05 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.973036+0000", 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7249999999999997e-06 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.752655+0000", 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.779618806 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.752670+0000", 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4953999999999999e-05 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757503+0000", 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0048330990000000004 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757517+0000", 2023-09-30T09:49:05.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4469e-05 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136401 8.9 8:9625ec8a:::benchmark_data_smithi019_478679_object43953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:20.665965+0000", 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 405.09687747300001, 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5133121200000001, 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.814 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136401 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.665965+0000", 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.815 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.665965+0000", 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.665967+0000", 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.468e-06 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.665972+0000", 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7140000000000001e-06 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.665985+0000", 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3376e-05 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.665993+0000", 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5610000000000001e-06 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.666054+0000", 2023-09-30T09:49:05.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1525000000000005e-05 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.666066+0000", 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1488000000000001e-05 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.151275+0000", 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4852094980000001 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.818 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.153844+0000", 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025691310000000001 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.158027+0000", 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041827710000000001 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.819 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.168930+0000", 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010903021000000001 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.170034+0000", 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001104011 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.172215+0000", 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021810559999999998 2023-09-30T09:49:05.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.173704+0000", 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001489403 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.179278+0000", 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055730969999999999 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.821 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136423 8.9 8:9584df07:::benchmark_data_smithi019_478679_object43975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:20.696962+0000", 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 405.06588089799999, 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.520990077, 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136423 2023-09-30T09:49:05.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.696962+0000", 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.696962+0000", 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.696963+0000", 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4410000000000001e-06 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.696970+0000", 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0959999999999997e-06 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.696985+0000", 2023-09-30T09:49:05.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5525000000000001e-05 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.696993+0000", 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9719999999999992e-06 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.697058+0000", 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4858999999999997e-05 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:20.697068+0000", 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8770000000000005e-06 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.184855+0000", 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4877874110000002 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.187405+0000", 2023-09-30T09:49:05.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002549707 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.191474+0000", 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040689239999999998 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208385+0000", 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016911223999999999 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.209154+0000", 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00076936300000000001 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.211026+0000", 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018713359999999999 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.828 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.212480+0000", 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001453785 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.217952+0000", 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0054725570000000003 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.829 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136472 8.9 8:955d25e8:::benchmark_data_smithi019_478679_object44024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.066374+0000", 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.69646929800001, 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1592290759999999, 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136472 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.066374+0000", 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.066374+0000", 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.066375+0000", 2023-09-30T09:49:05.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5990000000000001e-06 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.066381+0000", 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4829999999999999e-06 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.066396+0000", 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5252000000000001e-05 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.066402+0000", 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2910000000000004e-06 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.066462+0000", 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9537000000000003e-05 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.066468+0000", 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7429999999999996e-06 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.194592+0000", 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1281236049999999 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.197507+0000", 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002915384 2023-09-30T09:49:05.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.201822+0000", 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043148309999999999 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.210135+0000", 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0083131260000000005 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.218154+0000", 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0080183870000000001 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.219021+0000", 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00086689099999999995 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.220441+0000", 2023-09-30T09:49:05.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0014205489999999999 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.225603+0000", 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0051613980000000002 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:136521 8.11 e441/414 8:89afbfc6:::benchmark_data_smithi019_478679_object44073:head v 441'6633, mlcod=440'6631)", 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.493633+0000", 2023-09-30T09:49:05.837 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.26920956800001, 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7008001510000001, 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.493633+0000", 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.493633+0000", 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.493634+0000", 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4300000000000002e-07 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.493641+0000", 2023-09-30T09:49:05.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5150000000000003e-06 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.493653+0000", 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2058999999999999e-05 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.493661+0000", 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0930000000000003e-06 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.169264+0000", 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6756034 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.169296+0000", 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2116000000000002e-05 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.194384+0000", 2023-09-30T09:49:05.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.025087379999999999 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.194433+0000", 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9744999999999999e-05 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:136524 8.1 e441/435 8:82ffdc3d:::benchmark_data_smithi019_478679_object44076:head v 441'6521, mlcod=407'5601)", 2023-09-30T09:49:05.842 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.524130+0000", 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.23871314899998, 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6703375039999999, 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.524130+0000", 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.524130+0000", 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.524130+0000", 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6899999999999997e-07 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.524138+0000", 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1890000000000001e-06 2023-09-30T09:49:05.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.524152+0000", 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4151000000000001e-05 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.524160+0000", 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4020000000000002e-06 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.170355+0000", 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.646195007 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.170383+0000", 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7863000000000001e-05 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.194437+0000", 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.024053562000000001 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.194467+0000", 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0660999999999997e-05 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136530 8.9 8:969774af:::benchmark_data_smithi019_478679_object44082:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.569945+0000", 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.19289818999999, 2023-09-30T09:49:05.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6880902309999999, 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136530 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.569945+0000", 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.569945+0000", 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.569946+0000", 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.252e-06 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.849 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.569952+0000", 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5820000000000001e-06 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.569964+0000", 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.259e-05 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.569971+0000", 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3990000000000001e-06 2023-09-30T09:49:05.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.226981+0000", 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.657010834 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.232728+0000", 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0057466990000000001 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.237314+0000", 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0045856389999999999 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242502+0000", 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0051884779999999998 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243596+0000", 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0010939210000000001 2023-09-30T09:49:05.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.251027+0000", 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0074307640000000003 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.252284+0000", 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012571539999999999 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.258035+0000", 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0057509190000000002 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:136533 8.11 e441/414 8:8b46944d:::benchmark_data_smithi019_478679_object44085:head v 441'6634, mlcod=440'6631)", 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.581702+0000", 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 404.181141282, 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.612792153, 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.854 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.581702+0000", 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.581702+0000", 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.581702+0000", 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3099999999999997e-07 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.581709+0000", 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0690000000000004e-06 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.581720+0000", 2023-09-30T09:49:05.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0504e-05 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.581727+0000", 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8979999999999999e-06 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.179325+0000", 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.597598611 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.179350+0000", 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4735e-05 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.194470+0000", 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015119608 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.194494+0000", 2023-09-30T09:49:05.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3997000000000001e-05 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153588 8.19 8:9ada6c33:::benchmark_data_smithi019_478679_object61140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:01.934430+0000", 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 363.828412399, 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7179577620000002, 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:49:05.859 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153588 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.934430+0000", 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.860 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.934430+0000", 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.934432+0000", 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.531e-06 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.934437+0000", 2023-09-30T09:49:05.861 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7389999999999999e-06 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.934447+0000", 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0231000000000001e-05 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.934457+0000", 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9539999999999999e-06 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.938558+0000", 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041008670000000002 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.938737+0000", 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000178889 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,3,4", 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.939177+0000", 2023-09-30T09:49:05.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00044043400000000002 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.942328+0000", 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031510610000000001 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.943000+0000", 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00067205100000000001 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.948550+0000", 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0055497899999999998 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.981327+0000", 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.032777313000000002 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:01.981338+0000", 2023-09-30T09:49:05.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0764000000000001e-05 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.969094+0000", 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98775553999999999 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.969107+0000", 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3148e-05 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.652322+0000", 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.683215041 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.652388+0000", 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6408999999999994e-05 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153603 8.19 8:9d676f8e:::benchmark_data_smithi019_478679_object61155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.018948+0000", 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 363.74389468099997, 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0181555839999996, 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153603 2023-09-30T09:49:05.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.018948+0000", 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.018948+0000", 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.018950+0000", 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3999999999999999e-06 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.018955+0000", 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2809999999999999e-06 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.018970+0000", 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5597999999999999e-05 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.018979+0000", 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8040000000000007e-06 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.019030+0000", 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0547000000000002e-05 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.019038+0000", 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0989999999999999e-06 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.969131+0000", 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95009326699999996 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.872 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.969137+0000", 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2099999999999998e-06 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.652593+0000", 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6834559690000002 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.652703+0000", 2023-09-30T09:49:05.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000109447 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,6", 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.653095+0000", 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00039250999999999999 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.658870+0000", 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005774263 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.658936+0000", 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5934999999999998e-05 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.662724+0000", 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0037882329999999998 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.036802+0000", 2023-09-30T09:49:05.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.374077762 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.036873+0000", 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1321000000000004e-05 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.037104+0000", 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000230938 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.876 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153650 8.19 8:9a1a13f4:::benchmark_data_smithi019_478679_object61202:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:04.255929+0000", 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 361.50691421499999, 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7814235169999999, 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153650 2023-09-30T09:49:05.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.255929+0000", 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.255929+0000", 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.255930+0000", 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.35e-06 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.255935+0000", 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1819999999999997e-06 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.255947+0000", 2023-09-30T09:49:05.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2201000000000001e-05 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.255954+0000", 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.584e-06 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.256006+0000", 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2070999999999997e-05 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.256013+0000", 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9639999999999997e-06 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.654899+0000", 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.398885666 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.655024+0000", 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012493000000000001 2023-09-30T09:49:05.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,6", 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.655260+0000", 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00023599100000000001 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.658977+0000", 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003716932 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.658987+0000", 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0232e-05 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.662761+0000", 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0037744839999999998 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.037147+0000", 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.374386028 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.037204+0000", 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6428999999999998e-05 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.037352+0000", 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014847300000000001 2023-09-30T09:49:05.884 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153653 8.19 8:9f29c27f:::benchmark_data_smithi019_478679_object61205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:05.269960+0000", 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 360.492883205, 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7675840460000001, 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.885 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153653 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.269960+0000", 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.886 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.269960+0000", 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.269961+0000", 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.748e-06 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.269969+0000", 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9629999999999998e-06 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.269991+0000", 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1803000000000001e-05 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.270004+0000", 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2414e-05 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.270074+0000", 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0847999999999997e-05 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.270088+0000", 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3964e-05 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.656466+0000", 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.38637734099999999 2023-09-30T09:49:05.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.656617+0000", 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015149599999999999 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,4,6", 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.656936+0000", 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00031892599999999998 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.659010+0000", 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020738110000000001 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.660183+0000", 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011729380000000001 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.662802+0000", 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026187810000000001 2023-09-30T09:49:05.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.037381+0000", 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.374579679 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.037413+0000", 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2084e-05 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.892 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.037544+0000", 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013024999999999999 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2308 9.6 9:6c010bf7:::benchmark_data_smithi019_560486_object2307:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.073425+0000", 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 280.689417445, 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0051149229999998, 2023-09-30T09:49:05.893 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2308 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073425+0000", 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073425+0000", 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073427+0000", 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.53e-06 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073444+0000", 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7336000000000001e-05 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073457+0000", 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2767e-05 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073463+0000", 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7910000000000001e-06 2023-09-30T09:49:05.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073513+0000", 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9795e-05 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073528+0000", 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5732000000000001e-05 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.048589+0000", 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97506089299999998 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.048605+0000", 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6209e-05 2023-09-30T09:49:05.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.067517+0000", 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0189112579999999 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.067667+0000", 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014982999999999999 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.067825+0000", 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015861299999999999 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.069034+0000", 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001208771 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.900 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.069105+0000", 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0778000000000006e-05 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.078279+0000", 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091738259999999995 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.078342+0000", 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3880000000000005e-05 2023-09-30T09:49:05.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.078540+0000", 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019791399999999999 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:27891 10.8 e541/536 10:1fa30d81:::benchmark_data_smithi019_617837_object27890:head v 541'1693, mlcod=541'1691)", 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:49:03.653223+0000", 2023-09-30T09:49:05.902 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.1096201780000001, 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.26573897800000001, 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653223+0000", 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653223+0000", 2023-09-30T09:49:05.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653223+0000", 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7999999999999995e-07 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653245+0000", 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1165000000000001e-05 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653255+0000", 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0397000000000001e-05 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653262+0000", 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6959999999999999e-06 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.913747+0000", 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.26048544800000001 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.913774+0000", 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6580999999999998e-05 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.918909+0000", 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0051348599999999998 2023-09-30T09:49:05.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.918962+0000", 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3151e-05 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:27888 10.5 10:a8c437f8:::benchmark_data_smithi019_617837_object27887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e541)", 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:49:03.653244+0000", 2023-09-30T09:49:05.907 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 2.1095985800000001, 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.26456242000000002, 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 27888 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:05.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653244+0000", 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653244+0000", 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653246+0000", 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4169999999999999e-06 2023-09-30T09:49:05.909 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653281+0000", 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5114000000000003e-05 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653292+0000", 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0998999999999999e-05 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653307+0000", 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4773000000000001e-05 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653372+0000", 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5571e-05 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653507+0000", 2023-09-30T09:49:05.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013475999999999999 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.653637+0000", 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012967400000000001 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.914879+0000", 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.26124212299999999 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.914927+0000", 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8346000000000001e-05 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.917710+0000", 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0027828140000000002 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:49:05.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.917764+0000", 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3696999999999998e-05 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:03.917807+0000", 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3132e-05 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:05.914 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:49:05.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:05.916 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:05.916 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:49:05.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:49:06.017 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:06.055 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]': finished 2023-09-30T09:49:06.055 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]': finished 2023-09-30T09:49:06.055 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [7, 1]}]': finished 2023-09-30T09:49:06.055 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]': finished 2023-09-30T09:49:06.056 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 2]}]': finished 2023-09-30T09:49:06.056 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:06 smithi167 ceph-mon[192880]: failed to encode map e542 with expected crc 2023-09-30T09:49:06.056 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:06 smithi167 ceph-mon[192880]: failed to encode map e542 with expected crc 2023-09-30T09:49:06.056 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:06 smithi167 ceph-mon[192880]: osdmap e542: 8 total, 8 up, 7 in 2023-09-30T09:49:06.060 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:49:06.062 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:06.125 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:06.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:06.231 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:06.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]': finished 2023-09-30T09:49:06.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]': finished 2023-09-30T09:49:06.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [7, 1]}]': finished 2023-09-30T09:49:06.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]': finished 2023-09-30T09:49:06.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 2]}]': finished 2023-09-30T09:49:06.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[224043]: failed to encode map e542 with expected crc 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[224043]: failed to encode map e542 with expected crc 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[224043]: osdmap e542: 8 total, 8 up, 7 in 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]': finished 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]': finished 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [7, 1]}]': finished 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]': finished 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 2]}]': finished 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[216815]: failed to encode map e542 with expected crc 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[216815]: failed to encode map e542 with expected crc 2023-09-30T09:49:06.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:06 smithi019 ceph-mon[216815]: osdmap e542: 8 total, 8 up, 7 in 2023-09-30T09:49:06.307 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:06.715 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 15 16 32235 32219 134.19 99.125 0.00289631 0.0074473 2023-09-30T09:49:06.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:49:06.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:49:06.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:49:06.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:49:06.992 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:49:06.992 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:49:06.992 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:49:06.993 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:49:07.029 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:07.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:07.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:07.139 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:07.178 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:07.240 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:07.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:07.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[216815]: pgmap v1446: 119 pgs: 119 active+clean; 1.7 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 170 MiB/s wr, 2.72k op/s 2023-09-30T09:49:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[216815]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[216815]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[216815]: osdmap e543: 8 total, 8 up, 7 in 2023-09-30T09:49:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[216815]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[216815]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/966007385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:07.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[224043]: pgmap v1446: 119 pgs: 119 active+clean; 1.7 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 170 MiB/s wr, 2.72k op/s 2023-09-30T09:49:07.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[224043]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[224043]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[224043]: osdmap e543: 8 total, 8 up, 7 in 2023-09-30T09:49:07.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[224043]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[224043]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:07 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/966007385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:07.349 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:07.409 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:07 smithi167 ceph-mon[192880]: pgmap v1446: 119 pgs: 119 active+clean; 1.7 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 170 MiB/s wr, 2.72k op/s 2023-09-30T09:49:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:07 smithi167 ceph-mon[192880]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:07 smithi167 ceph-mon[192880]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:07 smithi167 ceph-mon[192880]: osdmap e543: 8 total, 8 up, 7 in 2023-09-30T09:49:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:07 smithi167 ceph-mon[192880]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:07 smithi167 ceph-mon[192880]: failed to encode map e543 with expected crc 2023-09-30T09:49:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:07 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/966007385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:07.714 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 16 16 33499 33483 130.741 79 0.00752733 0.00763275 2023-09-30T09:49:08.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:08 smithi019 ceph-mon[216815]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:08 smithi019 ceph-mon[216815]: osdmap e544: 8 total, 8 up, 7 in 2023-09-30T09:49:08.317 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:08 smithi019 ceph-mon[216815]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.317 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:08 smithi019 ceph-mon[216815]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:08 smithi019 ceph-mon[224043]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:08 smithi019 ceph-mon[224043]: osdmap e544: 8 total, 8 up, 7 in 2023-09-30T09:49:08.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:08 smithi019 ceph-mon[224043]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:08 smithi019 ceph-mon[224043]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.318 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:49:08.330 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:08.368 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:08.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:08.492 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:08.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:08 smithi167 ceph-mon[192880]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:08 smithi167 ceph-mon[192880]: osdmap e544: 8 total, 8 up, 7 in 2023-09-30T09:49:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:08 smithi167 ceph-mon[192880]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:08 smithi167 ceph-mon[192880]: failed to encode map e544 with expected crc 2023-09-30T09:49:08.528 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:08.572 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:08.588 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:08.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:08.695 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:08.716 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 17 16 34592 34576 127.049 68.3125 0.00958298 0.00786411 2023-09-30T09:49:08.727 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:08.790 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:08.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:08.897 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:08.935 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:08.998 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:09.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:09.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:09.074 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:09 smithi167 ceph-mon[192880]: pgmap v1449: 119 pgs: 119 active+clean; 1.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-30T09:49:09.074 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:09 smithi167 ceph-mon[192880]: failed to encode map e545 with expected crc 2023-09-30T09:49:09.074 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:09 smithi167 ceph-mon[192880]: failed to encode map e545 with expected crc 2023-09-30T09:49:09.074 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:09 smithi167 ceph-mon[192880]: osdmap e545: 8 total, 8 up, 7 in 2023-09-30T09:49:09.108 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:09.175 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:09 smithi019 ceph-mon[224043]: pgmap v1449: 119 pgs: 119 active+clean; 1.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-30T09:49:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:09 smithi019 ceph-mon[224043]: failed to encode map e545 with expected crc 2023-09-30T09:49:09.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:09 smithi019 ceph-mon[224043]: failed to encode map e545 with expected crc 2023-09-30T09:49:09.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:09 smithi019 ceph-mon[224043]: osdmap e545: 8 total, 8 up, 7 in 2023-09-30T09:49:09.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:09 smithi019 ceph-mon[216815]: pgmap v1449: 119 pgs: 119 active+clean; 1.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-30T09:49:09.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:09 smithi019 ceph-mon[216815]: failed to encode map e545 with expected crc 2023-09-30T09:49:09.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:09 smithi019 ceph-mon[216815]: failed to encode map e545 with expected crc 2023-09-30T09:49:09.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:09 smithi019 ceph-mon[216815]: osdmap e545: 8 total, 8 up, 7 in 2023-09-30T09:49:09.717 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 18 16 36217 36201 125.633 101.562 0.010757 0.00795321 2023-09-30T09:49:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:10 smithi019 ceph-mon[216815]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:10 smithi019 ceph-mon[216815]: osdmap e546: 8 total, 8 up, 7 in 2023-09-30T09:49:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:10 smithi019 ceph-mon[216815]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:10 smithi019 ceph-mon[216815]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:10 smithi019 ceph-mon[224043]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:10 smithi019 ceph-mon[224043]: osdmap e546: 8 total, 8 up, 7 in 2023-09-30T09:49:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:10 smithi019 ceph-mon[224043]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:10 smithi019 ceph-mon[224043]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.368 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:49:10.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:10 smithi167 ceph-mon[192880]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:10 smithi167 ceph-mon[192880]: osdmap e546: 8 total, 8 up, 7 in 2023-09-30T09:49:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:10 smithi167 ceph-mon[192880]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:10 smithi167 ceph-mon[192880]: failed to encode map e546 with expected crc 2023-09-30T09:49:10.717 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 19 16 37854 37838 124.404 102.312 0.00795716 0.00803304 2023-09-30T09:49:11.161 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:11.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:11.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:11.271 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:49:11.311 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:11.376 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:49:11.382 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:11 smithi167 ceph-mon[192880]: pgmap v1452: 119 pgs: 1 activating, 2 peering, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 3.5 KiB/s rd, 94 MiB/s wr, 1.51k op/s; 13 KiB/s, 0 objects/s recovering 2023-09-30T09:49:11.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:11.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:49:11.483 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:11 smithi019 ceph-mon[216815]: pgmap v1452: 119 pgs: 1 activating, 2 peering, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 3.5 KiB/s rd, 94 MiB/s wr, 1.51k op/s; 13 KiB/s, 0 objects/s recovering 2023-09-30T09:49:11.483 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:11 smithi019 ceph-mon[224043]: pgmap v1452: 119 pgs: 1 activating, 2 peering, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 3.5 KiB/s rd, 94 MiB/s wr, 1.51k op/s; 13 KiB/s, 0 objects/s recovering 2023-09-30T09:49:11.484 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:11.519 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:11.570 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:11.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:11.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:11.666 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:11.676 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:11.717 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:49:11.717746+0000 min lat: 0.00112729 max lat: 0.388787 avg lat: 0.00820484 2023-09-30T09:49:11.717 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:49:11.717 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 20 16 39008 38992 121.79 72.125 0.00429843 0.00820484 2023-09-30T09:49:11.777 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:11.789 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:11.993 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:11.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:12.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:12.100 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:49:12.146 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:12.212 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:49:12.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:12.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:49:12.317 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:12.355 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:12.428 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:12.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:12.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:12.537 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:49:12.631 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:12.717 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 21 16 40417 40401 120.184 88.0625 0.00748447 0.00831394 2023-09-30T09:49:13.148 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:49:13.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:13 smithi019 ceph-mon[216815]: pgmap v1453: 119 pgs: 1 activating, 2 peering, 116 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.4 KiB/s rd, 86 MiB/s wr, 1.37k op/s; 8.9 KiB/s, 0 objects/s recovering 2023-09-30T09:49:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/33048100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:13.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:13 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:13 smithi019 ceph-mon[224043]: pgmap v1453: 119 pgs: 1 activating, 2 peering, 116 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.4 KiB/s rd, 86 MiB/s wr, 1.37k op/s; 8.9 KiB/s, 0 objects/s recovering 2023-09-30T09:49:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/33048100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:13.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:13 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:13.401 INFO:teuthology.orchestra.run.smithi019.stderr:Error EINVAL: specified pg_num 81 < pg_num_max 32 2023-09-30T09:49:13.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:13 smithi167 ceph-mon[192880]: pgmap v1453: 119 pgs: 1 activating, 2 peering, 116 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.4 KiB/s rd, 86 MiB/s wr, 1.37k op/s; 8.9 KiB/s, 0 objects/s recovering 2023-09-30T09:49:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/33048100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:13 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:13.571 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:49:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:13.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:49:13.680 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:13.717 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 22 16 41649 41633 118.222 77 0.00979707 0.0084532 2023-09-30T09:49:13.719 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:13.802 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:13.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:13.907 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:13.940 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:49:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:49:13] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-09-30T09:49:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:49:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: osdmap e547: 8 total, 8 up, 7 in 2023-09-30T09:49:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/30426284' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-30T09:49:14.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: osdmap e547: 8 total, 8 up, 7 in 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/30426284' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-30T09:49:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:49:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: osdmap e547: 8 total, 8 up, 7 in 2023-09-30T09:49:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: failed to encode map e547 with expected crc 2023-09-30T09:49:14.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/30426284' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-30T09:49:14.717 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 23 16 42840 42824 116.319 74.4375 0.00690211 0.00859254 2023-09-30T09:49:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:15 smithi167 ceph-mon[192880]: pgmap v1455: 119 pgs: 1 activating, 1 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.3 KiB/s rd, 99 MiB/s wr, 1.59k op/s; 8.7 KiB/s, 0 objects/s recovering 2023-09-30T09:49:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:15 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:49:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:15 smithi167 ceph-mon[192880]: 1.1c scrub ok 2023-09-30T09:49:15.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:15 smithi019 ceph-mon[216815]: pgmap v1455: 119 pgs: 1 activating, 1 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.3 KiB/s rd, 99 MiB/s wr, 1.59k op/s; 8.7 KiB/s, 0 objects/s recovering 2023-09-30T09:49:15.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:15 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:49:15.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:15 smithi019 ceph-mon[216815]: 1.1c scrub ok 2023-09-30T09:49:15.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:15 smithi019 ceph-mon[224043]: pgmap v1455: 119 pgs: 1 activating, 1 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.3 KiB/s rd, 99 MiB/s wr, 1.59k op/s; 8.7 KiB/s, 0 objects/s recovering 2023-09-30T09:49:15.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:15 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:49:15.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:15 smithi019 ceph-mon[224043]: 1.1c scrub ok 2023-09-30T09:49:15.718 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 24 16 43798 43782 113.967 59.875 0.00597865 0.0087698 2023-09-30T09:49:15.962 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:15.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:15.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:16.073 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:16.110 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:16.721 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 25 16 45205 45189 112.91 87.9375 0.0076561 0.00885117 2023-09-30T09:49:16.920 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:16.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:16.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:49:17.025 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:17.114 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:17.341 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:17.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:17.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:17.447 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:17.458 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:49:17.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:17 smithi019 ceph-mon[216815]: pgmap v1456: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-30T09:49:17.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:17 smithi019 ceph-mon[216815]: 1.f scrub starts 2023-09-30T09:49:17.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:17 smithi019 ceph-mon[216815]: 1.f scrub ok 2023-09-30T09:49:17.477 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:17 smithi019 ceph-mon[224043]: pgmap v1456: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-30T09:49:17.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:17 smithi019 ceph-mon[224043]: 1.f scrub starts 2023-09-30T09:49:17.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:17 smithi019 ceph-mon[224043]: 1.f scrub ok 2023-09-30T09:49:17.487 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:17 smithi167 ceph-mon[192880]: pgmap v1456: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-30T09:49:17.487 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:17 smithi167 ceph-mon[192880]: 1.f scrub starts 2023-09-30T09:49:17.487 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:17 smithi167 ceph-mon[192880]: 1.f scrub ok 2023-09-30T09:49:17.492 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:17.508 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:49:17.555 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:17.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:17.560 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:49:17.604 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:49:17.646 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:49:17.660 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:17.684 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:49:17.699 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:17.723 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 26 16 46534 46518 111.757 83.0625 0.0133415 0.00894223 2023-09-30T09:49:17.727 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:49:17.761 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:49:17.768 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:17.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:17.778 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:49:17.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:49:17.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:49:17.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:49:17.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:49:17.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:49:17.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:49:17.779 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:49:17.838 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:49:17.838 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:49:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:46663 10.d e547/536 10:b7df1abd:::benchmark_data_smithi019_617837_object46662:head v 547'2867, mlcod=547'2865)", 2023-09-30T09:49:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:49:17.832311+0000", 2023-09-30T09:49:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 0.0021615879999999999, 2023-09-30T09:49:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021971930000000001, 2023-09-30T09:49:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:49:17.839 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:49:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:49:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:49:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:17.832311+0000", 2023-09-30T09:49:17.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:17.832311+0000", 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:17.832312+0000", 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.345e-06 2023-09-30T09:49:17.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:17.832351+0000", 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8395000000000001e-05 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:17.832367+0000", 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6249e-05 2023-09-30T09:49:17.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:17.832376+0000", 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8659999999999995e-06 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:17.832436+0000", 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0482999999999997e-05 2023-09-30T09:49:17.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:49:17.832460+0000", 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3286e-05 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:49:17.844 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2023-09-30T09:49:17.845 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 1 2023-09-30T09:49:17.845 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:49:17.876 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:17.918 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:17.988 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:17.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:18.009 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:49:18.095 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:18.174 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:18.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:18 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3400116795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:18 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3400116795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:18.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:18.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:18 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3400116795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:18.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:18.725 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 27 16 47887 47871 110.75 84.5625 0.00863165 0.00902347 2023-09-30T09:49:19.266 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:19.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:19.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:19.375 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:19.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:19 smithi167 ceph-mon[192880]: pgmap v1457: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-30T09:49:19.522 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:19 smithi019 ceph-mon[216815]: pgmap v1457: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-30T09:49:19.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:19 smithi019 ceph-mon[224043]: pgmap v1457: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-30T09:49:19.527 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:19.723 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 28 16 48986 48970 109.249 68.6875 0.00786225 0.00914869 2023-09-30T09:49:19.923 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:19.933 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:49:19.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:19.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:20.038 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:49:20.090 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:20.095 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:20.242 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:49:20.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:20.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:49:20.346 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:49:20.447 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:20.575 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:49:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:20.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:49:20.686 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:49:20.720 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:20.722 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 29 16 50209 50193 108.117 76.4375 0.0183924 0.00924397 2023-09-30T09:49:20.797 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:49:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:20.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:49:20.905 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:20.942 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:21.008 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:21.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:21.114 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:21.170 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:21.222 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:21.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:21.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:21.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:21 smithi167 ceph-mon[192880]: pgmap v1458: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-30T09:49:21.325 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:21.351 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:21.392 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:21.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:21.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:21.497 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:21.546 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:21.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:21 smithi019 ceph-mon[216815]: pgmap v1458: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-30T09:49:21.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:21 smithi019 ceph-mon[224043]: pgmap v1458: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-30T09:49:21.613 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:21.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:21.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:21.720 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:21.726 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 30 16 51281 51265 106.742 67 0.0111934 0.00936326 2023-09-30T09:49:21.786 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:21.885 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:49:21.907 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:21.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:21.987 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3] out_osds: [6] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:49:21.987 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:49:21.988 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-30T09:49:21.988 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-30T09:49:22.024 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:22.045 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:22.259 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:22.426 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:22.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:22.544 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:22.577 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:22.632 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:22.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:22.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:22.728 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 31 16 52312 52296 105.366 64.4375 0.0131534 0.00948281 2023-09-30T09:49:22.739 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:22.779 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:22.837 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:22.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:22.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:22.945 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:22.980 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:23.040 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:23.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:23.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:23.147 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:23.184 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:23.253 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:23.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:23.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:23.363 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:23.406 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:23.452 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:23.457 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:23 smithi167 ceph-mon[192880]: pgmap v1459: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-30T09:49:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:23.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:23.559 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:23.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:23 smithi019 ceph-mon[216815]: pgmap v1459: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-30T09:49:23.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:23 smithi019 ceph-mon[224043]: pgmap v1459: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-30T09:49:23.585 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:23.627 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:23.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:23.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:23.730 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 32 16 53227 53211 103.861 57.1875 0.0088609 0.00962406 2023-09-30T09:49:23.733 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:23.801 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:49:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:49:23] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-09-30T09:49:24.131 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:24.157 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:24.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:24.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:24.220 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:49:24.264 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:24.288 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:24.345 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:24.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/887226559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:24.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:24 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:24.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:24 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:24.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/887226559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:24.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:24 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:24.462 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:24 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/887226559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:24 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:24.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:24 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:24.524 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:24.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:24.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:24.630 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:24.656 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:24.697 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:24.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:24.728 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 33 16 54269 54253 102.688 65.125 0.0026083 0.00968816 2023-09-30T09:49:24.802 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:24.868 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:25.220 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[216815]: pgmap v1460: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-30T09:49:25.220 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:49:25.220 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:49:25.220 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:25.220 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[216815]: failed to encode map e548 with expected crc 2023-09-30T09:49:25.220 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[216815]: osdmap e548: 8 total, 8 up, 7 in 2023-09-30T09:49:25.235 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:25.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:25.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:25.346 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:25.386 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:25.445 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:25.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:25.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:25 smithi167 ceph-mon[192880]: pgmap v1460: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-30T09:49:25.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:25 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:49:25.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:25 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:49:25.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:25 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:25.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:25 smithi167 ceph-mon[192880]: failed to encode map e548 with expected crc 2023-09-30T09:49:25.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:25 smithi167 ceph-mon[192880]: osdmap e548: 8 total, 8 up, 7 in 2023-09-30T09:49:25.552 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:49:25.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[224043]: pgmap v1460: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-30T09:49:25.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:49:25.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:49:25.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:25.580 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[224043]: failed to encode map e548 with expected crc 2023-09-30T09:49:25.580 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:25 smithi019 ceph-mon[224043]: osdmap e548: 8 total, 8 up, 7 in 2023-09-30T09:49:25.598 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:25.673 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:49:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:25.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:49:25.728 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 34 16 55492 55476 101.916 76.4375 0.0166418 0.00980675 2023-09-30T09:49:25.783 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:25.824 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:25.886 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:25.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:25.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:25.992 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:26.075 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:49:26.075 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:49:26.075 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:49:26.075 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:49:26.075 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:49:26.087 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:26.234 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:26 smithi167 ceph-mon[192880]: failed to encode map e548 with expected crc 2023-09-30T09:49:26.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:26 smithi019 ceph-mon[216815]: failed to encode map e548 with expected crc 2023-09-30T09:49:26.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:26 smithi019 ceph-mon[224043]: failed to encode map e548 with expected crc 2023-09-30T09:49:26.728 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 35 16 56742 56726 101.236 78.125 0.00746559 0.00987343 2023-09-30T09:49:27.038 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:27.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:27.144 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:27.215 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:27.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:27 smithi019 ceph-mon[216815]: pgmap v1462: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-30T09:49:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:27 smithi019 ceph-mon[224043]: pgmap v1462: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-30T09:49:27.430 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:49:27.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:27 smithi167 ceph-mon[192880]: pgmap v1462: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-30T09:49:27.728 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 36 16 57484 57468 99.7131 46.375 0.0102452 0.0100239 2023-09-30T09:49:28.621 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:49:28.679 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:28.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:28.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:28.728 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 37 16 58492 58476 98.7212 63 0.0208529 0.0101208 2023-09-30T09:49:28.794 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:28.892 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:29.079 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:29.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:29.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:29.185 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:29.236 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:29.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:29 smithi019 ceph-mon[216815]: pgmap v1463: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-30T09:49:29.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:29 smithi019 ceph-mon[224043]: pgmap v1463: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-30T09:49:29.290 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:29.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:29.400 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:29.406 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:49:29.438 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:29.464 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:49:29.503 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:29.511 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:29 smithi167 ceph-mon[192880]: pgmap v1463: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-30T09:49:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:29.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:29.514 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:49:29.570 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:49:29.575 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:29.613 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:29.617 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:49:29.645 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:29.659 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:49:29.697 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:49:29.700 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:29.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:29.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:29.728 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 38 16 59452 59436 97.7025 60 0.0238638 0.0102292 2023-09-30T09:49:29.737 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:49:29.748 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:49:29.749 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:49:29.749 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:49:29.749 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:49:29.749 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:49:29.749 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:49:29.749 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:49:29.749 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:49:29.809 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:29.856 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:29.926 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:30.237 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:30.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:30.285 INFO:teuthology.orchestra.run.smithi019.stderr:marked in osd.6. 2023-09-30T09:49:30.343 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:30.368 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:30.410 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:30.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:30.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:30.515 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:30.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1596333586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:30.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:30 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4016786777' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-30T09:49:30.541 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:30 smithi019 conmon[216791]: 2023-09-30T09:49:30.112+0000 7faf52971700 -1 mon.a@0(leader).osd e548 definitely_dead 0 2023-09-30T09:49:30.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1596333586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:30.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:30 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4016786777' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-30T09:49:30.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1596333586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:30.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:30 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4016786777' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-30T09:49:30.544 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:30.728 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 39 16 60059 60043 96.1706 37.9375 0.00248146 0.010277 2023-09-30T09:49:31.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:31 smithi167 ceph-mon[192880]: pgmap v1464: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-30T09:49:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:31 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4016786777' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-30T09:49:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:31 smithi167 ceph-mon[192880]: failed to encode map e549 with expected crc 2023-09-30T09:49:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:31 smithi167 ceph-mon[192880]: failed to encode map e549 with expected crc 2023-09-30T09:49:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:31 smithi167 ceph-mon[192880]: osdmap e549: 8 total, 8 up, 8 in 2023-09-30T09:49:31.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[216815]: pgmap v1464: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-30T09:49:31.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4016786777' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-30T09:49:31.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[216815]: failed to encode map e549 with expected crc 2023-09-30T09:49:31.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[216815]: failed to encode map e549 with expected crc 2023-09-30T09:49:31.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[216815]: osdmap e549: 8 total, 8 up, 8 in 2023-09-30T09:49:31.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[224043]: pgmap v1464: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-30T09:49:31.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4016786777' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-30T09:49:31.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[224043]: failed to encode map e549 with expected crc 2023-09-30T09:49:31.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[224043]: failed to encode map e549 with expected crc 2023-09-30T09:49:31.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:31 smithi019 ceph-mon[224043]: osdmap e549: 8 total, 8 up, 8 in 2023-09-30T09:49:31.729 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:49:31.729245+0000 min lat: 0.00112729 max lat: 0.388787 avg lat: 0.010277 2023-09-30T09:49:31.729 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:49:31.729 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 40 16 60059 60043 93.7673 0 - 0.010277 2023-09-30T09:49:32.277 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:32.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:32.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:32.381 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:32.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: osdmap e550: 8 total, 8 up, 8 in 2023-09-30T09:49:32.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.399 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[216815]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.399 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.399 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.399 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.399 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.399 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.399 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.400 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: osdmap e550: 8 total, 8 up, 8 in 2023-09-30T09:49:32.400 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.400 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:32 smithi019 ceph-mon[224043]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.419 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:32.523 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:32.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:32.627 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:32.699 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:32.729 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 41 16 60146 60130 91.6138 2.71875 0.0216417 0.0109096 2023-09-30T09:49:32.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: osdmap e550: 8 total, 8 up, 8 in 2023-09-30T09:49:32.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: failed to encode map e550 with expected crc 2023-09-30T09:49:32.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:32 smithi167 ceph-mon[192880]: failed to encode map e550 with expected crc 2023-09-30T09:49:33.545 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: pgmap v1467: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s 2023-09-30T09:49:33.545 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: osdmap e551: 8 total, 8 up, 8 in 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:33.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: pgmap v1467: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: osdmap e551: 8 total, 8 up, 8 in 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:33.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:33.597 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-30T09:49:33.729 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 42 16 60929 60913 90.598 48.9375 0.0112144 0.0110322 2023-09-30T09:49:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: pgmap v1467: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s 2023-09-30T09:49:33.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: osdmap e551: 8 total, 8 up, 8 in 2023-09-30T09:49:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: failed to encode map e551 with expected crc 2023-09-30T09:49:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:33.775 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:33.814 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:49:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:49:33] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-09-30T09:49:33.896 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:33.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:33.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:34.006 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:34.086 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:34.124 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:49:34.261 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:34.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:34.329 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:34 smithi019 ceph-mon[216815]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.329 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:34 smithi019 ceph-mon[216815]: osdmap e552: 8 total, 8 up, 8 in 2023-09-30T09:49:34.330 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:34 smithi019 ceph-mon[224043]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.330 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:34 smithi019 ceph-mon[224043]: osdmap e552: 8 total, 8 up, 8 in 2023-09-30T09:49:34.330 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:34 smithi019 ceph-mon[224043]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.331 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:34 smithi019 ceph-mon[224043]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.378 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:34.426 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:34.498 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:34.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:34.558 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:49:34.558 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:49:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:34 smithi019 ceph-mon[216815]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:34 smithi019 ceph-mon[216815]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.603 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:34.690 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:34.734 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 43 16 61702 61686 89.6101 48.3125 0.0169342 0.0111531 2023-09-30T09:49:34.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:34 smithi167 ceph-mon[192880]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:34 smithi167 ceph-mon[192880]: osdmap e552: 8 total, 8 up, 8 in 2023-09-30T09:49:34.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:34 smithi167 ceph-mon[192880]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:34 smithi167 ceph-mon[192880]: failed to encode map e552 with expected crc 2023-09-30T09:49:34.843 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:34.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:34.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:34.963 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:49:35.032 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:35.116 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:49:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:35.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:49:35.221 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:35.247 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:35.294 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:35.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:35.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:35.400 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:35.459 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:35.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: pgmap v1470: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 5 remapped+peering, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 370 op/s; 3772/179082 objects misplaced (2.106%); 960 KiB/s, 14 objects/s recovering 2023-09-30T09:49:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: osdmap e553: 8 total, 8 up, 8 in 2023-09-30T09:49:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1376592708' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:49:35.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:49:35.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: pgmap v1470: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 5 remapped+peering, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 370 op/s; 3772/179082 objects misplaced (2.106%); 960 KiB/s, 14 objects/s recovering 2023-09-30T09:49:35.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: osdmap e553: 8 total, 8 up, 8 in 2023-09-30T09:49:35.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1376592708' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:49:35.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:35 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:49:35.731 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 44 16 62371 62355 88.5242 41.8125 0.0191967 0.01129 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: pgmap v1470: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 5 remapped+peering, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 370 op/s; 3772/179082 objects misplaced (2.106%); 960 KiB/s, 14 objects/s recovering 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: osdmap e553: 8 total, 8 up, 8 in 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: failed to encode map e553 with expected crc 2023-09-30T09:49:35.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1376592708' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:49:35.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:35 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:49:35.812 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:49:35.813 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:35.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:35.861 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:49:35.928 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:35.997 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:36.065 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:36.085 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:36.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:36.201 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:36.264 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:36.389 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:36.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:36.473 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:49:36.506 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:36.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:36 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:36 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:49:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:36 smithi167 ceph-mon[192880]: osdmap e554: 8 total, 8 up, 8 in 2023-09-30T09:49:36.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:36 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:36.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:36 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:49:36.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:36 smithi019 ceph-mon[216815]: osdmap e554: 8 total, 8 up, 8 in 2023-09-30T09:49:36.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:36 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:36.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:36 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:49:36.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:36 smithi019 ceph-mon[224043]: osdmap e554: 8 total, 8 up, 8 in 2023-09-30T09:49:36.535 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:36.731 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 45 16 63087 63071 87.5518 44.75 0.0129655 0.0114152 2023-09-30T09:49:36.910 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:36.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:36.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:37.016 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:37.078 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:37.178 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:37.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:37.284 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:37.340 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:37 smithi167 ceph-mon[192880]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.340 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:37 smithi167 ceph-mon[192880]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.340 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:37 smithi167 ceph-mon[192880]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.341 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:37 smithi167 ceph-mon[192880]: pgmap v1473: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 peering, 4 remapped+peering, 106 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 7512/182550 objects degraded (4.115%); 7566/182550 objects misplaced (4.145%); 976 KiB/s, 6 objects/s recovering 2023-09-30T09:49:37.341 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:37 smithi167 ceph-mon[192880]: Health check failed: Degraded data redundancy: 7512/182550 objects degraded (4.115%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:37.345 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:37.455 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:37.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:37.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:37.559 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[216815]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[216815]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[216815]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[216815]: pgmap v1473: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 peering, 4 remapped+peering, 106 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 7512/182550 objects degraded (4.115%); 7566/182550 objects misplaced (4.145%); 976 KiB/s, 6 objects/s recovering 2023-09-30T09:49:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[216815]: Health check failed: Degraded data redundancy: 7512/182550 objects degraded (4.115%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[224043]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[224043]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[224043]: failed to encode map e554 with expected crc 2023-09-30T09:49:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[224043]: pgmap v1473: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 peering, 4 remapped+peering, 106 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 7512/182550 objects degraded (4.115%); 7566/182550 objects misplaced (4.145%); 976 KiB/s, 6 objects/s recovering 2023-09-30T09:49:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:37 smithi019 ceph-mon[224043]: Health check failed: Degraded data redundancy: 7512/182550 objects degraded (4.115%), 2 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:37.642 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:37.732 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 46 16 63985 63969 86.8674 56.125 0.00988962 0.0115054 2023-09-30T09:49:37.742 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:37.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:37.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:37.846 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:37.873 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:37.918 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:37.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:37.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:38.021 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:38.086 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:38.395 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:38.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:38.500 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:38.566 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:38.597 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:49:38.598 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:49:38.598 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-09-30T09:49:38.598 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:49:38.667 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:38.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:38.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:38.734 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 47 16 64708 64692 85.9773 45.1875 0.0177065 0.0116245 2023-09-30T09:49:38.775 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:38.899 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:39.067 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:49:39.112 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:39.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:39.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:39.221 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:49:39.276 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:39.294 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:49:39.334 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:49:39.358 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:49:39.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:39.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:49:39.379 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:49:39.421 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:49:39.461 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:39.462 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:49:39.464 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:39.506 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:49:39.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:39 smithi167 ceph-mon[192880]: pgmap v1474: 119 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 peering, 107 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 31 MiB/s wr, 497 op/s; 18652/186243 objects degraded (10.015%); 7566/186243 objects misplaced (4.062%); 701 KiB/s, 4 objects/s recovering 2023-09-30T09:49:39.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/647980852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:39.543 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:39 smithi019 ceph-mon[216815]: pgmap v1474: 119 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 peering, 107 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 31 MiB/s wr, 497 op/s; 18652/186243 objects degraded (10.015%); 7566/186243 objects misplaced (4.062%); 701 KiB/s, 4 objects/s recovering 2023-09-30T09:49:39.544 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/647980852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:39.544 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:39 smithi019 ceph-mon[224043]: pgmap v1474: 119 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 peering, 107 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 31 MiB/s wr, 497 op/s; 18652/186243 objects degraded (10.015%); 7566/186243 objects misplaced (4.062%); 701 KiB/s, 4 objects/s recovering 2023-09-30T09:49:39.544 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/647980852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:39.544 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:49:39.549 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:39.589 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:49:39.626 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:49:39.627 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:49:39.627 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:49:39.627 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:49:39.627 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:49:39.627 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:49:39.627 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:49:39.627 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:49:39.721 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:39.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:39.734 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 48 16 65473 65457 85.1825 47.8125 0.017203 0.0117342 2023-09-30T09:49:39.835 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:39.923 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:40.090 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:40.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:40.198 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:40.269 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:40.734 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 49 16 66370 66354 84.5883 56.0625 0.022995 0.0118157 2023-09-30T09:49:41.735 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 50 16 67057 67041 83.7556 42.9375 0.0219953 0.0119343 2023-09-30T09:49:41.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:41 smithi167 ceph-mon[192880]: pgmap v1475: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 18652/192651 objects degraded (9.682%); 7345/192651 objects misplaced (3.813%); 2.7 MiB/s, 37 objects/s recovering 2023-09-30T09:49:41.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:41 smithi019 ceph-mon[224043]: pgmap v1475: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 18652/192651 objects degraded (9.682%); 7345/192651 objects misplaced (3.813%); 2.7 MiB/s, 37 objects/s recovering 2023-09-30T09:49:41.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:41 smithi019 ceph-mon[216815]: pgmap v1475: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 18652/192651 objects degraded (9.682%); 7345/192651 objects misplaced (3.813%); 2.7 MiB/s, 37 objects/s recovering 2023-09-30T09:49:41.962 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:42.005 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:42.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:42.114 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:42.196 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:42.334 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:42.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:42.443 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:42.510 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:42.615 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:42.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:42.692 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:42 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:49:42.721 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:42.736 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 51 16 67849 67833 83.0834 49.5 0.0184709 0.0120304 2023-09-30T09:49:42.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:42 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:49:42.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:42 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:49:42.822 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:42.961 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:42.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:42.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:42.967 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:49:42.967 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:68008 10.c e554/536 10:3c07d9fa:::benchmark_data_smithi019_617837_object68007:head v 554'4221, mlcod=554'4217)", 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:49:42.961553+0000", 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0047516210000000001, 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047760459999999999, 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:49:42.968 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.961553+0000", 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.961553+0000", 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.961552+0000", 2023-09-30T09:49:42.969 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.961580+0000", 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8133e-05 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.961591+0000", 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0985000000000001e-05 2023-09-30T09:49:42.970 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.961598+0000", 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3439999999999997e-06 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.964217+0000", 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026189469999999999 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.971 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.964241+0000", 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4134999999999998e-05 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:68000 10.e 10:71bb67ae:::benchmark_data_smithi019_617837_object67999:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e554)", 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:49:42.939413+0000", 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.026891414999999998, 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.026991593000000001, 2023-09-30T09:49:42.972 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 68000 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.939413+0000", 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.973 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.939413+0000", 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.939412+0000", 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.974 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.939450+0000", 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8003999999999997e-05 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.939462+0000", 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2636999999999999e-05 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.939468+0000", 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0480000000000004e-06 2023-09-30T09:49:42.975 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.953756+0000", 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014287631 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.953918+0000", 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000161614 2023-09-30T09:49:42.976 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.954113+0000", 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019510099999999999 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.962937+0000", 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.008824765 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.977 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.962967+0000", 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9492000000000002e-05 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:68004 10.c e554/536 10:328b5df3:::benchmark_data_smithi019_617837_object68003:head v 554'4219, mlcod=554'4217)", 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:49:42.950680+0000", 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.015624123, 2023-09-30T09:49:42.978 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015811314, 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.950680+0000", 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.950680+0000", 2023-09-30T09:49:42.979 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.950680+0000", 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.950709+0000", 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9739000000000001e-05 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.980 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.950726+0000", 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6558e-05 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.950733+0000", 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7839999999999998e-06 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.951637+0000", 2023-09-30T09:49:42.981 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00090478599999999998 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.951660+0000", 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2308999999999999e-05 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.982 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:68012 10.4 e554/536 10:28bc84cc:::benchmark_data_smithi019_617837_object68011:head v 554'4333, mlcod=554'4331)", 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:49:42.952307+0000", 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.013997684, 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014242134, 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.952307+0000", 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.983 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:49:42.985 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.952307+0000", 2023-09-30T09:49:42.985 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.985 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.985 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.985 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.952306+0000", 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.952350+0000", 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3656999999999999e-05 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.952367+0000", 2023-09-30T09:49:42.986 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7282000000000002e-05 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.952375+0000", 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6369999999999999e-06 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.954111+0000", 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017368010000000001 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.987 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.954133+0000", 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1151000000000001e-05 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:68006 10.c e554/536 10:3f22260b:::benchmark_data_smithi019_617837_object68005:head v 554'4220, mlcod=554'4217)", 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:49:42.953477+0000", 2023-09-30T09:49:42.988 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.012827257, 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013127262000000001, 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.953477+0000", 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.989 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.953477+0000", 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.953477+0000", 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.990 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.953504+0000", 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7858e-05 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.953521+0000", 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6642000000000001e-05 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.953528+0000", 2023-09-30T09:49:42.991 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8410000000000002e-06 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.956551+0000", 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003022907 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.956569+0000", 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8308000000000002e-05 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:49:42.992 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:68013 10.d e554/550 10:b756fad6:::benchmark_data_smithi019_617837_object68012:head v 554'4217, mlcod=0'0)", 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:49:42.957824+0000", 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0084808799999999997, 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088352450000000003, 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:49:42.993 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.957824+0000", 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.957824+0000", 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.957818+0000", 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999943 2023-09-30T09:49:42.994 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.957888+0000", 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0648999999999994e-05 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.957905+0000", 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6365e-05 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.995 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.957912+0000", 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.323e-06 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.962681+0000", 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047690739999999999 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.962715+0000", 2023-09-30T09:49:42.996 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4178999999999999e-05 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:68014 10.d e554/550 10:bd607c71:::benchmark_data_smithi019_617837_object68013:head v 554'4218, mlcod=0'0)", 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:49:42.958468+0000", 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0078366540000000002, 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0082542329999999997, 2023-09-30T09:49:42.997 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.958468+0000", 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.958468+0000", 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:49:42.998 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.958467+0000", 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.958587+0000", 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119364 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:42.999 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.958600+0000", 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3587999999999999e-05 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.958607+0000", 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3709999999999996e-06 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.965314+0000", 2023-09-30T09:49:43.000 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0067074630000000003 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:49:42.965343+0000", 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8759999999999999e-05 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 7 2023-09-30T09:49:43.001 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:49:43.066 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:43.141 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:43.219 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:49:43.229 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:49:43.334 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:43.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:43.444 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:43.502 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:43 smithi019 ceph-mon[216815]: pgmap v1476: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 37 MiB/s wr, 595 op/s; 18652/193389 objects degraded (9.645%); 7345/193389 objects misplaced (3.798%); 1.7 MiB/s, 23 objects/s recovering 2023-09-30T09:49:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:43 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18652/193389 objects degraded (9.645%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:43 smithi019 ceph-mon[216815]: from='client.36651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:49:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:43 smithi019 ceph-mon[224043]: pgmap v1476: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 37 MiB/s wr, 595 op/s; 18652/193389 objects degraded (9.645%); 7345/193389 objects misplaced (3.798%); 1.7 MiB/s, 23 objects/s recovering 2023-09-30T09:49:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:43 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18652/193389 objects degraded (9.645%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:43 smithi019 ceph-mon[224043]: from='client.36651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:49:43.737 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 52 16 68702 68686 82.5089 53.3125 0.0106591 0.0121141 2023-09-30T09:49:43.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:43 smithi167 ceph-mon[192880]: pgmap v1476: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 37 MiB/s wr, 595 op/s; 18652/193389 objects degraded (9.645%); 7345/193389 objects misplaced (3.798%); 1.7 MiB/s, 23 objects/s recovering 2023-09-30T09:49:43.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:43 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18652/193389 objects degraded (9.645%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:43.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:43 smithi167 ceph-mon[192880]: from='client.36651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:49:43.928 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:49:44.014 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:44.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:44.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:44.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:49:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:49:43] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-09-30T09:49:44.127 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:44.238 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:44.592 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:49:44.646 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:44.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:44.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:44.737 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 53 16 69535 69519 81.9346 52.0625 0.0128764 0.0121991 2023-09-30T09:49:44.752 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:44.787 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1476,"stamp":"2023-09-30T09:49:41.953899+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4227602343,"num_objects":64463,"num_object_clones":0,"num_object_copies":193389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18652,"num_objects_misplaced":7345,"num_objects_unfound":0,"num_objects_dirty":64463,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":65342,"num_write_kb":4136565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":597,"num_bytes_recovered":31489032,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70457,"ondisk_log_size":70457,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17566432,"kb_used_data":12467544,"kb_used_omap":30,"kb_used_meta":5098721,"kb_avail":732394784,"statfs":{"total":767960285184,"available":749972258816,"internally_reserved":0,"allocated":12766765056,"data_stored":12746825094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":5221090658},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":312541184,"num_objects":4769,"num_object_clones":0,"num_object_copies":14307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18652,"num_objects_misplaced":3573,"num_objects_unfound":0,"num_objects_dirty":4769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4769,"num_write_kb":305216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":14614617,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002617"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":977,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.309557+0000","last_change":"2023-09-30T09:44:26.055524+0000","last_active":"2023-09-30T09:49:34.309557+0000","last_peered":"2023-09-30T09:49:34.309557+0000","last_clean":"2023-09-30T09:49:34.309557+0000","last_became_active":"2023-09-30T09:44:26.055161+0000","last_became_peered":"2023-09-30T09:44:26.055161+0000","last_unstale":"2023-09-30T09:49:34.309557+0000","last_undegraded":"2023-09-30T09:49:34.309557+0000","last_fullsized":"2023-09-30T09:49:34.309557+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:11:08.494168+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":571,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317447+0000","last_change":"2023-09-30T09:33:33.604040+0000","last_active":"2023-09-30T09:49:34.317447+0000","last_peered":"2023-09-30T09:49:34.317447+0000","last_clean":"2023-09-30T09:49:34.317447+0000","last_became_active":"2023-09-30T09:33:33.603717+0000","last_became_peered":"2023-09-30T09:33:33.603717+0000","last_unstale":"2023-09-30T09:49:34.317447+0000","last_undegraded":"2023-09-30T09:49:34.317447+0000","last_fullsized":"2023-09-30T09:49:34.317447+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:51:44.287867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":468,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.318359+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:49:34.318359+0000","last_peered":"2023-09-30T09:49:34.318359+0000","last_clean":"2023-09-30T09:49:34.318359+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:49:34.318359+0000","last_undegraded":"2023-09-30T09:49:34.318359+0000","last_fullsized":"2023-09-30T09:49:34.318359+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":979,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.309674+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:49:34.309674+0000","last_peered":"2023-09-30T09:49:34.309674+0000","last_clean":"2023-09-30T09:49:34.309674+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:49:34.309674+0000","last_undegraded":"2023-09-30T09:49:34.309674+0000","last_fullsized":"2023-09-30T09:49:34.309674+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":119,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.318612+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:49:34.318612+0000","last_peered":"2023-09-30T09:49:34.318612+0000","last_clean":"2023-09-30T09:49:34.318612+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:49:34.318612+0000","last_undegraded":"2023-09-30T09:49:34.318612+0000","last_fullsized":"2023-09-30T09:49:34.318612+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":161,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.308040+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:49:34.308040+0000","last_peered":"2023-09-30T09:49:34.308040+0000","last_clean":"2023-09-30T09:49:34.308040+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:49:34.308040+0000","last_undegraded":"2023-09-30T09:49:34.308040+0000","last_fullsized":"2023-09-30T09:49:34.308040+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":786,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317460+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:49:34.317460+0000","last_peered":"2023-09-30T09:49:34.317460+0000","last_clean":"2023-09-30T09:49:34.317460+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:49:34.317460+0000","last_undegraded":"2023-09-30T09:49:34.317460+0000","last_fullsized":"2023-09-30T09:49:34.317460+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":60,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.307413+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:49:34.307413+0000","last_peered":"2023-09-30T09:49:34.307413+0000","last_clean":"2023-09-30T09:49:34.307413+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:49:34.307413+0000","last_undegraded":"2023-09-30T09:49:34.307413+0000","last_fullsized":"2023-09-30T09:49:34.307413+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.314075+0000","last_change":"2023-09-30T09:49:31.310014+0000","last_active":"2023-09-30T09:49:34.314075+0000","last_peered":"2023-09-30T09:49:34.314075+0000","last_clean":"2023-09-30T09:49:34.314075+0000","last_became_active":"2023-09-30T09:49:31.309121+0000","last_became_peered":"2023-09-30T09:49:31.309121+0000","last_unstale":"2023-09-30T09:49:34.314075+0000","last_undegraded":"2023-09-30T09:49:34.314075+0000","last_fullsized":"2023-09-30T09:49:34.314075+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:46.598805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":955,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312888+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:49:34.312888+0000","last_peered":"2023-09-30T09:49:34.312888+0000","last_clean":"2023-09-30T09:49:34.312888+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:49:34.312888+0000","last_undegraded":"2023-09-30T09:49:34.312888+0000","last_fullsized":"2023-09-30T09:49:34.312888+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":175,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.307691+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:49:34.307691+0000","last_peered":"2023-09-30T09:49:34.307691+0000","last_clean":"2023-09-30T09:49:34.307691+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:49:34.307691+0000","last_undegraded":"2023-09-30T09:49:34.307691+0000","last_fullsized":"2023-09-30T09:49:34.307691+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":151,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.316978+0000","last_change":"2023-09-30T09:49:31.295632+0000","last_active":"2023-09-30T09:49:34.316978+0000","last_peered":"2023-09-30T09:49:34.316978+0000","last_clean":"2023-09-30T09:49:34.316978+0000","last_became_active":"2023-09-30T09:49:31.295269+0000","last_became_peered":"2023-09-30T09:49:31.295269+0000","last_unstale":"2023-09-30T09:49:34.316978+0000","last_undegraded":"2023-09-30T09:49:34.316978+0000","last_fullsized":"2023-09-30T09:49:34.316978+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:32.754585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":745,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.311372+0000","last_change":"2023-09-30T09:38:30.733527+0000","last_active":"2023-09-30T09:49:34.311372+0000","last_peered":"2023-09-30T09:49:34.311372+0000","last_clean":"2023-09-30T09:49:34.311372+0000","last_became_active":"2023-09-30T09:38:30.733239+0000","last_became_peered":"2023-09-30T09:38:30.733239+0000","last_unstale":"2023-09-30T09:49:34.311372+0000","last_undegraded":"2023-09-30T09:49:34.311372+0000","last_fullsized":"2023-09-30T09:49:34.311372+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:05:31.042179+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":374,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.310433+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:49:34.310433+0000","last_peered":"2023-09-30T09:49:34.310433+0000","last_clean":"2023-09-30T09:49:34.310433+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:49:34.310433+0000","last_undegraded":"2023-09-30T09:49:34.310433+0000","last_fullsized":"2023-09-30T09:49:34.310433+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":530,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312096+0000","last_change":"2023-09-30T09:49:31.309241+0000","last_active":"2023-09-30T09:49:34.312096+0000","last_peered":"2023-09-30T09:49:34.312096+0000","last_clean":"2023-09-30T09:49:34.312096+0000","last_became_active":"2023-09-30T09:49:31.308703+0000","last_became_peered":"2023-09-30T09:49:31.308703+0000","last_unstale":"2023-09-30T09:49:34.312096+0000","last_undegraded":"2023-09-30T09:49:34.312096+0000","last_fullsized":"2023-09-30T09:49:34.312096+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:06.877644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":723,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.336441+0000","last_change":"2023-09-30T09:49:34.336174+0000","last_active":"2023-09-30T09:49:34.336441+0000","last_peered":"2023-09-30T09:49:34.336441+0000","last_clean":"2023-09-30T09:49:34.336441+0000","last_became_active":"2023-09-30T09:49:34.335879+0000","last_became_peered":"2023-09-30T09:49:34.335879+0000","last_unstale":"2023-09-30T09:49:34.336441+0000","last_undegraded":"2023-09-30T09:49:34.336441+0000","last_fullsized":"2023-09-30T09:49:34.336441+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:42.613890+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":268,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313050+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:49:34.313050+0000","last_peered":"2023-09-30T09:49:34.313050+0000","last_clean":"2023-09-30T09:49:34.313050+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:49:34.313050+0000","last_undegraded":"2023-09-30T09:49:34.313050+0000","last_fullsized":"2023-09-30T09:49:34.313050+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313092+0000","last_change":"2023-09-30T09:49:31.307558+0000","last_active":"2023-09-30T09:49:34.313092+0000","last_peered":"2023-09-30T09:49:34.313092+0000","last_clean":"2023-09-30T09:49:34.313092+0000","last_became_active":"2023-09-30T09:49:31.304494+0000","last_became_peered":"2023-09-30T09:49:31.304494+0000","last_unstale":"2023-09-30T09:49:34.313092+0000","last_undegraded":"2023-09-30T09:49:34.313092+0000","last_fullsized":"2023-09-30T09:49:34.313092+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:15.622860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":610,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.308103+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:49:34.308103+0000","last_peered":"2023-09-30T09:49:34.308103+0000","last_clean":"2023-09-30T09:49:34.308103+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:49:34.308103+0000","last_undegraded":"2023-09-30T09:49:34.308103+0000","last_fullsized":"2023-09-30T09:49:34.308103+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":637,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317991+0000","last_change":"2023-09-30T09:49:31.318169+0000","last_active":"2023-09-30T09:49:34.317991+0000","last_peered":"2023-09-30T09:49:34.317991+0000","last_clean":"2023-09-30T09:49:34.317991+0000","last_became_active":"2023-09-30T09:49:31.304043+0000","last_became_peered":"2023-09-30T09:49:31.304043+0000","last_unstale":"2023-09-30T09:49:34.317991+0000","last_undegraded":"2023-09-30T09:49:34.317991+0000","last_fullsized":"2023-09-30T09:49:34.317991+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:39.078569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28332,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":595,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317836+0000","last_change":"2023-09-30T09:49:31.304624+0000","last_active":"2023-09-30T09:49:34.317836+0000","last_peered":"2023-09-30T09:49:34.317836+0000","last_clean":"2023-09-30T09:49:34.317836+0000","last_became_active":"2023-09-30T09:49:31.304106+0000","last_became_peered":"2023-09-30T09:49:31.304106+0000","last_unstale":"2023-09-30T09:49:34.317836+0000","last_undegraded":"2023-09-30T09:49:34.317836+0000","last_fullsized":"2023-09-30T09:49:34.317836+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:42.343641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":809,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.308610+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:49:34.308610+0000","last_peered":"2023-09-30T09:49:34.308610+0000","last_clean":"2023-09-30T09:49:34.308610+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:49:34.308610+0000","last_undegraded":"2023-09-30T09:49:34.308610+0000","last_fullsized":"2023-09-30T09:49:34.308610+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":204,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312469+0000","last_change":"2023-09-30T09:38:27.703628+0000","last_active":"2023-09-30T09:49:34.312469+0000","last_peered":"2023-09-30T09:49:34.312469+0000","last_clean":"2023-09-30T09:49:34.312469+0000","last_became_active":"2023-09-30T09:38:27.679639+0000","last_became_peered":"2023-09-30T09:38:27.679639+0000","last_unstale":"2023-09-30T09:49:34.312469+0000","last_undegraded":"2023-09-30T09:49:34.312469+0000","last_fullsized":"2023-09-30T09:49:34.312469+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:11:53.937720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":531,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312523+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:49:34.312523+0000","last_peered":"2023-09-30T09:49:34.312523+0000","last_clean":"2023-09-30T09:49:34.312523+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:49:34.312523+0000","last_undegraded":"2023-09-30T09:49:34.312523+0000","last_fullsized":"2023-09-30T09:49:34.312523+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":915,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.332842+0000","last_change":"2023-09-30T09:49:34.332543+0000","last_active":"2023-09-30T09:49:34.332842+0000","last_peered":"2023-09-30T09:49:34.332842+0000","last_clean":"2023-09-30T09:49:34.332842+0000","last_became_active":"2023-09-30T09:49:34.332215+0000","last_became_peered":"2023-09-30T09:49:34.332215+0000","last_unstale":"2023-09-30T09:49:34.332842+0000","last_undegraded":"2023-09-30T09:49:34.332842+0000","last_fullsized":"2023-09-30T09:49:34.332842+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.827671+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":135,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.319643+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:49:34.319643+0000","last_peered":"2023-09-30T09:49:34.319643+0000","last_clean":"2023-09-30T09:49:34.319643+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:49:34.319643+0000","last_undegraded":"2023-09-30T09:49:34.319643+0000","last_fullsized":"2023-09-30T09:49:34.319643+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":229,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.310165+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:49:34.310165+0000","last_peered":"2023-09-30T09:49:34.310165+0000","last_clean":"2023-09-30T09:49:34.310165+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:49:34.310165+0000","last_undegraded":"2023-09-30T09:49:34.310165+0000","last_fullsized":"2023-09-30T09:49:34.310165+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":956,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317365+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:49:34.317365+0000","last_peered":"2023-09-30T09:49:34.317365+0000","last_clean":"2023-09-30T09:49:34.317365+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:49:34.317365+0000","last_undegraded":"2023-09-30T09:49:34.317365+0000","last_fullsized":"2023-09-30T09:49:34.317365+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":258,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312184+0000","last_change":"2023-09-30T09:49:31.309022+0000","last_active":"2023-09-30T09:49:34.312184+0000","last_peered":"2023-09-30T09:49:34.312184+0000","last_clean":"2023-09-30T09:49:34.312184+0000","last_became_active":"2023-09-30T09:49:31.308236+0000","last_became_peered":"2023-09-30T09:49:31.308236+0000","last_unstale":"2023-09-30T09:49:34.312184+0000","last_undegraded":"2023-09-30T09:49:34.312184+0000","last_fullsized":"2023-09-30T09:49:34.312184+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:39.779954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":143,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:35.870327+0000","last_change":"2023-09-30T09:49:06.060170+0000","last_active":"2023-09-30T09:49:35.870327+0000","last_peered":"2023-09-30T09:49:35.870327+0000","last_clean":"2023-09-30T09:49:35.870327+0000","last_became_active":"2023-09-30T09:49:06.059837+0000","last_became_peered":"2023-09-30T09:49:06.059837+0000","last_unstale":"2023-09-30T09:49:35.870327+0000","last_undegraded":"2023-09-30T09:49:35.870327+0000","last_fullsized":"2023-09-30T09:49:35.870327+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:51:50.308197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":1004,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.332996+0000","last_change":"2023-09-30T09:49:34.332561+0000","last_active":"2023-09-30T09:49:34.332996+0000","last_peered":"2023-09-30T09:49:34.332996+0000","last_clean":"2023-09-30T09:49:34.332996+0000","last_became_active":"2023-09-30T09:49:34.332170+0000","last_became_peered":"2023-09-30T09:49:34.332170+0000","last_unstale":"2023-09-30T09:49:34.332996+0000","last_undegraded":"2023-09-30T09:49:34.332996+0000","last_fullsized":"2023-09-30T09:49:34.332996+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:22.548068+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":24,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313262+0000","last_change":"2023-09-30T09:49:31.325085+0000","last_active":"2023-09-30T09:49:34.313262+0000","last_peered":"2023-09-30T09:49:34.313262+0000","last_clean":"2023-09-30T09:49:34.313262+0000","last_became_active":"2023-09-30T09:49:31.299012+0000","last_became_peered":"2023-09-30T09:49:31.299012+0000","last_unstale":"2023-09-30T09:49:34.313262+0000","last_undegraded":"2023-09-30T09:49:34.313262+0000","last_fullsized":"2023-09-30T09:49:34.313262+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:28:16.366701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":119,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.320054+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:49:34.320054+0000","last_peered":"2023-09-30T09:49:34.320054+0000","last_clean":"2023-09-30T09:49:34.320054+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:49:34.320054+0000","last_undegraded":"2023-09-30T09:49:34.320054+0000","last_fullsized":"2023-09-30T09:49:34.320054+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"554'3837","reported_seq":3866,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:37.024745+0000","last_change":"2023-09-30T09:48:40.945814+0000","last_active":"2023-09-30T09:49:37.024745+0000","last_peered":"2023-09-30T09:49:37.024745+0000","last_clean":"2023-09-30T09:49:37.024745+0000","last_became_active":"2023-09-30T09:48:40.945431+0000","last_became_peered":"2023-09-30T09:48:40.945431+0000","last_unstale":"2023-09-30T09:49:37.024745+0000","last_undegraded":"2023-09-30T09:49:37.024745+0000","last_fullsized":"2023-09-30T09:49:37.024745+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":3837,"log_dups_size":0,"ondisk_log_size":3837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:36:56.353301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251461632,"num_objects":3837,"num_object_clones":0,"num_object_copies":11511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3837,"num_write_kb":245568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":930,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312968+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:49:34.312968+0000","last_peered":"2023-09-30T09:49:34.312968+0000","last_clean":"2023-09-30T09:49:34.312968+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:49:34.312968+0000","last_undegraded":"2023-09-30T09:49:34.312968+0000","last_fullsized":"2023-09-30T09:49:34.312968+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":230,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313900+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:49:34.313900+0000","last_peered":"2023-09-30T09:49:34.313900+0000","last_clean":"2023-09-30T09:49:34.313900+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:49:34.313900+0000","last_undegraded":"2023-09-30T09:49:34.313900+0000","last_fullsized":"2023-09-30T09:49:34.313900+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":195,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.308925+0000","last_change":"2023-09-30T09:49:31.304898+0000","last_active":"2023-09-30T09:49:34.308925+0000","last_peered":"2023-09-30T09:49:34.308925+0000","last_clean":"2023-09-30T09:49:34.308925+0000","last_became_active":"2023-09-30T09:49:31.304532+0000","last_became_peered":"2023-09-30T09:49:31.304532+0000","last_unstale":"2023-09-30T09:49:34.308925+0000","last_undegraded":"2023-09-30T09:49:34.308925+0000","last_fullsized":"2023-09-30T09:49:34.308925+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:03:34.425387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"554'4127","reported_seq":4162,"reported_epoch":554,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-30T09:49:39.073773+0000","last_change":"2023-09-30T09:49:32.662986+0000","last_active":"2023-09-30T09:49:39.073773+0000","last_peered":"2023-09-30T09:49:39.073773+0000","last_clean":"2023-09-30T09:49:30.238890+0000","last_became_active":"2023-09-30T09:49:32.625304+0000","last_became_peered":"2023-09-30T09:49:32.625304+0000","last_unstale":"2023-09-30T09:49:39.073773+0000","last_undegraded":"2023-09-30T09:49:39.073773+0000","last_fullsized":"2023-09-30T09:49:39.073773+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4127,"log_dups_size":0,"ondisk_log_size":4127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:28:55.359568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270467072,"num_objects":4127,"num_object_clones":0,"num_object_copies":12381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3794,"num_objects_unfound":0,"num_objects_dirty":4127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4127,"num_write_kb":264128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":333}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":984,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.343447+0000","last_change":"2023-09-30T09:49:34.340125+0000","last_active":"2023-09-30T09:49:34.343447+0000","last_peered":"2023-09-30T09:49:34.343447+0000","last_clean":"2023-09-30T09:49:34.343447+0000","last_became_active":"2023-09-30T09:49:34.339638+0000","last_became_peered":"2023-09-30T09:49:34.339638+0000","last_unstale":"2023-09-30T09:49:34.343447+0000","last_undegraded":"2023-09-30T09:49:34.343447+0000","last_fullsized":"2023-09-30T09:49:34.343447+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:43:43.258960+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":200,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313399+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:49:34.313399+0000","last_peered":"2023-09-30T09:49:34.313399+0000","last_clean":"2023-09-30T09:49:34.313399+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:49:34.313399+0000","last_undegraded":"2023-09-30T09:49:34.313399+0000","last_fullsized":"2023-09-30T09:49:34.313399+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":604,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.319407+0000","last_change":"2023-09-30T09:42:48.651858+0000","last_active":"2023-09-30T09:49:34.319407+0000","last_peered":"2023-09-30T09:49:34.319407+0000","last_clean":"2023-09-30T09:49:34.319407+0000","last_became_active":"2023-09-30T09:42:48.651583+0000","last_became_peered":"2023-09-30T09:42:48.651583+0000","last_unstale":"2023-09-30T09:49:34.319407+0000","last_undegraded":"2023-09-30T09:49:34.319407+0000","last_fullsized":"2023-09-30T09:49:34.319407+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:23:36.523618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"554'4034","reported_seq":4063,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:36.922422+0000","last_change":"2023-09-30T09:48:40.935965+0000","last_active":"2023-09-30T09:49:36.922422+0000","last_peered":"2023-09-30T09:49:36.922422+0000","last_clean":"2023-09-30T09:49:36.922422+0000","last_became_active":"2023-09-30T09:48:40.935590+0000","last_became_peered":"2023-09-30T09:48:40.935590+0000","last_unstale":"2023-09-30T09:49:36.922422+0000","last_undegraded":"2023-09-30T09:49:36.922422+0000","last_fullsized":"2023-09-30T09:49:36.922422+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4034,"log_dups_size":0,"ondisk_log_size":4034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:25:22.798805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264372224,"num_objects":4034,"num_object_clones":0,"num_object_copies":12102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4034,"num_write_kb":258176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":610,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.344584+0000","last_change":"2023-09-30T09:49:34.341774+0000","last_active":"2023-09-30T09:49:34.344584+0000","last_peered":"2023-09-30T09:49:34.344584+0000","last_clean":"2023-09-30T09:49:34.344584+0000","last_became_active":"2023-09-30T09:49:34.341458+0000","last_became_peered":"2023-09-30T09:49:34.341458+0000","last_unstale":"2023-09-30T09:49:34.344584+0000","last_undegraded":"2023-09-30T09:49:34.344584+0000","last_fullsized":"2023-09-30T09:49:34.344584+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:16.116795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:16.116795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:52:42.222190+0000","scrub_duration":0.045979904000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":295,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312753+0000","last_change":"2023-09-30T09:49:31.319964+0000","last_active":"2023-09-30T09:49:34.312753+0000","last_peered":"2023-09-30T09:49:34.312753+0000","last_clean":"2023-09-30T09:49:34.312753+0000","last_became_active":"2023-09-30T09:49:31.302322+0000","last_became_peered":"2023-09-30T09:49:31.302322+0000","last_unstale":"2023-09-30T09:49:34.312753+0000","last_undegraded":"2023-09-30T09:49:34.312753+0000","last_fullsized":"2023-09-30T09:49:34.312753+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:41:54.782188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313165+0000","last_change":"2023-09-30T09:49:31.307771+0000","last_active":"2023-09-30T09:49:34.313165+0000","last_peered":"2023-09-30T09:49:34.313165+0000","last_clean":"2023-09-30T09:49:34.313165+0000","last_became_active":"2023-09-30T09:49:31.306732+0000","last_became_peered":"2023-09-30T09:49:31.306732+0000","last_unstale":"2023-09-30T09:49:34.313165+0000","last_undegraded":"2023-09-30T09:49:34.313165+0000","last_fullsized":"2023-09-30T09:49:34.313165+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:15:54.549553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"554'4066","reported_seq":4095,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:39.086042+0000","last_change":"2023-09-30T09:48:40.937574+0000","last_active":"2023-09-30T09:49:39.086042+0000","last_peered":"2023-09-30T09:49:39.086042+0000","last_clean":"2023-09-30T09:49:39.086042+0000","last_became_active":"2023-09-30T09:48:40.937127+0000","last_became_peered":"2023-09-30T09:48:40.937127+0000","last_unstale":"2023-09-30T09:49:39.086042+0000","last_undegraded":"2023-09-30T09:49:39.086042+0000","last_fullsized":"2023-09-30T09:49:39.086042+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4066,"log_dups_size":0,"ondisk_log_size":4066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:21:24.051843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266469376,"num_objects":4066,"num_object_clones":0,"num_object_copies":12198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4066,"num_write_kb":260224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":788,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.318453+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:49:34.318453+0000","last_peered":"2023-09-30T09:49:34.318453+0000","last_clean":"2023-09-30T09:49:34.318453+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:49:34.318453+0000","last_undegraded":"2023-09-30T09:49:34.318453+0000","last_fullsized":"2023-09-30T09:49:34.318453+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":509,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.310357+0000","last_change":"2023-09-30T09:49:31.301460+0000","last_active":"2023-09-30T09:49:34.310357+0000","last_peered":"2023-09-30T09:49:34.310357+0000","last_clean":"2023-09-30T09:49:34.310357+0000","last_became_active":"2023-09-30T09:49:31.301114+0000","last_became_peered":"2023-09-30T09:49:31.301114+0000","last_unstale":"2023-09-30T09:49:34.310357+0000","last_undegraded":"2023-09-30T09:49:34.310357+0000","last_fullsized":"2023-09-30T09:49:34.310357+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:42:01.066974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":155,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313378+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:49:34.313378+0000","last_peered":"2023-09-30T09:49:34.313378+0000","last_clean":"2023-09-30T09:49:34.313378+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:49:34.313378+0000","last_undegraded":"2023-09-30T09:49:34.313378+0000","last_fullsized":"2023-09-30T09:49:34.313378+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"554'4100","reported_seq":4125,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:38.844339+0000","last_change":"2023-09-30T09:48:40.937693+0000","last_active":"2023-09-30T09:49:38.844339+0000","last_peered":"2023-09-30T09:49:38.844339+0000","last_clean":"2023-09-30T09:49:38.844339+0000","last_became_active":"2023-09-30T09:48:40.937364+0000","last_became_peered":"2023-09-30T09:48:40.937364+0000","last_unstale":"2023-09-30T09:49:38.844339+0000","last_undegraded":"2023-09-30T09:49:38.844339+0000","last_fullsized":"2023-09-30T09:49:38.844339+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4100,"log_dups_size":0,"ondisk_log_size":4100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:58.223575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268697600,"num_objects":4100,"num_object_clones":0,"num_object_copies":12300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4100,"num_write_kb":262400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1036,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312124+0000","last_change":"2023-09-30T09:42:53.666796+0000","last_active":"2023-09-30T09:49:34.312124+0000","last_peered":"2023-09-30T09:49:34.312124+0000","last_clean":"2023-09-30T09:49:34.312124+0000","last_became_active":"2023-09-30T09:42:53.666453+0000","last_became_peered":"2023-09-30T09:42:53.666453+0000","last_unstale":"2023-09-30T09:49:34.312124+0000","last_undegraded":"2023-09-30T09:49:34.312124+0000","last_fullsized":"2023-09-30T09:49:34.312124+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:25.668290+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":114,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313169+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:49:34.313169+0000","last_peered":"2023-09-30T09:49:34.313169+0000","last_clean":"2023-09-30T09:49:34.313169+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:49:34.313169+0000","last_undegraded":"2023-09-30T09:49:34.313169+0000","last_fullsized":"2023-09-30T09:49:34.313169+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":571,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317983+0000","last_change":"2023-09-30T09:49:31.300551+0000","last_active":"2023-09-30T09:49:34.317983+0000","last_peered":"2023-09-30T09:49:34.317983+0000","last_clean":"2023-09-30T09:49:34.317983+0000","last_became_active":"2023-09-30T09:49:31.300231+0000","last_became_peered":"2023-09-30T09:49:31.300231+0000","last_unstale":"2023-09-30T09:49:34.317983+0000","last_undegraded":"2023-09-30T09:49:34.317983+0000","last_fullsized":"2023-09-30T09:49:34.317983+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:45.832564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"554'4102","reported_seq":4131,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:39.067307+0000","last_change":"2023-09-30T09:48:40.945351+0000","last_active":"2023-09-30T09:49:39.067307+0000","last_peered":"2023-09-30T09:49:39.067307+0000","last_clean":"2023-09-30T09:49:39.067307+0000","last_became_active":"2023-09-30T09:48:40.945009+0000","last_became_peered":"2023-09-30T09:48:40.945009+0000","last_unstale":"2023-09-30T09:49:39.067307+0000","last_undegraded":"2023-09-30T09:49:39.067307+0000","last_fullsized":"2023-09-30T09:49:39.067307+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4102,"log_dups_size":0,"ondisk_log_size":4102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:06.181942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268828672,"num_objects":4102,"num_object_clones":0,"num_object_copies":12306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4102,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":1036,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.309845+0000","last_change":"2023-09-30T09:44:27.071252+0000","last_active":"2023-09-30T09:49:34.309845+0000","last_peered":"2023-09-30T09:49:34.309845+0000","last_clean":"2023-09-30T09:49:34.309845+0000","last_became_active":"2023-09-30T09:44:27.070831+0000","last_became_peered":"2023-09-30T09:44:27.070831+0000","last_unstale":"2023-09-30T09:49:34.309845+0000","last_undegraded":"2023-09-30T09:49:34.309845+0000","last_fullsized":"2023-09-30T09:49:34.309845+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:00:06.908417+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313246+0000","last_change":"2023-09-30T09:49:31.297788+0000","last_active":"2023-09-30T09:49:34.313246+0000","last_peered":"2023-09-30T09:49:34.313246+0000","last_clean":"2023-09-30T09:49:34.313246+0000","last_became_active":"2023-09-30T09:49:31.297417+0000","last_became_peered":"2023-09-30T09:49:31.297417+0000","last_unstale":"2023-09-30T09:49:34.313246+0000","last_undegraded":"2023-09-30T09:49:34.313246+0000","last_fullsized":"2023-09-30T09:49:34.313246+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:56:00.311991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":247,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317996+0000","last_change":"2023-09-30T09:34:37.832993+0000","last_active":"2023-09-30T09:49:34.317996+0000","last_peered":"2023-09-30T09:49:34.317996+0000","last_clean":"2023-09-30T09:49:34.317996+0000","last_became_active":"2023-09-30T09:34:37.832771+0000","last_became_peered":"2023-09-30T09:34:37.832771+0000","last_unstale":"2023-09-30T09:49:34.317996+0000","last_undegraded":"2023-09-30T09:49:34.317996+0000","last_fullsized":"2023-09-30T09:49:34.317996+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:40.151123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"554'4165","reported_seq":4194,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:49:40.547894+0000","last_change":"2023-09-30T09:49:32.639291+0000","last_active":"2023-09-30T09:49:40.547894+0000","last_peered":"2023-09-30T09:49:40.547894+0000","last_clean":"2023-09-30T09:49:30.245902+0000","last_became_active":"2023-09-30T09:49:32.625551+0000","last_became_peered":"2023-09-30T09:49:32.625551+0000","last_unstale":"2023-09-30T09:49:40.547894+0000","last_undegraded":"2023-09-30T09:49:32.309304+0000","last_fullsized":"2023-09-30T09:49:32.283287+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4165,"log_dups_size":0,"ondisk_log_size":4165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.949758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272957440,"num_objects":4165,"num_object_clones":0,"num_object_copies":12495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4165,"num_write_kb":266560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":392},{"shards":"1,4,7","objects":3773}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":894,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.337886+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:49:34.337886+0000","last_peered":"2023-09-30T09:49:34.337886+0000","last_clean":"2023-09-30T09:49:34.337886+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:49:34.337886+0000","last_undegraded":"2023-09-30T09:49:34.337886+0000","last_fullsized":"2023-09-30T09:49:34.337886+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":558,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.319305+0000","last_change":"2023-09-30T09:38:27.680547+0000","last_active":"2023-09-30T09:49:34.319305+0000","last_peered":"2023-09-30T09:49:34.319305+0000","last_clean":"2023-09-30T09:49:34.319305+0000","last_became_active":"2023-09-30T09:38:27.680258+0000","last_became_peered":"2023-09-30T09:38:27.680258+0000","last_unstale":"2023-09-30T09:49:34.319305+0000","last_undegraded":"2023-09-30T09:49:34.319305+0000","last_fullsized":"2023-09-30T09:49:34.319305+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:16.195316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":585,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.318550+0000","last_change":"2023-09-30T09:49:31.311818+0000","last_active":"2023-09-30T09:49:34.318550+0000","last_peered":"2023-09-30T09:49:34.318550+0000","last_clean":"2023-09-30T09:49:34.318550+0000","last_became_active":"2023-09-30T09:49:31.311481+0000","last_became_peered":"2023-09-30T09:49:31.311481+0000","last_unstale":"2023-09-30T09:49:34.318550+0000","last_undegraded":"2023-09-30T09:49:34.318550+0000","last_fullsized":"2023-09-30T09:49:34.318550+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:54.941072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"554'4098","reported_seq":4123,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:38.885048+0000","last_change":"2023-09-30T09:48:40.940915+0000","last_active":"2023-09-30T09:49:38.885048+0000","last_peered":"2023-09-30T09:49:38.885048+0000","last_clean":"2023-09-30T09:49:38.885048+0000","last_became_active":"2023-09-30T09:48:40.940606+0000","last_became_peered":"2023-09-30T09:48:40.940606+0000","last_unstale":"2023-09-30T09:49:38.885048+0000","last_undegraded":"2023-09-30T09:49:38.885048+0000","last_fullsized":"2023-09-30T09:49:38.885048+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4098,"log_dups_size":0,"ondisk_log_size":4098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:11.151884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268566528,"num_objects":4098,"num_object_clones":0,"num_object_copies":12294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4098,"num_write_kb":262272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":786,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.328839+0000","last_change":"2023-09-30T09:49:34.328398+0000","last_active":"2023-09-30T09:49:34.328839+0000","last_peered":"2023-09-30T09:49:34.328839+0000","last_clean":"2023-09-30T09:49:34.328839+0000","last_became_active":"2023-09-30T09:49:34.328106+0000","last_became_peered":"2023-09-30T09:49:34.328106+0000","last_unstale":"2023-09-30T09:49:34.328839+0000","last_undegraded":"2023-09-30T09:49:34.328839+0000","last_fullsized":"2023-09-30T09:49:34.328839+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:10:51.441736+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":120,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.316897+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:49:34.316897+0000","last_peered":"2023-09-30T09:49:34.316897+0000","last_clean":"2023-09-30T09:49:34.316897+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:49:34.316897+0000","last_undegraded":"2023-09-30T09:49:34.316897+0000","last_fullsized":"2023-09-30T09:49:34.316897+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":238,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.318568+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:49:34.318568+0000","last_peered":"2023-09-30T09:49:34.318568+0000","last_clean":"2023-09-30T09:49:34.318568+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:49:34.318568+0000","last_undegraded":"2023-09-30T09:49:34.318568+0000","last_fullsized":"2023-09-30T09:49:34.318568+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"554'4037","reported_seq":4063,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:38.916965+0000","last_change":"2023-09-30T09:48:40.942045+0000","last_active":"2023-09-30T09:49:38.916965+0000","last_peered":"2023-09-30T09:49:38.916965+0000","last_clean":"2023-09-30T09:49:38.916965+0000","last_became_active":"2023-09-30T09:48:40.941786+0000","last_became_peered":"2023-09-30T09:48:40.941786+0000","last_unstale":"2023-09-30T09:49:38.916965+0000","last_undegraded":"2023-09-30T09:49:38.916965+0000","last_fullsized":"2023-09-30T09:49:38.916965+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4037,"log_dups_size":0,"ondisk_log_size":4037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:03:11.354244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264568832,"num_objects":4037,"num_object_clones":0,"num_object_copies":12111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4037,"num_write_kb":258368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":717,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.334645+0000","last_change":"2023-09-30T09:49:34.334275+0000","last_active":"2023-09-30T09:49:34.334645+0000","last_peered":"2023-09-30T09:49:34.334645+0000","last_clean":"2023-09-30T09:49:34.334645+0000","last_became_active":"2023-09-30T09:49:34.333939+0000","last_became_peered":"2023-09-30T09:49:34.333939+0000","last_unstale":"2023-09-30T09:49:34.334645+0000","last_undegraded":"2023-09-30T09:49:34.334645+0000","last_fullsized":"2023-09-30T09:49:34.334645+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:47.674459+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":786,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.307258+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:49:34.307258+0000","last_peered":"2023-09-30T09:49:34.307258+0000","last_clean":"2023-09-30T09:49:34.307258+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:49:34.307258+0000","last_undegraded":"2023-09-30T09:49:34.307258+0000","last_fullsized":"2023-09-30T09:49:34.307258+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":482,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.310256+0000","last_change":"2023-09-30T09:44:27.070100+0000","last_active":"2023-09-30T09:49:34.310256+0000","last_peered":"2023-09-30T09:49:34.310256+0000","last_clean":"2023-09-30T09:49:34.310256+0000","last_became_active":"2023-09-30T09:44:27.069503+0000","last_became_peered":"2023-09-30T09:44:27.069503+0000","last_unstale":"2023-09-30T09:49:34.310256+0000","last_undegraded":"2023-09-30T09:49:34.310256+0000","last_fullsized":"2023-09-30T09:49:34.310256+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:45:03.840617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":171,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.309514+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:49:34.309514+0000","last_peered":"2023-09-30T09:49:34.309514+0000","last_clean":"2023-09-30T09:49:34.309514+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:49:34.309514+0000","last_undegraded":"2023-09-30T09:49:34.309514+0000","last_fullsized":"2023-09-30T09:49:34.309514+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"554'4051","reported_seq":4088,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:49:37.039435+0000","last_change":"2023-09-30T09:49:32.640373+0000","last_active":"2023-09-30T09:49:37.039435+0000","last_peered":"2023-09-30T09:49:37.039435+0000","last_clean":"2023-09-30T09:49:30.256832+0000","last_became_active":"2023-09-30T09:49:32.627115+0000","last_became_peered":"2023-09-30T09:49:32.627115+0000","last_unstale":"2023-09-30T09:49:37.039435+0000","last_undegraded":"2023-09-30T09:49:32.411417+0000","last_fullsized":"2023-09-30T09:49:32.292861+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4051,"log_dups_size":0,"ondisk_log_size":4051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:17:32.550227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265486336,"num_objects":4051,"num_object_clones":0,"num_object_copies":12153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3827,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4051,"num_write_kb":259264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":3827},{"shards":"0,5","objects":224}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":995,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.311205+0000","last_change":"2023-09-30T09:44:27.075749+0000","last_active":"2023-09-30T09:49:34.311205+0000","last_peered":"2023-09-30T09:49:34.311205+0000","last_clean":"2023-09-30T09:49:34.311205+0000","last_became_active":"2023-09-30T09:44:27.075382+0000","last_became_peered":"2023-09-30T09:44:27.075382+0000","last_unstale":"2023-09-30T09:49:34.311205+0000","last_undegraded":"2023-09-30T09:49:34.311205+0000","last_fullsized":"2023-09-30T09:49:34.311205+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:22.398249+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":24,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313015+0000","last_change":"2023-09-30T09:49:31.328925+0000","last_active":"2023-09-30T09:49:34.313015+0000","last_peered":"2023-09-30T09:49:34.313015+0000","last_clean":"2023-09-30T09:49:34.313015+0000","last_became_active":"2023-09-30T09:49:31.305343+0000","last_became_peered":"2023-09-30T09:49:31.305343+0000","last_unstale":"2023-09-30T09:49:34.313015+0000","last_undegraded":"2023-09-30T09:49:34.313015+0000","last_fullsized":"2023-09-30T09:49:34.313015+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:23:40.829180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":181,"num_read_kb":1204,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":592,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317924+0000","last_change":"2023-09-30T09:49:31.304908+0000","last_active":"2023-09-30T09:49:34.317924+0000","last_peered":"2023-09-30T09:49:34.317924+0000","last_clean":"2023-09-30T09:49:34.317924+0000","last_became_active":"2023-09-30T09:49:31.304558+0000","last_became_peered":"2023-09-30T09:49:31.304558+0000","last_unstale":"2023-09-30T09:49:34.317924+0000","last_undegraded":"2023-09-30T09:49:34.317924+0000","last_fullsized":"2023-09-30T09:49:34.317924+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:03:56.889045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":314,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.311473+0000","last_change":"2023-09-30T09:44:27.067129+0000","last_active":"2023-09-30T09:49:34.311473+0000","last_peered":"2023-09-30T09:49:34.311473+0000","last_clean":"2023-09-30T09:49:34.311473+0000","last_became_active":"2023-09-30T09:44:27.066795+0000","last_became_peered":"2023-09-30T09:44:27.066795+0000","last_unstale":"2023-09-30T09:49:34.311473+0000","last_undegraded":"2023-09-30T09:49:34.311473+0000","last_fullsized":"2023-09-30T09:49:34.311473+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:32.377566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"554'4014","reported_seq":4040,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:39.322910+0000","last_change":"2023-09-30T09:48:40.944781+0000","last_active":"2023-09-30T09:49:39.322910+0000","last_peered":"2023-09-30T09:49:39.322910+0000","last_clean":"2023-09-30T09:49:39.322910+0000","last_became_active":"2023-09-30T09:48:40.944445+0000","last_became_peered":"2023-09-30T09:48:40.944445+0000","last_unstale":"2023-09-30T09:49:39.322910+0000","last_undegraded":"2023-09-30T09:49:39.322910+0000","last_fullsized":"2023-09-30T09:49:39.322910+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4014,"log_dups_size":0,"ondisk_log_size":4014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:26:02.476879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263061504,"num_objects":4014,"num_object_clones":0,"num_object_copies":12042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4014,"num_write_kb":256896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":973,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312238+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:49:34.312238+0000","last_peered":"2023-09-30T09:49:34.312238+0000","last_clean":"2023-09-30T09:49:34.312238+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:49:34.312238+0000","last_undegraded":"2023-09-30T09:49:34.312238+0000","last_fullsized":"2023-09-30T09:49:34.312238+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":823,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.310119+0000","last_change":"2023-09-30T09:43:04.009141+0000","last_active":"2023-09-30T09:49:34.310119+0000","last_peered":"2023-09-30T09:49:34.310119+0000","last_clean":"2023-09-30T09:49:34.310119+0000","last_became_active":"2023-09-30T09:43:04.008534+0000","last_became_peered":"2023-09-30T09:43:04.008534+0000","last_unstale":"2023-09-30T09:49:34.310119+0000","last_undegraded":"2023-09-30T09:49:34.310119+0000","last_fullsized":"2023-09-30T09:49:34.310119+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:04:49.721867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":334,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.309792+0000","last_change":"2023-09-30T09:49:31.291771+0000","last_active":"2023-09-30T09:49:34.309792+0000","last_peered":"2023-09-30T09:49:34.309792+0000","last_clean":"2023-09-30T09:49:34.309792+0000","last_became_active":"2023-09-30T09:49:31.291444+0000","last_became_peered":"2023-09-30T09:49:31.291444+0000","last_unstale":"2023-09-30T09:49:34.309792+0000","last_undegraded":"2023-09-30T09:49:34.309792+0000","last_fullsized":"2023-09-30T09:49:34.309792+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:55:31.883799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":403,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.311298+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:49:34.311298+0000","last_peered":"2023-09-30T09:49:34.311298+0000","last_clean":"2023-09-30T09:49:34.311298+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:49:34.311298+0000","last_undegraded":"2023-09-30T09:49:34.311298+0000","last_fullsized":"2023-09-30T09:49:34.311298+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"554'3819","reported_seq":3850,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:49:37.014953+0000","last_change":"2023-09-30T09:49:32.578088+0000","last_active":"2023-09-30T09:49:37.014953+0000","last_peered":"2023-09-30T09:49:37.014953+0000","last_clean":"2023-09-30T09:49:30.256457+0000","last_became_active":"2023-09-30T09:49:32.572702+0000","last_became_peered":"2023-09-30T09:49:32.572702+0000","last_unstale":"2023-09-30T09:49:37.014953+0000","last_undegraded":"2023-09-30T09:49:32.331615+0000","last_fullsized":"2023-09-30T09:49:32.294545+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":3819,"log_dups_size":0,"ondisk_log_size":3819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:29.549011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250281984,"num_objects":3819,"num_object_clones":0,"num_object_copies":11457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3819,"num_write_kb":244416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[0,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1","objects":192},{"shards":"0,1,4","objects":3627}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1115,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.309957+0000","last_change":"2023-09-30T09:44:26.065885+0000","last_active":"2023-09-30T09:49:34.309957+0000","last_peered":"2023-09-30T09:49:34.309957+0000","last_clean":"2023-09-30T09:49:34.309957+0000","last_became_active":"2023-09-30T09:44:26.063551+0000","last_became_peered":"2023-09-30T09:44:26.063551+0000","last_unstale":"2023-09-30T09:49:34.309957+0000","last_undegraded":"2023-09-30T09:49:34.309957+0000","last_fullsized":"2023-09-30T09:49:34.309957+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:20:33.354865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":956,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.333364+0000","last_change":"2023-09-30T09:49:34.332831+0000","last_active":"2023-09-30T09:49:34.333364+0000","last_peered":"2023-09-30T09:49:34.333364+0000","last_clean":"2023-09-30T09:49:34.333364+0000","last_became_active":"2023-09-30T09:49:34.332429+0000","last_became_peered":"2023-09-30T09:49:34.332429+0000","last_unstale":"2023-09-30T09:49:34.333364+0000","last_undegraded":"2023-09-30T09:49:34.333364+0000","last_fullsized":"2023-09-30T09:49:34.333364+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:40:57.365967+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":150,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.309594+0000","last_change":"2023-09-30T09:49:31.302048+0000","last_active":"2023-09-30T09:49:34.309594+0000","last_peered":"2023-09-30T09:49:34.309594+0000","last_clean":"2023-09-30T09:49:34.309594+0000","last_became_active":"2023-09-30T09:49:31.301086+0000","last_became_peered":"2023-09-30T09:49:31.301086+0000","last_unstale":"2023-09-30T09:49:34.309594+0000","last_undegraded":"2023-09-30T09:49:34.309594+0000","last_fullsized":"2023-09-30T09:49:34.309594+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:22.422063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313762+0000","last_change":"2023-09-30T09:49:31.311312+0000","last_active":"2023-09-30T09:49:34.313762+0000","last_peered":"2023-09-30T09:49:34.313762+0000","last_clean":"2023-09-30T09:49:34.313762+0000","last_became_active":"2023-09-30T09:49:31.310980+0000","last_became_peered":"2023-09-30T09:49:31.310980+0000","last_unstale":"2023-09-30T09:49:34.313762+0000","last_undegraded":"2023-09-30T09:49:34.313762+0000","last_fullsized":"2023-09-30T09:49:34.313762+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:45:00.790620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"554'3909","reported_seq":3938,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:37.024340+0000","last_change":"2023-09-30T09:48:40.941098+0000","last_active":"2023-09-30T09:49:37.024340+0000","last_peered":"2023-09-30T09:49:37.024340+0000","last_clean":"2023-09-30T09:49:37.024340+0000","last_became_active":"2023-09-30T09:48:40.940802+0000","last_became_peered":"2023-09-30T09:48:40.940802+0000","last_unstale":"2023-09-30T09:49:37.024340+0000","last_undegraded":"2023-09-30T09:49:37.024340+0000","last_fullsized":"2023-09-30T09:49:37.024340+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":3909,"log_dups_size":0,"ondisk_log_size":3909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:33:38.250788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256180224,"num_objects":3909,"num_object_clones":0,"num_object_copies":11727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3909,"num_write_kb":250176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":728,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.344591+0000","last_change":"2023-09-30T09:49:34.341842+0000","last_active":"2023-09-30T09:49:34.344591+0000","last_peered":"2023-09-30T09:49:34.344591+0000","last_clean":"2023-09-30T09:49:34.344591+0000","last_became_active":"2023-09-30T09:49:34.341485+0000","last_became_peered":"2023-09-30T09:49:34.341485+0000","last_unstale":"2023-09-30T09:49:34.344591+0000","last_undegraded":"2023-09-30T09:49:34.344591+0000","last_fullsized":"2023-09-30T09:49:34.344591+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:39.230227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.230227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:22:05.176236+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":962,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313392+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:49:34.313392+0000","last_peered":"2023-09-30T09:49:34.313392+0000","last_clean":"2023-09-30T09:49:34.313392+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:49:34.313392+0000","last_undegraded":"2023-09-30T09:49:34.313392+0000","last_fullsized":"2023-09-30T09:49:34.313392+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":168,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.308745+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:49:34.308745+0000","last_peered":"2023-09-30T09:49:34.308745+0000","last_clean":"2023-09-30T09:49:34.308745+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:49:34.308745+0000","last_undegraded":"2023-09-30T09:49:34.308745+0000","last_fullsized":"2023-09-30T09:49:34.308745+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":60,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.307320+0000","last_change":"2023-09-30T09:45:26.076414+0000","last_active":"2023-09-30T09:49:34.307320+0000","last_peered":"2023-09-30T09:49:34.307320+0000","last_clean":"2023-09-30T09:49:34.307320+0000","last_became_active":"2023-09-30T09:45:26.076082+0000","last_became_peered":"2023-09-30T09:45:26.076082+0000","last_unstale":"2023-09-30T09:49:34.307320+0000","last_undegraded":"2023-09-30T09:49:34.307320+0000","last_fullsized":"2023-09-30T09:49:34.307320+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:39.014971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"554'3896","reported_seq":3925,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:49:37.012121+0000","last_change":"2023-09-30T09:49:32.577789+0000","last_active":"2023-09-30T09:49:37.012121+0000","last_peered":"2023-09-30T09:49:37.012121+0000","last_clean":"2023-09-30T09:49:30.256680+0000","last_became_active":"2023-09-30T09:49:32.571991+0000","last_became_peered":"2023-09-30T09:49:32.571991+0000","last_unstale":"2023-09-30T09:49:37.012121+0000","last_undegraded":"2023-09-30T09:49:32.328908+0000","last_fullsized":"2023-09-30T09:49:32.294282+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":3896,"log_dups_size":0,"ondisk_log_size":3896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:35:37.439373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255328256,"num_objects":3896,"num_object_clones":0,"num_object_copies":11688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3896,"num_write_kb":249344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[0,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":3686},{"shards":"0,4","objects":210}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":752,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.334635+0000","last_change":"2023-09-30T09:49:34.334164+0000","last_active":"2023-09-30T09:49:34.334635+0000","last_peered":"2023-09-30T09:49:34.334635+0000","last_clean":"2023-09-30T09:49:34.334635+0000","last_became_active":"2023-09-30T09:49:34.333875+0000","last_became_peered":"2023-09-30T09:49:34.333875+0000","last_unstale":"2023-09-30T09:49:34.334635+0000","last_undegraded":"2023-09-30T09:49:34.334635+0000","last_fullsized":"2023-09-30T09:49:34.334635+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:04:31.062153+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1102,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313130+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:49:34.313130+0000","last_peered":"2023-09-30T09:49:34.313130+0000","last_clean":"2023-09-30T09:49:34.313130+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:49:34.313130+0000","last_undegraded":"2023-09-30T09:49:34.313130+0000","last_fullsized":"2023-09-30T09:49:34.313130+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":298,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.311281+0000","last_change":"2023-09-30T09:44:26.057275+0000","last_active":"2023-09-30T09:49:34.311281+0000","last_peered":"2023-09-30T09:49:34.311281+0000","last_clean":"2023-09-30T09:49:34.311281+0000","last_became_active":"2023-09-30T09:44:26.056881+0000","last_became_peered":"2023-09-30T09:44:26.056881+0000","last_unstale":"2023-09-30T09:49:34.311281+0000","last_undegraded":"2023-09-30T09:49:34.311281+0000","last_fullsized":"2023-09-30T09:49:34.311281+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:18:45.959094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":168,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313537+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:49:34.313537+0000","last_peered":"2023-09-30T09:49:34.313537+0000","last_clean":"2023-09-30T09:49:34.313537+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:49:34.313537+0000","last_undegraded":"2023-09-30T09:49:34.313537+0000","last_fullsized":"2023-09-30T09:49:34.313537+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"554'4138","reported_seq":4724,"reported_epoch":554,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:49:38.908616+0000","last_change":"2023-09-30T09:49:32.684900+0000","last_active":"2023-09-30T09:49:38.908616+0000","last_peered":"2023-09-30T09:49:38.908616+0000","last_clean":"2023-09-30T09:49:30.268920+0000","last_became_active":"2023-09-30T09:49:32.623433+0000","last_became_peered":"2023-09-30T09:49:32.623433+0000","last_unstale":"2023-09-30T09:49:38.908616+0000","last_undegraded":"2023-09-30T09:49:38.908616+0000","last_fullsized":"2023-09-30T09:49:32.293118+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4138,"log_dups_size":0,"ondisk_log_size":4138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:26:52.366788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271187968,"num_objects":4138,"num_object_clones":0,"num_object_copies":12414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3551,"num_objects_unfound":0,"num_objects_dirty":4138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4138,"num_write_kb":264832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":18087936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,2],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"2,4","objects":587},{"shards":"2,4,5","objects":3551}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1112,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.330005+0000","last_change":"2023-09-30T09:49:34.329215+0000","last_active":"2023-09-30T09:49:34.330005+0000","last_peered":"2023-09-30T09:49:34.330005+0000","last_clean":"2023-09-30T09:49:34.330005+0000","last_became_active":"2023-09-30T09:49:34.328794+0000","last_became_peered":"2023-09-30T09:49:34.328794+0000","last_unstale":"2023-09-30T09:49:34.330005+0000","last_undegraded":"2023-09-30T09:49:34.330005+0000","last_fullsized":"2023-09-30T09:49:34.330005+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:23:43.764904+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":585,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.312438+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:49:34.312438+0000","last_peered":"2023-09-30T09:49:34.312438+0000","last_clean":"2023-09-30T09:49:34.312438+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:49:34.312438+0000","last_undegraded":"2023-09-30T09:49:34.312438+0000","last_fullsized":"2023-09-30T09:49:34.312438+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1232,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.340662+0000","last_change":"2023-09-30T09:49:34.337949+0000","last_active":"2023-09-30T09:49:34.340662+0000","last_peered":"2023-09-30T09:49:34.340662+0000","last_clean":"2023-09-30T09:49:34.340662+0000","last_became_active":"2023-09-30T09:49:34.337417+0000","last_became_peered":"2023-09-30T09:49:34.337417+0000","last_unstale":"2023-09-30T09:49:34.340662+0000","last_undegraded":"2023-09-30T09:49:34.340662+0000","last_fullsized":"2023-09-30T09:49:34.340662+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:43:30.663146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":589,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313243+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:49:34.313243+0000","last_peered":"2023-09-30T09:49:34.313243+0000","last_clean":"2023-09-30T09:49:34.313243+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:49:34.313243+0000","last_undegraded":"2023-09-30T09:49:34.313243+0000","last_fullsized":"2023-09-30T09:49:34.313243+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"554'4042","reported_seq":4074,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:49:39.076014+0000","last_change":"2023-09-30T09:49:32.635139+0000","last_active":"2023-09-30T09:49:39.076014+0000","last_peered":"2023-09-30T09:49:39.076014+0000","last_clean":"2023-09-30T09:49:30.258329+0000","last_became_active":"2023-09-30T09:49:32.624683+0000","last_became_peered":"2023-09-30T09:49:32.624683+0000","last_unstale":"2023-09-30T09:49:39.076014+0000","last_undegraded":"2023-09-30T09:49:32.315710+0000","last_fullsized":"2023-09-30T09:49:32.286769+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4042,"log_dups_size":0,"ondisk_log_size":4042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:41:37.331470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264896512,"num_objects":4042,"num_object_clones":0,"num_object_copies":12126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3739,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4042,"num_write_kb":258688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3","objects":303},{"shards":"2,3,4","objects":3739}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":952,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317499+0000","last_change":"2023-09-30T09:49:09.080572+0000","last_active":"2023-09-30T09:49:34.317499+0000","last_peered":"2023-09-30T09:49:34.317499+0000","last_clean":"2023-09-30T09:49:34.317499+0000","last_became_active":"2023-09-30T09:49:09.080212+0000","last_became_peered":"2023-09-30T09:49:09.080212+0000","last_unstale":"2023-09-30T09:49:34.317499+0000","last_undegraded":"2023-09-30T09:49:34.317499+0000","last_fullsized":"2023-09-30T09:49:34.317499+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:02:52.327438+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":28,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.314605+0000","last_change":"2023-09-30T09:49:31.332895+0000","last_active":"2023-09-30T09:49:34.314605+0000","last_peered":"2023-09-30T09:49:34.314605+0000","last_clean":"2023-09-30T09:49:34.314605+0000","last_became_active":"2023-09-30T09:49:31.309595+0000","last_became_peered":"2023-09-30T09:49:31.309595+0000","last_unstale":"2023-09-30T09:49:34.314605+0000","last_undegraded":"2023-09-30T09:49:34.314605+0000","last_fullsized":"2023-09-30T09:49:34.314605+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:15:40.397678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":131,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.307970+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:49:34.307970+0000","last_peered":"2023-09-30T09:49:34.307970+0000","last_clean":"2023-09-30T09:49:34.307970+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:49:34.307970+0000","last_undegraded":"2023-09-30T09:49:34.307970+0000","last_fullsized":"2023-09-30T09:49:34.307970+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":574,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.319346+0000","last_change":"2023-09-30T09:49:31.304154+0000","last_active":"2023-09-30T09:49:34.319346+0000","last_peered":"2023-09-30T09:49:34.319346+0000","last_clean":"2023-09-30T09:49:34.319346+0000","last_became_active":"2023-09-30T09:49:31.303819+0000","last_became_peered":"2023-09-30T09:49:31.303819+0000","last_unstale":"2023-09-30T09:49:34.319346+0000","last_undegraded":"2023-09-30T09:49:34.319346+0000","last_fullsized":"2023-09-30T09:49:34.319346+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:17:41.151778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":902,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.338954+0000","last_change":"2023-09-30T09:49:34.338310+0000","last_active":"2023-09-30T09:49:34.338954+0000","last_peered":"2023-09-30T09:49:34.338954+0000","last_clean":"2023-09-30T09:49:34.338954+0000","last_became_active":"2023-09-30T09:49:34.337868+0000","last_became_peered":"2023-09-30T09:49:34.337868+0000","last_unstale":"2023-09-30T09:49:34.338954+0000","last_undegraded":"2023-09-30T09:49:34.338954+0000","last_fullsized":"2023-09-30T09:49:34.338954+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:13:34.757539+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":60,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.313466+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:49:34.313466+0000","last_peered":"2023-09-30T09:49:34.313466+0000","last_clean":"2023-09-30T09:49:34.313466+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:49:34.313466+0000","last_undegraded":"2023-09-30T09:49:34.313466+0000","last_fullsized":"2023-09-30T09:49:34.313466+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":260,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.311180+0000","last_change":"2023-09-30T09:42:48.667593+0000","last_active":"2023-09-30T09:49:34.311180+0000","last_peered":"2023-09-30T09:49:34.311180+0000","last_clean":"2023-09-30T09:49:34.311180+0000","last_became_active":"2023-09-30T09:42:48.656355+0000","last_became_peered":"2023-09-30T09:42:48.656355+0000","last_unstale":"2023-09-30T09:49:34.311180+0000","last_undegraded":"2023-09-30T09:49:34.311180+0000","last_fullsized":"2023-09-30T09:49:34.311180+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:19:35.767476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":610,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.334607+0000","last_change":"2023-09-30T09:49:34.334144+0000","last_active":"2023-09-30T09:49:34.334607+0000","last_peered":"2023-09-30T09:49:34.334607+0000","last_clean":"2023-09-30T09:49:34.334607+0000","last_became_active":"2023-09-30T09:49:34.333802+0000","last_became_peered":"2023-09-30T09:49:34.333802+0000","last_unstale":"2023-09-30T09:49:34.334607+0000","last_undegraded":"2023-09-30T09:49:34.334607+0000","last_fullsized":"2023-09-30T09:49:34.334607+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040664730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":482,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.310267+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:49:34.310267+0000","last_peered":"2023-09-30T09:49:34.310267+0000","last_clean":"2023-09-30T09:49:34.310267+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:49:34.310267+0000","last_undegraded":"2023-09-30T09:49:34.310267+0000","last_fullsized":"2023-09-30T09:49:34.310267+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":267,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317497+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:49:34.317497+0000","last_peered":"2023-09-30T09:49:34.317497+0000","last_clean":"2023-09-30T09:49:34.317497+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:49:34.317497+0000","last_undegraded":"2023-09-30T09:49:34.317497+0000","last_fullsized":"2023-09-30T09:49:34.317497+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":452,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.310392+0000","last_change":"2023-09-30T09:49:31.313229+0000","last_active":"2023-09-30T09:49:34.310392+0000","last_peered":"2023-09-30T09:49:34.310392+0000","last_clean":"2023-09-30T09:49:34.310392+0000","last_became_active":"2023-09-30T09:49:31.301137+0000","last_became_peered":"2023-09-30T09:49:31.301137+0000","last_unstale":"2023-09-30T09:49:34.310392+0000","last_undegraded":"2023-09-30T09:49:34.310392+0000","last_fullsized":"2023-09-30T09:49:34.310392+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:57.362379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":287,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317340+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:49:34.317340+0000","last_peered":"2023-09-30T09:49:34.317340+0000","last_clean":"2023-09-30T09:49:34.317340+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:49:34.317340+0000","last_undegraded":"2023-09-30T09:49:34.317340+0000","last_fullsized":"2023-09-30T09:49:34.317340+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":688,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.341148+0000","last_change":"2023-09-30T09:49:34.338827+0000","last_active":"2023-09-30T09:49:34.341148+0000","last_peered":"2023-09-30T09:49:34.341148+0000","last_clean":"2023-09-30T09:49:34.341148+0000","last_became_active":"2023-09-30T09:49:34.338526+0000","last_became_peered":"2023-09-30T09:49:34.338526+0000","last_unstale":"2023-09-30T09:49:34.341148+0000","last_undegraded":"2023-09-30T09:49:34.341148+0000","last_fullsized":"2023-09-30T09:49:34.341148+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:14.071789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:14.071789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:05:27.153303+0000","scrub_duration":0.0062798769999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.314274+0000","last_change":"2023-09-30T09:49:31.292436+0000","last_active":"2023-09-30T09:49:34.314274+0000","last_peered":"2023-09-30T09:49:34.314274+0000","last_clean":"2023-09-30T09:49:34.314274+0000","last_became_active":"2023-09-30T09:49:31.292081+0000","last_became_peered":"2023-09-30T09:49:31.292081+0000","last_unstale":"2023-09-30T09:49:34.314274+0000","last_undegraded":"2023-09-30T09:49:34.314274+0000","last_fullsized":"2023-09-30T09:49:34.314274+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:20:56.434538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":506,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.309983+0000","last_change":"2023-09-30T09:43:45.116512+0000","last_active":"2023-09-30T09:49:34.309983+0000","last_peered":"2023-09-30T09:49:34.309983+0000","last_clean":"2023-09-30T09:49:34.309983+0000","last_became_active":"2023-09-30T09:43:45.116208+0000","last_became_peered":"2023-09-30T09:43:45.116208+0000","last_unstale":"2023-09-30T09:49:34.309983+0000","last_undegraded":"2023-09-30T09:49:34.309983+0000","last_fullsized":"2023-09-30T09:49:34.309983+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:20:44.825936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":923,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.318083+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:49:34.318083+0000","last_peered":"2023-09-30T09:49:34.318083+0000","last_clean":"2023-09-30T09:49:34.318083+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:49:34.318083+0000","last_undegraded":"2023-09-30T09:49:34.318083+0000","last_fullsized":"2023-09-30T09:49:34.318083+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":67,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-30T09:49:35.862278+0000","last_change":"2023-09-30T09:45:25.707599+0000","last_active":"2023-09-30T09:49:35.862278+0000","last_peered":"2023-09-30T09:49:35.862278+0000","last_clean":"2023-09-30T09:49:35.862278+0000","last_became_active":"2023-09-30T09:45:25.697219+0000","last_became_peered":"2023-09-30T09:45:25.697219+0000","last_unstale":"2023-09-30T09:49:35.862278+0000","last_undegraded":"2023-09-30T09:49:35.862278+0000","last_fullsized":"2023-09-30T09:49:35.862278+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:33.779826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":588,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-30T09:49:34.317175+0000","last_change":"2023-09-30T09:49:31.299412+0000","last_active":"2023-09-30T09:49:34.317175+0000","last_peered":"2023-09-30T09:49:34.317175+0000","last_clean":"2023-09-30T09:49:34.317175+0000","last_became_active":"2023-09-30T09:49:31.299025+0000","last_became_peered":"2023-09-30T09:49:31.299025+0000","last_unstale":"2023-09-30T09:49:34.317175+0000","last_undegraded":"2023-09-30T09:49:34.317175+0000","last_fullsized":"2023-09-30T09:49:34.317175+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:54.965178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4222812160,"num_objects":64435,"num_object_clones":0,"num_object_copies":193305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18652,"num_objects_misplaced":7345,"num_objects_unfound":0,"num_objects_dirty":64435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64435,"num_write_kb":4123840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":278,"num_bytes_recovered":18219008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12770148352,"data_stored":12770148352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":64435,"ondisk_log_size":64435,"up":48,"acting":42,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":31714,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5942,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":202,"num_bytes_recovered":12582950,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234455,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1856304,"kb_used_data":1278952,"kb_used_omap":0,"kb_used_meta":577280,"kb_avail":91888848,"statfs":{"total":95995035648,"available":94094180352,"internally_reserved":0,"allocated":1309646848,"data_stored":1307835271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":591134720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:49:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":5.4710000000000001,"15min":5.181},"min":{"1min":0.28299999999999997,"5min":0.23300000000000001,"15min":0.22900000000000001},"max":{"1min":4.7430000000000003,"5min":110.467,"15min":215.416},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":5.343,"15min":5.109},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":2.1309999999999998,"5min":109.593,"15min":215.041},"last":0.26900000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:49:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":5.0949999999999998,"15min":4.0170000000000003},"min":{"1min":0.32300000000000001,"5min":0.27000000000000002,"15min":0.25},"max":{"1min":5.6070000000000002,"5min":157.578,"15min":157.578},"last":0.97899999999999998},{"interface":"front","average":{"1min":1.1200000000000001,"5min":5.0890000000000004,"15min":4.0759999999999996},"min":{"1min":0.29599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.8079999999999998,"5min":157.589,"15min":157.589},"last":1.7769999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:48:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9810000000000001,"5min":2.2949999999999999,"15min":3.7930000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.568,"5min":19.399000000000001,"15min":117.14},"last":1.7490000000000001},{"interface":"front","average":{"1min":1.9139999999999999,"5min":2.2770000000000001,"15min":3.7589999999999999},"min":{"1min":0.374,"5min":0.29599999999999999,"15min":0.28399999999999997},"max":{"1min":10.083,"5min":18.603000000000002,"15min":117.044},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 30 09:49:19 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":6.3239999999999998,"15min":7.8090000000000002},"min":{"1min":0.372,"5min":0.34200000000000003,"15min":0.314},"max":{"1min":5.1520000000000001,"5min":154.06700000000001,"15min":277.57799999999997},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.98699999999999999,"5min":6.7380000000000004,"15min":8.1039999999999992},"min":{"1min":0.436,"5min":0.35899999999999999,"15min":0.32200000000000001},"max":{"1min":5.2960000000000003,"5min":154.24799999999999,"15min":278.69299999999998},"last":0.376}]},{"osd":4,"last update":"Sat Sep 30 09:48:44 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.581,"15min":1.6579999999999999},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.23799999999999999},"max":{"1min":10.789,"5min":25.192,"15min":175.72},"last":0.85299999999999998},{"interface":"front","average":{"1min":1.395,"5min":1.4059999999999999,"15min":1.5489999999999999},"min":{"1min":0.35299999999999998,"5min":0.28999999999999998,"15min":0.252},"max":{"1min":10.231,"5min":24.521000000000001,"15min":179.048},"last":0.66300000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:49:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":3.3399999999999999,"15min":9.1080000000000005},"min":{"1min":0.36199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.79200000000000004,"5min":197.75700000000001,"15min":197.75700000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.64200000000000002,"5min":3.5139999999999998,"15min":9.0559999999999992},"min":{"1min":0.39500000000000002,"5min":0.309,"15min":0.309},"max":{"1min":1.8200000000000001,"5min":197.63300000000001,"15min":197.63300000000001},"last":1.7250000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:48:44 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.214,"15min":2.2170000000000001},"min":{"1min":0.376,"5min":0.30399999999999999,"15min":0.27100000000000002},"max":{"1min":4.9109999999999996,"5min":7.5830000000000002,"15min":206.244},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.1319999999999999,"15min":2.1709999999999998},"min":{"1min":0.35699999999999998,"5min":0.31900000000000001,"15min":0.29199999999999998},"max":{"1min":3.9279999999999999,"5min":5.8520000000000003,"15min":202.78800000000001},"last":0.32200000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496093,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":728156,"kb_used_data":177236,"kb_used_omap":0,"kb_used_meta":550912,"kb_avail":93016996,"statfs":{"total":95995035648,"available":95249403904,"internally_reserved":0,"allocated":181489664,"data_stored":179697308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":564133888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:49:16 2023","interfaces":[{"interface":"back","average":{"1min":4.2320000000000002,"5min":3.5550000000000002,"15min":5.7069999999999999},"min":{"1min":0.34000000000000002,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":77.834000000000003,"5min":79.504000000000005,"15min":187.19800000000001},"last":2.4929999999999999},{"interface":"front","average":{"1min":4.1390000000000002,"5min":3.5379999999999998,"15min":5.6630000000000003},"min":{"1min":0.248,"5min":0.248,"15min":0.22500000000000001},"max":{"1min":77.445999999999998,"5min":78.813000000000002,"15min":187.46700000000001},"last":1.1519999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":1.496,"15min":1.9399999999999999},"min":{"1min":0.29699999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.0659999999999998,"5min":13.494999999999999,"15min":69.569999999999993},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.659,"15min":2.129},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.198,"5min":8.5869999999999997,"15min":70.471999999999994},"last":1.123}]},{"osd":2,"last update":"Sat Sep 30 09:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.8680000000000001,"15min":4.008},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":3.4079999999999999,"5min":11.432,"15min":207.02500000000001},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.7829999999999999,"15min":4.1310000000000002},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":3.9670000000000001,"5min":16.474,"15min":207.114},"last":0.64700000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:49:16 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":2.0670000000000002,"15min":3.806},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":2.556,"5min":49.234000000000002,"15min":212.44800000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.70499999999999996,"5min":2.2000000000000002,"15min":4.0640000000000001},"min":{"1min":0.35899999999999999,"5min":0.33800000000000002,"15min":0.31900000000000001},"max":{"1min":2.5049999999999999,"5min":48.363,"15min":210.84299999999999},"last":1.1799999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:49:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.893,"15min":1.268},"min":{"1min":0.28699999999999998,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":0.97799999999999998,"5min":63.798000000000002,"15min":63.798000000000002},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":1.8,"15min":1.202},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":0.86099999999999999,"5min":63.133000000000003,"15min":63.133000000000003},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.637,"15min":1.911},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.2949999999999999,"5min":14.974,"15min":14.974},"last":1.5760000000000001},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.825,"15min":2.3599999999999999},"min":{"1min":0.36499999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.2970000000000002,"5min":16.443000000000001,"15min":16.443000000000001},"last":5.944}]},{"osd":7,"last update":"Sat Sep 30 09:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.2629999999999999,"15min":1.5329999999999999},"min":{"1min":0.32600000000000001,"5min":0.28599999999999998,"15min":0.26500000000000001},"max":{"1min":2.8929999999999998,"5min":11.276999999999999,"15min":138.191},"last":1.056},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.2989999999999999,"15min":1.45},"min":{"1min":0.41499999999999998,"5min":0.29599999999999999,"15min":0.27500000000000002},"max":{"1min":1.002,"5min":11.521000000000001,"15min":140.33500000000001},"last":0.58199999999999996}]}]},{"osd":1,"up_from":301,"seq":1292785156291,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1472876,"kb_used_data":791396,"kb_used_omap":9,"kb_used_meta":681462,"kb_avail":92272276,"statfs":{"total":95995035648,"available":94486810624,"internally_reserved":0,"allocated":810389504,"data_stored":807269369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":697817106},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:49:02 2023","interfaces":[{"interface":"back","average":{"1min":3.786,"5min":4.7009999999999996,"15min":3.9950000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.17100000000000001},"max":{"1min":55.438000000000002,"5min":142.08500000000001,"15min":142.08500000000001},"last":0.441},{"interface":"front","average":{"1min":3.8610000000000002,"5min":4.7350000000000003,"15min":3.8660000000000001},"min":{"1min":0.36799999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":55.329000000000001,"5min":142.137,"15min":142.137},"last":3.335}]},{"osd":2,"last update":"Sat Sep 30 09:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":3.508,"15min":2.7290000000000001},"min":{"1min":0.28100000000000003,"5min":0.22500000000000001,"15min":0.184},"max":{"1min":4.1639999999999997,"5min":64.759,"15min":64.759},"last":1.22},{"interface":"front","average":{"1min":1.361,"5min":3.5489999999999999,"15min":2.972},"min":{"1min":0.35099999999999998,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":8.391,"5min":68.373999999999995,"15min":68.373999999999995},"last":1.1799999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":6.2539999999999996,"15min":5.4980000000000002},"min":{"1min":0.33500000000000002,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":22.206,"5min":199.41,"15min":199.41},"last":3.0339999999999998},{"interface":"front","average":{"1min":0.99399999999999999,"5min":5.6840000000000002,"15min":5.2389999999999999},"min":{"1min":0.34100000000000003,"5min":0.27300000000000002,"15min":0.23999999999999999},"max":{"1min":4.2439999999999998,"5min":199.58099999999999,"15min":199.58099999999999},"last":1.25}]},{"osd":4,"last update":"Sat Sep 30 09:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":2.1379999999999999,"15min":2.3079999999999998},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":8.2880000000000003,"5min":48.604999999999997,"15min":142.77099999999999},"last":1.149},{"interface":"front","average":{"1min":2.1240000000000001,"5min":2.1240000000000001,"15min":2.5779999999999998},"min":{"1min":0.39000000000000001,"5min":0.36699999999999999,"15min":0.30499999999999999},"max":{"1min":22.373000000000001,"5min":22.373000000000001,"15min":144.898},"last":6.0389999999999997}]},{"osd":5,"last update":"Sat Sep 30 09:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":2.3170000000000002,"15min":2.0609999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.552,"5min":22.259,"15min":22.259},"last":1.425},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.581,"15min":1.3420000000000001},"min":{"1min":0.433,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.8069999999999999,"5min":15.250999999999999,"15min":15.250999999999999},"last":1.115}]},{"osd":6,"last update":"Sat Sep 30 09:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":1.9790000000000001,"15min":3.536},"min":{"1min":0.48199999999999998,"5min":0.38700000000000001,"15min":0.29899999999999999},"max":{"1min":8.3559999999999999,"5min":30.986999999999998,"15min":209.221},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.8979999999999999,"15min":3.4700000000000002},"min":{"1min":0.49199999999999999,"5min":0.45400000000000001,"15min":0.33900000000000002},"max":{"1min":8.3230000000000004,"5min":28.710999999999999,"15min":208.43600000000001},"last":7.1100000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.5880000000000001,"15min":2.476},"min":{"1min":0.47899999999999998,"5min":0.36299999999999999,"15min":0.28299999999999997},"max":{"1min":4.1239999999999997,"5min":9.3879999999999999,"15min":165.59399999999999},"last":1.681},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.6399999999999999,"15min":2.5169999999999999},"min":{"1min":0.56499999999999995,"5min":0.39100000000000001,"15min":0.316},"max":{"1min":4.1970000000000001,"5min":7.5460000000000003,"15min":165.833},"last":2.222}]}]},{"osd":0,"up_from":94,"seq":403726926189,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3302640,"kb_used_data":2531496,"kb_used_omap":5,"kb_used_meta":771130,"kb_avail":90442512,"statfs":{"total":95995035648,"available":92613132288,"internally_reserved":0,"allocated":2592251904,"data_stored":2588898162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":789637373},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.3180000000000001,"15min":1.9490000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.17799999999999999},"max":{"1min":5.9950000000000001,"5min":13.308999999999999,"15min":57.482999999999997},"last":0.246},{"interface":"front","average":{"1min":2.387,"5min":2.4329999999999998,"15min":3.1850000000000001},"min":{"1min":0.22500000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":23.503,"5min":23.503,"15min":138.92400000000001},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:48:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3740000000000001,"5min":3.0819999999999999,"15min":3.7069999999999999},"min":{"1min":0.31900000000000001,"5min":0.246,"15min":0.20799999999999999},"max":{"1min":23.547999999999998,"5min":31.521000000000001,"15min":127.73399999999999},"last":173.185},{"interface":"front","average":{"1min":2.323,"5min":3.363,"15min":3.9700000000000002},"min":{"1min":0.307,"5min":0.249,"15min":0.249},"max":{"1min":23.789000000000001,"5min":31.478999999999999,"15min":127.69},"last":173.22499999999999}]},{"osd":3,"last update":"Sat Sep 30 09:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":3.7799999999999998,"15min":5.4660000000000002},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.7679999999999998,"5min":80.126999999999995,"15min":218.06999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.2130000000000001,"5min":2.7189999999999999,"15min":4.1980000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.2359999999999998,"5min":80.838999999999999,"15min":218.63200000000001},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:49:16 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.905,"15min":2.2109999999999999},"min":{"1min":0.35299999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.226,"5min":23.658000000000001,"15min":72.403999999999996},"last":0.441},{"interface":"front","average":{"1min":1.097,"5min":2.1429999999999998,"15min":2.899},"min":{"1min":0.36899999999999999,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":3.5339999999999998,"5min":11.617000000000001,"15min":136.262},"last":2.1829999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:49:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":2.2650000000000001,"15min":1.4690000000000001},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":5.2329999999999997,"5min":13.162000000000001,"15min":13.162000000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.367,"15min":1.155},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":11.608000000000001,"5min":13.422000000000001,"15min":13.422000000000001},"last":1.966}]},{"osd":6,"last update":"Sat Sep 30 09:49:16 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":3.7410000000000001,"15min":3.8980000000000001},"min":{"1min":0.45900000000000002,"5min":0.38200000000000001,"15min":0.33200000000000002},"max":{"1min":2.8769999999999998,"5min":240.946,"15min":240.946},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":4.6870000000000003,"15min":4.4390000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.34999999999999998},"max":{"1min":4.6989999999999998,"5min":240.977,"15min":240.977},"last":0.88500000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:49:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":2.8359999999999999,"15min":2.5760000000000001},"min":{"1min":0.41499999999999998,"5min":0.40300000000000002,"15min":0.36099999999999999},"max":{"1min":3.468,"5min":146.828,"15min":146.828},"last":1.4350000000000001},{"interface":"front","average":{"1min":0.93100000000000005,"5min":3.9319999999999999,"15min":3.1219999999999999},"min":{"1min":0.441,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":2.4319999999999999,"5min":146.876,"15min":146.876},"last":1.5649999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599130,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2929288,"kb_used_data":2064572,"kb_used_omap":5,"kb_used_meta":864698,"kb_avail":90815864,"statfs":{"total":95995035648,"available":92995444736,"internally_reserved":0,"allocated":2114121728,"data_stored":2111037006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":885451405},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":7.0640000000000001,"15min":4.0270000000000001},"min":{"1min":0.20399999999999999,"5min":0.189,"15min":0.189},"max":{"1min":1.619,"5min":202.40899999999999,"15min":202.40899999999999},"last":2.02},{"interface":"front","average":{"1min":0.49199999999999999,"5min":7.3330000000000002,"15min":4.0910000000000002},"min":{"1min":0.26400000000000001,"5min":0.224,"15min":0.217},"max":{"1min":1.2609999999999999,"5min":202.38800000000001,"15min":202.38800000000001},"last":0.222}]},{"osd":1,"last update":"Sat Sep 30 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":5.6559999999999997,"5min":7.2460000000000004,"15min":4.5289999999999999},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":99.638999999999996,"5min":202.369,"15min":202.369},"last":1.115},{"interface":"front","average":{"1min":5.6269999999999998,"5min":7.4859999999999998,"15min":4.4020000000000001},"min":{"1min":0.29599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":102.131,"5min":202.54499999999999,"15min":202.54499999999999},"last":7.0010000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:49:13 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":5.9980000000000002,"15min":5.5090000000000003},"min":{"1min":0.29499999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.0499999999999998,"5min":202.59,"15min":202.59},"last":6.9100000000000001},{"interface":"front","average":{"1min":0.79000000000000004,"5min":6.109,"15min":5.4109999999999996},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.224},"max":{"1min":2.5630000000000002,"5min":202.34200000000001,"15min":202.34200000000001},"last":2.0659999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":5.976,"15min":3.27},"min":{"1min":0.34899999999999998,"5min":0.32800000000000001,"15min":0.30599999999999999},"max":{"1min":0.94199999999999995,"5min":247.655,"15min":247.655},"last":6.8550000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":6.2910000000000004,"15min":3.2959999999999998},"min":{"1min":0.35799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.82799999999999996,"5min":248.012,"15min":248.012},"last":5.7990000000000004}]},{"osd":5,"last update":"Sat Sep 30 09:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":6.1550000000000002,"15min":2.7690000000000001},"min":{"1min":0.42499999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.9610000000000001,"5min":243.86500000000001,"15min":243.86500000000001},"last":6.96},{"interface":"front","average":{"1min":0.88600000000000001,"5min":6.46,"15min":3.0299999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.5960000000000001,"5min":244.02199999999999,"15min":244.02199999999999},"last":1.893}]},{"osd":6,"last update":"Sat Sep 30 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.5990000000000002,"15min":2.9009999999999998},"min":{"1min":0.42399999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.1489999999999991,"5min":202.70599999999999,"15min":202.70599999999999},"last":2.238},{"interface":"front","average":{"1min":0.82299999999999995,"5min":3.714,"15min":3.323},"min":{"1min":0.433,"5min":0.372,"15min":0.34100000000000003},"max":{"1min":3.3839999999999999,"5min":202.916,"15min":202.916},"last":3.2490000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":7.7859999999999996,"15min":3.6760000000000002},"min":{"1min":0.47499999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.9740000000000002,"5min":244.911,"15min":244.911},"last":5.8319999999999999},{"interface":"front","average":{"1min":1.206,"5min":7.742,"15min":4.0490000000000004},"min":{"1min":0.45700000000000002,"5min":0.40799999999999997,"15min":0.40100000000000002},"max":{"1min":5.4279999999999999,"5min":244.66,"15min":244.66},"last":0.53300000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435603,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2471156,"kb_used_data":1534952,"kb_used_omap":2,"kb_used_meta":936189,"kb_avail":91273996,"statfs":{"total":95995035648,"available":93464571904,"internally_reserved":0,"allocated":1571790848,"data_stored":1568605470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":958657895},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":3.6560000000000001,"15min":4.633},"min":{"1min":0.22700000000000001,"5min":0.191,"15min":0.183},"max":{"1min":8.0190000000000001,"5min":45.957999999999998,"15min":205.73099999999999},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.294,"5min":3.528,"15min":4.5979999999999999},"min":{"1min":0.26900000000000002,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":4.806,"5min":44.298999999999999,"15min":205.495},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:48:55 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":2.7839999999999998,"15min":4.984},"min":{"1min":0.32000000000000001,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":15.784000000000001,"5min":19.725000000000001,"15min":273.99599999999998},"last":3.3660000000000001},{"interface":"front","average":{"1min":1.2370000000000001,"5min":2.6419999999999999,"15min":4.9050000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.20699999999999999},"max":{"1min":8.4269999999999996,"5min":19.638999999999999,"15min":274.21600000000001},"last":3.044}]},{"osd":2,"last update":"Sat Sep 30 09:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":2.3460000000000001,"15min":5.7610000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":3.2090000000000001,"5min":14.9,"15min":206.69},"last":0.503},{"interface":"front","average":{"1min":1.016,"5min":2.5219999999999998,"15min":5.6799999999999997},"min":{"1min":0.248,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.0579999999999998,"5min":14.228999999999999,"15min":209.22800000000001},"last":0.443}]},{"osd":4,"last update":"Sat Sep 30 09:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":2.4630000000000001,"15min":2.1659999999999999},"min":{"1min":0.29199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.8810000000000002,"5min":81.677000000000007,"15min":81.677000000000007},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.1020000000000001,"5min":2.8290000000000002,"15min":2.1749999999999998},"min":{"1min":0.39500000000000002,"5min":0.33200000000000002,"15min":0.29499999999999998},"max":{"1min":4.8460000000000001,"5min":80.623000000000005,"15min":80.623000000000005},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.837,"15min":1.6299999999999999},"min":{"1min":0.33700000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3130000000000002,"5min":11.701000000000001,"15min":11.701000000000001},"last":3.081},{"interface":"front","average":{"1min":1.5529999999999999,"5min":2.0880000000000001,"15min":1.3300000000000001},"min":{"1min":0.47499999999999998,"5min":0.311,"15min":0.311},"max":{"1min":6.4870000000000001,"5min":15.715,"15min":15.715},"last":3.395}]},{"osd":6,"last update":"Sat Sep 30 09:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.835,"15min":2.0150000000000001},"min":{"1min":0.40600000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.9790000000000001,"5min":10.837999999999999,"15min":60.698},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.1830000000000001,"5min":2.2749999999999999,"15min":2.1240000000000001},"min":{"1min":0.44800000000000001,"5min":0.377,"15min":0.33400000000000002},"max":{"1min":4.915,"5min":15.666,"15min":15.666},"last":3.2789999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.7970000000000002,"15min":2.6219999999999999},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.5299999999999998,"5min":213.548,"15min":213.548},"last":3.1520000000000001},{"interface":"front","average":{"1min":1.304,"5min":3.8159999999999998,"15min":2.6339999999999999},"min":{"1min":0.45800000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.9370000000000003,"5min":213.583,"15min":213.583},"last":3.98}]}]},{"osd":4,"up_from":30,"seq":128849019366,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3072288,"kb_used_data":2536600,"kb_used_omap":0,"kb_used_meta":535680,"kb_avail":90672864,"statfs":{"total":95995035648,"available":92849012736,"internally_reserved":0,"allocated":2597478400,"data_stored":2595682063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":548536320},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:49:03 2023","interfaces":[{"interface":"back","average":{"1min":6.9260000000000002,"5min":3.9300000000000002,"15min":3.2050000000000001},"min":{"1min":0.33400000000000002,"5min":0.25,"15min":0.21199999999999999},"max":{"1min":107.43000000000001,"5min":107.43000000000001,"15min":107.43000000000001},"last":0.313},{"interface":"front","average":{"1min":7.1840000000000002,"5min":4.2450000000000001,"15min":3.4409999999999998},"min":{"1min":0.317,"5min":0.26600000000000001,"15min":0.24299999999999999},"max":{"1min":107.373,"5min":107.373,"15min":107.373},"last":1.359}]},{"osd":1,"last update":"Sat Sep 30 09:49:03 2023","interfaces":[{"interface":"back","average":{"1min":7.2569999999999997,"5min":4.1619999999999999,"15min":3.2519999999999998},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.25},"max":{"1min":108.551,"5min":108.551,"15min":160.69999999999999},"last":0.438},{"interface":"front","average":{"1min":7.1340000000000003,"5min":4.407,"15min":3.2799999999999998},"min":{"1min":0.32100000000000001,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":108.884,"5min":108.884,"15min":160.50700000000001},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":4.0339999999999998,"15min":3.4980000000000002},"min":{"1min":0.378,"5min":0.33600000000000002,"15min":0.27300000000000002},"max":{"1min":6.5270000000000001,"5min":117.858,"15min":142.88999999999999},"last":2.4119999999999999},{"interface":"front","average":{"1min":1.296,"5min":4.2539999999999996,"15min":3.5720000000000001},"min":{"1min":0.42099999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.9850000000000003,"5min":117.871,"15min":142.90700000000001},"last":0.41099999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:49:03 2023","interfaces":[{"interface":"back","average":{"1min":6.976,"5min":3.472,"15min":2.5830000000000002},"min":{"1min":0.41599999999999998,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":105.142,"5min":105.142,"15min":105.142},"last":0.38300000000000001},{"interface":"front","average":{"1min":7.0430000000000001,"5min":3.3460000000000001,"15min":2.5579999999999998},"min":{"1min":0.40200000000000002,"5min":0.36699999999999999,"15min":0.29299999999999998},"max":{"1min":105.157,"5min":105.157,"15min":105.157},"last":1.4019999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":3.1760000000000002,"15min":8.2140000000000004},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.5750000000000002,"5min":140.11099999999999,"15min":140.11099999999999},"last":1.4359999999999999},{"interface":"front","average":{"1min":0.66700000000000004,"5min":3.016,"15min":8.1820000000000004},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.7040000000000002,"5min":140.065,"15min":140.065},"last":2.3719999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:49:28 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.198,"15min":1.5640000000000001},"min":{"1min":0.32400000000000001,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":4.1820000000000004,"5min":9.1470000000000002,"15min":182.10300000000001},"last":1.032},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.98199999999999998,"15min":1.5109999999999999},"min":{"1min":0.33900000000000002,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":2.641,"5min":7.173,"15min":181.76300000000001},"last":1.6259999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":2.7629999999999999,"15min":2.3290000000000002},"min":{"1min":0.375,"5min":0.29999999999999999,"15min":0.27600000000000002},"max":{"1min":5.1260000000000003,"5min":161.72999999999999,"15min":161.72999999999999},"last":6.2060000000000004},{"interface":"front","average":{"1min":0.86499999999999999,"5min":2.7829999999999999,"15min":2.278},"min":{"1min":0.35799999999999998,"5min":0.34999999999999998,"15min":0.27000000000000002},"max":{"1min":3.3570000000000002,"5min":162.60900000000001,"15min":162.60900000000001},"last":1.4510000000000001}]}]},{"osd":5,"up_from":489,"seq":2100239007808,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1733724,"kb_used_data":1552340,"kb_used_omap":6,"kb_used_meta":181369,"kb_avail":92011428,"statfs":{"total":95995035648,"available":94219702272,"internally_reserved":0,"allocated":1589596160,"data_stored":1587800445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":185721951},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:49:32 2023","interfaces":[{"interface":"back","average":{"1min":3.4020000000000001,"5min":8.7479999999999993,"15min":20.456},"min":{"1min":0.224,"5min":0.22,"15min":0.22},"max":{"1min":52.621000000000002,"5min":281.10599999999999,"15min":281.10599999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":4.1719999999999997,"5min":8.9559999999999995,"15min":20.718},"min":{"1min":0.27100000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":53.295999999999999,"5min":282.07400000000001,"15min":282.07400000000001},"last":0.57199999999999995}]},{"osd":1,"last update":"Sat Sep 30 09:49:32 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":5.5439999999999996,"15min":11.794},"min":{"1min":0.29999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.0249999999999999,"5min":273.48200000000003,"15min":273.48200000000003},"last":1.6970000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":5.202,"15min":11.137},"min":{"1min":0.28799999999999998,"5min":0.255,"15min":0.255},"max":{"1min":1.927,"5min":273.50200000000001,"15min":273.50200000000001},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:49:32 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":6.2649999999999997,"15min":2.7149999999999999},"min":{"1min":0.34899999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.1219999999999999,"5min":194.14400000000001,"15min":194.14400000000001},"last":2.5430000000000001},{"interface":"front","average":{"1min":0.96899999999999997,"5min":6.2850000000000001,"15min":3.1219999999999999},"min":{"1min":0.37,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.863,"5min":198.083,"15min":198.083},"last":0.439}]},{"osd":3,"last update":"Sat Sep 30 09:49:32 2023","interfaces":[{"interface":"back","average":{"1min":5.9880000000000004,"5min":6.758,"15min":12.085000000000001},"min":{"1min":0.42799999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":98.484999999999999,"5min":274.01799999999997,"15min":274.01799999999997},"last":9.4320000000000004},{"interface":"front","average":{"1min":5.8079999999999998,"5min":4.2030000000000003,"15min":2.5550000000000002},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":98.504000000000005,"5min":98.504000000000005,"15min":98.504000000000005},"last":2.8140000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:49:32 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.093,"15min":0.999},"min":{"1min":0.32700000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.0659999999999998,"5min":7.3360000000000003,"15min":7.3360000000000003},"last":6.0890000000000004},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.232,"15min":1.1739999999999999},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.317},"max":{"1min":2.6619999999999999,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:49:32 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.389,"15min":1.5589999999999999},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.8859999999999999,"5min":14.371,"15min":14.371},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.85399999999999998,"5min":1.3919999999999999,"15min":1.3899999999999999},"min":{"1min":0.44800000000000001,"5min":0.316,"15min":0.316},"max":{"1min":3.343,"5min":14.454000000000001,"15min":14.454000000000001},"last":0.503}]},{"osd":7,"last update":"Sat Sep 30 09:49:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":1.3380000000000001,"15min":1.3080000000000001},"min":{"1min":0.377,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.9590000000000001,"5min":9.7560000000000002,"15min":9.7560000000000002},"last":5.7889999999999997},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.3520000000000001,"15min":1.0620000000000001},"min":{"1min":0.47299999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.2729999999999997,"5min":11.000999999999999,"15min":11.000999999999999},"last":0.54300000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2588344320,"data_stored":2588344320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":808058880,"data_stored":808058880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2118844416,"data_stored":2118844416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1572274176,"data_stored":1572274176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2612920320,"data_stored":2612920320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1586954240,"data_stored":1586954240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":177471488,"data_stored":177471488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1305280512,"data_stored":1305280512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:49:44.795 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 16 2023-09-30T09:49:44.849 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:44 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:44.849 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:44 smithi167 ceph-mon[192880]: pgmap v1477: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 18652/200184 objects degraded (9.317%); 7140/200184 objects misplaced (3.567%); 2.8 MiB/s, 40 objects/s recovering 2023-09-30T09:49:44.861 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:44.955 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:44 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:44.955 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:44 smithi019 ceph-mon[216815]: pgmap v1477: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 18652/200184 objects degraded (9.317%); 7140/200184 objects misplaced (3.567%); 2.8 MiB/s, 40 objects/s recovering 2023-09-30T09:49:44.959 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:44 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:44.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:44 smithi019 ceph-mon[224043]: pgmap v1477: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 18652/200184 objects degraded (9.317%); 7140/200184 objects misplaced (3.567%); 2.8 MiB/s, 40 objects/s recovering 2023-09-30T09:49:44.965 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:44.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:44.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:45.073 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:45.176 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:45.189 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:45.323 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:45.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:45.350 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:49:45.428 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:45.513 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:45.663 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:45.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:45.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:45.737 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 54 16 70225 70209 81.2161 43.125 0.0272163 0.0123068 2023-09-30T09:49:45.769 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:45.851 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:45.863 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:49:45.896 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:45 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2911132759' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:45.896 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:45 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:45.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:45 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2911132759' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:45.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:45 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:46.222 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:45 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2911132759' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:46.223 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:45 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:49:46.353 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:46.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:46.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:46.462 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:46.531 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:46.737 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 55 16 71029 71013 80.6533 50.25 0.0249784 0.0123931 2023-09-30T09:49:47.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:47.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:47.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[216815]: osdmap e555: 8 total, 8 up, 8 in 2023-09-30T09:49:47.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[216815]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[216815]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[216815]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[216815]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[216815]: pgmap v1479: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s; 18652/205395 objects degraded (9.081%); 7140/205395 objects misplaced (3.476%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:49:47.260 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:47.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:47.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[224043]: osdmap e555: 8 total, 8 up, 8 in 2023-09-30T09:49:47.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[224043]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[224043]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[224043]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[224043]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.261 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:46 smithi019 ceph-mon[224043]: pgmap v1479: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s; 18652/205395 objects degraded (9.081%); 7140/205395 objects misplaced (3.476%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:49:47.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:46 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:47.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:46 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:49:47.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:46 smithi167 ceph-mon[192880]: osdmap e555: 8 total, 8 up, 8 in 2023-09-30T09:49:47.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:46 smithi167 ceph-mon[192880]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:46 smithi167 ceph-mon[192880]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:46 smithi167 ceph-mon[192880]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:46 smithi167 ceph-mon[192880]: failed to encode map e555 with expected crc 2023-09-30T09:49:47.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:46 smithi167 ceph-mon[192880]: pgmap v1479: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s; 18652/205395 objects degraded (9.081%); 7140/205395 objects misplaced (3.476%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:49:47.696 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:47.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:47.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:47.737 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 56 16 71498 71482 79.7369 29.3125 0.0121693 0.0125363 2023-09-30T09:49:47.815 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:47.857 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:48.052 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:48.065 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:48 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18652/209769 objects degraded (8.892%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:48.065 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:48.065 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:48.080 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:48.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:48.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:48.190 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:48.270 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:48 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18652/209769 objects degraded (8.892%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:48.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:48 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18652/209769 objects degraded (8.892%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:48.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:48.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:49:48.486 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:49:48.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:48.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:49:48.594 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:49:48.611 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:49:48.611 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:49:48.611 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:49:48.612 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:49:48.612 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:49:48.662 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:48.737 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 57 16 72195 72179 79.1024 43.5625 0.0218045 0.012637 2023-09-30T09:49:49.008 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:49:49.042 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:49:49.081 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:49:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:49.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:49:49.089 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:49:49.123 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:49:49.162 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:49:49.187 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:49.200 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:49:49.238 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:49:49.249 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:49.258 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:49:49.260 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:49:49.260 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:49:49.260 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:49:49.260 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:49:49.260 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:49:49.260 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:49:49.261 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:49:49.261 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:49:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:49 smithi019 ceph-mon[216815]: pgmap v1480: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 18652/209769 objects degraded (8.892%); 7140/209769 objects misplaced (3.404%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:49:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:49 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4058526045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:49 smithi019 ceph-mon[224043]: pgmap v1480: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 18652/209769 objects degraded (8.892%); 7140/209769 objects misplaced (3.404%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:49:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:49 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4058526045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:49 smithi167 ceph-mon[192880]: pgmap v1480: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 18652/209769 objects degraded (8.892%); 7140/209769 objects misplaced (3.404%); 2.7 MiB/s, 42 objects/s recovering 2023-09-30T09:49:49.546 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:49 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4058526045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:49.738 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 58 16 72710 72694 78.2925 32.1875 0.0285816 0.0127678 2023-09-30T09:49:50.010 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:49:50.032 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:50.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:50.072 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:49:50.140 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:49:50.229 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:50.448 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:49:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:50.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:49:50.565 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:50.660 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:50.738 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 59 16 73455 73439 77.7548 46.5625 0.0287746 0.0128544 2023-09-30T09:49:50.854 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:50.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:50.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:50.915 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:49:50.964 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:51.040 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:51.046 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:51.132 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:51.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:51.246 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:49:51.285 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:51 smithi167 ceph-mon[192880]: pgmap v1481: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 18652/215571 objects degraded (8.652%); 6914/215571 objects misplaced (3.207%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:49:51.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:51 smithi019 ceph-mon[216815]: pgmap v1481: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 18652/215571 objects degraded (8.652%); 6914/215571 objects misplaced (3.207%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:49:51.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:51 smithi019 ceph-mon[224043]: pgmap v1481: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 18652/215571 objects degraded (8.652%); 6914/215571 objects misplaced (3.207%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:49:51.340 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:51.698 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:49:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:51.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:49:51.738 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:49:51.738946+0000 min lat: 0.00112729 max lat: 2.39662 avg lat: 0.0129412 2023-09-30T09:49:51.738 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:49:51.738 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 60 16 74196 74180 77.231 46.3125 0.0179704 0.0129412 2023-09-30T09:49:51.807 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:51.874 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:52.074 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:52.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:52.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:52.122 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 10 pgp_num to 16 2023-09-30T09:49:52.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2814411115' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:49:52.155 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:49:52.156 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2814411115' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:49:52.156 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:49:52.185 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:52.248 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2814411115' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:49:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:49:52.575 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:49:52.738 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 61 16 74906 74890 76.6925 44.375 0.012728 0.0130333 2023-09-30T09:49:53.453 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:53.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:53.465 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[216815]: pgmap v1482: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 18652/216102 objects degraded (8.631%); 6914/216102 objects misplaced (3.199%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:49:53.465 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:49:53.465 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[216815]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.465 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[216815]: osdmap e556: 8 total, 8 up, 8 in 2023-09-30T09:49:53.465 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[216815]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.465 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[216815]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[224043]: pgmap v1482: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 18652/216102 objects degraded (8.631%); 6914/216102 objects misplaced (3.199%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:49:53.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:49:53.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[224043]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[224043]: osdmap e556: 8 total, 8 up, 8 in 2023-09-30T09:49:53.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[224043]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:53 smithi019 ceph-mon[224043]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:53 smithi167 ceph-mon[192880]: pgmap v1482: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 18652/216102 objects degraded (8.631%); 6914/216102 objects misplaced (3.199%); 2.7 MiB/s, 43 objects/s recovering 2023-09-30T09:49:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:53 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:49:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:53 smithi167 ceph-mon[192880]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:53 smithi167 ceph-mon[192880]: osdmap e556: 8 total, 8 up, 8 in 2023-09-30T09:49:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:53 smithi167 ceph-mon[192880]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:53 smithi167 ceph-mon[192880]: failed to encode map e556 with expected crc 2023-09-30T09:49:53.568 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:53.599 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:53.745 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 62 16 75615 75599 76.1704 44.3125 0.0293391 0.0131225 2023-09-30T09:49:53.829 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:53.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:53.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:53.936 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:53.999 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:54.045 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:49:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:49:53] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-09-30T09:49:54.105 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:49:54.105 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:49:54.538 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:54.723 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:54.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:54.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:54.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:54 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:54.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:54 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18652/216102 objects degraded (8.631%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:54.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:54 smithi019 ceph-mon[216815]: pgmap v1484: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 18652/221721 objects degraded (8.412%); 6692/221721 objects misplaced (3.018%); 3.4 MiB/s, 55 objects/s recovering 2023-09-30T09:49:54.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:54.733 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:54.733 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:54 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:54.733 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:54 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18652/216102 objects degraded (8.631%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:54.734 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:54 smithi019 ceph-mon[224043]: pgmap v1484: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 18652/221721 objects degraded (8.412%); 6692/221721 objects misplaced (3.018%); 3.4 MiB/s, 55 objects/s recovering 2023-09-30T09:49:54.738 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 63 16 76347 76331 75.6877 45.75 0.012897 0.0132062 2023-09-30T09:49:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:54 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:54 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18652/216102 objects degraded (8.631%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:54.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:54 smithi167 ceph-mon[192880]: pgmap v1484: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 18652/221721 objects degraded (8.412%); 6692/221721 objects misplaced (3.018%); 3.4 MiB/s, 55 objects/s recovering 2023-09-30T09:49:54.834 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:54.867 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:54.963 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:54.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:55.073 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:55.149 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:55.367 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:49:55.739 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 64 16 77040 77024 75.1819 43.3125 0.0249503 0.0132951 2023-09-30T09:49:55.904 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:49:55.954 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:49:56.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2456791468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:56.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2456791468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:56.148 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:56.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:56.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:56.191 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2456791468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:49:56.254 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:49:56.355 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:56.739 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 65 16 77758 77742 74.7158 44.875 0.0201094 0.0133788 2023-09-30T09:49:56.826 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:56 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:56.826 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2456791468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:49:56.826 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:56 smithi167 ceph-mon[192880]: failed to encode map e557 with expected crc 2023-09-30T09:49:56.826 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:56 smithi167 ceph-mon[192880]: osdmap e557: 8 total, 8 up, 8 in 2023-09-30T09:49:56.826 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:56 smithi167 ceph-mon[192880]: failed to encode map e557 with expected crc 2023-09-30T09:49:56.826 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:56 smithi167 ceph-mon[192880]: pgmap v1486: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 18652/226284 objects degraded (8.243%); 6692/226284 objects misplaced (2.957%); 3.5 MiB/s, 55 objects/s recovering 2023-09-30T09:49:57.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:57.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2456791468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:49:57.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[216815]: failed to encode map e557 with expected crc 2023-09-30T09:49:57.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[216815]: osdmap e557: 8 total, 8 up, 8 in 2023-09-30T09:49:57.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[216815]: failed to encode map e557 with expected crc 2023-09-30T09:49:57.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[216815]: pgmap v1486: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 18652/226284 objects degraded (8.243%); 6692/226284 objects misplaced (2.957%); 3.5 MiB/s, 55 objects/s recovering 2023-09-30T09:49:57.038 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:49:57.038 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2456791468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:49:57.038 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[224043]: failed to encode map e557 with expected crc 2023-09-30T09:49:57.038 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[224043]: osdmap e557: 8 total, 8 up, 8 in 2023-09-30T09:49:57.038 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[224043]: failed to encode map e557 with expected crc 2023-09-30T09:49:57.038 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:56 smithi019 ceph-mon[224043]: pgmap v1486: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 18652/226284 objects degraded (8.243%); 6692/226284 objects misplaced (2.957%); 3.5 MiB/s, 55 objects/s recovering 2023-09-30T09:49:57.580 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:49:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:57.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:49:57.688 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:57.742 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 66 16 78434 78418 74.224 42.25 0.0139991 0.0134657 2023-09-30T09:49:57.778 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:57.931 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:57.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:57.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:58.041 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:49:58.136 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:58.241 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:49:58.256 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:49:58.257 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:49:58.279 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:49:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:58.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:49:58.289 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:49:58.330 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:49:58.375 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:49:58.385 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:58.407 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:49:58.447 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:49:58.470 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:58.501 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:49:58.538 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:49:58.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:49:58.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:49:58.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:49:58.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:49:58.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:49:58.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:49:58.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:49:58.540 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:49:58.572 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:58.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:58.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:58.677 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:49:58.739 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:58.739 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 67 16 79163 79147 73.7963 45.5625 0.0093572 0.0135454 2023-09-30T09:49:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[224043]: pgmap v1487: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 18652/230073 objects degraded (8.107%); 6692/230073 objects misplaced (2.909%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:49:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3623765891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:58.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:49:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:49:58.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[216815]: pgmap v1487: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 18652/230073 objects degraded (8.107%); 6692/230073 objects misplaced (2.909%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:49:58.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3623765891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:58.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:49:58.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:49:58.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:58 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.866 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:49:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:58.878 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.879 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.879 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:58 smithi167 ceph-mon[192880]: pgmap v1487: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 18652/230073 objects degraded (8.107%); 6692/230073 objects misplaced (2.909%); 1.7 MiB/s, 27 objects/s recovering 2023-09-30T09:49:58.879 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:58 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3623765891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:49:58.879 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:49:58.879 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:49:58.879 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:58 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:49:58.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:49:58.980 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:59.036 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:59.082 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:59.083 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:49:59.083 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:49:59.084 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-30T09:49:59.084 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:49:59.084 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:49:59.084 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-30T09:49:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:59.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:59.186 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:49:59.219 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:59.341 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:49:59.344 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:49:59.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:59.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:49:59.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:49:59.511 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:59.626 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:49:59 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:59.627 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:49:59 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:49:59.691 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:49:59.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:49:59.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:49:59.739 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 68 16 79815 79799 73.3105 40.75 0.0171382 0.0136346 2023-09-30T09:49:59.764 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:49:59.808 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:49:59.871 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:49:59.908 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:49:59.994 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:50:00.025 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:00.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:00.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:00.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:49:59 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:00.135 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:00.206 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:00.318 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:00.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:00.427 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:00.494 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:00.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: pgmap v1488: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 18652/236106 objects degraded (7.900%); 6464/236106 objects misplaced (2.738%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:50:00.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded 2023-09-30T09:50:00.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:50:00.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: [WRN] PG_DEGRADED: Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded 2023-09-30T09:50:00.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: pg 10.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: pg 10.a is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: pg 10.b is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: pg 10.d is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[216815]: pg 10.f is active+recovery_wait+undersized+degraded+remapped, acting [2,3] 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: pgmap v1488: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 18652/236106 objects degraded (7.900%); 6464/236106 objects misplaced (2.738%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: [WRN] PG_DEGRADED: Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded 2023-09-30T09:50:00.739 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: pg 10.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-09-30T09:50:00.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: pg 10.a is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2023-09-30T09:50:00.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: pg 10.b is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2023-09-30T09:50:00.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: pg 10.d is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-09-30T09:50:00.740 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:00 smithi019 ceph-mon[224043]: pg 10.f is active+recovery_wait+undersized+degraded+remapped, acting [2,3] 2023-09-30T09:50:00.740 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 69 16 80552 80536 72.9155 46.0625 0.011929 0.0137084 2023-09-30T09:50:01.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: pgmap v1488: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 18652/236106 objects degraded (7.900%); 6464/236106 objects misplaced (2.738%); 3.5 MiB/s, 56 objects/s recovering 2023-09-30T09:50:01.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded 2023-09-30T09:50:01.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-30T09:50:01.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: [WRN] PG_DEGRADED: Degraded data redundancy: 18652/230073 objects degraded (8.107%), 5 pgs degraded 2023-09-30T09:50:01.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: pg 10.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-09-30T09:50:01.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: pg 10.a is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2023-09-30T09:50:01.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: pg 10.b is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2023-09-30T09:50:01.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: pg 10.d is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-09-30T09:50:01.024 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:00 smithi167 ceph-mon[192880]: pg 10.f is active+recovery_wait+undersized+degraded+remapped, acting [2,3] 2023-09-30T09:50:01.655 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:01.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:01.694 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:01.706 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:50:01.744 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 70 16 81185 81169 72.4392 39.5625 0.0226821 0.0137982 2023-09-30T09:50:01.775 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:01.864 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:01.986 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:01.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:01.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:02.097 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:02.195 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:02.413 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:02.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:02.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:02.501 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:50:02.501 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:50:02.501 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:50:02.501 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:50:02.501 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:50:02.523 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:02.531 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:50:02.531 INFO:teuthology.orchestra.run.smithi019.stdout: "success": "" 2023-09-30T09:50:02.532 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:50:02.628 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:02.741 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 71 16 81890 81874 72.0396 44.0625 0.0202151 0.0138754 2023-09-30T09:50:02.803 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:02.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:02.915 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:02.966 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:03.011 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:03.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:03.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:03.115 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:03.143 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:03 smithi167 ceph-mon[192880]: pgmap v1489: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 18652/236718 objects degraded (7.879%); 6464/236718 objects misplaced (2.731%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:50:03.143 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:03.146 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:03.190 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:03.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:03.295 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:03.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:03 smithi019 ceph-mon[216815]: pgmap v1489: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 18652/236718 objects degraded (7.879%); 6464/236718 objects misplaced (2.731%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:50:03.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:03 smithi019 ceph-mon[224043]: pgmap v1489: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 18652/236718 objects degraded (7.879%); 6464/236718 objects misplaced (2.731%); 2.9 MiB/s, 45 objects/s recovering 2023-09-30T09:50:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:03.383 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:03.739 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 72 16 82460 82444 71.534 35.625 0.032792 0.0139743 2023-09-30T09:50:03.797 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:03.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:03.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:03.812 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:50:03.904 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:03.976 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:50:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:50:03] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-09-30T09:50:03.980 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:04.038 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:04 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18652/236718 objects degraded (7.879%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:04.082 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:04.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:04.185 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:04.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:04 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18652/236718 objects degraded (7.879%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:04.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:04 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18652/236718 objects degraded (7.879%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:04.264 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:04.427 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:04.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:04.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:04.467 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:04.534 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:04.587 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:50:04.598 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:04.719 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:04.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:04.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:04.739 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 73 16 83240 83224 71.2219 48.75 0.0154588 0.0140337 2023-09-30T09:50:04.833 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:04.917 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:05.042 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:05.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:05.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:05.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:05 smithi019 ceph-mon[216815]: pgmap v1490: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 18652/242508 objects degraded (7.691%); 6180/242508 objects misplaced (2.548%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:50:05.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:05 smithi019 ceph-mon[224043]: pgmap v1490: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 18652/242508 objects degraded (7.691%); 6180/242508 objects misplaced (2.548%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:50:05.156 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:05.196 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:05 smithi167 ceph-mon[192880]: pgmap v1490: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 18652/242508 objects degraded (7.691%); 6180/242508 objects misplaced (2.548%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:50:05.210 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:05.459 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:05.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:05.565 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:05.653 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:05.740 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 74 16 84016 84000 70.9149 48.5 0.0171343 0.0140948 2023-09-30T09:50:05.758 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:05.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:05.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:05.867 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:05.903 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:05.945 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:05.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:06.049 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:06.061 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:50:06.079 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:06 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1622836270' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:06.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:06 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:06 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1622836270' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:06.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:06 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:06 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1622836270' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:06.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:06 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:06.476 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:06.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:06.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:06.584 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:06.630 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:06.740 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 75 16 84645 84629 70.4936 39.3125 0.00547458 0.0141801 2023-09-30T09:50:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:06.997 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:06.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:07.083 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[224043]: pgmap v1491: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s; 18652/247122 objects degraded (7.548%); 6180/247122 objects misplaced (2.501%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:07.083 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:50:07.083 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:07.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[224043]: failed to encode map e558 with expected crc 2023-09-30T09:50:07.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[224043]: failed to encode map e558 with expected crc 2023-09-30T09:50:07.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[224043]: osdmap e558: 8 total, 8 up, 8 in 2023-09-30T09:50:07.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[216815]: pgmap v1491: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s; 18652/247122 objects degraded (7.548%); 6180/247122 objects misplaced (2.501%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:07.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:50:07.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:07.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[216815]: failed to encode map e558 with expected crc 2023-09-30T09:50:07.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[216815]: failed to encode map e558 with expected crc 2023-09-30T09:50:07.088 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:07 smithi019 ceph-mon[216815]: osdmap e558: 8 total, 8 up, 8 in 2023-09-30T09:50:07.098 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:07.185 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:07 smithi167 ceph-mon[192880]: pgmap v1491: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s; 18652/247122 objects degraded (7.548%); 6180/247122 objects misplaced (2.501%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:07.185 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:07 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:50:07.185 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:07 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:07.185 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:07 smithi167 ceph-mon[192880]: failed to encode map e558 with expected crc 2023-09-30T09:50:07.185 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:07 smithi167 ceph-mon[192880]: failed to encode map e558 with expected crc 2023-09-30T09:50:07.185 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:07 smithi167 ceph-mon[192880]: osdmap e558: 8 total, 8 up, 8 in 2023-09-30T09:50:07.190 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:07.269 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:50:07.269 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:50:07.269 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:50:07.269 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:50:07.269 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.269 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2300 9.b 9:dee21a30:::benchmark_data_smithi019_560486_object2299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.031346+0000", 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 342.22992838900001, 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0658693860000001, 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2300 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.270 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031346+0000", 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031346+0000", 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.271 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031346+0000", 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031390+0000", 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3908e-05 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031403+0000", 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2999e-05 2023-09-30T09:50:07.272 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031408+0000", 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3580000000000002e-06 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031464+0000", 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5831999999999997e-05 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.273 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031593+0000", 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129018 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2", 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031732+0000", 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013851099999999999 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.033586+0000", 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001854337 2023-09-30T09:50:07.274 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064449+0000", 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.030863595000000001 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064457+0000", 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8189999999999992e-06 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.275 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052844+0000", 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.988386451 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052857+0000", 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2961e-05 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.090028+0000", 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.037171329 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.276 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.091452+0000", 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014238129999999999 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097130+0000", 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0056783509999999999 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097216+0000", 2023-09-30T09:50:07.277 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5702e-05 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2301 9.b 9:d16e4333:::benchmark_data_smithi019_560486_object2300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.058107+0000", 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 342.20316805300001, 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0393504029999998, 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.278 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2301 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058107+0000", 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.279 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058107+0000", 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058105+0000", 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058212+0000", 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000106388 2023-09-30T09:50:07.280 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058246+0000", 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4666999999999999e-05 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058256+0000", 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6940000000000007e-06 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.281 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064479+0000", 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062230080000000004 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064486+0000", 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6390000000000003e-06 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052884+0000", 2023-09-30T09:50:07.282 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98839818000000002 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052892+0000", 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8879999999999997e-06 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.091815+0000", 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.038923182 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.283 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.091981+0000", 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016666200000000001 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.092165+0000", 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018395999999999999 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094158+0000", 2023-09-30T09:50:07.284 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001992382 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096529+0000", 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023708689999999998 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097255+0000", 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00072591599999999995 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.285 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097301+0000", 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6601000000000003e-05 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097457+0000", 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015600400000000001 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.286 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2307 9.b 9:d81a73d3:::benchmark_data_smithi019_560486_object2306:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.058514+0000", 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 342.20276137500002, 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0391732079999998, 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2307 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.287 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058514+0000", 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058514+0000", 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.288 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058513+0000", 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058562+0000", 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8896000000000001e-05 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058569+0000", 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1640000000000004e-06 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.289 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058574+0000", 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4900000000000002e-06 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064506+0000", 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0059326819999999999 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:07.290 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064512+0000", 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1140000000000002e-06 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052928+0000", 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98841563700000001 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052936+0000", 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8849999999999999e-06 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.291 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.092555+0000", 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.039618919 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.092702+0000", 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000147109 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:50:07.292 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.092859+0000", 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000157472 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094391+0000", 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015314 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096558+0000", 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002167123 2023-09-30T09:50:07.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097502+0000", 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00094386399999999999 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097546+0000", 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4351000000000003e-05 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.294 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097687+0000", 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014054399999999999 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79004 9.0 9:0d8925f6:::benchmark_data_smithi019_560486_object79003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 276.727783182, 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.377661042, 2023-09-30T09:50:07.295 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79004 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.296 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533491+0000", 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.297 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533548+0000", 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7012999999999998e-05 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533561+0000", 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2352e-05 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533565+0000", 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8060000000000002e-06 2023-09-30T09:50:07.298 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533608+0000", 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2279e-05 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533742+0000", 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013385 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:50:07.299 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533885+0000", 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014328400000000001 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.538697+0000", 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004812584 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.540380+0000", 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001682944 2023-09-30T09:50:07.300 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911056+0000", 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.37067566800000001 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911112+0000", 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6187999999999998e-05 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.301 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911153+0000", 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0515000000000001e-05 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79005 9.0 9:01cc776d:::benchmark_data_smithi019_560486_object79004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 276.72769680300001, 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.377676597, 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.302 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79005 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.303 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533627+0000", 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9091000000000001e-05 2023-09-30T09:50:07.304 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533634+0000", 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3649999999999998e-06 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533638+0000", 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2409999999999997e-06 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.305 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536549+0000", 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029111250000000001 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536664+0000", 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011423500000000001 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536750+0000", 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5932000000000003e-05 2023-09-30T09:50:07.306 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.538813+0000", 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020632480000000002 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.544406+0000", 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055927779999999996 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.307 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911189+0000", 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.366783097 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911227+0000", 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8250999999999997e-05 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911255+0000", 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7742e-05 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.308 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79017 9.0 9:0d388969:::benchmark_data_smithi019_560486_object79016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 276.71259626, 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.37037067400000001, 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.309 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79017 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:50:07.310 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548677+0000", 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548770+0000", 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2744999999999994e-05 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.311 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548786+0000", 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6557000000000001e-05 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548793+0000", 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4880000000000004e-06 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.312 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548838+0000", 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5550000000000003e-05 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548961+0000", 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012252199999999999 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.549114+0000", 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015329499999999999 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.313 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.554511+0000", 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0053968749999999998 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.560437+0000", 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0059257829999999996 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.314 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.918969+0000", 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.35853247300000002 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.919009+0000", 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9582999999999997e-05 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.919049+0000", 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0509999999999997e-05 2023-09-30T09:50:07.315 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:172370 9.1 9:8087334a:::benchmark_data_smithi019_560486_object75625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 144.26215849499999, 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28865046500000002, 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.316 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 172370 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.317 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999119+0000", 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.807e-06 2023-09-30T09:50:07.318 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999125+0000", 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.959e-06 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999129+0000", 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4599999999999996e-06 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999170+0000", 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0299999999999997e-05 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999273+0000", 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010336999999999999 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999396+0000", 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012268500000000001 2023-09-30T09:50:07.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.005183+0000", 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057871479999999998 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.007181+0000", 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019979009999999998 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287675+0000", 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28049398800000003 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287725+0000", 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0720999999999998e-05 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287767+0000", 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1523999999999997e-05 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:172371 9.1 9:8d82cc29:::benchmark_data_smithi019_560486_object75626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 144.26213276499999, 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.29074772500000001, 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.323 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 172371 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999144+0000", 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.424e-06 2023-09-30T09:50:07.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999150+0000", 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8680000000000004e-06 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999154+0000", 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7950000000000001e-06 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.326 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003063+0000", 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0039091020000000002 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003142+0000", 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9460000000000002e-05 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003210+0000", 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7366000000000005e-05 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.005317+0000", 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021074150000000001 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.007204+0000", 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018863300000000001 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289803+0000", 2023-09-30T09:50:07.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28259912500000001 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289853+0000", 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0512e-05 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289890+0000", 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6699999999999998e-05 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.329 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187021 9.1 9:8424d638:::benchmark_data_smithi019_560486_object90276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 123.095162847, 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34549460300000001, 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.330 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187021 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:50:07.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166111+0000", 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166115+0000", 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2400000000000001e-06 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166124+0000", 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4719999999999995e-06 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166128+0000", 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3390000000000003e-06 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166156+0000", 2023-09-30T09:50:07.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7353999999999999e-05 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166240+0000", 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4368000000000003e-05 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166360+0000", 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119819 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.174444+0000", 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0080842840000000006 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.174458+0000", 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.397e-05 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511510+0000", 2023-09-30T09:50:07.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33705242400000002 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511561+0000", 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0382e-05 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511607+0000", 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5847000000000003e-05 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.336 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187029 9.1 9:8b8b6a3e:::benchmark_data_smithi019_560486_object90284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 123.094087875, 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34455740800000001, 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.337 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187029 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167186+0000", 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167191+0000", 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2349999999999999e-06 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167203+0000", 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1552999999999999e-05 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167209+0000", 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0039999999999996e-06 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170626+0000", 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034176219999999999 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170739+0000", 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112713 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170847+0000", 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000107892 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.175569+0000", 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047219749999999998 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511470+0000", 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33590098299999999 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511656+0000", 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018579499999999999 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511698+0000", 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2017000000000001e-05 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511744+0000", 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6542999999999998e-05 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187032 9.0 9:01af9635:::benchmark_data_smithi019_560486_object90287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 123.08878296499999, 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33105306600000001, 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187032 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.344 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172491+0000", 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172496+0000", 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4219999999999999e-06 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172506+0000", 2023-09-30T09:50:07.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8460000000000003e-06 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172511+0000", 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6589999999999998e-06 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172557+0000", 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6353999999999997e-05 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172666+0000", 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109045 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172769+0000", 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010234900000000001 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176428+0000", 2023-09-30T09:50:07.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036596440000000001 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176537+0000", 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108589 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503453+0000", 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32691612199999998 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503501+0000", 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8403999999999999e-05 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503545+0000", 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3650999999999999e-05 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.350 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187034 9.1 9:85c6d82b:::benchmark_data_smithi019_560486_object90289:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 123.084607831, 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34325798600000001, 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187034 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176666+0000", 2023-09-30T09:50:07.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176672+0000", 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0789999999999997e-06 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176683+0000", 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1401e-05 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176689+0000", 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8710000000000002e-06 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503342+0000", 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32665281600000001 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503509+0000", 2023-09-30T09:50:07.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016727300000000001 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503648+0000", 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013894800000000001 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511477+0000", 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0078290040000000005 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511790+0000", 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031249499999999998 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519837+0000", 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.008047874 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519883+0000", 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5791999999999998e-05 2023-09-30T09:50:07.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519925+0000", 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1763e-05 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187035 9.1 9:8a5829f5:::benchmark_data_smithi019_560486_object90290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 123.084572419, 2023-09-30T09:50:07.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34335321800000002, 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187035 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:50:07.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176705+0000", 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5059999999999998e-06 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176710+0000", 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7910000000000001e-06 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176714+0000", 2023-09-30T09:50:07.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7469999999999999e-06 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507398+0000", 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33068423200000002 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507518+0000", 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119772 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507653+0000", 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000134466 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519800+0000", 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012147956999999999 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519964+0000", 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016390900000000001 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519994+0000", 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9394999999999999e-05 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.520026+0000", 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2098999999999999e-05 2023-09-30T09:50:07.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.520056+0000", 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9793000000000001e-05 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop_reply(client.46214.0:187032 9.0 e526/483 ondisk, result = 0)", 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 123.083244192, 2023-09-30T09:50:07.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32552107499999999, 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:50:07.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178030+0000", 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178036+0000", 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.468e-06 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503359+0000", 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32532316900000002 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503369+0000", 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0635999999999999e-05 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503435+0000", 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6155000000000006e-05 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503444+0000", 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5150000000000006e-06 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503552+0000", 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010775100000000001 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.368 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:80825 10.e e557/550 10:73bacab1:::benchmark_data_smithi019_617837_object80824:head v 557'5093, mlcod=541'1292)", 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:01.241419+0000", 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 6.0198556730000004, 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.108384336, 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.241419+0000", 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.241419+0000", 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.241418+0000", 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:07.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.241449+0000", 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0719999999999997e-05 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.241460+0000", 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1474000000000001e-05 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.241467+0000", 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0449999999999997e-06 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.241613+0000", 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014632100000000001 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.241637+0000", 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3957000000000001e-05 2023-09-30T09:50:07.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.349752+0000", 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.108115512 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.349804+0000", 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1218000000000003e-05 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGPush(10.e 557/550 [PushOp(10:730a0e3e:::benchmark_data_smithi019_617837_object20747:head, version: 541'1293, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:730a0e3e:::benchmark_data_smithi019_617837_object20747:head@541'1293, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:01.244006+0000", 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 6.0172684580000002, 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.105738259, 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244006+0000", 2023-09-30T09:50:07.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244006+0000", 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244006+0000", 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244033+0000", 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7443000000000001e-05 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244047+0000", 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.393e-05 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244055+0000", 2023-09-30T09:50:07.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5480000000000004e-06 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.246349+0000", 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022941179999999999 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.246358+0000", 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9390000000000003e-06 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.349745+0000", 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.103387118 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:80843 10.f e557/550 10:fc9e491f:::benchmark_data_smithi019_617837_object80842:head v 557'4992, mlcod=0'0)", 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:01.244959+0000", 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 6.0163156989999997, 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.11643253200000001, 2023-09-30T09:50:07.378 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244959+0000", 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244959+0000", 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244959+0000", 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245134+0000", 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000175005 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245139+0000", 2023-09-30T09:50:07.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.113e-06 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245141+0000", 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5339999999999998e-06 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245185+0000", 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3699999999999998e-05 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245201+0000", 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6341000000000002e-05 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.361341+0000", 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.11613952900000001 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.361392+0000", 2023-09-30T09:50:07.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0943999999999999e-05 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:80837 10.b e557/550 10:d22a24f6:::benchmark_data_smithi019_617837_object80836:head v 557'4884, mlcod=0'0)", 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:01.244975+0000", 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 6.016299633, 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.108470097, 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.383 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244975+0000", 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244975+0000", 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.244975+0000", 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967296 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245071+0000", 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5808000000000002e-05 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245080+0000", 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3470000000000005e-06 2023-09-30T09:50:07.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245085+0000", 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3290000000000004e-06 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245173+0000", 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8814999999999996e-05 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245211+0000", 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7577000000000003e-05 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.353409+0000", 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.108197616 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.353445+0000", 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6816999999999997e-05 2023-09-30T09:50:07.387 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:80840 10.d e557/550 10:b077d62f:::benchmark_data_smithi019_617837_object80839:head v 557'5042, mlcod=0'0)", 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:01.245096+0000", 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 6.0161791789999999, 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.10828781699999999, 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245096+0000", 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245096+0000", 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245096+0000", 2023-09-30T09:50:07.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967296 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245224+0000", 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012856799999999999 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245231+0000", 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7089999999999996e-06 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245235+0000", 2023-09-30T09:50:07.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1110000000000001e-06 2023-09-30T09:50:07.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245284+0000", 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9499000000000002e-05 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.245304+0000", 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9647999999999999e-05 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.353330+0000", 2023-09-30T09:50:07.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.10802599 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.353384+0000", 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3479000000000001e-05 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:80852 10.7 e557/550 10:ed7522d5:::benchmark_data_smithi019_617837_object80851:head v 557'5127, mlcod=0'0)", 2023-09-30T09:50:07.393 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:01.253311+0000", 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 6.0079639589999996, 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.108108659, 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.253311+0000", 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.253311+0000", 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.253310+0000", 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.253373+0000", 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2788999999999998e-05 2023-09-30T09:50:07.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.349755+0000", 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.096382412000000001 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.349764+0000", 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1959999999999997e-06 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.349810+0000", 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6158000000000001e-05 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.349837+0000", 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6585999999999999e-05 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.361396+0000", 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011559036999999999 2023-09-30T09:50:07.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:01.361420+0000", 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3578e-05 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:07.398 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:50:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:07.399 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:07.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:07.500 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:07.577 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:07.654 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:50:07.679 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:07.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:07.699 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:50:07.741 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 76 16 85402 85386 70.1881 47.3125 0.0211634 0.0142421 2023-09-30T09:50:07.742 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:50:07.774 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:50:07.789 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:07.815 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:50:07.856 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:07.857 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:50:07.886 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:50:07.902 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:07.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:07.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:07.917 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:50:07.919 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:50:07.919 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:50:07.919 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:50:07.919 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:50:07.919 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:50:07.919 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:50:07.919 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:50:07.919 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:50:08.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:08.041 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:08.220 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:08.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2681697511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:08.230 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2681697511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:08.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2681697511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:08.332 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:08.359 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:08.461 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:50:08.465 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:08.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:08.572 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:08.615 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:08.726 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:08.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:08.742 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 77 16 85866 85850 69.6523 29 0.0121533 0.0143516 2023-09-30T09:50:08.751 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:50:08.751 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:50:08.752 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:50:08.752 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2023-09-30T09:50:08.752 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 0 2023-09-30T09:50:08.835 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:08.906 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:09.032 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:09.035 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:50:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:09.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:09.111 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:50:09.153 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:09.218 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:09.311 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:09.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:09.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:09 smithi019 ceph-mon[216815]: pgmap v1493: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 18652/250977 objects degraded (7.432%); 6180/250977 objects misplaced (2.462%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:50:09.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18652/250977 objects degraded (7.432%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:09.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:09 smithi019 ceph-mon[224043]: pgmap v1493: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 18652/250977 objects degraded (7.432%); 6180/250977 objects misplaced (2.462%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:50:09.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18652/250977 objects degraded (7.432%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:09.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:09 smithi167 ceph-mon[192880]: pgmap v1493: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 18652/250977 objects degraded (7.432%); 6180/250977 objects misplaced (2.462%); 3.2 MiB/s, 51 objects/s recovering 2023-09-30T09:50:09.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18652/250977 objects degraded (7.432%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:09.416 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:09.532 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:09.566 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:09.653 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:09.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:09.745 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 78 16 86582 86566 69.3304 44.75 0.0184059 0.0144185 2023-09-30T09:50:09.779 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:09.857 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:09.959 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:09.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:10.076 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:10.134 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:10.143 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:10.193 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:10.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:10.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:10.296 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:10.326 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:10.745 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 79 16 87052 87036 68.8248 29.375 0.228573 0.0145168 2023-09-30T09:50:11.021 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:50:11.061 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:11.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:11.167 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:11.228 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:11 smithi167 ceph-mon[192880]: pgmap v1494: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 18652/256662 objects degraded (7.267%); 5955/256662 objects misplaced (2.320%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:11.234 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:11.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:11 smithi019 ceph-mon[224043]: pgmap v1494: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 18652/256662 objects degraded (7.267%); 5955/256662 objects misplaced (2.320%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:11.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:11 smithi019 ceph-mon[216815]: pgmap v1494: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 18652/256662 objects degraded (7.267%); 5955/256662 objects misplaced (2.320%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:11.357 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:11.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:11.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:11.462 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:11.538 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:11.606 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:50:11.606 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:50:11.664 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:11.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:11.748 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:2023-09-30T09:50:11.747143+0000 min lat: 0.00112729 max lat: 2.39662 avg lat: 0.0145851 2023-09-30T09:50:11.748 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-30T09:50:11.748 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 80 16 87781 87765 68.5328 45.5625 0.0195698 0.0145851 2023-09-30T09:50:11.779 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:11.843 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:11.973 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:11.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:12.087 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:12.166 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:12.754 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 81 16 88460 88444 68.2049 42.4375 0.0259152 0.014656 2023-09-30T09:50:12.837 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:50:12.908 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:12.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:13.016 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:13.067 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:13.124 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:13 smithi167 ceph-mon[192880]: pgmap v1495: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s; 18652/257304 objects degraded (7.249%); 5955/257304 objects misplaced (2.314%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:13.124 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:13 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/531873501' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-30T09:50:13.124 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:13 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-30T09:50:13.124 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:13.153 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.6 primary-affinity to 0 (802) 2023-09-30T09:50:13.178 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:13 smithi019 ceph-mon[216815]: pgmap v1495: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s; 18652/257304 objects degraded (7.249%); 5955/257304 objects misplaced (2.314%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:13.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:13 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/531873501' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-30T09:50:13.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:13 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-30T09:50:13.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:13 smithi019 ceph-mon[224043]: pgmap v1495: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s; 18652/257304 objects degraded (7.249%); 5955/257304 objects misplaced (2.314%); 3.2 MiB/s, 50 objects/s recovering 2023-09-30T09:50:13.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:13 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/531873501' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-30T09:50:13.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:13 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-30T09:50:13.194 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:13.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:13.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:13.299 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:13.326 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:50:13.327 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:13.378 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:13.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:13.482 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:13.509 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:13.556 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:13.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:13.661 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:13.689 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:13.754 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 82 16 88754 88738 67.5975 18.375 0.00448357 0.0146773 2023-09-30T09:50:13.790 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:13.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:13.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:13.894 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:13.932 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:14.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:50:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:50:13] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-09-30T09:50:14.181 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:14.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:14.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:14.284 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:14.315 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:14.401 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-30T09:50:14.401 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: osdmap e559: 8 total, 8 up, 8 in 2023-09-30T09:50:14.401 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.401 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.401 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.401 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 18652/257304 objects degraded (7.249%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[224043]: osdmap e560: 8 total, 8 up, 8 in 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-30T09:50:14.402 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: osdmap e559: 8 total, 8 up, 8 in 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 18652/257304 objects degraded (7.249%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:14.403 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:14 smithi019 ceph-mon[216815]: osdmap e560: 8 total, 8 up, 8 in 2023-09-30T09:50:14.514 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:14.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:14.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:14.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-30T09:50:14.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: osdmap e559: 8 total, 8 up, 8 in 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: failed to encode map e559 with expected crc 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 18652/257304 objects degraded (7.249%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:14.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:14 smithi167 ceph-mon[192880]: osdmap e560: 8 total, 8 up, 8 in 2023-09-30T09:50:14.619 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:14.649 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:14.698 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:14.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:14.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:14.754 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 83 16 88754 88738 66.7831 0 - 0.0146773 2023-09-30T09:50:14.802 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:14.831 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:15.145 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:15.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:15.256 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:15.302 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:15.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.385 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: pgmap v1498: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 11193/262830 objects degraded (4.259%); 5616/262830 objects misplaced (2.137%); 4.5 MiB/s, 74 objects/s recovering 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: osdmap e561: 8 total, 8 up, 8 in 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/608692127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.386 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: pgmap v1498: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 11193/262830 objects degraded (4.259%); 5616/262830 objects misplaced (2.137%); 4.5 MiB/s, 74 objects/s recovering 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: osdmap e561: 8 total, 8 up, 8 in 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/608692127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:15.387 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:15 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:15.437 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:15.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:15.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:15.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: failed to encode map e560 with expected crc 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: pgmap v1498: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 11193/262830 objects degraded (4.259%); 5616/262830 objects misplaced (2.137%); 4.5 MiB/s, 74 objects/s recovering 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: osdmap e561: 8 total, 8 up, 8 in 2023-09-30T09:50:15.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/608692127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:15.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:15 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:15.547 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:15.643 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:15.756 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 84 16 89340 89324 66.4226 18.3125 0.0237651 0.01505 2023-09-30T09:50:15.888 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:50:15.902 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:15.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:15.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:16.016 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:50:16.016 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:50:16.017 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:16.079 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:16.246 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.246 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.246 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.246 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.246 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.246 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.247 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.247 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:16.247 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:50:16.247 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:16 smithi167 ceph-mon[192880]: osdmap e562: 8 total, 8 up, 8 in 2023-09-30T09:50:16.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[224043]: osdmap e562: 8 total, 8 up, 8 in 2023-09-30T09:50:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: failed to encode map e561 with expected crc 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:50:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:16 smithi019 ceph-mon[216815]: osdmap e562: 8 total, 8 up, 8 in 2023-09-30T09:50:16.376 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:50:16.376 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:50:16.376 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:50:16.376 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:50:16.376 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:50:16.376 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:50:16.407 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:16.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:16.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:16.412 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:50:16.452 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:50:16.458 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:50:16.458 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:50:16.458 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:50:16.459 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:50:16.459 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:50:16.459 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:50:16.459 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:50:16.459 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:50:16.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:16.601 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:16.724 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:16.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:16.756 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 85 16 90071 90055 66.1788 45.6875 0.0196759 0.0150989 2023-09-30T09:50:16.828 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:16.854 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:16.896 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:16.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:16.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:16.999 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:17.051 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:17.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[216815]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[216815]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[216815]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[216815]: pgmap v1501: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 265 KiB/s rd, 41 MiB/s wr, 669 op/s; 11193/265227 objects degraded (4.220%); 1822/265227 objects misplaced (0.687%); 5.3 MiB/s, 93 objects/s recovering 2023-09-30T09:50:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2256481800' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[224043]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[224043]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[224043]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[224043]: pgmap v1501: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 265 KiB/s rd, 41 MiB/s wr, 669 op/s; 11193/265227 objects degraded (4.220%); 1822/265227 objects misplaced (0.687%); 5.3 MiB/s, 93 objects/s recovering 2023-09-30T09:50:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2256481800' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:17.392 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:17 smithi167 ceph-mon[192880]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.392 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:17 smithi167 ceph-mon[192880]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.392 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:17 smithi167 ceph-mon[192880]: failed to encode map e562 with expected crc 2023-09-30T09:50:17.392 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:17 smithi167 ceph-mon[192880]: pgmap v1501: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 265 KiB/s rd, 41 MiB/s wr, 669 op/s; 11193/265227 objects degraded (4.220%); 1822/265227 objects misplaced (0.687%); 5.3 MiB/s, 93 objects/s recovering 2023-09-30T09:50:17.393 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2256481800' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:17.412 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:50:17.756 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 86 16 90525 90509 65.7394 28.375 0.033188 0.0152061 2023-09-30T09:50:17.770 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:17.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:17.876 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:17.946 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:18.064 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:18.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:18.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:18.146 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:50:18.176 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:18.290 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:18.477 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:18.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:18.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:18.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:18.587 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:18.678 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:18.757 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 87 16 91286 91270 65.5305 47.5625 0.0199697 0.0152531 2023-09-30T09:50:18.946 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:18.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:18.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:19.053 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:19.113 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:19.182 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:19.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:19.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:19.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:19 smithi167 ceph-mon[192880]: pgmap v1502: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 218 KiB/s rd, 28 MiB/s wr, 452 op/s; 11193/267246 objects degraded (4.188%); 1822/267246 objects misplaced (0.682%); 4.4 MiB/s, 76 objects/s recovering 2023-09-30T09:50:19.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 11193/267246 objects degraded (4.188%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:19.289 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:19.314 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 2, 7, 0, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:50:19.315 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:50:19.315 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 4, 5, 2, 7, 0, 3, 6] 2023-09-30T09:50:19.315 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-30T09:50:19.383 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:19.465 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:19.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:19.551 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:50:19.569 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:19.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:19 smithi019 ceph-mon[224043]: pgmap v1502: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 218 KiB/s rd, 28 MiB/s wr, 452 op/s; 11193/267246 objects degraded (4.188%); 1822/267246 objects misplaced (0.682%); 4.4 MiB/s, 76 objects/s recovering 2023-09-30T09:50:19.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 11193/267246 objects degraded (4.188%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:19 smithi019 ceph-mon[216815]: pgmap v1502: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 218 KiB/s rd, 28 MiB/s wr, 452 op/s; 11193/267246 objects degraded (4.188%); 1822/267246 objects misplaced (0.682%); 4.4 MiB/s, 76 objects/s recovering 2023-09-30T09:50:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 11193/267246 objects degraded (4.188%), 3 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:19.665 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:19.757 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 88 16 91974 91958 65.2745 43 0.0164366 0.0153154 2023-09-30T09:50:19.773 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:19.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:19.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:19.861 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:50:19.882 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:20.024 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:20.050 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:20.397 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:20.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:20.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:20.502 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:20.590 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:20.644 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:20.758 INFO:tasks.radosbench.radosbench.0.smithi019.stdout: 89 16 92664 92648 65.0257 43.125 0.0158183 0.0153723 2023-09-30T09:50:20.785 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:50:20.785 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:50:20.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:50:20.785 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:50:20.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.785 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126845 8.c 8:36dee011:::benchmark_data_smithi019_478679_object34397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:50:20.785 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 515.00840854299997, 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.93607846500000003, 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126845 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771859+0000", 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771858+0000", 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771863+0000", 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6930000000000004e-06 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771879+0000", 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6004000000000001e-05 2023-09-30T09:50:20.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771886+0000", 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4760000000000003e-06 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771947+0000", 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1393e-05 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.771958+0000", 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0495e-05 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.687778+0000", 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.91581990400000002 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.687930+0000", 2023-09-30T09:50:20.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015184700000000001 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.688083+0000", 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000153113 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.694570+0000", 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064871449999999997 2023-09-30T09:50:20.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695773+0000", 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012035189999999999 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707761+0000", 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011987507999999999 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707817+0000", 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6196999999999997e-05 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.707937+0000", 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000120059 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.793 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:126846 8.c 8:38abd25d:::benchmark_data_smithi019_478679_object34398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 515.00006991700002, 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92797822600000002, 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 126846 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780197+0000", 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780196+0000", 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780202+0000", 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0950000000000001e-06 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780217+0000", 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4602e-05 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780223+0000", 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6860000000000003e-06 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780256+0000", 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2988999999999999e-05 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:45.780263+0000", 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1359999999999999e-06 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691354+0000", 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.911091706 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691496+0000", 2023-09-30T09:50:20.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000142058 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.691644+0000", 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014777599999999999 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695526+0000", 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003882108 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.695780+0000", 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000253804 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708058+0000", 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012278213 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708097+0000", 2023-09-30T09:50:20.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8503e-05 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:41:46.708175+0000", 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8568999999999999e-05 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.801 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:132587 8.d 8:b57b0da1:::benchmark_data_smithi019_478679_object40139:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e435)", 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 492.87419844200002, 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.74966547400000005, 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 132587 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.802 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906069+0000", 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906068+0000", 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906071+0000", 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1879999999999998e-06 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906078+0000", 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7440000000000001e-06 2023-09-30T09:50:20.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.906083+0000", 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5639999999999998e-06 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.914573+0000", 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0084897930000000007 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:07.914581+0000", 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8370000000000003e-06 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644509+0000", 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.72992777600000003 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644642+0000", 2023-09-30T09:50:20.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013305200000000001 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2", 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.644719+0000", 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6363000000000001e-05 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655647+0000", 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010928208 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655652+0000", 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.967e-06 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655686+0000", 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4452999999999997e-05 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:42:08.655734+0000", 2023-09-30T09:50:20.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7939000000000003e-05 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:176082 8.18 8:1f93b3c9:::benchmark_data_smithi019_478679_object83634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 388.49803919599998, 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.35076756999999997, 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.809 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 176082 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282228+0000", 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282230+0000", 2023-09-30T09:50:20.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6960000000000001e-06 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282237+0000", 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7569999999999998e-06 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.282241+0000", 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9249999999999997e-06 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.284838+0000", 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002597125 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.284993+0000", 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015470999999999999 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.285408+0000", 2023-09-30T09:50:20.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00041504500000000001 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.287512+0000", 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021044219999999999 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.500557+0000", 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21304519899999999 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632421+0000", 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.13186376399999999 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632470+0000", 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8624000000000001e-05 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632996+0000", 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052566699999999995 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:176090 8.8 8:17545422:::benchmark_data_smithi019_478679_object83642:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 388.49103103700003, 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34340390900000001, 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.816 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 176090 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289236+0000", 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289239+0000", 2023-09-30T09:50:20.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.264e-06 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289246+0000", 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8859999999999998e-06 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289250+0000", 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4379999999999997e-06 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289291+0000", 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1014999999999999e-05 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289420+0000", 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128823 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,2,3", 2023-09-30T09:50:20.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.289844+0000", 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00042338899999999998 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.292151+0000", 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023077509999999998 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.294495+0000", 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023442430000000002 2023-09-30T09:50:20.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.500522+0000", 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.20602649000000001 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632458+0000", 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.13193581900000001 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632514+0000", 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5893000000000002e-05 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:52.632640+0000", 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000126327 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.823 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2297 9.d 9:b0cd7046:::benchmark_data_smithi019_560486_object2296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 355.74878568499997, 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0633178289999998, 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2297 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031516+0000", 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.464e-05 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031523+0000", 2023-09-30T09:50:20.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8779999999999998e-06 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031528+0000", 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6970000000000001e-06 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032166+0000", 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00063765200000000005 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032280+0000", 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000113811 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032419+0000", 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013967100000000001 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.033675+0000", 2023-09-30T09:50:20.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001255363 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053674+0000", 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.019999847000000001 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053684+0000", 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9990000000000003e-06 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.059238+0000", 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.00555335 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.059246+0000", 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4940000000000003e-06 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.086761+0000", 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.027514383 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.088028+0000", 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012668429999999999 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094723+0000", 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0066955230000000001 2023-09-30T09:50:20.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094799+0000", 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6225999999999998e-05 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2293 9.a 9:552d93c3:::benchmark_data_smithi019_560486_object2292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:50:20.832 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 354.73519934699999, 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.051156384, 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2293 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045066+0000", 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:20.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045217+0000", 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150221 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045234+0000", 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7676000000000002e-05 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.835 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045243+0000", 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0429999999999996e-06 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045361+0000", 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011739900000000001 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045370+0000", 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0049999999999993e-06 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058484+0000", 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013114657 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058494+0000", 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3559999999999999e-06 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.085277+0000", 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.026783555 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.087303+0000", 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020253559999999999 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096095+0000", 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0087920210000000006 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096224+0000", 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129453 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2299 9.a 9:5ff10925:::benchmark_data_smithi019_560486_object2298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 354.73501020100002, 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0514241090000001, 2023-09-30T09:50:20.839 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2299 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045256+0000", 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045294+0000", 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7457999999999999e-05 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045304+0000", 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0271999999999999e-05 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045310+0000", 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7760000000000003e-06 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045439+0000", 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128909 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045445+0000", 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0229999999999998e-06 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058514+0000", 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013069717999999999 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058520+0000", 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2920000000000003e-06 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.087342+0000", 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.028822465 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.088285+0000", 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00094316300000000001 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096601+0000", 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0083155179999999992 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096681+0000", 2023-09-30T09:50:20.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0162999999999998e-05 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5657 9.d 9:b0e10006:::benchmark_data_smithi019_560486_object5656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 350.10357553, 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.006709437, 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.846 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5657 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676689+0000", 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676814+0000", 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012537599999999999 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676838+0000", 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3623000000000001e-05 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676847+0000", 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2590000000000006e-06 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676916+0000", 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8670000000000005e-05 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676927+0000", 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0849000000000001e-05 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680187+0000", 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0032605560000001 2023-09-30T09:50:20.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680317+0000", 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129453 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680486+0000", 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016957800000000001 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.851 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682922+0000", 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024357670000000001 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683200+0000", 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027740300000000001 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683261+0000", 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1591e-05 2023-09-30T09:50:20.852 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683346+0000", 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4541000000000003e-05 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683401+0000", 2023-09-30T09:50:20.853 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5316999999999999e-05 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5664 9.d 9:b51be770:::benchmark_data_smithi019_560486_object5663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 350.10241937299998, 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.005731259, 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.854 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5664 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:50:20.855 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677847+0000", 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677904+0000", 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7519999999999998e-05 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677923+0000", 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8369e-05 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677930+0000", 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4680000000000003e-06 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677977+0000", 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7170999999999999e-05 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677988+0000", 2023-09-30T09:50:20.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0855e-05 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680921+0000", 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0029331560000001 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681034+0000", 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011288199999999999 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.859 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681184+0000", 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150094 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682930+0000", 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017458339999999999 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.860 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683222+0000", 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00029221800000000001 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683451+0000", 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00022835100000000001 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683528+0000", 2023-09-30T09:50:20.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6852000000000002e-05 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683579+0000", 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1418999999999997e-05 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5673 9.d 9:b96c3c13:::benchmark_data_smithi019_560486_object5672:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:50:20.862 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 350.100481495, 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0071881890000001, 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5673 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679785+0000", 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:20.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679870+0000", 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4948000000000001e-05 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679884+0000", 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4348000000000001e-05 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679891+0000", 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9589999999999998e-06 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679948+0000", 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7120999999999997e-05 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679955+0000", 2023-09-30T09:50:20.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7870000000000004e-06 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681621+0000", 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.001666497 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681730+0000", 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010867 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681870+0000", 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013995699999999999 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682935+0000", 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010647829999999999 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.685992+0000", 2023-09-30T09:50:20.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030569799999999999 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686739+0000", 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00074698799999999997 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686937+0000", 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000197858 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686974+0000", 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7280999999999998e-05 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5686 9.d 9:b2765571:::benchmark_data_smithi019_560486_object5685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 350.094963017, 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.002027996, 2023-09-30T09:50:20.870 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5686 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685339+0000", 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5487000000000001e-05 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685347+0000", 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.072e-06 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685353+0000", 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8860000000000001e-06 2023-09-30T09:50:20.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685417+0000", 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4487999999999996e-05 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685433+0000", 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5132e-05 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682219+0000", 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99678628499999999 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682328+0000", 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109515 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682483+0000", 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015417100000000001 2023-09-30T09:50:20.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686027+0000", 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035446449999999999 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687236+0000", 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012086670000000001 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687247+0000", 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.136e-05 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687295+0000", 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8118999999999998e-05 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687332+0000", 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6742999999999997e-05 2023-09-30T09:50:20.877 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:83148 10.c 10:3b69819a:::benchmark_data_smithi019_617837_object83147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:04.610882+0000", 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 16.169384699999998, 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.049684473, 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.878 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 83148 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.610882+0000", 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.879 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.610882+0000", 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.610881+0000", 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.610932+0000", 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1038000000000002e-05 2023-09-30T09:50:20.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.610949+0000", 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7078000000000001e-05 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.610956+0000", 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9489999999999999e-06 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.615721+0000", 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047647139999999998 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.615888+0000", 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016756 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.616044+0000", 2023-09-30T09:50:20.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015555500000000001 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.619931+0000", 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0038869289999999999 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.623373+0000", 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034424910000000002 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.660485+0000", 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.037112184999999999 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.660524+0000", 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8859999999999997e-05 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:04.660567+0000", 2023-09-30T09:50:20.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2769000000000003e-05 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:90243 10.c 10:3db9a0b7:::benchmark_data_smithi019_617837_object90242:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:17.195970+0000", 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 3.584297641, 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.046849962000000002, 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.885 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 90243 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.195970+0000", 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.195970+0000", 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.195968+0000", 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.196097+0000", 2023-09-30T09:50:20.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012889299999999999 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.196116+0000", 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8519999999999999e-05 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.196128+0000", 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2428e-05 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204702+0000", 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0085741540000000005 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204857+0000", 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015486300000000001 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.205008+0000", 2023-09-30T09:50:20.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150724 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.221473+0000", 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01646545 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.231631+0000", 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010157631 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242752+0000", 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011121107999999999 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242795+0000", 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3408999999999997e-05 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242819+0000", 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4147000000000001e-05 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:90244 10.c 10:351cc1b5:::benchmark_data_smithi019_617837_object90243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:17.204413+0000", 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 3.575854584, 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.038490413000000001, 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.892 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 90244 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204413+0000", 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.893 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204413+0000", 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204411+0000", 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204464+0000", 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3430999999999997e-05 2023-09-30T09:50:20.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204486+0000", 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1835999999999999e-05 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204494+0000", 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6129999999999998e-06 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.208191+0000", 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036970430000000001 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.208323+0000", 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013188499999999999 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.208448+0000", 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012486299999999999 2023-09-30T09:50:20.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.221477+0000", 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013029351999999999 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.231665+0000", 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010187860999999999 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242852+0000", 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011187144 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242881+0000", 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9043999999999998e-05 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242903+0000", 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1883e-05 2023-09-30T09:50:20.898 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:90246 10.c 10:323e0bde:::benchmark_data_smithi019_617837_object90245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:17.204500+0000", 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 3.5757668300000001, 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.038495024000000003, 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.899 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 90246 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204500+0000", 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204500+0000", 2023-09-30T09:50:20.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204500+0000", 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204541+0000", 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1096000000000001e-05 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204549+0000", 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9589999999999995e-06 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.204554+0000", 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6530000000000002e-06 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.211334+0000", 2023-09-30T09:50:20.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0067808900000000004 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.211508+0000", 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000173504 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.211665+0000", 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015661000000000001 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.221488+0000", 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0098236360000000002 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.231697+0000", 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010208956999999999 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242937+0000", 2023-09-30T09:50:20.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011239727999999999 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242964+0000", 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7066000000000001e-05 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.242995+0000", 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1389000000000001e-05 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.905 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:90275 10.c 10:341da16c:::benchmark_data_smithi019_617837_object90274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:17.239406+0000", 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 3.540861601, 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21586704000000001, 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:20.906 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 90275 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239406+0000", 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239406+0000", 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239404+0000", 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239449+0000", 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5130000000000003e-05 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239471+0000", 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2171000000000002e-05 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239481+0000", 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8560000000000002e-06 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239530+0000", 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8631999999999998e-05 2023-09-30T09:50:20.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239672+0000", 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014218400000000001 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.239822+0000", 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015036699999999999 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.243170+0000", 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003347708 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.243323+0000", 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015291400000000001 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.455189+0000", 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21186595499999999 2023-09-30T09:50:20.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.455234+0000", 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5484e-05 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:17.455273+0000", 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8231e-05 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.912 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:90969 10.c 10:3854a414:::benchmark_data_smithi019_617837_object90968:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:18.355134+0000", 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.4251331070000002, 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.044360754000000002, 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:20.913 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 90969 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.355134+0000", 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.355134+0000", 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.355132+0000", 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.358735+0000", 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036028990000000001 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.358760+0000", 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4875000000000002e-05 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.358769+0000", 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6449999999999994e-06 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.361465+0000", 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026968360000000002 2023-09-30T09:50:20.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.361618+0000", 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015223100000000001 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.361774+0000", 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015580600000000001 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.365299+0000", 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035258239999999999 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.368692+0000", 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003393119 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.399392+0000", 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.030699321000000002 2023-09-30T09:50:20.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.399451+0000", 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8785e-05 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.399495+0000", 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.422e-05 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:91199 10.c 10:345f2913:::benchmark_data_smithi019_617837_object91198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:18.634965+0000", 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 2.1453022019999999, 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.041203337, 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 91199 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.920 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.634965+0000", 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.634965+0000", 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.634964+0000", 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.635030+0000", 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5203999999999994e-05 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.635036+0000", 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3890000000000002e-06 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.635039+0000", 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2799999999999999e-06 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.645348+0000", 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010308387 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.645472+0000", 2023-09-30T09:50:20.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000124244 2023-09-30T09:50:20.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.645611+0000", 2023-09-30T09:50:20.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000138792 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.659654+0000", 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01404306 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.660163+0000", 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00050958899999999996 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.676069+0000", 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015905826000000001 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.676125+0000", 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5457000000000001e-05 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:18.676168+0000", 2023-09-30T09:50:20.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3683000000000002e-05 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:92239 10.c 10:3eae1d30:::benchmark_data_smithi019_617837_object92238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:20.150331+0000", 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.62993657999999997, 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.040309855999999998, 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:20.927 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 92239 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150331+0000", 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150331+0000", 2023-09-30T09:50:20.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150329+0000", 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150356+0000", 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6520999999999999e-05 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150370+0000", 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4688000000000001e-05 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150384+0000", 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3458000000000001e-05 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150479+0000", 2023-09-30T09:50:20.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4971999999999998e-05 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150622+0000", 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000143416 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,4", 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.150772+0000", 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014957100000000001 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.162689+0000", 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011917452 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.168567+0000", 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0058776180000000003 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.190532+0000", 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.021965433999999999 2023-09-30T09:50:20.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.190594+0000", 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1639999999999999e-05 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:20.190640+0000", 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6650000000000002e-05 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.933 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:20.934 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:20.934 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:50:20.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:20.935 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:20.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:21.035 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:21.088 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:21.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:21 smithi019 ceph-mon[224043]: pgmap v1503: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 173 KiB/s rd, 43 MiB/s wr, 692 op/s; 22446/273405 objects degraded (8.210%); 1471/273405 objects misplaced (0.538%); 7.0 MiB/s, 117 objects/s recovering 2023-09-30T09:50:21.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:21 smithi019 ceph-mon[216815]: pgmap v1503: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 173 KiB/s rd, 43 MiB/s wr, 692 op/s; 22446/273405 objects degraded (8.210%); 1471/273405 objects misplaced (0.538%); 7.0 MiB/s, 117 objects/s recovering 2023-09-30T09:50:21.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:21 smithi167 ceph-mon[192880]: pgmap v1503: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 173 KiB/s rd, 43 MiB/s wr, 692 op/s; 22446/273405 objects degraded (8.210%); 1471/273405 objects misplaced (0.538%); 7.0 MiB/s, 117 objects/s recovering 2023-09-30T09:50:21.762 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total time run: 90.0113 2023-09-30T09:50:21.762 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Total writes made: 93397 2023-09-30T09:50:21.762 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Write size: 65536 2023-09-30T09:50:21.762 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Object size: 65536 2023-09-30T09:50:21.762 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Bandwidth (MB/sec): 64.8509 2023-09-30T09:50:21.762 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Bandwidth: 38.4614 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max bandwidth (MB/sec): 176.625 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min bandwidth (MB/sec): 0 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average IOPS: 1037 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev IOPS: 615.391 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max IOPS: 2826 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min IOPS: 0 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Average Latency(s): 0.0154156 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Stddev Latency(s): 0.0432629 2023-09-30T09:50:21.763 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Max latency(s): 2.39662 2023-09-30T09:50:21.764 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Min latency(s): 0.00112729 2023-09-30T09:50:21.768 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Cleaning up (deleting benchmark objects) 2023-09-30T09:50:21.860 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:21.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:21.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:21.951 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:50:21.966 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:21.978 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:22.026 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:22.135 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:22.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:22.248 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:22.342 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:22.691 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:22.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:22.796 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:22.844 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:23.278 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:23.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:23.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:23 smithi019 ceph-mon[216815]: pgmap v1504: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 133 KiB/s rd, 28 MiB/s wr, 445 op/s; 22446/273405 objects degraded (8.210%); 1471/273405 objects misplaced (0.538%); 2.7 MiB/s, 44 objects/s recovering 2023-09-30T09:50:23.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:23 smithi019 ceph-mon[224043]: pgmap v1504: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 133 KiB/s rd, 28 MiB/s wr, 445 op/s; 22446/273405 objects degraded (8.210%); 1471/273405 objects misplaced (0.538%); 2.7 MiB/s, 44 objects/s recovering 2023-09-30T09:50:23.391 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:23.431 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:23.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:23 smithi167 ceph-mon[192880]: pgmap v1504: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 133 KiB/s rd, 28 MiB/s wr, 445 op/s; 22446/273405 objects degraded (8.210%); 1471/273405 objects misplaced (0.538%); 2.7 MiB/s, 44 objects/s recovering 2023-09-30T09:50:23.952 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:23.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:23.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:24.060 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:24.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:50:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:50:23] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-09-30T09:50:24.089 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:24.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22446/273405 objects degraded (8.210%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:24.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22446/273405 objects degraded (8.210%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:24.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22446/273405 objects degraded (8.210%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:24.752 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:24.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:24.876 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:24.959 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:24.966 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:25.069 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:25.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:25.186 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:25.251 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:25.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:25 smithi019 conmon[216791]: 2023-09-30T09:50:25.082+0000 7faf52971700 -1 mon.a@0(leader).osd e562 definitely_dead 0 2023-09-30T09:50:25.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:25 smithi019 ceph-mon[216815]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 111 B/s rd, 31 MiB/s wr, 563 op/s; 22223/277395 objects degraded (8.011%); 1116/277395 objects misplaced (0.402%); 6.4 MiB/s, 101 objects/s recovering 2023-09-30T09:50:25.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:25 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3920775500' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:50:25.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:25 smithi019 ceph-mon[216815]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-30T09:50:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:25 smithi019 ceph-mon[224043]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 111 B/s rd, 31 MiB/s wr, 563 op/s; 22223/277395 objects degraded (8.011%); 1116/277395 objects misplaced (0.402%); 6.4 MiB/s, 101 objects/s recovering 2023-09-30T09:50:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:25 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3920775500' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:50:25.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:25 smithi019 ceph-mon[224043]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-30T09:50:25.423 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:25.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:25.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:25 smithi167 ceph-mon[192880]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 111 B/s rd, 31 MiB/s wr, 563 op/s; 22223/277395 objects degraded (8.011%); 1116/277395 objects misplaced (0.402%); 6.4 MiB/s, 101 objects/s recovering 2023-09-30T09:50:25.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:25 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3920775500' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-30T09:50:25.530 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:25 smithi167 ceph-mon[192880]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-30T09:50:25.531 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:25.582 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:25.878 INFO:teuthology.orchestra.run.smithi019.stderr:marked out osd.2. 2023-09-30T09:50:25.895 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:50:25.896 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:25.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:26.000 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:26.037 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:26.229 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:50:26.229 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:50:26.229 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:50:26.229 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:50:26.230 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:50:26.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:50:26.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:50:26.231 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:50:26.258 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:26.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:26.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:26.363 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:26.392 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1575012593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:26 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:26 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3920775500' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-30T09:50:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:26 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:26 smithi167 ceph-mon[192880]: osdmap e563: 8 total, 8 up, 7 in 2023-09-30T09:50:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:26 smithi167 ceph-mon[192880]: failed to encode map e563 with expected crc 2023-09-30T09:50:26.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/185323208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:26.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1575012593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3920775500' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[224043]: osdmap e563: 8 total, 8 up, 7 in 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[224043]: failed to encode map e563 with expected crc 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/185323208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1575012593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:26.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3920775500' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-30T09:50:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[216815]: osdmap e563: 8 total, 8 up, 7 in 2023-09-30T09:50:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[216815]: failed to encode map e563 with expected crc 2023-09-30T09:50:26.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/185323208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:27.290 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:27.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:27.395 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:27.422 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:27.469 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:27.475 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:27 smithi167 ceph-mon[192880]: failed to encode map e563 with expected crc 2023-09-30T09:50:27.475 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:27 smithi167 ceph-mon[192880]: failed to encode map e563 with expected crc 2023-09-30T09:50:27.475 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:27 smithi167 ceph-mon[192880]: pgmap v1507: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 590 op/s; 22121/277605 objects degraded (7.969%); 1116/277605 objects misplaced (0.402%); 6.4 MiB/s, 102 objects/s recovering 2023-09-30T09:50:27.476 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:27 smithi167 ceph-mon[192880]: osdmap e564: 8 total, 8 up, 7 in 2023-09-30T09:50:27.476 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:27 smithi167 ceph-mon[192880]: failed to encode map e564 with expected crc 2023-09-30T09:50:27.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:27.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:27.577 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:27.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[216815]: failed to encode map e563 with expected crc 2023-09-30T09:50:27.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[216815]: failed to encode map e563 with expected crc 2023-09-30T09:50:27.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[216815]: pgmap v1507: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 590 op/s; 22121/277605 objects degraded (7.969%); 1116/277605 objects misplaced (0.402%); 6.4 MiB/s, 102 objects/s recovering 2023-09-30T09:50:27.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[216815]: osdmap e564: 8 total, 8 up, 7 in 2023-09-30T09:50:27.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[216815]: failed to encode map e564 with expected crc 2023-09-30T09:50:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[224043]: failed to encode map e563 with expected crc 2023-09-30T09:50:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[224043]: failed to encode map e563 with expected crc 2023-09-30T09:50:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[224043]: pgmap v1507: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 112 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 590 op/s; 22121/277605 objects degraded (7.969%); 1116/277605 objects misplaced (0.402%); 6.4 MiB/s, 102 objects/s recovering 2023-09-30T09:50:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[224043]: osdmap e564: 8 total, 8 up, 7 in 2023-09-30T09:50:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:27 smithi019 ceph-mon[224043]: failed to encode map e564 with expected crc 2023-09-30T09:50:27.606 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:28.407 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:28.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:28.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:28.513 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:28 smithi167 ceph-mon[192880]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:28 smithi167 ceph-mon[192880]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:28 smithi167 ceph-mon[192880]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:28 smithi167 ceph-mon[192880]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:28 smithi167 ceph-mon[192880]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:28 smithi167 ceph-mon[192880]: failed to encode map e565 with expected crc 2023-09-30T09:50:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:28 smithi167 ceph-mon[192880]: osdmap e565: 8 total, 8 up, 7 in 2023-09-30T09:50:28.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:28 smithi167 ceph-mon[192880]: failed to encode map e565 with expected crc 2023-09-30T09:50:28.555 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:28.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[224043]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[224043]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[224043]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[224043]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[224043]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[224043]: failed to encode map e565 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[224043]: osdmap e565: 8 total, 8 up, 7 in 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[224043]: failed to encode map e565 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[216815]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[216815]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[216815]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[216815]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[216815]: failed to encode map e564 with expected crc 2023-09-30T09:50:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[216815]: failed to encode map e565 with expected crc 2023-09-30T09:50:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[216815]: osdmap e565: 8 total, 8 up, 7 in 2023-09-30T09:50:28.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:28 smithi019 ceph-mon[216815]: failed to encode map e565 with expected crc 2023-09-30T09:50:28.719 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:28.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:28.835 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:28.880 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:29.024 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:29.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:29.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:29.156 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:29.210 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.210 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.210 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: pgmap v1510: 119 pgs: 3 remapped+peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 24 MiB/s wr, 649 op/s; 21757/275457 objects degraded (7.899%); 1116/275457 objects misplaced (0.405%); 11 MiB/s, 170 objects/s recovering 2023-09-30T09:50:29.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 22121/277605 objects degraded (7.969%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:29.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: failed to encode map e566 with expected crc 2023-09-30T09:50:29.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: osdmap e566: 8 total, 8 up, 7 in 2023-09-30T09:50:29.211 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:29 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:50:29.257 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:29.375 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:29.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:29.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:29.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: pgmap v1510: 119 pgs: 3 remapped+peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 24 MiB/s wr, 649 op/s; 21757/275457 objects degraded (7.899%); 1116/275457 objects misplaced (0.405%); 11 MiB/s, 170 objects/s recovering 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 22121/277605 objects degraded (7.969%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: failed to encode map e566 with expected crc 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: osdmap e566: 8 total, 8 up, 7 in 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: failed to encode map e565 with expected crc 2023-09-30T09:50:29.469 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: pgmap v1510: 119 pgs: 3 remapped+peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 24 MiB/s wr, 649 op/s; 21757/275457 objects degraded (7.899%); 1116/275457 objects misplaced (0.405%); 11 MiB/s, 170 objects/s recovering 2023-09-30T09:50:29.469 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 22121/277605 objects degraded (7.969%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:29.469 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: failed to encode map e566 with expected crc 2023-09-30T09:50:29.469 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: osdmap e566: 8 total, 8 up, 7 in 2023-09-30T09:50:29.469 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:29 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:50:29.493 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:29.578 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:29.762 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:29.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:29.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:29.838 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:50:29.838 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:50:29.866 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:29.921 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:30.054 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:30.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:30.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:30.160 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:30.195 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:30.265 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:30.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[216815]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[216815]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[216815]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[216815]: osdmap e567: 8 total, 8 up, 7 in 2023-09-30T09:50:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[216815]: failed to encode map e567 with expected crc 2023-09-30T09:50:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[224043]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[224043]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[224043]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[224043]: osdmap e567: 8 total, 8 up, 7 in 2023-09-30T09:50:30.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:30 smithi019 ceph-mon[224043]: failed to encode map e567 with expected crc 2023-09-30T09:50:30.369 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:30.447 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:30.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:30 smithi167 ceph-mon[192880]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:30 smithi167 ceph-mon[192880]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:30 smithi167 ceph-mon[192880]: failed to encode map e566 with expected crc 2023-09-30T09:50:30.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:30 smithi167 ceph-mon[192880]: osdmap e567: 8 total, 8 up, 7 in 2023-09-30T09:50:30.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:30 smithi167 ceph-mon[192880]: failed to encode map e567 with expected crc 2023-09-30T09:50:30.593 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:30.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:30.719 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:30.764 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:30.789 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:50:30.834 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:30.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:30.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:30.941 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:30.977 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:31.056 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:31.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:31.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:31.165 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:31.259 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:31.277 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[224043]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[224043]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[224043]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[224043]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[224043]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[224043]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[224043]: pgmap v1513: 119 pgs: 7 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 active+recovery_wait+degraded, 97 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 397 op/s; 34932/272865 objects degraded (12.802%); 3383/272865 objects misplaced (1.240%); 16 MiB/s, 2 keys/s, 244 objects/s recovering 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[216815]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[216815]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[216815]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[216815]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[216815]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[216815]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:31 smithi019 ceph-mon[216815]: pgmap v1513: 119 pgs: 7 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 active+recovery_wait+degraded, 97 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 397 op/s; 34932/272865 objects degraded (12.802%); 3383/272865 objects misplaced (1.240%); 16 MiB/s, 2 keys/s, 244 objects/s recovering 2023-09-30T09:50:31.399 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:31.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:31.449 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:50:31.522 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:31.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:31 smithi167 ceph-mon[192880]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:31 smithi167 ceph-mon[192880]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:31 smithi167 ceph-mon[192880]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:31 smithi167 ceph-mon[192880]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:31 smithi167 ceph-mon[192880]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:31 smithi167 ceph-mon[192880]: failed to encode map e567 with expected crc 2023-09-30T09:50:31.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:31 smithi167 ceph-mon[192880]: pgmap v1513: 119 pgs: 7 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 active+recovery_wait+degraded, 97 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 397 op/s; 34932/272865 objects degraded (12.802%); 3383/272865 objects misplaced (1.240%); 16 MiB/s, 2 keys/s, 244 objects/s recovering 2023-09-30T09:50:31.578 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:31.653 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:50:31.773 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:31.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:31.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:31.882 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:31.938 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:32.020 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:32.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:32.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:32.128 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:32.184 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:32.230 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:32.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:32.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:32.335 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:32.377 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:32.779 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:32.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:32.814 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:32.876 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:32.892 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:32.945 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:33.117 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:33.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:33.224 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:33.290 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:33.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:33 smithi019 ceph-mon[224043]: pgmap v1514: 119 pgs: 7 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 active+recovery_wait+degraded, 97 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.7 KiB/s rd, 7.2 KiB/s wr, 310 op/s; 34932/272865 objects degraded (12.802%); 3383/272865 objects misplaced (1.240%); 12 MiB/s, 1 keys/s, 191 objects/s recovering 2023-09-30T09:50:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:50:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:33 smithi019 ceph-mon[216815]: pgmap v1514: 119 pgs: 7 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 active+recovery_wait+degraded, 97 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.7 KiB/s rd, 7.2 KiB/s wr, 310 op/s; 34932/272865 objects degraded (12.802%); 3383/272865 objects misplaced (1.240%); 12 MiB/s, 1 keys/s, 191 objects/s recovering 2023-09-30T09:50:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:50:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:33.350 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:50:33.351 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:50:33.449 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:33.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:33 smithi167 ceph-mon[192880]: pgmap v1514: 119 pgs: 7 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 active+recovery_wait+degraded, 97 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.7 KiB/s rd, 7.2 KiB/s wr, 310 op/s; 34932/272865 objects degraded (12.802%); 3383/272865 objects misplaced (1.240%); 12 MiB/s, 1 keys/s, 191 objects/s recovering 2023-09-30T09:50:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:50:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:33.568 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:33.624 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:33.807 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:50:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:50:33] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-30T09:50:34.495 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6] out_osds: [2] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:50:34.496 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:50:34.496 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-30T09:50:34.496 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-30T09:50:34.523 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:34 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 34932/272865 objects degraded (12.802%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:34.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:34 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 34932/272865 objects degraded (12.802%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:34.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:34 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 34932/272865 objects degraded (12.802%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:34.544 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:34.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:34.648 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:34.726 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:34.880 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:34.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:35.001 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:35.035 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:35.423 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:35.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:35.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:35.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:35 smithi019 ceph-mon[224043]: pgmap v1515: 119 pgs: 7 peering, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 97 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.0 KiB/s wr, 292 op/s; 46324/270126 objects degraded (17.149%); 3383/270126 objects misplaced (1.252%); 8.1 MiB/s, 1 keys/s, 130 objects/s recovering 2023-09-30T09:50:35.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:35 smithi019 ceph-mon[224043]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:50:35.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:35 smithi019 ceph-mon[216815]: pgmap v1515: 119 pgs: 7 peering, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 97 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.0 KiB/s wr, 292 op/s; 46324/270126 objects degraded (17.149%); 3383/270126 objects misplaced (1.252%); 8.1 MiB/s, 1 keys/s, 130 objects/s recovering 2023-09-30T09:50:35.512 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:35 smithi019 ceph-mon[216815]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:50:35.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:35 smithi167 ceph-mon[192880]: pgmap v1515: 119 pgs: 7 peering, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 97 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.0 KiB/s wr, 292 op/s; 46324/270126 objects degraded (17.149%); 3383/270126 objects misplaced (1.252%); 8.1 MiB/s, 1 keys/s, 130 objects/s recovering 2023-09-30T09:50:35.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:35 smithi167 ceph-mon[192880]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:50:35.528 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:35.566 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:35.869 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:50:36.134 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:36.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:36.241 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:36.243 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:36.299 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:36.383 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:36 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:50:36.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:36 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:50:36.388 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:36.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:36.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:36.504 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:36.569 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:36.726 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:36.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:36.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:36.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:36 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-30T09:50:36.831 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:36.882 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:37.098 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:50:37.125 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:37.126 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:50:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:37.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:37.167 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:50:37.198 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:50:37.229 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:50:37.233 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:37.269 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:50:37.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:50:37.346 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:50:37.346 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:50:37.351 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:50:37.351 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:50:37.351 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:50:37.351 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:50:37.351 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:50:37.351 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:50:37.351 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:50:37.351 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:50:37.366 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:37.520 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:37.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:37.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:37 smithi019 ceph-mon[216815]: pgmap v1516: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 399 op/s; 46018/265899 objects degraded (17.307%); 3213/265899 objects misplaced (1.208%); 9.9 MiB/s, 1 keys/s, 159 objects/s recovering 2023-09-30T09:50:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2054012187' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:50:37.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1960900501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:37 smithi019 ceph-mon[224043]: pgmap v1516: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 399 op/s; 46018/265899 objects degraded (17.307%); 3213/265899 objects misplaced (1.208%); 9.9 MiB/s, 1 keys/s, 159 objects/s recovering 2023-09-30T09:50:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2054012187' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:50:37.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1960900501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:37.639 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:37.670 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:37.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:37 smithi167 ceph-mon[192880]: pgmap v1516: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 399 op/s; 46018/265899 objects degraded (17.307%); 3213/265899 objects misplaced (1.208%); 9.9 MiB/s, 1 keys/s, 159 objects/s recovering 2023-09-30T09:50:37.779 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2054012187' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:50:37.779 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1960900501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:38.400 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:38.401 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2054012187' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:50:38.401 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[216815]: osdmap e568: 8 total, 8 up, 7 in 2023-09-30T09:50:38.401 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[216815]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.401 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[216815]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.401 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[216815]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.401 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[216815]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.401 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[216815]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:38.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2054012187' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:50:38.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[224043]: osdmap e568: 8 total, 8 up, 7 in 2023-09-30T09:50:38.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[224043]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[224043]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[224043]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.403 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[224043]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.403 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:38 smithi019 ceph-mon[224043]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:38 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:50:38.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:38 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2054012187' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:50:38.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:38 smithi167 ceph-mon[192880]: osdmap e568: 8 total, 8 up, 7 in 2023-09-30T09:50:38.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:38 smithi167 ceph-mon[192880]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:38 smithi167 ceph-mon[192880]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:38 smithi167 ceph-mon[192880]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:38 smithi167 ceph-mon[192880]: failed to encode map e568 with expected crc 2023-09-30T09:50:38.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:38 smithi167 ceph-mon[192880]: failed to encode map e568 with expected crc 2023-09-30T09:50:39.083 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:39.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:39.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:39.187 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:39.233 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:39.251 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:39.379 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:39.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:39.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:39.483 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:39.535 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:39.754 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:39.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:39.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:39.760 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:39 smithi019 ceph-mon[216815]: pgmap v1518: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 45717/263181 objects degraded (17.371%); 3213/263181 objects misplaced (1.221%); 7.3 MiB/s, 116 objects/s recovering 2023-09-30T09:50:39.760 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:39 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 45717/263181 objects degraded (17.371%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:39.760 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:39 smithi019 ceph-mon[224043]: pgmap v1518: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 45717/263181 objects degraded (17.371%); 3213/263181 objects misplaced (1.221%); 7.3 MiB/s, 116 objects/s recovering 2023-09-30T09:50:39.760 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:39 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 45717/263181 objects degraded (17.371%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:39.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:39 smithi167 ceph-mon[192880]: pgmap v1518: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 45717/263181 objects degraded (17.371%); 3213/263181 objects misplaced (1.221%); 7.3 MiB/s, 116 objects/s recovering 2023-09-30T09:50:39.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:39 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 45717/263181 objects degraded (17.371%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:39.861 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:39.972 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:40.092 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:40.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:40.203 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:40.239 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:40.300 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:40.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:40 smithi019 conmon[216791]: 2023-09-30T09:50:40.041+0000 7faf52971700 -1 mon.a@0(leader).osd e568 definitely_dead 0 2023-09-30T09:50:40.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:40.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:40.373 INFO:teuthology.orchestra.run.smithi019.stderr:marked in osd.2. 2023-09-30T09:50:40.407 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:40.442 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:40.511 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:40.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:40.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:40.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2578874358' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:50:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2578874358' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:50:40.616 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:40.644 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:40.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2578874358' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-30T09:50:40.841 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:40.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:40.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:40.945 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:40.973 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:41.115 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:50:41.231 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:50:41.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[216815]: pgmap v1519: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 45397/258513 objects degraded (17.561%); 3027/258513 objects misplaced (1.171%); 9.1 MiB/s, 144 objects/s recovering 2023-09-30T09:50:41.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2578874358' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-30T09:50:41.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[216815]: failed to encode map e569 with expected crc 2023-09-30T09:50:41.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[216815]: failed to encode map e569 with expected crc 2023-09-30T09:50:41.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[216815]: osdmap e569: 8 total, 8 up, 8 in 2023-09-30T09:50:41.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[224043]: pgmap v1519: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 45397/258513 objects degraded (17.561%); 3027/258513 objects misplaced (1.171%); 9.1 MiB/s, 144 objects/s recovering 2023-09-30T09:50:41.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2578874358' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-30T09:50:41.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[224043]: failed to encode map e569 with expected crc 2023-09-30T09:50:41.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[224043]: failed to encode map e569 with expected crc 2023-09-30T09:50:41.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:41 smithi019 ceph-mon[224043]: osdmap e569: 8 total, 8 up, 8 in 2023-09-30T09:50:41.629 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-30T09:50:41.670 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:41.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:41.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:41 smithi167 ceph-mon[192880]: pgmap v1519: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 45397/258513 objects degraded (17.561%); 3027/258513 objects misplaced (1.171%); 9.1 MiB/s, 144 objects/s recovering 2023-09-30T09:50:41.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2578874358' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-30T09:50:41.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:41 smithi167 ceph-mon[192880]: failed to encode map e569 with expected crc 2023-09-30T09:50:41.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:41 smithi167 ceph-mon[192880]: failed to encode map e569 with expected crc 2023-09-30T09:50:41.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:41 smithi167 ceph-mon[192880]: osdmap e569: 8 total, 8 up, 8 in 2023-09-30T09:50:41.776 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:41.806 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:41.862 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:41.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:41.948 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:41.969 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:41.988 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:41.997 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153598 8.13 8:ca06820b:::benchmark_data_smithi019_478679_object61150:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.972895+0000", 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 459.056297734, 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1570760339999999, 2023-09-30T09:50:42.032 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153598 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972895+0000", 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.033 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972895+0000", 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972897+0000", 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.576e-06 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.034 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972901+0000", 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7929999999999999e-06 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972913+0000", 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2913e-05 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972924+0000", 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0859999999999999e-05 2023-09-30T09:50:42.035 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975474+0000", 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025492499999999999 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975483+0000", 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8919999999999995e-06 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.036 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.120688+0000", 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.145204678 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.120833+0000", 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000144955 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.121225+0000", 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00039230599999999998 2023-09-30T09:50:42.037 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.124776+0000", 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035502789999999999 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.126836+0000", 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002060615 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.038 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.129791+0000", 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029550119999999999 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.129855+0000", 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3948000000000005e-05 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.129971+0000", 2023-09-30T09:50:42.039 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115957 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153604 8.13 8:cb19032d:::benchmark_data_smithi019_478679_object61156:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.972944+0000", 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 459.056248815, 2023-09-30T09:50:42.040 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1685259029999999, 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153604 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972944+0000", 2023-09-30T09:50:42.041 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972944+0000", 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972945+0000", 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7899999999999999e-07 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.042 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972947+0000", 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.537e-06 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972956+0000", 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6179999999999995e-06 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.043 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972962+0000", 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8050000000000003e-06 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975510+0000", 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0025486850000000002 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975517+0000", 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2069999999999998e-06 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.044 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.123223+0000", 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1477051220000001 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.123354+0000", 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013182600000000001 2023-09-30T09:50:42.045 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.123631+0000", 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027705399999999999 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.126497+0000", 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028658239999999999 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.046 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.128367+0000", 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018693570000000001 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141250+0000", 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012883321999999999 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141335+0000", 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5402999999999995e-05 2023-09-30T09:50:42.047 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141470+0000", 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000134664 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153612 8.13 8:ce4cdea7:::benchmark_data_smithi019_478679_object61164:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:50:42.048 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.972969+0000", 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 459.05622409300003, 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.168764243, 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153612 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.049 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972969+0000", 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972969+0000", 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972969+0000", 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7e-07 2023-09-30T09:50:42.050 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972971+0000", 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.21e-06 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972978+0000", 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5980000000000002e-06 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.051 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.972983+0000", 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4360000000000001e-06 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975544+0000", 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002560756 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:50:42.052 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.975552+0000", 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7160000000000003e-06 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.124813+0000", 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.149260663 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.124944+0000", 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013185399999999999 2023-09-30T09:50:42.053 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.125194+0000", 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024942299999999997 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.126591+0000", 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0013973080000000001 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.054 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.129736+0000", 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003144441 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141566+0000", 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011830827 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141626+0000", 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.003e-05 2023-09-30T09:50:42.055 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.141733+0000", 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000106611 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153642 8.f 8:f10f80bc:::benchmark_data_smithi019_478679_object61194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:04.196235+0000", 2023-09-30T09:50:42.056 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 457.83295773100002, 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.064478372, 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153642 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196235+0000", 2023-09-30T09:50:42.057 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196235+0000", 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196237+0000", 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.48e-06 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.058 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196242+0000", 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4079999999999997e-06 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196259+0000", 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.736e-05 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196269+0000", 2023-09-30T09:50:42.059 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4630000000000007e-06 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196353+0000", 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4209000000000006e-05 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196537+0000", 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018348999999999999 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.060 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3,4", 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.196924+0000", 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00038737299999999999 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.198566+0000", 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0016421809999999999 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.061 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.198914+0000", 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00034747599999999999 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.200146+0000", 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012326959999999999 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260516+0000", 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0603693430000001 2023-09-30T09:50:42.062 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260584+0000", 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8002000000000004e-05 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260714+0000", 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012989100000000001 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.063 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153648 8.1f 8:fd9e162a:::benchmark_data_smithi019_478679_object61200:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:04.245434+0000", 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 457.78375906999997, 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0155153640000001, 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:42.064 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153648 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245434+0000", 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245434+0000", 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.065 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245435+0000", 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.587e-06 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245440+0000", 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6800000000000001e-06 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.066 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245450+0000", 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5419999999999994e-06 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245455+0000", 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2909999999999998e-06 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245502+0000", 2023-09-30T09:50:42.067 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6783999999999998e-05 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245625+0000", 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012290099999999999 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3,4", 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.245887+0000", 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000262174 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.068 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.246902+0000", 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001015143 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.247809+0000", 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00090754099999999999 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.248648+0000", 2023-09-30T09:50:42.069 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00083857300000000001 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260755+0000", 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0121074299999999 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260810+0000", 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4120999999999999e-05 2023-09-30T09:50:42.070 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.260949+0000", 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013959700000000001 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153659 8.1f 8:fcf7805c:::benchmark_data_smithi019_478679_object61211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:05.668494+0000", 2023-09-30T09:50:42.071 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 456.360699372, 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.490733206, 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153659 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.072 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668494+0000", 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668494+0000", 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668495+0000", 2023-09-30T09:50:42.073 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.827e-06 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668503+0000", 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8760000000000005e-06 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668525+0000", 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2130999999999999e-05 2023-09-30T09:50:42.074 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668537+0000", 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1717e-05 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668603+0000", 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5463000000000007e-05 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.075 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668720+0000", 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000117565 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3,4", 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.668956+0000", 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000236214 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.076 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.670527+0000", 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015708300000000001 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.670834+0000", 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00030689600000000002 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.671351+0000", 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00051731800000000001 2023-09-30T09:50:42.077 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.158999+0000", 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.487647202 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.159082+0000", 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3016999999999995e-05 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.078 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.159227+0000", 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014515 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2291 9.f 9:f6cf5fc3:::benchmark_data_smithi019_560486_object2290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.043373+0000", 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 375.98581943599999, 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.042291273, 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.079 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2291 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043373+0000", 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.080 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.081 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.081 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043373+0000", 2023-09-30T09:50:42.081 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.081 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.081 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.081 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.081 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043376+0000", 2023-09-30T09:50:42.081 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2680000000000001e-06 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043413+0000", 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7765000000000002e-05 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043430+0000", 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6107999999999999e-05 2023-09-30T09:50:42.082 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043443+0000", 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3502e-05 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045256+0000", 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001812857 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.083 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045267+0000", 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1302e-05 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085561+0000", 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0402934189999999 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.084 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085665+0000", 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000104052 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2294 9.f 9:fece5195:::benchmark_data_smithi019_560486_object2293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.044894+0000", 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 375.98429910700003, 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.041032036, 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:50:42.085 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2294 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.044894+0000", 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.086 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.044894+0000", 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.044895+0000", 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0610000000000001e-06 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.044933+0000", 2023-09-30T09:50:42.087 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8374999999999998e-05 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045261+0000", 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000328241 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045274+0000", 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2401e-05 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.088 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045324+0000", 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0433000000000003e-05 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045333+0000", 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0960000000000007e-06 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085856+0000", 2023-09-30T09:50:42.089 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0405229039999999 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085926+0000", 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9524999999999996e-05 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2296 9.f 9:f85e51b4:::benchmark_data_smithi019_560486_object2295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:50:42.090 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.045296+0000", 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 375.98389695899999, 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0445070430000001, 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2296 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.091 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045296+0000", 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045296+0000", 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.092 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045297+0000", 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8000000000000004e-07 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045332+0000", 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5549000000000001e-05 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045341+0000", 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9090000000000006e-06 2023-09-30T09:50:42.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045348+0000", 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8619999999999997e-06 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045476+0000", 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000127904 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:50:42.094 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045487+0000", 2023-09-30T09:50:42.095 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0564000000000001e-05 2023-09-30T09:50:42.095 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.095 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.095 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.097 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085950+0000", 2023-09-30T09:50:42.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0404637800000001 2023-09-30T09:50:42.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.097 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.097 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.097 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.086071+0000", 2023-09-30T09:50:42.097 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012108 2023-09-30T09:50:42.097 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.086237+0000", 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016526400000000001 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087626+0000", 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001388937 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.098 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087772+0000", 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014617800000000001 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.089501+0000", 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00172934 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.099 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.089571+0000", 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9715999999999995e-05 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.089803+0000", 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00023207999999999999 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.100 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2303 9.f 9:f68e18ee:::benchmark_data_smithi019_560486_object2302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.045368+0000", 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 375.983825217, 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0475383970000001, 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2303 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.101 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045368+0000", 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045368+0000", 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.102 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045368+0000", 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9800000000000003e-07 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045403+0000", 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4819e-05 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045415+0000", 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1474999999999999e-05 2023-09-30T09:50:42.103 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045423+0000", 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0399999999999993e-06 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045530+0000", 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000107414 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:50:42.104 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045541+0000", 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0934999999999999e-05 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.086823+0000", 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0412817050000001 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.086978+0000", 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015566199999999999 2023-09-30T09:50:42.105 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087167+0000", 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018856400000000001 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087731+0000", 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00056380500000000001 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.106 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.088006+0000", 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027526999999999999 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.092619+0000", 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0046127850000000003 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.092680+0000", 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0841000000000002e-05 2023-09-30T09:50:42.107 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.092906+0000", 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022638399999999999 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:39903 9.2 e499/483 9:495196ac:::benchmark_data_smithi019_560486_object39902:head v 499'2452, mlcod=499'2450)", 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:52.939778+0000", 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 349.08941443700002, 2023-09-30T09:50:42.108 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27977527899999999, 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939778+0000", 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939778+0000", 2023-09-30T09:50:42.109 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939779+0000", 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8999999999999995e-07 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939800+0000", 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0455e-05 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.110 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939810+0000", 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0236e-05 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939817+0000", 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5590000000000002e-06 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939893+0000", 2023-09-30T09:50:42.111 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6860000000000006e-05 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.939912+0000", 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8742000000000001e-05 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.219523+0000", 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27961119400000001 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.112 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.219554+0000", 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0443e-05 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:39905 9.3 e499/483 9:c278c8fc:::benchmark_data_smithi019_560486_object39904:head v 499'2617, mlcod=499'2614)", 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:52.940206+0000", 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 349.08898660699998, 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27906742200000001, 2023-09-30T09:50:42.113 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.940206+0000", 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.940206+0000", 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.940207+0000", 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3600000000000003e-07 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:52.940306+0000", 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9411000000000001e-05 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.115 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.216799+0000", 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.276492403 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.216814+0000", 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4929999999999999e-05 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.216879+0000", 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5248999999999998e-05 2023-09-30T09:50:42.116 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.216906+0000", 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7217e-05 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.219237+0000", 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023303759999999999 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.117 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:53.219274+0000", 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7200000000000003e-05 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184146 9.a e525/492 9:549708fe:::benchmark_data_smithi019_560486_object87401:head v 525'11572, mlcod=525'11568)", 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.145546+0000", 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.883646497, 2023-09-30T09:50:42.118 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21242733599999999, 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145546+0000", 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145546+0000", 2023-09-30T09:50:42.119 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145547+0000", 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4e-07 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145553+0000", 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5319999999999997e-06 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.120 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145561+0000", 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2900000000000002e-06 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.145569+0000", 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3110000000000006e-06 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.148760+0000", 2023-09-30T09:50:42.121 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031910350000000001 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.148782+0000", 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1480000000000001e-05 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.154088+0000", 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0053063429999999998 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.122 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357974+0000", 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20388550499999999 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184156 9.0 e525/483 9:0180b523:::benchmark_data_smithi019_560486_object87411:head v 525'11358, mlcod=525'11356)", 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.149851+0000", 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.87934155400001, 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20812422999999999, 2023-09-30T09:50:42.123 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149851+0000", 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149851+0000", 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.124 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149852+0000", 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9000000000000005e-07 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149861+0000", 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2600000000000005e-06 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.125 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149873+0000", 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2109e-05 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149881+0000", 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2230000000000007e-06 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149954+0000", 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2893999999999997e-05 2023-09-30T09:50:42.126 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.149973+0000", 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9381999999999998e-05 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.154102+0000", 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0041292680000000002 2023-09-30T09:50:42.127 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357976+0000", 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.203873104 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184153 9.2 e525/483 9:46c9a4a1:::benchmark_data_smithi019_560486_object87408:head v 525'11373, mlcod=525'11371)", 2023-09-30T09:50:42.128 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.153325+0000", 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.87586827999999, 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.220288503, 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153325+0000", 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.129 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153325+0000", 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153326+0000", 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.116e-06 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153333+0000", 2023-09-30T09:50:42.130 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0319999999999998e-06 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153344+0000", 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1558e-05 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153352+0000", 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6899999999999992e-06 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.131 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153409+0000", 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6789999999999997e-05 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153428+0000", 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9531e-05 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.132 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.373581+0000", 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22015299399999999 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.373613+0000", 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1792000000000003e-05 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.133 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184159 9.d e525/495 9:b38dcb7a:::benchmark_data_smithi019_560486_object87414:head v 525'11418, mlcod=525'11416)", 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.153388+0000", 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.87580485500001, 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21351836799999999, 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153388+0000", 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.134 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153388+0000", 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153389+0000", 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3399999999999999e-07 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153392+0000", 2023-09-30T09:50:42.135 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7069999999999998e-06 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153402+0000", 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4210000000000001e-06 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153407+0000", 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.95e-06 2023-09-30T09:50:42.136 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357959+0000", 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20455210500000001 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357986+0000", 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7469000000000001e-05 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.137 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.366863+0000", 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088767259999999997 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.366906+0000", 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3356000000000003e-05 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.138 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46214.0:184160 9.a e525/492 9:532ad83b:::benchmark_data_smithi019_560486_object87415:head v 525'11573, mlcod=525'11569)", 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:48:00.153436+0000", 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 161.87575650100001, 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22014215200000001, 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153436+0000", 2023-09-30T09:50:42.139 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153436+0000", 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153437+0000", 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4700000000000002e-07 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.140 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.153440+0000", 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4309999999999998e-06 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357934+0000", 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20449425500000001 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.357950+0000", 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.579e-05 2023-09-30T09:50:42.141 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.358005+0000", 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4580000000000003e-05 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.358025+0000", 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.037e-05 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.142 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.373535+0000", 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.015509441000000001 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:48:00.373579+0000", 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3937999999999998e-05 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.143 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:84149 10.0 10:03a2d2cb:::benchmark_data_smithi019_617837_object84148:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:05.882442+0000", 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.146750597, 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.275497981, 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 84149 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.144 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.882442+0000", 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.882442+0000", 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.145 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.882444+0000", 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.711e-06 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.882497+0000", 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2701000000000002e-05 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.882510+0000", 2023-09-30T09:50:42.146 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2988e-05 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.882518+0000", 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2260000000000005e-06 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.883830+0000", 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0013124499999999999 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.147 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.883984+0000", 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000153358 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6", 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.884144+0000", 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016073500000000001 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887122+0000", 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002977974 2023-09-30T09:50:42.148 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.901342+0000", 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014219488000000001 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.155408+0000", 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25406590000000001 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.149 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.155488+0000", 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0140000000000002e-05 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.157940+0000", 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024523100000000001 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.150 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:84152 10.0 10:0d01d84d:::benchmark_data_smithi019_617837_object84151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:05.887416+0000", 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.141776985, 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.273359878, 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 84152 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.151 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887416+0000", 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887416+0000", 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887417+0000", 2023-09-30T09:50:42.152 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5820000000000001e-06 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887444+0000", 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6129e-05 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887457+0000", 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3077e-05 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.153 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887465+0000", 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5699999999999993e-06 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887532+0000", 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6252000000000002e-05 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.154 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887694+0000", 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016245100000000001 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6", 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.887819+0000", 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012499399999999999 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.155 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.894567+0000", 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0067484529999999997 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.901383+0000", 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0068158649999999999 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.158021+0000", 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25663778199999998 2023-09-30T09:50:42.156 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.158091+0000", 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0346999999999996e-05 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.160776+0000", 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002684376 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.157 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:84154 10.0 10:0be5089c:::benchmark_data_smithi019_617837_object84153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:05.888236+0000", 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.140956434000003, 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.27558076500000001, 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:42.158 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 84154 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.888236+0000", 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.888236+0000", 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:50:42.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.888237+0000", 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1699999999999997e-07 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.888255+0000", 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7156e-05 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.160 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.888264+0000", 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2469999999999997e-06 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.888272+0000", 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5429999999999994e-06 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.891399+0000", 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031262249999999998 2023-09-30T09:50:42.161 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.891570+0000", 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000171733 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6", 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.891738+0000", 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000167317 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.162 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.894844+0000", 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031064259999999998 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:05.901413+0000", 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.006568473 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.160844+0000", 2023-09-30T09:50:42.163 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.25943114 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.160906+0000", 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1880999999999999e-05 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:06.163817+0000", 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029117069999999999 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.164 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.165 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:50:42.165 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:50:42.165 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:50:42.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:42.166 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:42.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:42.267 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:42.297 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:42.346 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:42.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:42.451 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:42.483 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[224043]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[224043]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[224043]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[224043]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[224043]: osdmap e570: 8 total, 8 up, 8 in 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[224043]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[224043]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[224043]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[216815]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[216815]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[216815]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[216815]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[216815]: osdmap e570: 8 total, 8 up, 8 in 2023-09-30T09:50:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[216815]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[216815]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:42 smithi019 ceph-mon[216815]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:42 smithi167 ceph-mon[192880]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:42 smithi167 ceph-mon[192880]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:42 smithi167 ceph-mon[192880]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:42 smithi167 ceph-mon[192880]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:42 smithi167 ceph-mon[192880]: osdmap e570: 8 total, 8 up, 8 in 2023-09-30T09:50:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:42 smithi167 ceph-mon[192880]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:42 smithi167 ceph-mon[192880]: failed to encode map e570 with expected crc 2023-09-30T09:50:42.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:42 smithi167 ceph-mon[192880]: failed to encode map e570 with expected crc 2023-09-30T09:50:43.359 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:43.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:43.464 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:43.520 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:43.631 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:43.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:43.742 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:43.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:43 smithi167 ceph-mon[192880]: pgmap v1522: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 45397/258513 objects degraded (17.561%); 3027/258513 objects misplaced (1.171%); 8.4 MiB/s, 134 objects/s recovering 2023-09-30T09:50:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:43 smithi167 ceph-mon[192880]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:43 smithi167 ceph-mon[192880]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:43 smithi167 ceph-mon[192880]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:43 smithi167 ceph-mon[192880]: osdmap e571: 8 total, 8 up, 8 in 2023-09-30T09:50:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:43 smithi167 ceph-mon[192880]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:43 smithi167 ceph-mon[192880]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:43 smithi167 ceph-mon[192880]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.807 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:50:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:50:43] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-30T09:50:43.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[224043]: pgmap v1522: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 45397/258513 objects degraded (17.561%); 3027/258513 objects misplaced (1.171%); 8.4 MiB/s, 134 objects/s recovering 2023-09-30T09:50:43.807 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[224043]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[224043]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[224043]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[224043]: osdmap e571: 8 total, 8 up, 8 in 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[224043]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[224043]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[224043]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[216815]: pgmap v1522: 119 pgs: 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 104 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 45397/258513 objects degraded (17.561%); 3027/258513 objects misplaced (1.171%); 8.4 MiB/s, 134 objects/s recovering 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[216815]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[216815]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[216815]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[216815]: osdmap e571: 8 total, 8 up, 8 in 2023-09-30T09:50:43.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[216815]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[216815]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:43 smithi019 ceph-mon[216815]: failed to encode map e571 with expected crc 2023-09-30T09:50:43.816 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:44.205 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:50:44.508 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: osdmap e572: 8 total, 8 up, 8 in 2023-09-30T09:50:44.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 45397/258513 objects degraded (17.561%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:44.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[224043]: pgmap v1525: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 8 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 104 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 28563/255810 objects degraded (11.166%); 6179/255810 objects misplaced (2.415%); 4.3 MiB/s, 2 keys/s, 69 objects/s recovering 2023-09-30T09:50:44.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: osdmap e572: 8 total, 8 up, 8 in 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 45397/258513 objects degraded (17.561%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:44.511 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:44 smithi019 ceph-mon[216815]: pgmap v1525: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 8 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 104 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 28563/255810 objects degraded (11.166%); 6179/255810 objects misplaced (2.415%); 4.3 MiB/s, 2 keys/s, 69 objects/s recovering 2023-09-30T09:50:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: osdmap e572: 8 total, 8 up, 8 in 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: failed to encode map e572 with expected crc 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 45397/258513 objects degraded (17.561%), 14 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:44.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:44 smithi167 ceph-mon[192880]: pgmap v1525: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 8 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 104 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 28563/255810 objects degraded (11.166%); 6179/255810 objects misplaced (2.415%); 4.3 MiB/s, 2 keys/s, 69 objects/s recovering 2023-09-30T09:50:44.556 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:44.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:44.664 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:44.768 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:44.816 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:50:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:45 smithi167 ceph-mon[192880]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:45 smithi167 ceph-mon[192880]: osdmap e573: 8 total, 8 up, 8 in 2023-09-30T09:50:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:45 smithi167 ceph-mon[192880]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:45 smithi167 ceph-mon[192880]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:45 smithi167 ceph-mon[192880]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:45 smithi167 ceph-mon[192880]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:45 smithi167 ceph-mon[192880]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[224043]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[224043]: osdmap e573: 8 total, 8 up, 8 in 2023-09-30T09:50:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[224043]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[224043]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[224043]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[224043]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[224043]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[216815]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[216815]: osdmap e573: 8 total, 8 up, 8 in 2023-09-30T09:50:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[216815]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.836 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[216815]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[216815]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[216815]: failed to encode map e573 with expected crc 2023-09-30T09:50:45.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:45 smithi019 ceph-mon[216815]: failed to encode map e573 with expected crc 2023-09-30T09:50:46.150 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:46.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:46.255 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:46.345 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:46.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:46 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:50:46.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:46 smithi019 ceph-mon[216815]: pgmap v1527: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 102 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 18334/253116 objects degraded (7.243%); 3152/253116 objects misplaced (1.245%); 10 MiB/s, 2 keys/s, 166 objects/s recovering 2023-09-30T09:50:46.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:46 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:50:46.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:46 smithi019 ceph-mon[224043]: pgmap v1527: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 102 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 18334/253116 objects degraded (7.243%); 3152/253116 objects misplaced (1.245%); 10 MiB/s, 2 keys/s, 166 objects/s recovering 2023-09-30T09:50:46.486 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:46.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:46.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:46.544 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:50:46.576 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:50:46.599 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:46.615 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:50:46.629 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:50:46.630 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:50:46.630 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-30T09:50:46.630 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:50:46.650 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:50:46.673 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:46 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:50:46.673 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:46 smithi167 ceph-mon[192880]: pgmap v1527: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 102 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 18334/253116 objects degraded (7.243%); 3152/253116 objects misplaced (1.245%); 10 MiB/s, 2 keys/s, 166 objects/s recovering 2023-09-30T09:50:46.681 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:46.687 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:50:46.737 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:50:46.771 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:50:46.811 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:50:46.814 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:46.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:46.842 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:50:46.843 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:50:46.843 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:50:46.843 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:50:46.843 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:50:46.843 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:50:46.843 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:50:46.843 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:50:46.921 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:47.018 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:47.132 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:47.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:47.241 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:47.318 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:47.392 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:47.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:47.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:47.502 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:47.549 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:47 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:50:47.549 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:47 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1721012382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:47.552 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:47.666 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:47.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:47.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:47.771 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:47.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:47 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:50:47.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:47 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1721012382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:47.809 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:47 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:50:47.809 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:47 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1721012382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:47.812 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:48.393 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:48.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:48.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:48.493 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:50:48.501 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:48.520 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:48.553 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:48.648 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:48.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:48.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1815873799' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:48.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:48.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[224043]: pgmap v1528: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 102 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 17971/249918 objects degraded (7.191%); 3152/249918 objects misplaced (1.261%); 12 MiB/s, 1 keys/s, 188 objects/s recovering 2023-09-30T09:50:48.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:50:48.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:48.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1815873799' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:48.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:48.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[216815]: pgmap v1528: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 102 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 17971/249918 objects degraded (7.191%); 3152/249918 objects misplaced (1.261%); 12 MiB/s, 1 keys/s, 188 objects/s recovering 2023-09-30T09:50:48.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:50:48.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:48.764 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:48.781 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1815873799' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:48.781 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:50:48.781 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:48 smithi167 ceph-mon[192880]: pgmap v1528: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 102 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 17971/249918 objects degraded (7.191%); 3152/249918 objects misplaced (1.261%); 12 MiB/s, 1 keys/s, 188 objects/s recovering 2023-09-30T09:50:48.781 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:50:48.781 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:50:48.833 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:48.938 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:48.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:49.050 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:49.129 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:49.234 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:49.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:49.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:49.343 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:49.404 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:49.513 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:49.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:49.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:49.618 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:49 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:50:49.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:49.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:49 smithi167 ceph-mon[192880]: osdmap e574: 8 total, 8 up, 8 in 2023-09-30T09:50:49.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:49 smithi167 ceph-mon[192880]: failed to encode map e574 with expected crc 2023-09-30T09:50:49.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:49 smithi167 ceph-mon[192880]: failed to encode map e574 with expected crc 2023-09-30T09:50:49.665 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:49 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 17971/249918 objects degraded (7.191%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:49.667 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[224043]: osdmap e574: 8 total, 8 up, 8 in 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[224043]: failed to encode map e574 with expected crc 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[224043]: failed to encode map e574 with expected crc 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 17971/249918 objects degraded (7.191%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:50:49.752 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[216815]: osdmap e574: 8 total, 8 up, 8 in 2023-09-30T09:50:49.753 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[216815]: failed to encode map e574 with expected crc 2023-09-30T09:50:49.753 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[216815]: failed to encode map e574 with expected crc 2023-09-30T09:50:49.753 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:49 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 17971/249918 objects degraded (7.191%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:49.761 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:49.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:49.865 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:49.925 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:50.330 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:50.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:50.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:50.363 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:50:50.444 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:50.475 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:50 smithi019 ceph-mon[216815]: pgmap v1530: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 17584/244143 objects degraded (7.202%); 2944/244143 objects misplaced (1.206%); 14 MiB/s, 1 keys/s, 228 objects/s recovering 2023-09-30T09:50:50.497 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:50.580 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:50.582 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:50 smithi167 ceph-mon[192880]: pgmap v1530: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 17584/244143 objects degraded (7.202%); 2944/244143 objects misplaced (1.206%); 14 MiB/s, 1 keys/s, 228 objects/s recovering 2023-09-30T09:50:50.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:50.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:50.684 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:50.725 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:50 smithi019 ceph-mon[224043]: pgmap v1530: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 17584/244143 objects degraded (7.202%); 2944/244143 objects misplaced (1.206%); 14 MiB/s, 1 keys/s, 228 objects/s recovering 2023-09-30T09:50:50.728 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:51.021 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:50:51.021 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:50:51.081 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:51.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:51.160 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:51.189 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:50:51.260 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:51.639 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:50:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:51.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:50:51.748 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:51.817 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:50:51.833 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:52.062 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:50:52.114 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:50:52.143 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:50:52.245 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:52.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:52.352 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:50:52.408 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:52.728 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:50:53.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:53 smithi167 ceph-mon[192880]: pgmap v1531: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 17584/244143 objects degraded (7.202%); 2944/244143 objects misplaced (1.206%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:50:53.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:53 smithi167 ceph-mon[192880]: from='client.36738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:50:53.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:53 smithi019 ceph-mon[216815]: pgmap v1531: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 17584/244143 objects degraded (7.202%); 2944/244143 objects misplaced (1.206%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:50:53.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:53 smithi019 ceph-mon[216815]: from='client.36738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:50:53.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:53 smithi019 ceph-mon[224043]: pgmap v1531: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 17584/244143 objects degraded (7.202%); 2944/244143 objects misplaced (1.206%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:50:53.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:53 smithi019 ceph-mon[224043]: from='client.36738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:50:53.452 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:50:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:53.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:50:53.558 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:53.615 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:53.806 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:50:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:50:53] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.43.0" 2023-09-30T09:50:54.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:54 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:50:54.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:54 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 17584/244143 objects degraded (7.202%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:54.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:54 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:50:54.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:54 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 17584/244143 objects degraded (7.202%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:54.207 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:54.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:54.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:54.237 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1531,"stamp":"2023-09-30T09:50:51.987394+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5336283824,"num_objects":81381,"num_object_clones":0,"num_object_copies":244143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17584,"num_objects_misplaced":2944,"num_objects_unfound":0,"num_objects_dirty":81381,"num_whiteouts":0,"num_read":5989,"num_read_kb":13222,"num_write":106354,"num_write_kb":5990171,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7546,"num_bytes_recovered":485380640,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97659,"ondisk_log_size":97659,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":382,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21764768,"kb_used_data":16422104,"kb_used_omap":30,"kb_used_meta":5342497,"kb_avail":728196448,"statfs":{"total":767960285184,"available":745673162752,"internally_reserved":0,"allocated":16816234496,"data_stored":16795537549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":5470717282},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-254869504,"num_objects":-3889,"num_object_clones":0,"num_object_copies":-11667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-10979,"num_objects_misplaced":-3235,"num_objects_unfound":0,"num_objects_dirty":-3889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3889,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1449,"num_bytes_recovered":94961664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003704"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":1019,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.438993+0000","last_change":"2023-09-30T09:50:42.391750+0000","last_active":"2023-09-30T09:50:44.438993+0000","last_peered":"2023-09-30T09:50:44.438993+0000","last_clean":"2023-09-30T09:50:44.438993+0000","last_became_active":"2023-09-30T09:50:42.391304+0000","last_became_peered":"2023-09-30T09:50:42.391304+0000","last_unstale":"2023-09-30T09:50:44.438993+0000","last_undegraded":"2023-09-30T09:50:44.438993+0000","last_fullsized":"2023-09-30T09:50:44.438993+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:01.703933+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":611,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.454556+0000","last_change":"2023-09-30T09:50:41.402505+0000","last_active":"2023-09-30T09:50:44.454556+0000","last_peered":"2023-09-30T09:50:44.454556+0000","last_clean":"2023-09-30T09:50:44.454556+0000","last_became_active":"2023-09-30T09:50:41.402099+0000","last_became_peered":"2023-09-30T09:50:41.402099+0000","last_unstale":"2023-09-30T09:50:44.454556+0000","last_undegraded":"2023-09-30T09:50:44.454556+0000","last_fullsized":"2023-09-30T09:50:44.454556+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:45.502519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":488,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.459073+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:50:44.459073+0000","last_peered":"2023-09-30T09:50:44.459073+0000","last_clean":"2023-09-30T09:50:44.459073+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:50:44.459073+0000","last_undegraded":"2023-09-30T09:50:44.459073+0000","last_fullsized":"2023-09-30T09:50:44.459073+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":999,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.441771+0000","last_change":"2023-09-30T09:44:26.066362+0000","last_active":"2023-09-30T09:50:44.441771+0000","last_peered":"2023-09-30T09:50:44.441771+0000","last_clean":"2023-09-30T09:50:44.441771+0000","last_became_active":"2023-09-30T09:44:26.064211+0000","last_became_peered":"2023-09-30T09:44:26.064211+0000","last_unstale":"2023-09-30T09:50:44.441771+0000","last_undegraded":"2023-09-30T09:50:44.441771+0000","last_fullsized":"2023-09-30T09:50:44.441771+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:41:14.203282+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":139,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.411271+0000","last_change":"2023-09-30T09:42:48.648221+0000","last_active":"2023-09-30T09:50:43.411271+0000","last_peered":"2023-09-30T09:50:43.411271+0000","last_clean":"2023-09-30T09:50:43.411271+0000","last_became_active":"2023-09-30T09:42:48.643987+0000","last_became_peered":"2023-09-30T09:42:48.643987+0000","last_unstale":"2023-09-30T09:50:43.411271+0000","last_undegraded":"2023-09-30T09:50:43.411271+0000","last_fullsized":"2023-09-30T09:50:43.411271+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:10:27.607192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":181,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.421717+0000","last_change":"2023-09-30T09:42:48.649381+0000","last_active":"2023-09-30T09:50:43.421717+0000","last_peered":"2023-09-30T09:50:43.421717+0000","last_clean":"2023-09-30T09:50:43.421717+0000","last_became_active":"2023-09-30T09:42:48.649037+0000","last_became_peered":"2023-09-30T09:42:48.649037+0000","last_unstale":"2023-09-30T09:50:43.421717+0000","last_undegraded":"2023-09-30T09:50:43.421717+0000","last_fullsized":"2023-09-30T09:50:43.421717+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:36.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":806,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.454649+0000","last_change":"2023-09-30T09:38:30.740442+0000","last_active":"2023-09-30T09:50:44.454649+0000","last_peered":"2023-09-30T09:50:44.454649+0000","last_clean":"2023-09-30T09:50:44.454649+0000","last_became_active":"2023-09-30T09:38:30.739955+0000","last_became_peered":"2023-09-30T09:38:30.739955+0000","last_unstale":"2023-09-30T09:50:44.454649+0000","last_undegraded":"2023-09-30T09:50:44.454649+0000","last_fullsized":"2023-09-30T09:50:44.454649+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:23.349503+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.420990+0000","last_change":"2023-09-30T09:45:26.076097+0000","last_active":"2023-09-30T09:50:43.420990+0000","last_peered":"2023-09-30T09:50:43.420990+0000","last_clean":"2023-09-30T09:50:43.420990+0000","last_became_active":"2023-09-30T09:45:26.075730+0000","last_became_peered":"2023-09-30T09:45:26.075730+0000","last_unstale":"2023-09-30T09:50:43.420990+0000","last_undegraded":"2023-09-30T09:50:43.420990+0000","last_fullsized":"2023-09-30T09:50:43.420990+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:08.101394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":34,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.453985+0000","last_change":"2023-09-30T09:49:31.310014+0000","last_active":"2023-09-30T09:50:44.453985+0000","last_peered":"2023-09-30T09:50:44.453985+0000","last_clean":"2023-09-30T09:50:44.453985+0000","last_became_active":"2023-09-30T09:49:31.309121+0000","last_became_peered":"2023-09-30T09:49:31.309121+0000","last_unstale":"2023-09-30T09:50:44.453985+0000","last_undegraded":"2023-09-30T09:50:44.453985+0000","last_fullsized":"2023-09-30T09:50:44.453985+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:46.598805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":975,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435548+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:50:44.435548+0000","last_peered":"2023-09-30T09:50:44.435548+0000","last_clean":"2023-09-30T09:50:44.435548+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:50:44.435548+0000","last_undegraded":"2023-09-30T09:50:44.435548+0000","last_fullsized":"2023-09-30T09:50:44.435548+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":195,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422087+0000","last_change":"2023-09-30T09:44:27.079141+0000","last_active":"2023-09-30T09:50:43.422087+0000","last_peered":"2023-09-30T09:50:43.422087+0000","last_clean":"2023-09-30T09:50:43.422087+0000","last_became_active":"2023-09-30T09:44:27.078305+0000","last_became_peered":"2023-09-30T09:44:27.078305+0000","last_unstale":"2023-09-30T09:50:43.422087+0000","last_undegraded":"2023-09-30T09:50:43.422087+0000","last_fullsized":"2023-09-30T09:50:43.422087+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:16:10.608740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":171,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.406509+0000","last_change":"2023-09-30T09:49:31.295632+0000","last_active":"2023-09-30T09:50:43.406509+0000","last_peered":"2023-09-30T09:50:43.406509+0000","last_clean":"2023-09-30T09:50:43.406509+0000","last_became_active":"2023-09-30T09:49:31.295269+0000","last_became_peered":"2023-09-30T09:49:31.295269+0000","last_unstale":"2023-09-30T09:50:43.406509+0000","last_undegraded":"2023-09-30T09:50:43.406509+0000","last_fullsized":"2023-09-30T09:50:43.406509+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:32.754585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":561,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.460474+0000","last_change":"2023-09-30T09:50:44.460152+0000","last_active":"2023-09-30T09:50:44.460474+0000","last_peered":"2023-09-30T09:50:44.460474+0000","last_clean":"2023-09-30T09:50:44.460474+0000","last_became_active":"2023-09-30T09:50:44.459818+0000","last_became_peered":"2023-09-30T09:50:44.459818+0000","last_unstale":"2023-09-30T09:50:44.460474+0000","last_undegraded":"2023-09-30T09:50:44.460474+0000","last_fullsized":"2023-09-30T09:50:44.460474+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:58:57.798682+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":394,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.449796+0000","last_change":"2023-09-30T09:44:26.063719+0000","last_active":"2023-09-30T09:50:44.449796+0000","last_peered":"2023-09-30T09:50:44.449796+0000","last_clean":"2023-09-30T09:50:44.449796+0000","last_became_active":"2023-09-30T09:44:26.063392+0000","last_became_peered":"2023-09-30T09:44:26.063392+0000","last_unstale":"2023-09-30T09:50:44.449796+0000","last_undegraded":"2023-09-30T09:50:44.449796+0000","last_fullsized":"2023-09-30T09:50:44.449796+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:39:38.125519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":550,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.436010+0000","last_change":"2023-09-30T09:49:31.309241+0000","last_active":"2023-09-30T09:50:44.436010+0000","last_peered":"2023-09-30T09:50:44.436010+0000","last_clean":"2023-09-30T09:50:44.436010+0000","last_became_active":"2023-09-30T09:49:31.308703+0000","last_became_peered":"2023-09-30T09:49:31.308703+0000","last_unstale":"2023-09-30T09:50:44.436010+0000","last_undegraded":"2023-09-30T09:50:44.436010+0000","last_fullsized":"2023-09-30T09:50:44.436010+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:06.877644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":743,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.441319+0000","last_change":"2023-09-30T09:49:34.336174+0000","last_active":"2023-09-30T09:50:44.441319+0000","last_peered":"2023-09-30T09:50:44.441319+0000","last_clean":"2023-09-30T09:50:44.441319+0000","last_became_active":"2023-09-30T09:49:34.335879+0000","last_became_peered":"2023-09-30T09:49:34.335879+0000","last_unstale":"2023-09-30T09:50:44.441319+0000","last_undegraded":"2023-09-30T09:50:44.441319+0000","last_fullsized":"2023-09-30T09:50:44.441319+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:10:42.613890+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":288,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435721+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:50:44.435721+0000","last_peered":"2023-09-30T09:50:44.435721+0000","last_clean":"2023-09-30T09:50:44.435721+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:50:44.435721+0000","last_undegraded":"2023-09-30T09:50:44.435721+0000","last_fullsized":"2023-09-30T09:50:44.435721+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.434996+0000","last_change":"2023-09-30T09:50:41.399925+0000","last_active":"2023-09-30T09:50:44.434996+0000","last_peered":"2023-09-30T09:50:44.434996+0000","last_clean":"2023-09-30T09:50:44.434996+0000","last_became_active":"2023-09-30T09:50:41.399642+0000","last_became_peered":"2023-09-30T09:50:41.399642+0000","last_unstale":"2023-09-30T09:50:44.434996+0000","last_undegraded":"2023-09-30T09:50:44.434996+0000","last_fullsized":"2023-09-30T09:50:44.434996+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:45:44.698050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":630,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.421793+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:50:43.421793+0000","last_peered":"2023-09-30T09:50:43.421793+0000","last_clean":"2023-09-30T09:50:43.421793+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:50:43.421793+0000","last_undegraded":"2023-09-30T09:50:43.421793+0000","last_fullsized":"2023-09-30T09:50:43.421793+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":657,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.454970+0000","last_change":"2023-09-30T09:49:31.318169+0000","last_active":"2023-09-30T09:50:44.454970+0000","last_peered":"2023-09-30T09:50:44.454970+0000","last_clean":"2023-09-30T09:50:44.454970+0000","last_became_active":"2023-09-30T09:49:31.304043+0000","last_became_peered":"2023-09-30T09:49:31.304043+0000","last_unstale":"2023-09-30T09:50:44.454970+0000","last_undegraded":"2023-09-30T09:50:44.454970+0000","last_fullsized":"2023-09-30T09:50:44.454970+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:37:39.078569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28332,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":615,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.447014+0000","last_change":"2023-09-30T09:49:31.304624+0000","last_active":"2023-09-30T09:50:44.447014+0000","last_peered":"2023-09-30T09:50:44.447014+0000","last_clean":"2023-09-30T09:50:44.447014+0000","last_became_active":"2023-09-30T09:49:31.304106+0000","last_became_peered":"2023-09-30T09:49:31.304106+0000","last_unstale":"2023-09-30T09:50:44.447014+0000","last_undegraded":"2023-09-30T09:50:44.447014+0000","last_fullsized":"2023-09-30T09:50:44.447014+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:42.343641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":828,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422609+0000","last_change":"2023-09-30T09:44:27.079685+0000","last_active":"2023-09-30T09:50:43.422609+0000","last_peered":"2023-09-30T09:50:43.422609+0000","last_clean":"2023-09-30T09:50:43.422609+0000","last_became_active":"2023-09-30T09:44:27.079330+0000","last_became_peered":"2023-09-30T09:44:27.079330+0000","last_unstale":"2023-09-30T09:50:43.422609+0000","last_undegraded":"2023-09-30T09:50:43.422609+0000","last_fullsized":"2023-09-30T09:50:43.422609+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:47:07.602352+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":256,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.437039+0000","last_change":"2023-09-30T09:50:41.420912+0000","last_active":"2023-09-30T09:50:44.437039+0000","last_peered":"2023-09-30T09:50:44.437039+0000","last_clean":"2023-09-30T09:50:44.437039+0000","last_became_active":"2023-09-30T09:50:41.401425+0000","last_became_peered":"2023-09-30T09:50:41.401425+0000","last_unstale":"2023-09-30T09:50:44.437039+0000","last_undegraded":"2023-09-30T09:50:44.437039+0000","last_fullsized":"2023-09-30T09:50:44.437039+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:42:25.561844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":551,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.436583+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:50:44.436583+0000","last_peered":"2023-09-30T09:50:44.436583+0000","last_clean":"2023-09-30T09:50:44.436583+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:50:44.436583+0000","last_undegraded":"2023-09-30T09:50:44.436583+0000","last_fullsized":"2023-09-30T09:50:44.436583+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":935,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.438792+0000","last_change":"2023-09-30T09:49:34.332543+0000","last_active":"2023-09-30T09:50:44.438792+0000","last_peered":"2023-09-30T09:50:44.438792+0000","last_clean":"2023-09-30T09:50:44.438792+0000","last_became_active":"2023-09-30T09:49:34.332215+0000","last_became_peered":"2023-09-30T09:49:34.332215+0000","last_unstale":"2023-09-30T09:50:44.438792+0000","last_undegraded":"2023-09-30T09:50:44.438792+0000","last_fullsized":"2023-09-30T09:50:44.438792+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.827671+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":154,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.423518+0000","last_change":"2023-09-30T09:42:48.649921+0000","last_active":"2023-09-30T09:50:43.423518+0000","last_peered":"2023-09-30T09:50:43.423518+0000","last_clean":"2023-09-30T09:50:43.423518+0000","last_became_active":"2023-09-30T09:42:48.649533+0000","last_became_peered":"2023-09-30T09:42:48.649533+0000","last_unstale":"2023-09-30T09:50:43.423518+0000","last_undegraded":"2023-09-30T09:50:43.423518+0000","last_fullsized":"2023-09-30T09:50:43.423518+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:18.615100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":249,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.439397+0000","last_change":"2023-09-30T09:44:27.078937+0000","last_active":"2023-09-30T09:50:44.439397+0000","last_peered":"2023-09-30T09:50:44.439397+0000","last_clean":"2023-09-30T09:50:44.439397+0000","last_became_active":"2023-09-30T09:44:27.078583+0000","last_became_peered":"2023-09-30T09:44:27.078583+0000","last_unstale":"2023-09-30T09:50:44.439397+0000","last_undegraded":"2023-09-30T09:50:44.439397+0000","last_fullsized":"2023-09-30T09:50:44.439397+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:06:41.406457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":976,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.405969+0000","last_change":"2023-09-30T09:44:14.154303+0000","last_active":"2023-09-30T09:50:43.405969+0000","last_peered":"2023-09-30T09:50:43.405969+0000","last_clean":"2023-09-30T09:50:43.405969+0000","last_became_active":"2023-09-30T09:42:51.649145+0000","last_became_peered":"2023-09-30T09:42:51.649145+0000","last_unstale":"2023-09-30T09:50:43.405969+0000","last_undegraded":"2023-09-30T09:50:43.405969+0000","last_fullsized":"2023-09-30T09:50:43.405969+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:36.711705+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":278,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.436410+0000","last_change":"2023-09-30T09:49:31.309022+0000","last_active":"2023-09-30T09:50:44.436410+0000","last_peered":"2023-09-30T09:50:44.436410+0000","last_clean":"2023-09-30T09:50:44.436410+0000","last_became_active":"2023-09-30T09:49:31.308236+0000","last_became_peered":"2023-09-30T09:49:31.308236+0000","last_unstale":"2023-09-30T09:50:44.436410+0000","last_undegraded":"2023-09-30T09:50:44.436410+0000","last_fullsized":"2023-09-30T09:50:44.436410+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:39.779954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422223+0000","last_change":"2023-09-30T09:50:26.904206+0000","last_active":"2023-09-30T09:50:43.422223+0000","last_peered":"2023-09-30T09:50:43.422223+0000","last_clean":"2023-09-30T09:50:43.422223+0000","last_became_active":"2023-09-30T09:50:26.903736+0000","last_became_peered":"2023-09-30T09:50:26.903736+0000","last_unstale":"2023-09-30T09:50:43.422223+0000","last_undegraded":"2023-09-30T09:50:43.422223+0000","last_fullsized":"2023-09-30T09:50:43.422223+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:28:55.508526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":1024,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.440553+0000","last_change":"2023-09-30T09:49:34.332561+0000","last_active":"2023-09-30T09:50:44.440553+0000","last_peered":"2023-09-30T09:50:44.440553+0000","last_clean":"2023-09-30T09:50:44.440553+0000","last_became_active":"2023-09-30T09:49:34.332170+0000","last_became_peered":"2023-09-30T09:49:34.332170+0000","last_unstale":"2023-09-30T09:50:44.440553+0000","last_undegraded":"2023-09-30T09:50:44.440553+0000","last_fullsized":"2023-09-30T09:50:44.440553+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:22.548068+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":144,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.421355+0000","last_change":"2023-09-30T09:50:13.844550+0000","last_active":"2023-09-30T09:50:43.421355+0000","last_peered":"2023-09-30T09:50:43.421355+0000","last_clean":"2023-09-30T09:50:43.421355+0000","last_became_active":"2023-09-30T09:50:13.844163+0000","last_became_peered":"2023-09-30T09:50:13.844163+0000","last_unstale":"2023-09-30T09:50:43.421355+0000","last_undegraded":"2023-09-30T09:50:43.421355+0000","last_fullsized":"2023-09-30T09:50:43.421355+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:19:30.326933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":139,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.411675+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:50:43.411675+0000","last_peered":"2023-09-30T09:50:43.411675+0000","last_clean":"2023-09-30T09:50:43.411675+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:50:43.411675+0000","last_undegraded":"2023-09-30T09:50:43.411675+0000","last_fullsized":"2023-09-30T09:50:43.411675+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"573'6383","reported_seq":6432,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:47.330101+0000","last_change":"2023-09-30T09:48:40.945814+0000","last_active":"2023-09-30T09:50:47.330101+0000","last_peered":"2023-09-30T09:50:47.330101+0000","last_clean":"2023-09-30T09:50:47.330101+0000","last_became_active":"2023-09-30T09:48:40.945431+0000","last_became_peered":"2023-09-30T09:48:40.945431+0000","last_unstale":"2023-09-30T09:50:47.330101+0000","last_undegraded":"2023-09-30T09:50:47.330101+0000","last_fullsized":"2023-09-30T09:50:47.330101+0000","mapping_epoch":536,"log_start":"540'600","ondisk_log_start":"540'600","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:32.281640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329973760,"num_objects":5035,"num_object_clones":0,"num_object_copies":15105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6383,"num_write_kb":365376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":950,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435633+0000","last_change":"2023-09-30T09:44:13.360396+0000","last_active":"2023-09-30T09:50:44.435633+0000","last_peered":"2023-09-30T09:50:44.435633+0000","last_clean":"2023-09-30T09:50:44.435633+0000","last_became_active":"2023-09-30T09:38:30.736653+0000","last_became_peered":"2023-09-30T09:38:30.736653+0000","last_unstale":"2023-09-30T09:50:44.435633+0000","last_undegraded":"2023-09-30T09:50:44.435633+0000","last_fullsized":"2023-09-30T09:50:44.435633+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:56:23.965635+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":250,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.436213+0000","last_change":"2023-09-30T09:42:51.314194+0000","last_active":"2023-09-30T09:50:44.436213+0000","last_peered":"2023-09-30T09:50:44.436213+0000","last_clean":"2023-09-30T09:50:44.436213+0000","last_became_active":"2023-09-30T09:42:21.444646+0000","last_became_peered":"2023-09-30T09:42:21.444646+0000","last_unstale":"2023-09-30T09:50:44.436213+0000","last_undegraded":"2023-09-30T09:50:44.436213+0000","last_fullsized":"2023-09-30T09:50:44.436213+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:48:51.926274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":215,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.438834+0000","last_change":"2023-09-30T09:49:31.304898+0000","last_active":"2023-09-30T09:50:44.438834+0000","last_peered":"2023-09-30T09:50:44.438834+0000","last_clean":"2023-09-30T09:50:44.438834+0000","last_became_active":"2023-09-30T09:49:31.304532+0000","last_became_peered":"2023-09-30T09:49:31.304532+0000","last_unstale":"2023-09-30T09:50:44.438834+0000","last_undegraded":"2023-09-30T09:50:44.438834+0000","last_fullsized":"2023-09-30T09:50:44.438834+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:03:34.425387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"574'6694","reported_seq":8329,"reported_epoch":574,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:50:49.406361+0000","last_change":"2023-09-30T09:50:26.885356+0000","last_active":"2023-09-30T09:50:49.406361+0000","last_peered":"2023-09-30T09:50:49.406361+0000","last_clean":"2023-09-30T09:49:30.238890+0000","last_became_active":"2023-09-30T09:50:14.936098+0000","last_became_peered":"2023-09-30T09:50:14.936098+0000","last_unstale":"2023-09-30T09:50:49.406361+0000","last_undegraded":"2023-09-30T09:50:26.883555+0000","last_fullsized":"2023-09-30T09:50:49.406361+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6694,"log_dups_size":0,"ondisk_log_size":6694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:56:47.734109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335806464,"num_objects":5124,"num_object_clones":0,"num_object_copies":15372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3009,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6694,"num_write_kb":378176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":787,"num_bytes_recovered":51576832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":3009},{"shards":"3,5,6","objects":2115}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":1004,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.407579+0000","last_change":"2023-09-30T09:49:34.340125+0000","last_active":"2023-09-30T09:50:43.407579+0000","last_peered":"2023-09-30T09:50:43.407579+0000","last_clean":"2023-09-30T09:50:43.407579+0000","last_became_active":"2023-09-30T09:49:34.339638+0000","last_became_peered":"2023-09-30T09:49:34.339638+0000","last_unstale":"2023-09-30T09:50:43.407579+0000","last_undegraded":"2023-09-30T09:50:43.407579+0000","last_fullsized":"2023-09-30T09:50:43.407579+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:43:43.258960+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":220,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.437886+0000","last_change":"2023-09-30T09:38:27.706065+0000","last_active":"2023-09-30T09:50:44.437886+0000","last_peered":"2023-09-30T09:50:44.437886+0000","last_clean":"2023-09-30T09:50:44.437886+0000","last_became_active":"2023-09-30T09:38:27.680003+0000","last_became_peered":"2023-09-30T09:38:27.680003+0000","last_unstale":"2023-09-30T09:50:44.437886+0000","last_undegraded":"2023-09-30T09:50:44.437886+0000","last_fullsized":"2023-09-30T09:50:44.437886+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:15:30.360245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":634,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.453957+0000","last_change":"2023-09-30T09:50:26.903683+0000","last_active":"2023-09-30T09:50:44.453957+0000","last_peered":"2023-09-30T09:50:44.453957+0000","last_clean":"2023-09-30T09:50:44.453957+0000","last_became_active":"2023-09-30T09:50:26.903262+0000","last_became_peered":"2023-09-30T09:50:26.903262+0000","last_unstale":"2023-09-30T09:50:44.453957+0000","last_undegraded":"2023-09-30T09:50:44.453957+0000","last_fullsized":"2023-09-30T09:50:44.453957+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:40.724403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"573'6711","reported_seq":6760,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:47.322886+0000","last_change":"2023-09-30T09:48:40.935965+0000","last_active":"2023-09-30T09:50:47.322886+0000","last_peered":"2023-09-30T09:50:47.322886+0000","last_clean":"2023-09-30T09:50:47.322886+0000","last_became_active":"2023-09-30T09:48:40.935590+0000","last_became_peered":"2023-09-30T09:48:40.935590+0000","last_unstale":"2023-09-30T09:50:47.322886+0000","last_undegraded":"2023-09-30T09:50:47.322886+0000","last_fullsized":"2023-09-30T09:50:47.322886+0000","mapping_epoch":536,"log_start":"540'844","ondisk_log_start":"540'844","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:27.241032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341508096,"num_objects":5211,"num_object_clones":0,"num_object_copies":15633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6711,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":636,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.439461+0000","last_change":"2023-09-30T09:50:29.921334+0000","last_active":"2023-09-30T09:50:44.439461+0000","last_peered":"2023-09-30T09:50:44.439461+0000","last_clean":"2023-09-30T09:50:44.439461+0000","last_became_active":"2023-09-30T09:50:29.920976+0000","last_became_peered":"2023-09-30T09:50:29.920976+0000","last_unstale":"2023-09-30T09:50:44.439461+0000","last_undegraded":"2023-09-30T09:50:44.439461+0000","last_fullsized":"2023-09-30T09:50:44.439461+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:16.116795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:16.116795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:23:14.144582+0000","scrub_duration":0.045979904000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":315,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435410+0000","last_change":"2023-09-30T09:49:31.319964+0000","last_active":"2023-09-30T09:50:44.435410+0000","last_peered":"2023-09-30T09:50:44.435410+0000","last_clean":"2023-09-30T09:50:44.435410+0000","last_became_active":"2023-09-30T09:49:31.302322+0000","last_became_peered":"2023-09-30T09:49:31.302322+0000","last_unstale":"2023-09-30T09:50:44.435410+0000","last_undegraded":"2023-09-30T09:50:44.435410+0000","last_fullsized":"2023-09-30T09:50:44.435410+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:41:54.782188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":459,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.440458+0000","last_change":"2023-09-30T09:50:13.889514+0000","last_active":"2023-09-30T09:50:44.440458+0000","last_peered":"2023-09-30T09:50:44.440458+0000","last_clean":"2023-09-30T09:50:44.440458+0000","last_became_active":"2023-09-30T09:50:13.889033+0000","last_became_peered":"2023-09-30T09:50:13.889033+0000","last_unstale":"2023-09-30T09:50:44.440458+0000","last_undegraded":"2023-09-30T09:50:44.440458+0000","last_fullsized":"2023-09-30T09:50:44.440458+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:19.671881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"573'6524","reported_seq":6587,"reported_epoch":573,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:50:47.335562+0000","last_change":"2023-09-30T09:50:41.438723+0000","last_active":"2023-09-30T09:50:47.335562+0000","last_peered":"2023-09-30T09:50:47.335562+0000","last_clean":"2023-09-30T09:50:25.781288+0000","last_became_active":"2023-09-30T09:50:41.427901+0000","last_became_peered":"2023-09-30T09:50:41.427901+0000","last_unstale":"2023-09-30T09:50:47.335562+0000","last_undegraded":"2023-09-30T09:50:41.402143+0000","last_fullsized":"2023-09-30T09:50:41.380412+0000","mapping_epoch":569,"log_start":"540'696","ondisk_log_start":"540'696","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5828,"log_dups_size":0,"ondisk_log_size":5828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:37:35.221896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335020032,"num_objects":5112,"num_object_clones":0,"num_object_copies":15336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6524,"num_write_kb":372352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":5112}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":808,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.459535+0000","last_change":"2023-09-30T09:42:51.651362+0000","last_active":"2023-09-30T09:50:44.459535+0000","last_peered":"2023-09-30T09:50:44.459535+0000","last_clean":"2023-09-30T09:50:44.459535+0000","last_became_active":"2023-09-30T09:42:51.651003+0000","last_became_peered":"2023-09-30T09:42:51.651003+0000","last_unstale":"2023-09-30T09:50:44.459535+0000","last_undegraded":"2023-09-30T09:50:44.459535+0000","last_fullsized":"2023-09-30T09:50:44.459535+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:03:48.730026+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":529,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.439043+0000","last_change":"2023-09-30T09:49:31.301460+0000","last_active":"2023-09-30T09:50:44.439043+0000","last_peered":"2023-09-30T09:50:44.439043+0000","last_clean":"2023-09-30T09:50:44.439043+0000","last_became_active":"2023-09-30T09:49:31.301114+0000","last_became_peered":"2023-09-30T09:49:31.301114+0000","last_unstale":"2023-09-30T09:50:44.439043+0000","last_undegraded":"2023-09-30T09:50:44.439043+0000","last_fullsized":"2023-09-30T09:50:44.439043+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:42:01.066974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":174,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422609+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:50:43.422609+0000","last_peered":"2023-09-30T09:50:43.422609+0000","last_clean":"2023-09-30T09:50:43.422609+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:50:43.422609+0000","last_undegraded":"2023-09-30T09:50:43.422609+0000","last_fullsized":"2023-09-30T09:50:43.422609+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"574'6708","reported_seq":6753,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-30T09:50:49.026917+0000","last_change":"2023-09-30T09:48:40.937693+0000","last_active":"2023-09-30T09:50:49.026917+0000","last_peered":"2023-09-30T09:50:49.026917+0000","last_clean":"2023-09-30T09:50:49.026917+0000","last_became_active":"2023-09-30T09:48:40.937364+0000","last_became_peered":"2023-09-30T09:48:40.937364+0000","last_unstale":"2023-09-30T09:50:49.026917+0000","last_undegraded":"2023-09-30T09:50:49.026917+0000","last_fullsized":"2023-09-30T09:50:49.026917+0000","mapping_epoch":536,"log_start":"541'1784","ondisk_log_start":"541'1784","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4924,"log_dups_size":0,"ondisk_log_size":4924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:58.223575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332005376,"num_objects":5066,"num_object_clones":0,"num_object_copies":15198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6708,"num_write_kb":376768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1096,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.454120+0000","last_change":"2023-09-30T09:50:44.449513+0000","last_active":"2023-09-30T09:50:44.454120+0000","last_peered":"2023-09-30T09:50:44.454120+0000","last_clean":"2023-09-30T09:50:44.454120+0000","last_became_active":"2023-09-30T09:50:44.449149+0000","last_became_peered":"2023-09-30T09:50:44.449149+0000","last_unstale":"2023-09-30T09:50:44.454120+0000","last_undegraded":"2023-09-30T09:50:44.454120+0000","last_fullsized":"2023-09-30T09:50:44.454120+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:06:49.308121+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":133,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422362+0000","last_change":"2023-09-30T09:43:45.123798+0000","last_active":"2023-09-30T09:50:43.422362+0000","last_peered":"2023-09-30T09:50:43.422362+0000","last_clean":"2023-09-30T09:50:43.422362+0000","last_became_active":"2023-09-30T09:43:45.123448+0000","last_became_peered":"2023-09-30T09:43:45.123448+0000","last_unstale":"2023-09-30T09:50:43.422362+0000","last_undegraded":"2023-09-30T09:50:43.422362+0000","last_fullsized":"2023-09-30T09:50:43.422362+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:34.745292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":591,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.455048+0000","last_change":"2023-09-30T09:49:31.300551+0000","last_active":"2023-09-30T09:50:44.455048+0000","last_peered":"2023-09-30T09:50:44.455048+0000","last_clean":"2023-09-30T09:50:44.455048+0000","last_became_active":"2023-09-30T09:49:31.300231+0000","last_became_peered":"2023-09-30T09:49:31.300231+0000","last_unstale":"2023-09-30T09:50:44.455048+0000","last_undegraded":"2023-09-30T09:50:44.455048+0000","last_fullsized":"2023-09-30T09:50:44.455048+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:45.832564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"574'6719","reported_seq":6787,"reported_epoch":574,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:50:49.317567+0000","last_change":"2023-09-30T09:50:41.502055+0000","last_active":"2023-09-30T09:50:49.317567+0000","last_peered":"2023-09-30T09:50:49.317567+0000","last_clean":"2023-09-30T09:50:25.778933+0000","last_became_active":"2023-09-30T09:50:41.440200+0000","last_became_peered":"2023-09-30T09:50:41.440200+0000","last_unstale":"2023-09-30T09:50:49.317567+0000","last_undegraded":"2023-09-30T09:50:41.397850+0000","last_fullsized":"2023-09-30T09:50:41.381025+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:44:03.989308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332595200,"num_objects":5075,"num_object_clones":0,"num_object_copies":15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":390,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6719,"num_write_kb":377408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":5075}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":921,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.474064+0000","last_change":"2023-09-30T09:50:44.461471+0000","last_active":"2023-09-30T09:50:44.474064+0000","last_peered":"2023-09-30T09:50:44.474064+0000","last_clean":"2023-09-30T09:50:44.474064+0000","last_became_active":"2023-09-30T09:50:44.461074+0000","last_became_peered":"2023-09-30T09:50:44.461074+0000","last_unstale":"2023-09-30T09:50:44.474064+0000","last_undegraded":"2023-09-30T09:50:44.474064+0000","last_fullsized":"2023-09-30T09:50:44.474064+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:40:12.442532+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":416,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.454283+0000","last_change":"2023-09-30T09:50:13.887314+0000","last_active":"2023-09-30T09:50:44.454283+0000","last_peered":"2023-09-30T09:50:44.454283+0000","last_clean":"2023-09-30T09:50:44.454283+0000","last_became_active":"2023-09-30T09:50:13.887032+0000","last_became_peered":"2023-09-30T09:50:13.887032+0000","last_unstale":"2023-09-30T09:50:44.454283+0000","last_undegraded":"2023-09-30T09:50:44.454283+0000","last_fullsized":"2023-09-30T09:50:44.454283+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:27:26.626380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":287,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.446742+0000","last_change":"2023-09-30T09:50:41.401904+0000","last_active":"2023-09-30T09:50:44.446742+0000","last_peered":"2023-09-30T09:50:44.446742+0000","last_clean":"2023-09-30T09:50:44.446742+0000","last_became_active":"2023-09-30T09:50:41.401604+0000","last_became_peered":"2023-09-30T09:50:41.401604+0000","last_unstale":"2023-09-30T09:50:44.446742+0000","last_undegraded":"2023-09-30T09:50:44.446742+0000","last_fullsized":"2023-09-30T09:50:44.446742+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:59.806231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"574'6664","reported_seq":8241,"reported_epoch":574,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:50:49.022084+0000","last_change":"2023-09-30T09:50:14.992476+0000","last_active":"2023-09-30T09:50:49.022084+0000","last_peered":"2023-09-30T09:50:49.022084+0000","last_clean":"2023-09-30T09:49:30.245902+0000","last_became_active":"2023-09-30T09:50:14.976385+0000","last_became_peered":"2023-09-30T09:50:14.976385+0000","last_unstale":"2023-09-30T09:50:49.022084+0000","last_undegraded":"2023-09-30T09:50:14.843531+0000","last_fullsized":"2023-09-30T09:50:49.022084+0000","mapping_epoch":560,"log_start":"541'1782","ondisk_log_start":"541'1782","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:01:24.371528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336986112,"num_objects":5142,"num_object_clones":0,"num_object_copies":15426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3012,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6664,"num_write_kb":377792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":761,"num_bytes_recovered":49872896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4,6","objects":2130},{"shards":"1,4,7","objects":3012}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":913,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422695+0000","last_change":"2023-09-30T09:44:26.052604+0000","last_active":"2023-09-30T09:50:43.422695+0000","last_peered":"2023-09-30T09:50:43.422695+0000","last_clean":"2023-09-30T09:50:43.422695+0000","last_became_active":"2023-09-30T09:44:26.052231+0000","last_became_peered":"2023-09-30T09:44:26.052231+0000","last_unstale":"2023-09-30T09:50:43.422695+0000","last_undegraded":"2023-09-30T09:50:43.422695+0000","last_fullsized":"2023-09-30T09:50:43.422695+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:02:09.129127+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":598,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.453796+0000","last_change":"2023-09-30T09:50:41.399955+0000","last_active":"2023-09-30T09:50:44.453796+0000","last_peered":"2023-09-30T09:50:44.453796+0000","last_clean":"2023-09-30T09:50:44.453796+0000","last_became_active":"2023-09-30T09:50:41.399615+0000","last_became_peered":"2023-09-30T09:50:41.399615+0000","last_unstale":"2023-09-30T09:50:44.453796+0000","last_undegraded":"2023-09-30T09:50:44.453796+0000","last_fullsized":"2023-09-30T09:50:44.453796+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:14:56.117693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":605,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.459938+0000","last_change":"2023-09-30T09:49:31.311818+0000","last_active":"2023-09-30T09:50:44.459938+0000","last_peered":"2023-09-30T09:50:44.459938+0000","last_clean":"2023-09-30T09:50:44.459938+0000","last_became_active":"2023-09-30T09:49:31.311481+0000","last_became_peered":"2023-09-30T09:49:31.311481+0000","last_unstale":"2023-09-30T09:50:44.459938+0000","last_undegraded":"2023-09-30T09:50:44.459938+0000","last_fullsized":"2023-09-30T09:50:44.459938+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:54.941072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"574'6632","reported_seq":6677,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-30T09:50:48.991298+0000","last_change":"2023-09-30T09:48:40.940915+0000","last_active":"2023-09-30T09:50:48.991298+0000","last_peered":"2023-09-30T09:50:48.991298+0000","last_clean":"2023-09-30T09:50:48.991298+0000","last_became_active":"2023-09-30T09:48:40.940606+0000","last_became_peered":"2023-09-30T09:48:40.940606+0000","last_unstale":"2023-09-30T09:50:48.991298+0000","last_undegraded":"2023-09-30T09:50:48.991298+0000","last_fullsized":"2023-09-30T09:50:48.991298+0000","mapping_epoch":536,"log_start":"541'1734","ondisk_log_start":"541'1734","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4898,"log_dups_size":0,"ondisk_log_size":4898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:11.151884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330170368,"num_objects":5038,"num_object_clones":0,"num_object_copies":15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6632,"num_write_kb":373440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":772,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.470801+0000","last_change":"2023-09-30T09:50:44.462926+0000","last_active":"2023-09-30T09:50:44.470801+0000","last_peered":"2023-09-30T09:50:44.470801+0000","last_clean":"2023-09-30T09:50:44.470801+0000","last_became_active":"2023-09-30T09:50:44.462573+0000","last_became_peered":"2023-09-30T09:50:44.462573+0000","last_unstale":"2023-09-30T09:50:44.470801+0000","last_undegraded":"2023-09-30T09:50:44.470801+0000","last_fullsized":"2023-09-30T09:50:44.470801+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:28.805205+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":139,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.406424+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:50:43.406424+0000","last_peered":"2023-09-30T09:50:43.406424+0000","last_clean":"2023-09-30T09:50:43.406424+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:50:43.406424+0000","last_undegraded":"2023-09-30T09:50:43.406424+0000","last_fullsized":"2023-09-30T09:50:43.406424+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":258,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.447314+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:50:44.447314+0000","last_peered":"2023-09-30T09:50:44.447314+0000","last_clean":"2023-09-30T09:50:44.447314+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:50:44.447314+0000","last_undegraded":"2023-09-30T09:50:44.447314+0000","last_fullsized":"2023-09-30T09:50:44.447314+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"574'6612","reported_seq":6658,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-30T09:50:48.958370+0000","last_change":"2023-09-30T09:48:40.942045+0000","last_active":"2023-09-30T09:50:48.958370+0000","last_peered":"2023-09-30T09:50:48.958370+0000","last_clean":"2023-09-30T09:50:48.958370+0000","last_became_active":"2023-09-30T09:48:40.941786+0000","last_became_peered":"2023-09-30T09:48:40.941786+0000","last_unstale":"2023-09-30T09:50:48.958370+0000","last_undegraded":"2023-09-30T09:50:48.958370+0000","last_fullsized":"2023-09-30T09:50:48.958370+0000","mapping_epoch":536,"log_start":"540'300","ondisk_log_start":"540'300","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6312,"log_dups_size":0,"ondisk_log_size":6312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:03:11.354244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339476480,"num_objects":5180,"num_object_clones":0,"num_object_copies":15540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6612,"num_write_kb":377344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":814,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435917+0000","last_change":"2023-09-30T09:50:13.903010+0000","last_active":"2023-09-30T09:50:44.435917+0000","last_peered":"2023-09-30T09:50:44.435917+0000","last_clean":"2023-09-30T09:50:44.435917+0000","last_became_active":"2023-09-30T09:50:13.902355+0000","last_became_peered":"2023-09-30T09:50:13.902355+0000","last_unstale":"2023-09-30T09:50:44.435917+0000","last_undegraded":"2023-09-30T09:50:44.435917+0000","last_fullsized":"2023-09-30T09:50:44.435917+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:35.240830+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":805,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.421598+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:50:43.421598+0000","last_peered":"2023-09-30T09:50:43.421598+0000","last_clean":"2023-09-30T09:50:43.421598+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:50:43.421598+0000","last_undegraded":"2023-09-30T09:50:43.421598+0000","last_fullsized":"2023-09-30T09:50:43.421598+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":522,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.439533+0000","last_change":"2023-09-30T09:50:41.396047+0000","last_active":"2023-09-30T09:50:44.439533+0000","last_peered":"2023-09-30T09:50:44.439533+0000","last_clean":"2023-09-30T09:50:44.439533+0000","last_became_active":"2023-09-30T09:50:41.395627+0000","last_became_peered":"2023-09-30T09:50:41.395627+0000","last_unstale":"2023-09-30T09:50:44.439533+0000","last_undegraded":"2023-09-30T09:50:44.439533+0000","last_fullsized":"2023-09-30T09:50:44.439533+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:16.050977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":191,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.440147+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:50:44.440147+0000","last_peered":"2023-09-30T09:50:44.440147+0000","last_clean":"2023-09-30T09:50:44.440147+0000","last_became_active":"2023-09-30T09:44:26.063515+0000","last_became_peered":"2023-09-30T09:44:26.063515+0000","last_unstale":"2023-09-30T09:50:44.440147+0000","last_undegraded":"2023-09-30T09:50:44.440147+0000","last_fullsized":"2023-09-30T09:50:44.440147+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:32:39.729247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"573'6660","reported_seq":8157,"reported_epoch":573,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:50:47.367465+0000","last_change":"2023-09-30T09:50:26.877630+0000","last_active":"2023-09-30T09:50:47.367465+0000","last_peered":"2023-09-30T09:50:47.367465+0000","last_clean":"2023-09-30T09:49:30.256832+0000","last_became_active":"2023-09-30T09:50:14.857202+0000","last_became_peered":"2023-09-30T09:50:14.857202+0000","last_unstale":"2023-09-30T09:50:47.367465+0000","last_undegraded":"2023-09-30T09:50:26.875609+0000","last_fullsized":"2023-09-30T09:50:47.367465+0000","mapping_epoch":560,"log_start":"540'827","ondisk_log_start":"540'827","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5833,"log_dups_size":0,"ondisk_log_size":5833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:02.846741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342491136,"num_objects":5226,"num_object_clones":0,"num_object_copies":15678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3110,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6660,"num_write_kb":380352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":718,"num_bytes_recovered":47054848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":3110},{"shards":"0,5,6","objects":2116}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":1055,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.458700+0000","last_change":"2023-09-30T09:50:44.458338+0000","last_active":"2023-09-30T09:50:44.458700+0000","last_peered":"2023-09-30T09:50:44.458700+0000","last_clean":"2023-09-30T09:50:44.458700+0000","last_became_active":"2023-09-30T09:50:44.457992+0000","last_became_peered":"2023-09-30T09:50:44.457992+0000","last_unstale":"2023-09-30T09:50:44.458700+0000","last_undegraded":"2023-09-30T09:50:44.458700+0000","last_fullsized":"2023-09-30T09:50:44.458700+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:34:30.514024+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":371,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.440381+0000","last_change":"2023-09-30T09:50:13.889332+0000","last_active":"2023-09-30T09:50:44.440381+0000","last_peered":"2023-09-30T09:50:44.440381+0000","last_clean":"2023-09-30T09:50:44.440381+0000","last_became_active":"2023-09-30T09:50:13.888518+0000","last_became_peered":"2023-09-30T09:50:13.888518+0000","last_unstale":"2023-09-30T09:50:44.440381+0000","last_undegraded":"2023-09-30T09:50:44.440381+0000","last_fullsized":"2023-09-30T09:50:44.440381+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:40:20.465620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":219,"num_read_kb":2265,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":612,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.447080+0000","last_change":"2023-09-30T09:49:31.304908+0000","last_active":"2023-09-30T09:50:44.447080+0000","last_peered":"2023-09-30T09:50:44.447080+0000","last_clean":"2023-09-30T09:50:44.447080+0000","last_became_active":"2023-09-30T09:49:31.304558+0000","last_became_peered":"2023-09-30T09:49:31.304558+0000","last_unstale":"2023-09-30T09:50:44.447080+0000","last_undegraded":"2023-09-30T09:50:44.447080+0000","last_fullsized":"2023-09-30T09:50:44.447080+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:03:56.889045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":15,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435058+0000","last_change":"2023-09-30T09:50:41.410343+0000","last_active":"2023-09-30T09:50:44.435058+0000","last_peered":"2023-09-30T09:50:44.435058+0000","last_clean":"2023-09-30T09:50:44.435058+0000","last_became_active":"2023-09-30T09:50:41.409979+0000","last_became_peered":"2023-09-30T09:50:41.409979+0000","last_unstale":"2023-09-30T09:50:44.435058+0000","last_undegraded":"2023-09-30T09:50:44.435058+0000","last_fullsized":"2023-09-30T09:50:44.435058+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:30:21.395759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"574'6573","reported_seq":6638,"reported_epoch":574,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:50:49.405603+0000","last_change":"2023-09-30T09:50:41.537925+0000","last_active":"2023-09-30T09:50:49.405603+0000","last_peered":"2023-09-30T09:50:49.405603+0000","last_clean":"2023-09-30T09:50:25.759432+0000","last_became_active":"2023-09-30T09:50:41.446699+0000","last_became_peered":"2023-09-30T09:50:41.446699+0000","last_unstale":"2023-09-30T09:50:49.405603+0000","last_undegraded":"2023-09-30T09:50:41.400458+0000","last_fullsized":"2023-09-30T09:50:41.381677+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:24:01.265807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328400896,"num_objects":5011,"num_object_clones":0,"num_object_copies":15033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":355,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6573,"num_write_kb":370688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":5011}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":993,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.438231+0000","last_change":"2023-09-30T09:40:39.586604+0000","last_active":"2023-09-30T09:50:44.438231+0000","last_peered":"2023-09-30T09:50:44.438231+0000","last_clean":"2023-09-30T09:50:44.438231+0000","last_became_active":"2023-09-30T09:40:39.586260+0000","last_became_peered":"2023-09-30T09:40:39.586260+0000","last_unstale":"2023-09-30T09:50:44.438231+0000","last_undegraded":"2023-09-30T09:50:44.438231+0000","last_fullsized":"2023-09-30T09:50:44.438231+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:41:04.675845+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":846,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.434787+0000","last_change":"2023-09-30T09:50:42.391868+0000","last_active":"2023-09-30T09:50:44.434787+0000","last_peered":"2023-09-30T09:50:44.434787+0000","last_clean":"2023-09-30T09:50:44.434787+0000","last_became_active":"2023-09-30T09:50:42.391282+0000","last_became_peered":"2023-09-30T09:50:42.391282+0000","last_unstale":"2023-09-30T09:50:44.434787+0000","last_undegraded":"2023-09-30T09:50:44.434787+0000","last_fullsized":"2023-09-30T09:50:44.434787+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:14:17.298036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.434735+0000","last_change":"2023-09-30T09:50:41.394348+0000","last_active":"2023-09-30T09:50:44.434735+0000","last_peered":"2023-09-30T09:50:44.434735+0000","last_clean":"2023-09-30T09:50:44.434735+0000","last_became_active":"2023-09-30T09:50:41.394017+0000","last_became_peered":"2023-09-30T09:50:41.394017+0000","last_unstale":"2023-09-30T09:50:44.434735+0000","last_undegraded":"2023-09-30T09:50:44.434735+0000","last_fullsized":"2023-09-30T09:50:44.434735+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:37:30.133826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":423,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.441467+0000","last_change":"2023-09-30T09:44:27.078267+0000","last_active":"2023-09-30T09:50:44.441467+0000","last_peered":"2023-09-30T09:50:44.441467+0000","last_clean":"2023-09-30T09:50:44.441467+0000","last_became_active":"2023-09-30T09:44:27.077735+0000","last_became_peered":"2023-09-30T09:44:27.077735+0000","last_unstale":"2023-09-30T09:50:44.441467+0000","last_undegraded":"2023-09-30T09:50:44.441467+0000","last_fullsized":"2023-09-30T09:50:44.441467+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:29.239462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"573'6382","reported_seq":7889,"reported_epoch":573,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:50:47.364229+0000","last_change":"2023-09-30T09:50:14.988649+0000","last_active":"2023-09-30T09:50:47.364229+0000","last_peered":"2023-09-30T09:50:47.364229+0000","last_clean":"2023-09-30T09:49:30.256457+0000","last_became_active":"2023-09-30T09:50:14.978894+0000","last_became_peered":"2023-09-30T09:50:14.978894+0000","last_unstale":"2023-09-30T09:50:47.364229+0000","last_undegraded":"2023-09-30T09:50:14.842503+0000","last_fullsized":"2023-09-30T09:50:47.364229+0000","mapping_epoch":560,"log_start":"540'600","ondisk_log_start":"540'600","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5782,"log_dups_size":0,"ondisk_log_size":5782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:20:48.615096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323092480,"num_objects":4930,"num_object_clones":0,"num_object_copies":14790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2901,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6382,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":726,"num_bytes_recovered":47579136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2901},{"shards":"0,1,6","objects":2029}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1157,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.441923+0000","last_change":"2023-09-30T09:50:42.387920+0000","last_active":"2023-09-30T09:50:44.441923+0000","last_peered":"2023-09-30T09:50:44.441923+0000","last_clean":"2023-09-30T09:50:44.441923+0000","last_became_active":"2023-09-30T09:50:42.387121+0000","last_became_peered":"2023-09-30T09:50:42.387121+0000","last_unstale":"2023-09-30T09:50:44.441923+0000","last_undegraded":"2023-09-30T09:50:44.441923+0000","last_fullsized":"2023-09-30T09:50:44.441923+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:34:05.192090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":976,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435700+0000","last_change":"2023-09-30T09:49:34.332831+0000","last_active":"2023-09-30T09:50:44.435700+0000","last_peered":"2023-09-30T09:50:44.435700+0000","last_clean":"2023-09-30T09:50:44.435700+0000","last_became_active":"2023-09-30T09:49:34.332429+0000","last_became_peered":"2023-09-30T09:49:34.332429+0000","last_unstale":"2023-09-30T09:50:44.435700+0000","last_undegraded":"2023-09-30T09:50:44.435700+0000","last_fullsized":"2023-09-30T09:50:44.435700+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:40:57.365967+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":170,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.440226+0000","last_change":"2023-09-30T09:49:31.302048+0000","last_active":"2023-09-30T09:50:44.440226+0000","last_peered":"2023-09-30T09:50:44.440226+0000","last_clean":"2023-09-30T09:50:44.440226+0000","last_became_active":"2023-09-30T09:49:31.301086+0000","last_became_peered":"2023-09-30T09:49:31.301086+0000","last_unstale":"2023-09-30T09:50:44.440226+0000","last_undegraded":"2023-09-30T09:50:44.440226+0000","last_fullsized":"2023-09-30T09:50:44.440226+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:15:22.422063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":468,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.459048+0000","last_change":"2023-09-30T09:50:13.859597+0000","last_active":"2023-09-30T09:50:44.459048+0000","last_peered":"2023-09-30T09:50:44.459048+0000","last_clean":"2023-09-30T09:50:44.459048+0000","last_became_active":"2023-09-30T09:50:13.859342+0000","last_became_peered":"2023-09-30T09:50:13.859342+0000","last_unstale":"2023-09-30T09:50:44.459048+0000","last_undegraded":"2023-09-30T09:50:44.459048+0000","last_fullsized":"2023-09-30T09:50:44.459048+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:56:21.575332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"573'6424","reported_seq":6486,"reported_epoch":573,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:50:47.287857+0000","last_change":"2023-09-30T09:50:41.455795+0000","last_active":"2023-09-30T09:50:47.287857+0000","last_peered":"2023-09-30T09:50:47.287857+0000","last_clean":"2023-09-30T09:50:25.822312+0000","last_became_active":"2023-09-30T09:50:41.447697+0000","last_became_peered":"2023-09-30T09:50:41.447697+0000","last_unstale":"2023-09-30T09:50:47.287857+0000","last_undegraded":"2023-09-30T09:50:41.416619+0000","last_fullsized":"2023-09-30T09:50:41.380961+0000","mapping_epoch":569,"log_start":"540'600","ondisk_log_start":"540'600","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:54:49.550720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330170368,"num_objects":5038,"num_object_clones":0,"num_object_copies":15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":360,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6424,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5038}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":750,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.446895+0000","last_change":"2023-09-30T09:50:13.854891+0000","last_active":"2023-09-30T09:50:44.446895+0000","last_peered":"2023-09-30T09:50:44.446895+0000","last_clean":"2023-09-30T09:50:44.446895+0000","last_became_active":"2023-09-30T09:50:13.854623+0000","last_became_peered":"2023-09-30T09:50:13.854623+0000","last_unstale":"2023-09-30T09:50:44.446895+0000","last_undegraded":"2023-09-30T09:50:44.446895+0000","last_fullsized":"2023-09-30T09:50:44.446895+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:39.230227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.230227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:38.720805+0000","scrub_duration":0.0016926949999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":982,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435627+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:50:44.435627+0000","last_peered":"2023-09-30T09:50:44.435627+0000","last_clean":"2023-09-30T09:50:44.435627+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:50:44.435627+0000","last_undegraded":"2023-09-30T09:50:44.435627+0000","last_fullsized":"2023-09-30T09:50:44.435627+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":187,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422750+0000","last_change":"2023-09-30T09:42:48.649589+0000","last_active":"2023-09-30T09:50:43.422750+0000","last_peered":"2023-09-30T09:50:43.422750+0000","last_clean":"2023-09-30T09:50:43.422750+0000","last_became_active":"2023-09-30T09:42:48.649307+0000","last_became_peered":"2023-09-30T09:42:48.649307+0000","last_unstale":"2023-09-30T09:50:43.422750+0000","last_undegraded":"2023-09-30T09:50:43.422750+0000","last_fullsized":"2023-09-30T09:50:43.422750+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:17:16.658075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":96,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.421259+0000","last_change":"2023-09-30T09:50:41.413261+0000","last_active":"2023-09-30T09:50:43.421259+0000","last_peered":"2023-09-30T09:50:43.421259+0000","last_clean":"2023-09-30T09:50:43.421259+0000","last_became_active":"2023-09-30T09:50:41.412932+0000","last_became_peered":"2023-09-30T09:50:41.412932+0000","last_unstale":"2023-09-30T09:50:43.421259+0000","last_undegraded":"2023-09-30T09:50:43.421259+0000","last_fullsized":"2023-09-30T09:50:43.421259+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:38:45.187070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"574'6586","reported_seq":8510,"reported_epoch":574,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:50:48.980560+0000","last_change":"2023-09-30T09:50:42.541801+0000","last_active":"2023-09-30T09:50:48.980560+0000","last_peered":"2023-09-30T09:50:48.980560+0000","last_clean":"2023-09-30T09:49:30.256680+0000","last_became_active":"2023-09-30T09:50:14.855383+0000","last_became_peered":"2023-09-30T09:50:14.855383+0000","last_unstale":"2023-09-30T09:50:48.980560+0000","last_undegraded":"2023-09-30T09:50:42.539739+0000","last_fullsized":"2023-09-30T09:50:48.980560+0000","mapping_epoch":560,"log_start":"541'1718","ondisk_log_start":"541'1718","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4868,"log_dups_size":0,"ondisk_log_size":4868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:08:53.752022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330235928,"num_objects":5040,"num_object_clones":0,"num_object_copies":15120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5040,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6586,"num_write_kb":371969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":932,"num_bytes_recovered":61079552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":2754},{"shards":"0,4,6","objects":2286}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":768,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.440587+0000","last_change":"2023-09-30T09:50:13.844441+0000","last_active":"2023-09-30T09:50:44.440587+0000","last_peered":"2023-09-30T09:50:44.440587+0000","last_clean":"2023-09-30T09:50:44.440587+0000","last_became_active":"2023-09-30T09:50:13.844037+0000","last_became_peered":"2023-09-30T09:50:13.844037+0000","last_unstale":"2023-09-30T09:50:44.440587+0000","last_undegraded":"2023-09-30T09:50:44.440587+0000","last_fullsized":"2023-09-30T09:50:44.440587+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:39:45.897942+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1122,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435827+0000","last_change":"2023-09-30T09:42:21.444875+0000","last_active":"2023-09-30T09:50:44.435827+0000","last_peered":"2023-09-30T09:50:44.435827+0000","last_clean":"2023-09-30T09:50:44.435827+0000","last_became_active":"2023-09-30T09:42:21.444260+0000","last_became_peered":"2023-09-30T09:42:21.444260+0000","last_unstale":"2023-09-30T09:50:44.435827+0000","last_undegraded":"2023-09-30T09:50:44.435827+0000","last_fullsized":"2023-09-30T09:50:44.435827+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:13:47.960016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435556+0000","last_change":"2023-09-30T09:50:41.394281+0000","last_active":"2023-09-30T09:50:44.435556+0000","last_peered":"2023-09-30T09:50:44.435556+0000","last_clean":"2023-09-30T09:50:44.435556+0000","last_became_active":"2023-09-30T09:50:41.393945+0000","last_became_peered":"2023-09-30T09:50:41.393945+0000","last_unstale":"2023-09-30T09:50:44.435556+0000","last_undegraded":"2023-09-30T09:50:44.435556+0000","last_fullsized":"2023-09-30T09:50:44.435556+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:43:10.077265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":187,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422891+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:50:43.422891+0000","last_peered":"2023-09-30T09:50:43.422891+0000","last_clean":"2023-09-30T09:50:43.422891+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:50:43.422891+0000","last_undegraded":"2023-09-30T09:50:43.422891+0000","last_fullsized":"2023-09-30T09:50:43.422891+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"574'6621","reported_seq":12456,"reported_epoch":574,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:50:48.994236+0000","last_change":"2023-09-30T09:50:41.461233+0000","last_active":"2023-09-30T09:50:48.994236+0000","last_peered":"2023-09-30T09:50:48.994236+0000","last_clean":"2023-09-30T09:49:30.268920+0000","last_became_active":"2023-09-30T09:50:41.447971+0000","last_became_peered":"2023-09-30T09:50:41.447971+0000","last_unstale":"2023-09-30T09:50:48.994236+0000","last_undegraded":"2023-09-30T09:50:41.438231+0000","last_fullsized":"2023-09-30T09:50:41.391772+0000","mapping_epoch":569,"log_start":"541'1772","ondisk_log_start":"541'1772","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":4849,"log_dups_size":0,"ondisk_log_size":4849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:47:36.586019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335216640,"num_objects":5115,"num_object_clones":0,"num_object_copies":15345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1335,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6621,"num_write_kb":375552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2880,"num_bytes_recovered":188678144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6],"avail_no_missing":["4"],"object_location_counts":[{"shards":"2,4,5","objects":947},{"shards":"4,6","objects":4168}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1172,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.474239+0000","last_change":"2023-09-30T09:50:44.465676+0000","last_active":"2023-09-30T09:50:44.474239+0000","last_peered":"2023-09-30T09:50:44.474239+0000","last_clean":"2023-09-30T09:50:44.474239+0000","last_became_active":"2023-09-30T09:50:44.465298+0000","last_became_peered":"2023-09-30T09:50:44.465298+0000","last_unstale":"2023-09-30T09:50:44.474239+0000","last_undegraded":"2023-09-30T09:50:44.474239+0000","last_fullsized":"2023-09-30T09:50:44.474239+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:38:14.860229+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":605,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.436058+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:50:44.436058+0000","last_peered":"2023-09-30T09:50:44.436058+0000","last_clean":"2023-09-30T09:50:44.436058+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:50:44.436058+0000","last_undegraded":"2023-09-30T09:50:44.436058+0000","last_fullsized":"2023-09-30T09:50:44.436058+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1301,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.459868+0000","last_change":"2023-09-30T09:50:13.858344+0000","last_active":"2023-09-30T09:50:44.459868+0000","last_peered":"2023-09-30T09:50:44.459868+0000","last_clean":"2023-09-30T09:50:44.459868+0000","last_became_active":"2023-09-30T09:50:13.857963+0000","last_became_peered":"2023-09-30T09:50:13.857963+0000","last_unstale":"2023-09-30T09:50:44.459868+0000","last_undegraded":"2023-09-30T09:50:44.459868+0000","last_fullsized":"2023-09-30T09:50:44.459868+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:21.850425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":609,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435491+0000","last_change":"2023-09-30T09:45:25.698976+0000","last_active":"2023-09-30T09:50:44.435491+0000","last_peered":"2023-09-30T09:50:44.435491+0000","last_clean":"2023-09-30T09:50:44.435491+0000","last_became_active":"2023-09-30T09:45:25.698684+0000","last_became_peered":"2023-09-30T09:45:25.698684+0000","last_unstale":"2023-09-30T09:50:44.435491+0000","last_undegraded":"2023-09-30T09:50:44.435491+0000","last_fullsized":"2023-09-30T09:50:44.435491+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:57:09.507571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"573'6550","reported_seq":7423,"reported_epoch":573,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:50:48.452965+0000","last_change":"2023-09-30T09:50:42.544120+0000","last_active":"2023-09-30T09:50:48.452965+0000","last_peered":"2023-09-30T09:50:48.452965+0000","last_clean":"2023-09-30T09:49:30.258329+0000","last_became_active":"2023-09-30T09:50:42.462730+0000","last_became_peered":"2023-09-30T09:50:42.462730+0000","last_unstale":"2023-09-30T09:50:48.452965+0000","last_undegraded":"2023-09-30T09:50:48.452965+0000","last_fullsized":"2023-09-30T09:50:42.385426+0000","mapping_epoch":570,"log_start":"540'549","ondisk_log_start":"540'549","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:27.841717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328335360,"num_objects":5010,"num_object_clones":0,"num_object_copies":15030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2944,"num_objects_unfound":0,"num_objects_dirty":5010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6550,"num_write_kb":369920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":401,"num_bytes_recovered":26214400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3"],"object_location_counts":[{"shards":"2,3,4","objects":2944},{"shards":"3,6","objects":2066}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":971,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.407495+0000","last_change":"2023-09-30T09:49:09.080572+0000","last_active":"2023-09-30T09:50:43.407495+0000","last_peered":"2023-09-30T09:50:43.407495+0000","last_clean":"2023-09-30T09:50:43.407495+0000","last_became_active":"2023-09-30T09:49:09.080212+0000","last_became_peered":"2023-09-30T09:49:09.080212+0000","last_unstale":"2023-09-30T09:50:43.407495+0000","last_undegraded":"2023-09-30T09:50:43.407495+0000","last_fullsized":"2023-09-30T09:50:43.407495+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:02:52.327438+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":879,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.436138+0000","last_change":"2023-09-30T09:50:41.393850+0000","last_active":"2023-09-30T09:50:44.436138+0000","last_peered":"2023-09-30T09:50:44.436138+0000","last_clean":"2023-09-30T09:50:44.436138+0000","last_became_active":"2023-09-30T09:50:41.393481+0000","last_became_peered":"2023-09-30T09:50:41.393481+0000","last_unstale":"2023-09-30T09:50:44.436138+0000","last_undegraded":"2023-09-30T09:50:44.436138+0000","last_fullsized":"2023-09-30T09:50:44.436138+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:47:48.909491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":150,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.421646+0000","last_change":"2023-09-30T09:42:21.452541+0000","last_active":"2023-09-30T09:50:43.421646+0000","last_peered":"2023-09-30T09:50:43.421646+0000","last_clean":"2023-09-30T09:50:43.421646+0000","last_became_active":"2023-09-30T09:42:21.440390+0000","last_became_peered":"2023-09-30T09:42:21.440390+0000","last_unstale":"2023-09-30T09:50:43.421646+0000","last_undegraded":"2023-09-30T09:50:43.421646+0000","last_fullsized":"2023-09-30T09:50:43.421646+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:37:44.101526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":594,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.453877+0000","last_change":"2023-09-30T09:49:31.304154+0000","last_active":"2023-09-30T09:50:44.453877+0000","last_peered":"2023-09-30T09:50:44.453877+0000","last_clean":"2023-09-30T09:50:44.453877+0000","last_became_active":"2023-09-30T09:49:31.303819+0000","last_became_peered":"2023-09-30T09:49:31.303819+0000","last_unstale":"2023-09-30T09:50:44.453877+0000","last_undegraded":"2023-09-30T09:50:44.453877+0000","last_fullsized":"2023-09-30T09:50:44.453877+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:17:41.151778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":922,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422518+0000","last_change":"2023-09-30T09:49:34.338310+0000","last_active":"2023-09-30T09:50:43.422518+0000","last_peered":"2023-09-30T09:50:43.422518+0000","last_clean":"2023-09-30T09:50:43.422518+0000","last_became_active":"2023-09-30T09:49:34.337868+0000","last_became_peered":"2023-09-30T09:49:34.337868+0000","last_unstale":"2023-09-30T09:50:43.422518+0000","last_undegraded":"2023-09-30T09:50:43.422518+0000","last_fullsized":"2023-09-30T09:50:43.422518+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:13:34.757539+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":80,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.422814+0000","last_change":"2023-09-30T09:45:25.702675+0000","last_active":"2023-09-30T09:50:43.422814+0000","last_peered":"2023-09-30T09:50:43.422814+0000","last_clean":"2023-09-30T09:50:43.422814+0000","last_became_active":"2023-09-30T09:45:25.702150+0000","last_became_peered":"2023-09-30T09:45:25.702150+0000","last_unstale":"2023-09-30T09:50:43.422814+0000","last_undegraded":"2023-09-30T09:50:43.422814+0000","last_fullsized":"2023-09-30T09:50:43.422814+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:37.329181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":15,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435119+0000","last_change":"2023-09-30T09:50:41.398486+0000","last_active":"2023-09-30T09:50:44.435119+0000","last_peered":"2023-09-30T09:50:44.435119+0000","last_clean":"2023-09-30T09:50:44.435119+0000","last_became_active":"2023-09-30T09:50:41.398144+0000","last_became_peered":"2023-09-30T09:50:41.398144+0000","last_unstale":"2023-09-30T09:50:44.435119+0000","last_undegraded":"2023-09-30T09:50:44.435119+0000","last_fullsized":"2023-09-30T09:50:44.435119+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:48:35.523625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":720,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.437961+0000","last_change":"2023-09-30T09:50:13.849267+0000","last_active":"2023-09-30T09:50:44.437961+0000","last_peered":"2023-09-30T09:50:44.437961+0000","last_clean":"2023-09-30T09:50:44.437961+0000","last_became_active":"2023-09-30T09:50:13.848637+0000","last_became_peered":"2023-09-30T09:50:13.848637+0000","last_unstale":"2023-09-30T09:50:44.437961+0000","last_undegraded":"2023-09-30T09:50:44.437961+0000","last_fullsized":"2023-09-30T09:50:44.437961+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:24.775922+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":502,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.439086+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:50:44.439086+0000","last_peered":"2023-09-30T09:50:44.439086+0000","last_clean":"2023-09-30T09:50:44.439086+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:50:44.439086+0000","last_undegraded":"2023-09-30T09:50:44.439086+0000","last_fullsized":"2023-09-30T09:50:44.439086+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":287,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.446665+0000","last_change":"2023-09-30T09:42:48.649740+0000","last_active":"2023-09-30T09:50:44.446665+0000","last_peered":"2023-09-30T09:50:44.446665+0000","last_clean":"2023-09-30T09:50:44.446665+0000","last_became_active":"2023-09-30T09:42:48.649455+0000","last_became_peered":"2023-09-30T09:42:48.649455+0000","last_unstale":"2023-09-30T09:50:44.446665+0000","last_undegraded":"2023-09-30T09:50:44.446665+0000","last_fullsized":"2023-09-30T09:50:44.446665+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:59:25.156537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":472,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.439771+0000","last_change":"2023-09-30T09:49:31.313229+0000","last_active":"2023-09-30T09:50:44.439771+0000","last_peered":"2023-09-30T09:50:44.439771+0000","last_clean":"2023-09-30T09:50:44.439771+0000","last_became_active":"2023-09-30T09:49:31.301137+0000","last_became_peered":"2023-09-30T09:49:31.301137+0000","last_unstale":"2023-09-30T09:50:44.439771+0000","last_undegraded":"2023-09-30T09:50:44.439771+0000","last_fullsized":"2023-09-30T09:50:44.439771+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:57.362379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":307,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.454365+0000","last_change":"2023-09-30T09:42:48.645465+0000","last_active":"2023-09-30T09:50:44.454365+0000","last_peered":"2023-09-30T09:50:44.454365+0000","last_clean":"2023-09-30T09:50:44.454365+0000","last_became_active":"2023-09-30T09:42:48.644762+0000","last_became_peered":"2023-09-30T09:42:48.644762+0000","last_unstale":"2023-09-30T09:50:44.454365+0000","last_undegraded":"2023-09-30T09:50:44.454365+0000","last_fullsized":"2023-09-30T09:50:44.454365+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:26:35.003658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":680,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:46.480192+0000","last_change":"2023-09-30T09:50:46.480162+0000","last_active":"2023-09-30T09:50:46.480192+0000","last_peered":"2023-09-30T09:50:46.480192+0000","last_clean":"2023-09-30T09:50:46.480192+0000","last_became_active":"2023-09-30T09:50:29.883457+0000","last_became_peered":"2023-09-30T09:50:29.883457+0000","last_unstale":"2023-09-30T09:50:46.480192+0000","last_undegraded":"2023-09-30T09:50:46.480192+0000","last_fullsized":"2023-09-30T09:50:46.480192+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:14.071789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:14.071789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.435717+0000","last_change":"2023-09-30T09:50:41.394419+0000","last_active":"2023-09-30T09:50:44.435717+0000","last_peered":"2023-09-30T09:50:44.435717+0000","last_clean":"2023-09-30T09:50:44.435717+0000","last_became_active":"2023-09-30T09:50:41.394073+0000","last_became_peered":"2023-09-30T09:50:41.394073+0000","last_unstale":"2023-09-30T09:50:44.435717+0000","last_undegraded":"2023-09-30T09:50:44.435717+0000","last_fullsized":"2023-09-30T09:50:44.435717+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:22:22.271661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.434614+0000","last_change":"2023-09-30T09:50:41.404832+0000","last_active":"2023-09-30T09:50:44.434614+0000","last_peered":"2023-09-30T09:50:44.434614+0000","last_clean":"2023-09-30T09:50:44.434614+0000","last_became_active":"2023-09-30T09:50:41.404445+0000","last_became_peered":"2023-09-30T09:50:41.404445+0000","last_unstale":"2023-09-30T09:50:44.434614+0000","last_undegraded":"2023-09-30T09:50:44.434614+0000","last_fullsized":"2023-09-30T09:50:44.434614+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:27.041434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":943,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.446829+0000","last_change":"2023-09-30T09:44:14.528402+0000","last_active":"2023-09-30T09:50:44.446829+0000","last_peered":"2023-09-30T09:50:44.446829+0000","last_clean":"2023-09-30T09:50:44.446829+0000","last_became_active":"2023-09-30T09:43:45.121864+0000","last_became_peered":"2023-09-30T09:43:45.121864+0000","last_unstale":"2023-09-30T09:50:44.446829+0000","last_undegraded":"2023-09-30T09:50:44.446829+0000","last_fullsized":"2023-09-30T09:50:44.446829+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:24:45.616947+0000","scrub_duration":0.0018066,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":114,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-30T09:50:43.421451+0000","last_change":"2023-09-30T09:50:41.480353+0000","last_active":"2023-09-30T09:50:43.421451+0000","last_peered":"2023-09-30T09:50:43.421451+0000","last_clean":"2023-09-30T09:50:43.421451+0000","last_became_active":"2023-09-30T09:50:41.412459+0000","last_became_peered":"2023-09-30T09:50:41.412459+0000","last_unstale":"2023-09-30T09:50:43.421451+0000","last_undegraded":"2023-09-30T09:50:43.421451+0000","last_fullsized":"2023-09-30T09:50:43.421451+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:04:44.887755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":618,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-30T09:50:44.459275+0000","last_change":"2023-09-30T09:50:26.905427+0000","last_active":"2023-09-30T09:50:44.459275+0000","last_peered":"2023-09-30T09:50:44.459275+0000","last_clean":"2023-09-30T09:50:44.459275+0000","last_became_active":"2023-09-30T09:50:26.905100+0000","last_became_peered":"2023-09-30T09:50:26.905100+0000","last_unstale":"2023-09-30T09:50:44.459275+0000","last_undegraded":"2023-09-30T09:50:44.459275+0000","last_fullsized":"2023-09-30T09:50:44.459275+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:38.036208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5331484696,"num_objects":81353,"num_object_clones":0,"num_object_copies":244059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17584,"num_objects_misplaced":2944,"num_objects_unfound":0,"num_objects_dirty":81353,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":105443,"num_write_kb":5977409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7206,"num_bytes_recovered":472055808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16670208000,"data_stored":16670195784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91637,"ondisk_log_size":91637,"up":48,"acting":42,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":119,"num_bytes_recovered":32890,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":599313,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":709030,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234469,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2150800,"kb_used_data":1551816,"kb_used_omap":0,"kb_used_meta":598976,"kb_avail":91594352,"statfs":{"total":95995035648,"available":93792616448,"internally_reserved":0,"allocated":1589059584,"data_stored":1587179022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":613351424},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:50:09 2023","interfaces":[{"interface":"back","average":{"1min":3.556,"5min":6.0289999999999999,"15min":5.2859999999999996},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.22900000000000001},"max":{"1min":37.703000000000003,"5min":110.467,"15min":215.416},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.6219999999999999,"5min":5.9459999999999997,"15min":5.1710000000000003},"min":{"1min":0.247,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":37.619999999999997,"5min":109.593,"15min":215.041},"last":0.628}]},{"osd":1,"last update":"Sat Sep 30 09:50:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":5.133,"15min":3.911},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.25},"max":{"1min":6.6870000000000003,"5min":157.578,"15min":157.578},"last":3.153},{"interface":"front","average":{"1min":1.952,"5min":5.2359999999999998,"15min":3.9590000000000001},"min":{"1min":0.39400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.6709999999999994,"5min":157.589,"15min":157.589},"last":0.92100000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:50:46 2023","interfaces":[{"interface":"back","average":{"1min":3.3580000000000001,"5min":5.2610000000000001,"15min":4.6689999999999996},"min":{"1min":0.372,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.7330000000000005,"5min":187.74600000000001,"15min":187.74600000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.5510000000000002,"5min":4.875,"15min":4.5709999999999997},"min":{"1min":0.38500000000000001,"5min":0.29599999999999999,"15min":0.28399999999999997},"max":{"1min":12.125999999999999,"5min":187.583,"15min":187.583},"last":6.4130000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:50:24 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":5.0099999999999998,"15min":6.7560000000000002},"min":{"1min":0.47399999999999998,"5min":0.34200000000000003,"15min":0.314},"max":{"1min":8.8859999999999992,"5min":154.06700000000001,"15min":277.57799999999997},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.5609999999999999,"5min":5.4169999999999998,"15min":7.0220000000000002},"min":{"1min":0.376,"5min":0.35899999999999999,"15min":0.32200000000000001},"max":{"1min":11.108000000000001,"5min":154.24799999999999,"15min":278.69299999999998},"last":0.65900000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:50:46 2023","interfaces":[{"interface":"back","average":{"1min":14.741,"5min":4.0090000000000003,"15min":2.585},"min":{"1min":0.36899999999999999,"5min":0.27100000000000002,"15min":0.23799999999999999},"max":{"1min":238.511,"5min":238.511,"15min":238.511},"last":3.2360000000000002},{"interface":"front","average":{"1min":15.565,"5min":3.9849999999999999,"15min":2.5169999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.252},"max":{"1min":238.387,"5min":238.387,"15min":238.387},"last":0.68200000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:50:32 2023","interfaces":[{"interface":"back","average":{"1min":14.084,"5min":3.758,"15min":9.2479999999999993},"min":{"1min":0.40500000000000003,"5min":0.32700000000000001,"15min":0.28399999999999997},"max":{"1min":187.374,"5min":187.374,"15min":197.75700000000001},"last":0.80700000000000005},{"interface":"front","average":{"1min":12.972,"5min":3.7429999999999999,"15min":9.1319999999999997},"min":{"1min":0.40300000000000002,"5min":0.309,"15min":0.309},"max":{"1min":187.416,"5min":187.416,"15min":197.63300000000001},"last":0.88800000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:50:46 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":1.575,"15min":2.2149999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.6479999999999997,"5min":7.6479999999999997,"15min":206.244},"last":3.198},{"interface":"front","average":{"1min":2.766,"5min":1.361,"15min":2.1509999999999998},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":12.058,"5min":12.058,"15min":202.78800000000001},"last":5.3920000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496107,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1677004,"kb_used_data":1086468,"kb_used_omap":0,"kb_used_meta":590528,"kb_avail":92068148,"statfs":{"total":95995035648,"available":94277783552,"internally_reserved":0,"allocated":1112543232,"data_stored":1110673865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":604700672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":11.316000000000001,"5min":5.6310000000000002,"15min":6.4139999999999997},"min":{"1min":0.34999999999999998,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":192.511,"5min":192.511,"15min":192.511},"last":0.54900000000000004},{"interface":"front","average":{"1min":11.956,"5min":5.6959999999999997,"15min":6.3860000000000001},"min":{"1min":0.374,"5min":0.248,"15min":0.22500000000000001},"max":{"1min":192.346,"5min":192.346,"15min":192.346},"last":0.60699999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":7.306,"5min":2.802,"15min":2.3069999999999999},"min":{"1min":0.32700000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":103.038,"5min":103.038,"15min":103.038},"last":0.45300000000000001},{"interface":"front","average":{"1min":7.3630000000000004,"5min":2.9670000000000001,"15min":2.4820000000000002},"min":{"1min":0.42099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":103.05,"5min":103.05,"15min":103.05},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":7.7720000000000002,"5min":3.2879999999999998,"15min":4.4109999999999996},"min":{"1min":0.39200000000000002,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":109.88500000000001,"5min":109.88500000000001,"15min":207.02500000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":7.6470000000000002,"5min":3.1429999999999998,"15min":4.4569999999999999},"min":{"1min":0.371,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":105.81699999999999,"5min":105.81699999999999,"15min":207.114},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.74,"15min":3.831},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.6470000000000002,"5min":11.644,"15min":212.44800000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.9510000000000001,"5min":1.855,"15min":4.0049999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.31900000000000001},"max":{"1min":5.5490000000000004,"5min":8.8559999999999999,"15min":210.84299999999999},"last":0.46600000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.423,"15min":1.3520000000000001},"min":{"1min":0.29099999999999998,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":12.169,"5min":12.169,"15min":63.798000000000002},"last":30.568000000000001},{"interface":"front","average":{"1min":1.675,"5min":1.3580000000000001,"15min":1.2749999999999999},"min":{"1min":0.36299999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":8.9589999999999996,"5min":11.558999999999999,"15min":63.133000000000003},"last":29.27}]},{"osd":5,"last update":"Sat Sep 30 09:50:45 2023","interfaces":[{"interface":"back","average":{"1min":4.2489999999999997,"5min":1.802,"15min":1.966},"min":{"1min":0.38400000000000001,"5min":0.29199999999999998,"15min":0.27100000000000002},"max":{"1min":19.212,"5min":19.212,"15min":19.212},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.9820000000000002,"5min":1.6259999999999999,"15min":2.294},"min":{"1min":0.33400000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.731,"5min":12.731,"15min":16.443000000000001},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":1.407,"15min":1.5580000000000001},"min":{"1min":0.38400000000000001,"5min":0.30399999999999999,"15min":0.26500000000000001},"max":{"1min":6.8860000000000001,"5min":11.276999999999999,"15min":138.191},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.591,"5min":1.492,"15min":1.5029999999999999},"min":{"1min":0.40999999999999998,"5min":0.34300000000000003,"15min":0.27500000000000002},"max":{"1min":7.5060000000000002,"5min":11.521000000000001,"15min":140.33500000000001},"last":0.39900000000000002}]}]},{"osd":1,"up_from":301,"seq":1292785156305,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1666952,"kb_used_data":967424,"kb_used_omap":9,"kb_used_meta":699510,"kb_avail":92078200,"statfs":{"total":95995035648,"available":94288076800,"internally_reserved":0,"allocated":990642176,"data_stored":987407379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":716298258},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:50:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":4.7770000000000001,"15min":3.9180000000000001},"min":{"1min":0.23499999999999999,"5min":0.20699999999999999,"15min":0.17100000000000001},"max":{"1min":6.5919999999999996,"5min":142.08500000000001,"15min":142.08500000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.262,"5min":4.8250000000000002,"15min":3.7480000000000002},"min":{"1min":0.218,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.6900000000000004,"5min":142.137,"15min":142.137},"last":0.59399999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:50:05 2023","interfaces":[{"interface":"back","average":{"1min":5.0899999999999999,"5min":4.3369999999999997,"15min":2.8820000000000001},"min":{"1min":0.27700000000000002,"5min":0.23999999999999999,"15min":0.184},"max":{"1min":92.504999999999995,"5min":92.504999999999995,"15min":92.504999999999995},"last":0.29399999999999998},{"interface":"front","average":{"1min":5.6100000000000003,"5min":4.5039999999999996,"15min":3.1659999999999999},"min":{"1min":0.30199999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":91.022999999999996,"5min":91.022999999999996,"15min":91.022999999999996},"last":0.317}]},{"osd":3,"last update":"Sat Sep 30 09:50:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":3.9609999999999999,"15min":4.9889999999999999},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.23400000000000001},"max":{"1min":11.967000000000001,"5min":59.478000000000002,"15min":199.41},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.48,"5min":3.3069999999999999,"15min":4.6589999999999998},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.23999999999999999},"max":{"1min":5.9800000000000004,"5min":59.621000000000002,"15min":199.58099999999999},"last":0.49199999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:50:05 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":2.3399999999999999,"15min":2.302},"min":{"1min":0.311,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":7.0510000000000002,"5min":48.604999999999997,"15min":142.77099999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.575,"5min":2.117,"15min":2.597},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":6.7549999999999999,"5min":22.373000000000001,"15min":144.898},"last":1.1859999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:50:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1219999999999999,"5min":2.5550000000000002,"15min":2.141},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":12.387,"5min":22.259,"15min":22.259},"last":2.9020000000000001},{"interface":"front","average":{"1min":2.8159999999999998,"5min":1.8999999999999999,"15min":1.448},"min":{"1min":0.41899999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":12.41,"5min":15.250999999999999,"15min":15.250999999999999},"last":2.867}]},{"osd":6,"last update":"Sat Sep 30 09:50:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":2.1360000000000001,"15min":2.52},"min":{"1min":0.38900000000000001,"5min":0.38700000000000001,"15min":0.29899999999999999},"max":{"1min":7.4560000000000004,"5min":30.986999999999998,"15min":96.719999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.589,"5min":2.2160000000000002,"15min":2.508},"min":{"1min":0.48499999999999999,"5min":0.45400000000000001,"15min":0.33900000000000002},"max":{"1min":7.6310000000000002,"5min":28.710999999999999,"15min":96.356999999999999},"last":0.38800000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:50:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":1.6839999999999999,"15min":1.7569999999999999},"min":{"1min":0.41399999999999998,"5min":0.36299999999999999,"15min":0.28299999999999997},"max":{"1min":6.1559999999999997,"5min":9.3879999999999999,"15min":15.587},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.7350000000000001,"15min":1.788},"min":{"1min":0.442,"5min":0.39100000000000001,"15min":0.316},"max":{"1min":6.1269999999999998,"5min":7.5460000000000003,"15min":15.226000000000001},"last":0.57899999999999996}]}]},{"osd":0,"up_from":94,"seq":403726926203,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4138568,"kb_used_data":3307964,"kb_used_omap":5,"kb_used_meta":830586,"kb_avail":89606584,"statfs":{"total":95995035648,"available":91757142016,"internally_reserved":0,"allocated":3387355136,"data_stored":3383886947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":850520317},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:49:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":1.448,"15min":1.8620000000000001},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.17799999999999999},"max":{"1min":11.641,"5min":13.308999999999999,"15min":57.482999999999997},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.2629999999999999,"5min":2.4710000000000001,"15min":2.641},"min":{"1min":0.251,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":4.7839999999999998,"5min":23.503,"15min":57.140999999999998},"last":0.23300000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:49:49 2023","interfaces":[{"interface":"back","average":{"1min":11.657,"5min":5.0380000000000003,"15min":4.2960000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.20799999999999999},"max":{"1min":173.185,"5min":173.185,"15min":173.185},"last":3.5790000000000002},{"interface":"front","average":{"1min":11.535,"5min":5.2649999999999997,"15min":4.5979999999999999},"min":{"1min":0.34399999999999997,"5min":0.249,"15min":0.249},"max":{"1min":173.22499999999999,"5min":173.22499999999999,"15min":173.22499999999999},"last":3.5499999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":3.149,"15min":3.835},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.9939999999999998,"5min":28.015000000000001,"15min":125.562},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.8779999999999999,"5min":1.986,"15min":3.0720000000000001},"min":{"1min":0.34599999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":11.679,"5min":13.361000000000001,"15min":117.295},"last":0.32300000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":5.6260000000000003,"5min":2.8620000000000001,"15min":2.484},"min":{"1min":0.38300000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":67.945999999999998,"5min":67.945999999999998,"15min":72.403999999999996},"last":3.6110000000000002},{"interface":"front","average":{"1min":5.5919999999999996,"5min":3.0329999999999999,"15min":2.6600000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.29699999999999999},"max":{"1min":67.792000000000002,"5min":67.792000000000002,"15min":72.162999999999997},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":7.125,"5min":3.476,"15min":1.873},"min":{"1min":0.42199999999999999,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":71.376000000000005,"5min":71.376000000000005,"15min":71.376000000000005},"last":2.3119999999999998},{"interface":"front","average":{"1min":7.5090000000000003,"5min":2.6589999999999998,"15min":1.585},"min":{"1min":0.46700000000000003,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":71.706000000000003,"5min":71.706000000000003,"15min":71.706000000000003},"last":3.032}]},{"osd":6,"last update":"Sat Sep 30 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8210000000000002,"5min":4.0919999999999996,"15min":2.5579999999999998},"min":{"1min":0.53400000000000003,"5min":0.38200000000000001,"15min":0.33200000000000002},"max":{"1min":11.512,"5min":240.946,"15min":240.946},"last":0.378},{"interface":"front","average":{"1min":1.853,"5min":4.8499999999999996,"15min":3.024},"min":{"1min":0.443,"5min":0.41299999999999998,"15min":0.34999999999999998},"max":{"1min":4.9009999999999998,"5min":240.977,"15min":240.977},"last":3.4790000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7759999999999998,"5min":3.1869999999999998,"15min":2.266},"min":{"1min":0.45100000000000001,"5min":0.40300000000000002,"15min":0.36099999999999999},"max":{"1min":11.571,"5min":146.828,"15min":146.828},"last":2.2040000000000002},{"interface":"front","average":{"1min":1.4870000000000001,"5min":4.0350000000000001,"15min":2.762},"min":{"1min":0.41599999999999998,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":3.8540000000000001,"5min":146.876,"15min":146.876},"last":3.5190000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599144,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3575968,"kb_used_data":2693140,"kb_used_omap":5,"kb_used_meta":882746,"kb_avail":90169184,"statfs":{"total":95995035648,"available":92333244416,"internally_reserved":0,"allocated":2757775360,"data_stored":2754579035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":903932557},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:49:57 2023","interfaces":[{"interface":"back","average":{"1min":7.306,"5min":7.593,"15min":4.375},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":97.087000000000003,"5min":202.40899999999999,"15min":202.40899999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":8.3190000000000008,"5min":8.1210000000000004,"15min":4.5179999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.217},"max":{"1min":97.176000000000002,"5min":202.38800000000001,"15min":202.38800000000001},"last":0.437}]},{"osd":1,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":7.3360000000000003,"15min":4.3799999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":3.5600000000000001,"5min":202.369,"15min":202.369},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.6220000000000001,"5min":7.5049999999999999,"15min":4.306},"min":{"1min":0.29999999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.6920000000000002,"5min":202.54499999999999,"15min":202.54499999999999},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:50:18 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":5.4059999999999997,"15min":5.0289999999999999},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.9100000000000001,"5min":202.59,"15min":202.59},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.236,"5min":5.4589999999999996,"15min":4.96},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.224},"max":{"1min":3.5299999999999998,"5min":202.34200000000001,"15min":202.34200000000001},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":6.0700000000000003,"15min":3.0880000000000001},"min":{"1min":0.46100000000000002,"5min":0.32800000000000001,"15min":0.30599999999999999},"max":{"1min":6.8550000000000004,"5min":247.655,"15min":247.655},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.331,"5min":6.335,"15min":3.1160000000000001},"min":{"1min":0.29299999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.7990000000000004,"5min":248.012,"15min":248.012},"last":0.89200000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":2.7000000000000002,"5min":6.4800000000000004,"15min":2.8769999999999998},"min":{"1min":0.317,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.375,"5min":243.86500000000001,"15min":243.86500000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.726,"5min":6.742,"15min":3.1240000000000001},"min":{"1min":0.39500000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":12.353,"5min":244.02199999999999,"15min":244.02199999999999},"last":0.503}]},{"osd":6,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":3.6099999999999999,"15min":2.1509999999999998},"min":{"1min":0.32800000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.6789999999999998,"5min":202.70599999999999,"15min":202.70599999999999},"last":1.859},{"interface":"front","average":{"1min":1.601,"5min":3.7799999999999998,"15min":2.5409999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.34100000000000003},"max":{"1min":5.9329999999999998,"5min":202.916,"15min":202.916},"last":0.54200000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":1.698,"5min":7.8570000000000002,"15min":3.4889999999999999},"min":{"1min":0.38500000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.8319999999999999,"5min":244.911,"15min":244.911},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.3720000000000001,"5min":7.7770000000000001,"15min":3.7839999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.9740000000000002,"5min":244.66,"15min":244.66},"last":0.56999999999999995}]}]},{"osd":3,"up_from":109,"seq":468151435617,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2850004,"kb_used_data":1883276,"kb_used_omap":2,"kb_used_meta":966717,"kb_avail":90895148,"statfs":{"total":95995035648,"available":93076631552,"internally_reserved":0,"allocated":1928474624,"data_stored":1925161011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":989918567},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:50:21 2023","interfaces":[{"interface":"back","average":{"1min":8.3829999999999991,"5min":4.774,"15min":4.992},"min":{"1min":0.22900000000000001,"5min":0.191,"15min":0.183},"max":{"1min":152.429,"5min":152.429,"15min":205.73099999999999},"last":0.313},{"interface":"front","average":{"1min":9.5380000000000003,"5min":4.9009999999999998,"15min":4.9429999999999996},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":150.464,"5min":150.464,"15min":205.495},"last":0.22800000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:49:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1739999999999999,"5min":2.9279999999999999,"15min":3.7109999999999999},"min":{"1min":0.34499999999999997,"5min":0.26200000000000001,"15min":0.23000000000000001},"max":{"1min":6.5499999999999998,"5min":19.725000000000001,"15min":170.512},"last":1.605},{"interface":"front","average":{"1min":2.149,"5min":2.8610000000000002,"15min":3.7810000000000001},"min":{"1min":0.25700000000000001,"5min":0.248,"15min":0.20699999999999999},"max":{"1min":11.497999999999999,"5min":19.638999999999999,"15min":170.535},"last":0.72899999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:50:21 2023","interfaces":[{"interface":"back","average":{"1min":8.5169999999999995,"5min":3.6949999999999998,"15min":5.8449999999999998},"min":{"1min":0.36099999999999999,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":147.87,"5min":147.87,"15min":206.69},"last":3.524},{"interface":"front","average":{"1min":8.9410000000000007,"5min":4.0030000000000001,"15min":5.8710000000000004},"min":{"1min":0.33300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":148.04900000000001,"5min":148.04900000000001,"15min":209.22800000000001},"last":0.625}]},{"osd":4,"last update":"Sat Sep 30 09:50:21 2023","interfaces":[{"interface":"back","average":{"1min":10.205,"5min":3.613,"15min":2.673},"min":{"1min":0.29499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":172.298,"5min":172.298,"15min":172.298},"last":0.66500000000000004},{"interface":"front","average":{"1min":10.57,"5min":4.0890000000000004,"15min":2.6190000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":172.45099999999999,"5min":172.45099999999999,"15min":172.45099999999999},"last":1.4550000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:50:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0600000000000001,"5min":2.1440000000000001,"15min":1.732},"min":{"1min":0.36199999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.9339999999999993,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.9609999999999999,"5min":2.4900000000000002,"15min":1.464},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.311},"max":{"1min":9.1649999999999991,"5min":15.715,"15min":15.715},"last":1.5029999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:50:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":2.113,"15min":2.0019999999999998},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.977,"5min":10.977,"15min":60.698},"last":0.70699999999999996},{"interface":"front","average":{"1min":2.915,"5min":2.6619999999999999,"15min":2.044},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33400000000000002},"max":{"1min":6.5190000000000001,"5min":15.666,"15min":15.666},"last":1.5609999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:50:21 2023","interfaces":[{"interface":"back","average":{"1min":2.492,"5min":2.3050000000000002,"15min":2.5},"min":{"1min":0.44400000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.9199999999999999,"5min":11.753,"15min":213.548},"last":0.68799999999999994},{"interface":"front","average":{"1min":2.5590000000000002,"5min":2.399,"15min":2.6099999999999999},"min":{"1min":0.44600000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.936,"5min":10.936,"15min":213.583},"last":0.59099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019380,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3643780,"kb_used_data":3072060,"kb_used_omap":0,"kb_used_meta":571712,"kb_avail":90101372,"statfs":{"total":95995035648,"available":92263804928,"internally_reserved":0,"allocated":3145789440,"data_stored":3143920046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":585433088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:50:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":3.7440000000000002,"15min":3.2210000000000001},"min":{"1min":0.26400000000000001,"5min":0.26300000000000001,"15min":0.21199999999999999},"max":{"1min":4.6980000000000004,"5min":107.43000000000001,"15min":107.43000000000001},"last":1.167},{"interface":"front","average":{"1min":2.2210000000000001,"5min":4.2320000000000002,"15min":3.468},"min":{"1min":0.371,"5min":0.26600000000000001,"15min":0.24299999999999999},"max":{"1min":6.8719999999999999,"5min":107.373,"15min":107.373},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:50:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.423,"15min":2.7280000000000002},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.25},"max":{"1min":7.282,"5min":108.551,"15min":108.551},"last":1.502},{"interface":"front","average":{"1min":1.879,"5min":3.722,"15min":2.7679999999999998},"min":{"1min":0.29999999999999999,"5min":0.27300000000000002,"15min":0.26800000000000002},"max":{"1min":7.335,"5min":108.884,"15min":108.884},"last":1.2070000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":4.1399999999999997,"15min":3.5409999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27300000000000002},"max":{"1min":12.741,"5min":117.858,"15min":142.88999999999999},"last":1.6359999999999999},{"interface":"front","average":{"1min":2.1539999999999999,"5min":4.194,"15min":3.5649999999999999},"min":{"1min":0.41099999999999998,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":7.6639999999999997,"5min":117.871,"15min":142.90700000000001},"last":1.647}]},{"osd":3,"last update":"Sat Sep 30 09:50:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":3.7389999999999999,"15min":2.383},"min":{"1min":0.38100000000000001,"5min":0.35799999999999998,"15min":0.32700000000000001},"max":{"1min":7.3079999999999998,"5min":105.142,"15min":105.142},"last":1.4319999999999999},{"interface":"front","average":{"1min":1.9319999999999999,"5min":3.589,"15min":2.371},"min":{"1min":0.41899999999999998,"5min":0.36699999999999999,"15min":0.29299999999999998},"max":{"1min":7.0460000000000003,"5min":105.157,"15min":105.157},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:50:29 2023","interfaces":[{"interface":"back","average":{"1min":4.7160000000000002,"5min":1.972,"15min":7.8129999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":59.530999999999999,"5min":59.530999999999999,"15min":140.11099999999999},"last":5.2569999999999997},{"interface":"front","average":{"1min":5.2480000000000002,"5min":1.9119999999999999,"15min":7.8140000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":60.012,"5min":60.012,"15min":140.065},"last":1.2709999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:50:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1280000000000001,"5min":1.4530000000000001,"15min":1.643},"min":{"1min":0.318,"5min":0.318,"15min":0.25900000000000001},"max":{"1min":7.0949999999999998,"5min":9.1470000000000002,"15min":182.10300000000001},"last":1.468},{"interface":"front","average":{"1min":2.2679999999999998,"5min":1.2490000000000001,"15min":1.5980000000000001},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.253},"max":{"1min":11.523999999999999,"5min":11.523999999999999,"15min":181.76300000000001},"last":4.6689999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:49:56 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":1.5669999999999999,"15min":2.137},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.27600000000000002},"max":{"1min":12.676,"5min":12.771000000000001,"15min":161.72999999999999},"last":1.4159999999999999},{"interface":"front","average":{"1min":1.355,"5min":1.403,"15min":2.0299999999999998},"min":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.27000000000000002},"max":{"1min":4.7640000000000002,"5min":9.2970000000000006,"15min":162.60900000000001},"last":1.399}]}]},{"osd":5,"up_from":489,"seq":2100239007822,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2061692,"kb_used_data":1859956,"kb_used_omap":6,"kb_used_meta":201721,"kb_avail":91683460,"statfs":{"total":95995035648,"available":93883863040,"internally_reserved":0,"allocated":1904594944,"data_stored":1902730244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":206562399},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":6.3019999999999996,"5min":4.7469999999999999,"15min":19.122},"min":{"1min":0.26100000000000001,"5min":0.22,"15min":0.22},"max":{"1min":84.328999999999994,"5min":89.805999999999997,"15min":281.10599999999999},"last":78.969999999999999},{"interface":"front","average":{"1min":7.1020000000000003,"5min":5.0570000000000004,"15min":19.417999999999999},"min":{"1min":0.29699999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":87.887,"5min":88.313999999999993,"15min":282.07400000000001},"last":79.021000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":2.9470000000000001,"15min":10.929},"min":{"1min":0.27600000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.8300000000000001,"5min":76.828999999999994,"15min":273.48200000000003},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.726,"15min":10.311999999999999},"min":{"1min":0.31,"5min":0.255,"15min":0.255},"max":{"1min":5.5700000000000003,"5min":76.805000000000007,"15min":273.50200000000001},"last":0.40999999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":6.4539999999999997,"15min":2.778},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.723000000000001,"5min":194.14400000000001,"15min":194.14400000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":6.3390000000000004,"15min":3.1400000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.7939999999999996,"5min":198.083,"15min":198.083},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":4.3819999999999997,"15min":11.292999999999999},"min":{"1min":0.39300000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.4320000000000004,"5min":98.484999999999999,"15min":274.01799999999997},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.875,"5min":4.2309999999999999,"15min":2.5649999999999999},"min":{"1min":0.375,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.4160000000000004,"5min":98.504000000000005,"15min":98.504000000000005},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":10.712,"5min":3.0449999999999999,"15min":1.649},"min":{"1min":0.29499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":131.34100000000001,"5min":131.34100000000001,"15min":131.34100000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":10.773,"5min":3.1579999999999999,"15min":1.8160000000000001},"min":{"1min":0.372,"5min":0.317,"15min":0.317},"max":{"1min":131.17500000000001,"5min":131.17500000000001,"15min":131.17500000000001},"last":0.317}]},{"osd":6,"last update":"Sat Sep 30 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.5609999999999999,"15min":1.617},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":10.565,"5min":10.565,"15min":14.371},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.508,"5min":1.6160000000000001,"15min":1.4650000000000001},"min":{"1min":0.36199999999999999,"5min":0.316,"15min":0.316},"max":{"1min":12.406000000000001,"5min":12.406000000000001,"15min":14.454000000000001},"last":0.42799999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":3.073,"5min":1.694,"15min":1.427},"min":{"1min":0.39000000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.513999999999999,"5min":10.513999999999999,"15min":10.513999999999999},"last":0.372},{"interface":"front","average":{"1min":2.2759999999999998,"5min":1.623,"15min":1.1519999999999999},"min":{"1min":0.38500000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":11.539999999999999,"5min":11.539999999999999,"15min":11.539999999999999},"last":0.46300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3352301568,"data_stored":3352297496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":976814080,"data_stored":976814080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2743664640,"data_stored":2743664640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1914109952,"data_stored":1914109952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3122663424,"data_stored":3122659352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1884553216,"data_stored":1884553216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1103892480,"data_stored":1103888408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1572208640,"data_stored":1572208640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:50:54.246 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 81 2023-09-30T09:50:54.333 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:54.355 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:54 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:50:54.355 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:54 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 17584/244143 objects degraded (7.202%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:54.407 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:54.515 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:54.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:54.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:54.540 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:54.621 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:54.694 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:54.787 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:54.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:54.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:54.895 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:54.949 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:55.023 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:55.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:55.060 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:50:55.060 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:50:55.078 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:50:55.114 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:50:55.134 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:55.151 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:50:55.181 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:50:55.221 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:55.232 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:50:55.279 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:50:55.311 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:50:55.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:55 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:50:55.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:55 smithi019 ceph-mon[216815]: pgmap v1532: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 17225/239739 objects degraded (7.185%); 2769/239739 objects misplaced (1.155%); 9.8 MiB/s, 156 objects/s recovering 2023-09-30T09:50:55.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1525484900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:55.341 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:55 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:50:55.341 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:55 smithi167 ceph-mon[192880]: pgmap v1532: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 17225/239739 objects degraded (7.185%); 2769/239739 objects misplaced (1.155%); 9.8 MiB/s, 156 objects/s recovering 2023-09-30T09:50:55.341 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1525484900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:55.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:55 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:50:55.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:55 smithi019 ceph-mon[224043]: pgmap v1532: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 17225/239739 objects degraded (7.185%); 2769/239739 objects misplaced (1.155%); 9.8 MiB/s, 156 objects/s recovering 2023-09-30T09:50:55.343 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1525484900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:50:55.351 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:50:55.364 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:50:55.364 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:50:55.364 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:50:55.365 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:50:55.365 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:50:55.365 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:50:55.365 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:50:55.365 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:50:55.384 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:55.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:55.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:55.499 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:55.566 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:55.908 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:55.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:55.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:56.022 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:56.106 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:56.229 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:50:56.265 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:56.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:56.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:56.377 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:56.410 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:56.479 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:56.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:56.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:56.583 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:56.645 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:56.761 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:56.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:56.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:56.865 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:56.886 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:50:56.934 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:57.029 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:57.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:57.133 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:57.160 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:57.231 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:57 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:50:57.231 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:57 smithi167 ceph-mon[192880]: pgmap v1533: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 16896/235725 objects degraded (7.168%); 2769/235725 objects misplaced (1.175%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:50:57.285 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:57.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:57.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:57.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:57 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:50:57.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:57 smithi019 ceph-mon[216815]: pgmap v1533: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 16896/235725 objects degraded (7.168%); 2769/235725 objects misplaced (1.175%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:50:57.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:57 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:50:57.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:57 smithi019 ceph-mon[224043]: pgmap v1533: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 16896/235725 objects degraded (7.168%); 2769/235725 objects misplaced (1.175%); 11 MiB/s, 181 objects/s recovering 2023-09-30T09:50:57.393 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:50:57.462 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:57.630 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:50:57.664 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:50:57.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:57.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:50:57.768 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:50:57.822 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:57.924 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:50:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:57.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:50:58.029 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:50:58.055 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:58.097 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:50:58.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:58.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:50:58.201 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:50:58.293 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:58.768 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:50:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:58.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:50:58.876 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:58.906 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:59.001 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:59.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:59.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:59.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:59 smithi019 ceph-mon[216815]: pgmap v1534: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 16531/232707 objects degraded (7.104%); 2769/232707 objects misplaced (1.190%); 11 MiB/s, 182 objects/s recovering 2023-09-30T09:50:59.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:59 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:50:59.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:59 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 16531/232707 objects degraded (7.104%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:59.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:50:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2373804295' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:59.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:59 smithi019 ceph-mon[224043]: pgmap v1534: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 16531/232707 objects degraded (7.104%); 2769/232707 objects misplaced (1.190%); 11 MiB/s, 182 objects/s recovering 2023-09-30T09:50:59.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:59 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:50:59.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:59 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 16531/232707 objects degraded (7.104%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:59.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:50:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2373804295' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:59.106 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:50:59.186 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:59.190 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:59 smithi167 ceph-mon[192880]: pgmap v1534: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 16531/232707 objects degraded (7.104%); 2769/232707 objects misplaced (1.190%); 11 MiB/s, 182 objects/s recovering 2023-09-30T09:50:59.190 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:59 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:50:59.190 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:59 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 16531/232707 objects degraded (7.104%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:50:59.190 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:50:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2373804295' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:50:59.319 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2291 9.f 9.c3faf36f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 394.263797168, 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99786489, 2023-09-30T09:50:59.320 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2291 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051294+0000", 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051293+0000", 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.051362+0000", 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9276999999999994e-05 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052566+0000", 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012036099999999999 2023-09-30T09:50:59.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052575+0000", 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3910000000000004e-06 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061665+0000", 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0090901720000000005 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061671+0000", 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3539999999999996e-06 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049150+0000", 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98747858099999997 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049159+0000", 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9099999999999994e-06 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2293 9.a 9.c3c9b4aa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:50:59.326 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 394.26249704899999, 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98499129799999996, 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2293 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052594+0000", 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052593+0000", 2023-09-30T09:50:59.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052645+0000", 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1387000000000001e-05 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052655+0000", 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8919999999999995e-06 2023-09-30T09:50:59.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052660+0000", 2023-09-30T09:50:59.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4809999999999997e-06 2023-09-30T09:50:59.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.059982+0000", 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0073216380000000001 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.059988+0000", 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7399999999999998e-06 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037576+0000", 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97758752599999998 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037585+0000", 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4930000000000004e-06 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.332 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2294 9.f 9.a98a737f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 394.26238832799999, 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99648747699999995, 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.333 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2294 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052703+0000", 2023-09-30T09:50:59.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052702+0000", 2023-09-30T09:50:59.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:59.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052798+0000", 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5100999999999996e-05 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052809+0000", 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1833e-05 2023-09-30T09:50:59.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052815+0000", 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.356e-06 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061690+0000", 2023-09-30T09:50:59.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088752989999999997 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061694+0000", 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0940000000000001e-06 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049184+0000", 2023-09-30T09:50:59.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98748982500000004 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049190+0000", 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4779999999999996e-06 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2296 9.f 9.2d8a7a1f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 394.26226768200002, 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99639533999999996, 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.340 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2296 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052824+0000", 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052823+0000", 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052859+0000", 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6331999999999997e-05 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052867+0000", 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8970000000000008e-06 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052873+0000", 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3149999999999999e-06 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061712+0000", 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.008839431 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.061716+0000", 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7979999999999999e-06 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049212+0000", 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.987496598 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049219+0000", 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4300000000000003e-06 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2298 9.7 9.88338c87 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 394.26221350399999, 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99525176199999998, 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.345 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2298 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052878+0000", 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052877+0000", 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052906+0000", 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8574000000000001e-05 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052912+0000", 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5150000000000003e-06 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052917+0000", 2023-09-30T09:50:59.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1230000000000002e-06 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053439+0000", 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052202900000000003 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053447+0000", 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1620000000000008e-06 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.048119+0000", 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99467239399999996 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.048130+0000", 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0394000000000001e-05 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.350 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2299 9.a 9.a4908ffa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 394.262170158, 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.984693029, 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.351 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2299 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:50:59.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052921+0000", 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052947+0000", 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6486999999999999e-05 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052954+0000", 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6640000000000001e-06 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.052958+0000", 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.617e-06 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.060007+0000", 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0070489289999999998 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.060012+0000", 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7790000000000003e-06 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037607+0000", 2023-09-30T09:50:59.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.977594764 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.037614+0000", 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2309999999999999e-06 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2303 9.f 9.7718716f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 394.25163673700001, 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98609501700000002, 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2303 2023-09-30T09:50:59.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063455+0000", 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063453+0000", 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063499+0000", 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5393999999999999e-05 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063510+0000", 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1914e-05 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.063518+0000", 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9719999999999992e-06 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064775+0000", 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001256987 2023-09-30T09:50:59.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064783+0000", 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.267e-06 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049541+0000", 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98475831199999997 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.049550+0000", 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5299999999999996e-06 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5456 9.f 9:f741c261:::benchmark_data_smithi019_560486_object5455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.637107308, 2023-09-30T09:50:59.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99827741800000003, 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5456 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677984+0000", 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.677983+0000", 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678041+0000", 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7439999999999999e-05 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678050+0000", 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2380000000000003e-06 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678055+0000", 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2480000000000004e-06 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678212+0000", 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015715 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.678221+0000", 2023-09-30T09:50:59.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2140000000000002e-06 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666463+0000", 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98824183200000004 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666571+0000", 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000107722 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.666679+0000", 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108446 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669881+0000", 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032016509999999998 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676162+0000", 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062807169999999999 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676201+0000", 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9162000000000002e-05 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676235+0000", 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4459000000000003e-05 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676261+0000", 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5897999999999998e-05 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.370 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5488 9.f 9:f5c1ce8d:::benchmark_data_smithi019_560486_object5487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.62714049599998, 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98845970699999997, 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.371 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5488 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687951+0000", 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687950+0000", 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.687988+0000", 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8231e-05 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688001+0000", 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2577e-05 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688007+0000", 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3740000000000003e-06 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688066+0000", 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8805999999999999e-05 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.688075+0000", 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7499999999999998e-06 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667057+0000", 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97898119299999997 2023-09-30T09:50:59.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667125+0000", 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7909000000000003e-05 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667219+0000", 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4759000000000005e-05 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669890+0000", 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026711130000000001 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676296+0000", 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064055370000000002 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676351+0000", 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5433999999999998e-05 2023-09-30T09:50:59.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676386+0000", 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4400000000000003e-05 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676410+0000", 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4664e-05 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.378 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5500 9.f 9:fb77b28e:::benchmark_data_smithi019_560486_object5499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.62393040799998, 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98537109199999995, 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.379 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5500 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691161+0000", 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691160+0000", 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691215+0000", 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5435e-05 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691223+0000", 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9340000000000006e-06 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691227+0000", 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7459999999999998e-06 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691295+0000", 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8145999999999994e-05 2023-09-30T09:50:59.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.691307+0000", 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1516e-05 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667479+0000", 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97617223600000003 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667613+0000", 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013427400000000001 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.667811+0000", 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00019798 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669899+0000", 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002087977 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676442+0000", 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0065432470000000003 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676474+0000", 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1275000000000001e-05 2023-09-30T09:50:59.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676512+0000", 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7997000000000002e-05 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676532+0000", 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0285999999999999e-05 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.386 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5531 9.f 9:fcece89e:::benchmark_data_smithi019_560486_object5530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.61443959899998, 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97602003800000003, 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.387 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5531 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700652+0000", 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700650+0000", 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700746+0000", 2023-09-30T09:50:59.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5857000000000001e-05 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700758+0000", 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1983e-05 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700765+0000", 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8279999999999997e-06 2023-09-30T09:50:59.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700839+0000", 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4226000000000004e-05 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.700857+0000", 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7193e-05 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668290+0000", 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96743334299999995 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668421+0000", 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013117099999999999 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668566+0000", 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014448900000000001 2023-09-30T09:50:59.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671187+0000", 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002621529 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676578+0000", 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005391051 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676610+0000", 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2039999999999998e-05 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676647+0000", 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7292000000000001e-05 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676672+0000", 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4232000000000001e-05 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5545 9.f 9:fa26aa6f:::benchmark_data_smithi019_560486_object5544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.60949275500002, 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97120855500000003, 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.395 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5545 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705598+0000", 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705597+0000", 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705693+0000", 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5401000000000004e-05 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705706+0000", 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2665e-05 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705713+0000", 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9940000000000003e-06 2023-09-30T09:50:59.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705830+0000", 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011760900000000001 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.705836+0000", 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7610000000000004e-06 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.668931+0000", 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96309502499999999 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669015+0000", 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3947000000000001e-05 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669120+0000", 2023-09-30T09:50:59.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010467900000000001 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671197+0000", 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020777399999999998 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676699+0000", 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055018530000000001 2023-09-30T09:50:59.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676739+0000", 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9949e-05 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676786+0000", 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6755999999999998e-05 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676807+0000", 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1189e-05 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5551 9.f 9:f9ae74d4:::benchmark_data_smithi019_560486_object5550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.60857769500001, 2023-09-30T09:50:59.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97040703399999995, 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5551 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:50:59.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706514+0000", 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706513+0000", 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706568+0000", 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5432000000000001e-05 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706580+0000", 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1494999999999999e-05 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706586+0000", 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7530000000000004e-06 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706645+0000", 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8591000000000002e-05 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.706653+0000", 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5250000000000005e-06 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669422+0000", 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96276874499999998 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669547+0000", 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012500799999999999 2023-09-30T09:50:59.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669633+0000", 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5400000000000002e-05 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672244+0000", 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026115729999999998 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676836+0000", 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045915579999999999 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676856+0000", 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.02e-05 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676898+0000", 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1789000000000003e-05 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676921+0000", 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2900000000000001e-05 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.411 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5571 9.f 9:f04c103d:::benchmark_data_smithi019_560486_object5570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.60242252299997, 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96437061999999996, 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5571 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712669+0000", 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.712668+0000", 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713054+0000", 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00038651499999999998 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713063+0000", 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6440000000000006e-06 2023-09-30T09:50:59.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713068+0000", 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2120000000000002e-06 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713200+0000", 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000131977 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.713210+0000", 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4250000000000004e-06 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.669945+0000", 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95673509599999995 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670055+0000", 2023-09-30T09:50:59.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000110145 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670148+0000", 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3212000000000001e-05 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.672252+0000", 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021043170000000001 2023-09-30T09:50:59.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676960+0000", 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047072759999999998 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676992+0000", 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2808000000000002e-05 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677019+0000", 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6267000000000001e-05 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677039+0000", 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0647000000000001e-05 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5581 9.f 9:fa532eaf:::benchmark_data_smithi019_560486_object5580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.599353533, 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.96139814300000004, 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.420 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5581 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715738+0000", 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715737+0000", 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715765+0000", 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8323e-05 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715780+0000", 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5475999999999999e-05 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715787+0000", 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9149999999999999e-06 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715853+0000", 2023-09-30T09:50:59.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5413000000000005e-05 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.715862+0000", 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.594e-06 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670527+0000", 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95466473799999996 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670657+0000", 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012968699999999999 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.670839+0000", 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000182461 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673403+0000", 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002564036 2023-09-30T09:50:59.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677071+0000", 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036674810000000002 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677096+0000", 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5429999999999999e-05 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677119+0000", 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3090000000000001e-05 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677136+0000", 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6702e-05 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.427 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5591 9.f 9:fb1424fd:::benchmark_data_smithi019_560486_object5590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 389.59612148600002, 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95826697900000002, 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:50:59.428 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5591 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718970+0000", 2023-09-30T09:50:59.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.718969+0000", 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719014+0000", 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4433e-05 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719031+0000", 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7078000000000001e-05 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719036+0000", 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2240000000000003e-06 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719098+0000", 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2292999999999995e-05 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:29.719107+0000", 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3319999999999992e-06 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671243+0000", 2023-09-30T09:50:59.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95213663199999998 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671330+0000", 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.6706999999999995e-05 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.671432+0000", 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010215 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.673409+0000", 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019769060000000001 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677162+0000", 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037529379999999999 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677190+0000", 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7702e-05 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677216+0000", 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6894e-05 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677237+0000", 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0253000000000001e-05 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.435 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101653 10.9 10:9320278f:::benchmark_data_smithi019_617837_object8253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.437960+0000", 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.877131606999999, 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1004105500000001, 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:59.436 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101653 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437960+0000", 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437960+0000", 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437958+0000", 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437966+0000", 2023-09-30T09:50:59.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5440000000000001e-06 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437980+0000", 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4278e-05 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437987+0000", 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7479999999999996e-06 2023-09-30T09:50:59.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.438042+0000", 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4951999999999998e-05 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.438054+0000", 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1787000000000001e-05 2023-09-30T09:50:59.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462424+0000", 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.02437044 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462572+0000", 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000147677 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462700+0000", 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128238 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538056+0000", 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.075356266000000005 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538088+0000", 2023-09-30T09:50:59.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1581000000000001e-05 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538219+0000", 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013155900000000001 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538270+0000", 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0521e-05 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538370+0000", 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100263 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101657 10.3 10:ce53bcc2:::benchmark_data_smithi019_617837_object8257:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.470044+0000", 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.845047189999999, 2023-09-30T09:50:59.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0323604369999999, 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101657 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470044+0000", 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470044+0000", 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470043+0000", 2023-09-30T09:50:59.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470050+0000", 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6620000000000005e-06 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470068+0000", 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7569999999999999e-05 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470075+0000", 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7159999999999997e-06 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470126+0000", 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.109e-05 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470136+0000", 2023-09-30T09:50:59.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0567000000000001e-05 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.448589+0000", 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97845242099999996 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.448733+0000", 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014411799999999999 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.448876+0000", 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014285599999999999 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502174+0000", 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.053298092999999998 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502182+0000", 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3720000000000005e-06 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502244+0000", 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1347999999999996e-05 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502300+0000", 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6619000000000001e-05 2023-09-30T09:50:59.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502404+0000", 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010429699999999999 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101659 10.9 10:9101f776:::benchmark_data_smithi019_617837_object8259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.471630+0000", 2023-09-30T09:50:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.843460867000001, 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.066891738, 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101659 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471630+0000", 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471630+0000", 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471629+0000", 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471636+0000", 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5359999999999998e-06 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471649+0000", 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.331e-05 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471656+0000", 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2690000000000002e-06 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.472458+0000", 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00080184699999999995 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.472474+0000", 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5597999999999999e-05 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.468844+0000", 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99637029499999996 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.468934+0000", 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9730999999999999e-05 2023-09-30T09:50:59.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.469028+0000", 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4293e-05 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538118+0000", 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.069089388000000002 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538176+0000", 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8721999999999998e-05 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538383+0000", 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00020698299999999999 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538428+0000", 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4286000000000003e-05 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538522+0000", 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4582999999999999e-05 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.460 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101667 10.9 10:92ad1a50:::benchmark_data_smithi019_617837_object8267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.489225+0000", 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 18.825866680000001, 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.049433718, 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101667 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.461 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489225+0000", 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489225+0000", 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489223+0000", 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489230+0000", 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5749999999999997e-06 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489243+0000", 2023-09-30T09:50:59.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2921999999999999e-05 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489250+0000", 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6529999999999997e-06 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489314+0000", 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4048999999999996e-05 2023-09-30T09:50:59.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489323+0000", 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1260000000000004e-06 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.471681+0000", 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.982358697 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.471824+0000", 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014217400000000001 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.471949+0000", 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000125208 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538147+0000", 2023-09-30T09:50:59.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.066198513 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538210+0000", 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3071000000000007e-05 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538529+0000", 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031860800000000001 2023-09-30T09:50:59.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538569+0000", 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9814999999999997e-05 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538658+0000", 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9487000000000006e-05 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.468 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.469 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:50:59.469 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:50:59.469 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:50:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:59.470 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:50:59.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:50:59.571 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:50:59.612 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:50:59.649 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:50:59.854 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:50:59.866 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:50:59.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:50:59.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:50:59.971 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:00.008 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:00.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:00 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:00.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2373804295' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:51:00.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:00 smithi019 ceph-mon[216815]: osdmap e575: 8 total, 8 up, 8 in 2023-09-30T09:51:00.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:00 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:00.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2373804295' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:51:00.138 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:00 smithi019 ceph-mon[224043]: osdmap e575: 8 total, 8 up, 8 in 2023-09-30T09:51:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:00 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2373804295' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:51:00.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:00 smithi167 ceph-mon[192880]: osdmap e575: 8 total, 8 up, 8 in 2023-09-30T09:51:00.321 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:00.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:00.430 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:00.479 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:00.489 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:51:01.136 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:01.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:01.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:01.166 INFO:teuthology.orchestra.run.smithi019.stderr:Error EINVAL: specified pgp_num 81 > pg_num 31 2023-09-30T09:51:01.242 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:01.289 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:51:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[224043]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[224043]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[224043]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[224043]: pgmap v1536: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 16040/227952 objects degraded (7.037%); 2552/227952 objects misplaced (1.120%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:51:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:51:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[216815]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[216815]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[216815]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:01 smithi019 ceph-mon[216815]: pgmap v1536: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 16040/227952 objects degraded (7.037%); 2552/227952 objects misplaced (1.120%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:51:01.414 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:01.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:01.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:01.424 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:01 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:51:01.424 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:01 smithi167 ceph-mon[192880]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.424 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:01 smithi167 ceph-mon[192880]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.424 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:01 smithi167 ceph-mon[192880]: failed to encode map e575 with expected crc 2023-09-30T09:51:01.424 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:01 smithi167 ceph-mon[192880]: pgmap v1536: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 16040/227952 objects degraded (7.037%); 2552/227952 objects misplaced (1.120%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:51:01.523 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:01.579 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:01.686 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:01.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:01.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:01.801 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:01.849 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:02.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:02 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3236895907' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:51:02.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:02 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:51:02.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:02 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3236895907' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:51:02.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:02 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:51:02.519 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:02.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:02 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3236895907' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:51:02.801 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:02 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:51:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:02.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:02.803 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:51:02.804 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:02.903 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:02.971 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:03.070 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:03.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:03.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:03.177 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:03.224 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:03.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:03 smithi019 ceph-mon[224043]: pgmap v1537: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 16040/227952 objects degraded (7.037%); 2552/227952 objects misplaced (1.120%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:51:03.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:03.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:03 smithi019 ceph-mon[216815]: pgmap v1537: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 16040/227952 objects degraded (7.037%); 2552/227952 objects misplaced (1.120%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:51:03.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:03.251 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:51:03.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:03 smithi167 ceph-mon[192880]: pgmap v1537: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 16040/227952 objects degraded (7.037%); 2552/227952 objects misplaced (1.120%); 12 MiB/s, 193 objects/s recovering 2023-09-30T09:51:03.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:03.458 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:03.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:03.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:03.570 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:03.676 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:03.808 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:03.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:03.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:03.922 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:04.027 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:51:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:51:03] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-09-30T09:51:04.151 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:04.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:04.266 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:04.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:04 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 16040/227952 objects degraded (7.037%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:04.338 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:04 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 16040/227952 objects degraded (7.037%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:04 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 16040/227952 objects degraded (7.037%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:05.314 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:05.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:05.422 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:05.465 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:05 smithi019 ceph-mon[216815]: pgmap v1538: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 15711/223824 objects degraded (7.019%); 2354/223824 objects misplaced (1.052%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:51:05.465 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:05 smithi019 ceph-mon[224043]: pgmap v1538: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 15711/223824 objects degraded (7.019%); 2354/223824 objects misplaced (1.052%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:51:05.471 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:05.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:05 smithi167 ceph-mon[192880]: pgmap v1538: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 15711/223824 objects degraded (7.019%); 2354/223824 objects misplaced (1.052%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:51:05.831 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:05.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:05.947 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:06.013 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:06.311 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:06.392 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:06.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:06.486 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:51:06.503 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:06.520 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:51:06.559 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:51:06.593 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:06.604 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:51:06.605 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:51:06.605 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:51:06.645 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:51:06.694 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:51:06.735 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:51:06.784 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:51:06.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:51:06.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:51:06.786 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:51:06.787 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:51:06.787 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:51:06.787 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:51:06.787 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:51:06.787 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:51:06.860 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:06.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:06.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:06.979 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:07.045 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:07.183 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:07.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:07.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:07.301 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:07 smithi019 ceph-mon[216815]: pgmap v1539: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 15165/219405 objects degraded (6.912%); 2354/219405 objects misplaced (1.073%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:51:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:07.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:07 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2904410609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:07 smithi019 ceph-mon[224043]: pgmap v1539: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 15165/219405 objects degraded (6.912%); 2354/219405 objects misplaced (1.073%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:51:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:07.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:07 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2904410609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:07.351 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:07 smithi167 ceph-mon[192880]: pgmap v1539: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 15165/219405 objects degraded (6.912%); 2354/219405 objects misplaced (1.073%); 13 MiB/s, 214 objects/s recovering 2023-09-30T09:51:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:07 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2904410609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:07.804 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:51:07.804 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:51:07.804 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.6217454014648216 2023-09-30T09:51:07.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 0.6217454014648216 2023-09-30T09:51:09.509 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:09 smithi019 ceph-mon[216815]: pgmap v1540: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14809/216228 objects degraded (6.849%); 2354/216228 objects misplaced (1.089%); 13 MiB/s, 213 objects/s recovering 2023-09-30T09:51:09.509 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 14809/216228 objects degraded (6.849%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:09.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:09 smithi019 ceph-mon[224043]: pgmap v1540: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14809/216228 objects degraded (6.849%); 2354/216228 objects misplaced (1.089%); 13 MiB/s, 213 objects/s recovering 2023-09-30T09:51:09.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 14809/216228 objects degraded (6.849%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:09 smithi167 ceph-mon[192880]: pgmap v1540: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14809/216228 objects degraded (6.849%); 2354/216228 objects misplaced (1.089%); 13 MiB/s, 213 objects/s recovering 2023-09-30T09:51:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 14809/216228 objects degraded (6.849%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:09.535 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:51:09.758 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:09.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:09.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:09.868 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:09.926 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:10.020 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:10.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:10.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:10.129 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:10.200 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:10.240 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:10.521 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:10.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:10.538 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:10.639 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:10.734 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:11.286 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:51:11.298 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:11.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:11.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:11.406 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:11.465 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:11 smithi167 ceph-mon[192880]: pgmap v1541: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 14243/210618 objects degraded (6.762%); 2110/210618 objects misplaced (1.002%); 14 MiB/s, 220 objects/s recovering 2023-09-30T09:51:11.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/559540456' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:11.564 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:11.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:11.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:11.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:11 smithi019 ceph-mon[224043]: pgmap v1541: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 14243/210618 objects degraded (6.762%); 2110/210618 objects misplaced (1.002%); 14 MiB/s, 220 objects/s recovering 2023-09-30T09:51:11.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/559540456' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:11.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:11 smithi019 ceph-mon[216815]: pgmap v1541: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 14243/210618 objects degraded (6.762%); 2110/210618 objects misplaced (1.002%); 14 MiB/s, 220 objects/s recovering 2023-09-30T09:51:11.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/559540456' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:11.676 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:11.746 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:11.839 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:11.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:11.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:11.865 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:51:11.947 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:12.034 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:12.135 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:12.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:12.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:12.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:12.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/559540456' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:12.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[216815]: osdmap e576: 8 total, 8 up, 8 in 2023-09-30T09:51:12.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[216815]: failed to encode map e576 with expected crc 2023-09-30T09:51:12.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[216815]: failed to encode map e576 with expected crc 2023-09-30T09:51:12.247 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:12.306 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:12 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/559540456' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:12 smithi167 ceph-mon[192880]: osdmap e576: 8 total, 8 up, 8 in 2023-09-30T09:51:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:12 smithi167 ceph-mon[192880]: failed to encode map e576 with expected crc 2023-09-30T09:51:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:12 smithi167 ceph-mon[192880]: failed to encode map e576 with expected crc 2023-09-30T09:51:12.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:12.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/559540456' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:12.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[224043]: osdmap e576: 8 total, 8 up, 8 in 2023-09-30T09:51:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[224043]: failed to encode map e576 with expected crc 2023-09-30T09:51:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:12 smithi019 ceph-mon[224043]: failed to encode map e576 with expected crc 2023-09-30T09:51:13.526 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:13 smithi167 ceph-mon[192880]: pgmap v1543: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 14243/210618 objects degraded (6.762%); 2110/210618 objects misplaced (1.002%); 14 MiB/s, 223 objects/s recovering 2023-09-30T09:51:13.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:13 smithi019 ceph-mon[224043]: pgmap v1543: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 14243/210618 objects degraded (6.762%); 2110/210618 objects misplaced (1.002%); 14 MiB/s, 223 objects/s recovering 2023-09-30T09:51:13.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:13 smithi019 ceph-mon[216815]: pgmap v1543: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 14243/210618 objects degraded (6.762%); 2110/210618 objects misplaced (1.002%); 14 MiB/s, 223 objects/s recovering 2023-09-30T09:51:13.772 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:13.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:13.885 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:13.950 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:14.043 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:14.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:14.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:51:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:51:13] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-09-30T09:51:14.157 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:14.194 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:14.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 14243/210618 objects degraded (6.762%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:14.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 14243/210618 objects degraded (6.762%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:14.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 14243/210618 objects degraded (6.762%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:14.789 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:14.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:14.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:14.901 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:14.965 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:15.053 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:15.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:15.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:15.080 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:15.161 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:15.250 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:15.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:15 smithi019 ceph-mon[216815]: pgmap v1544: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 13818/205743 objects degraded (6.716%); 1870/205743 objects misplaced (0.909%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:51:15.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:15 smithi019 ceph-mon[224043]: pgmap v1544: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 13818/205743 objects degraded (6.716%); 1870/205743 objects misplaced (0.909%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:51:15.668 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:15.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:15.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:15.744 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:51:15.744 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:51:15.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:15 smithi167 ceph-mon[192880]: pgmap v1544: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 13818/205743 objects degraded (6.716%); 1870/205743 objects misplaced (0.909%); 15 MiB/s, 237 objects/s recovering 2023-09-30T09:51:15.783 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:15.817 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:15.882 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:15.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:15.989 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:16.034 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:16.137 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:16.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:16.253 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:16.316 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:16.643 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.5 to 0.621745 (9f2a) 2023-09-30T09:51:16.757 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:51:17.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:16 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:16 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:16 smithi167 ceph-mon[192880]: pgmap v1545: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13194/200892 objects degraded (6.568%); 1870/200892 objects misplaced (0.931%); 15 MiB/s, 245 objects/s recovering 2023-09-30T09:51:17.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3716862490' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.6217454014648216}]: dispatch 2023-09-30T09:51:17.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:16 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:16 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:16 smithi019 ceph-mon[216815]: pgmap v1545: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13194/200892 objects degraded (6.568%); 1870/200892 objects misplaced (0.931%); 15 MiB/s, 245 objects/s recovering 2023-09-30T09:51:17.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3716862490' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.6217454014648216}]: dispatch 2023-09-30T09:51:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:16 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:16 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:16 smithi019 ceph-mon[224043]: pgmap v1545: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 108 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13194/200892 objects degraded (6.568%); 1870/200892 objects misplaced (0.931%); 15 MiB/s, 245 objects/s recovering 2023-09-30T09:51:17.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3716862490' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.6217454014648216}]: dispatch 2023-09-30T09:51:18.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:17 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3716862490' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.6217454014648216}]': finished 2023-09-30T09:51:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:17 smithi167 ceph-mon[192880]: failed to encode map e577 with expected crc 2023-09-30T09:51:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:17 smithi167 ceph-mon[192880]: osdmap e577: 8 total, 8 up, 8 in 2023-09-30T09:51:18.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:17 smithi167 ceph-mon[192880]: failed to encode map e577 with expected crc 2023-09-30T09:51:18.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:17 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3716862490' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.6217454014648216}]': finished 2023-09-30T09:51:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:17 smithi019 ceph-mon[216815]: failed to encode map e577 with expected crc 2023-09-30T09:51:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:17 smithi019 ceph-mon[216815]: osdmap e577: 8 total, 8 up, 8 in 2023-09-30T09:51:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:17 smithi019 ceph-mon[216815]: failed to encode map e577 with expected crc 2023-09-30T09:51:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:17 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3716862490' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.6217454014648216}]': finished 2023-09-30T09:51:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:17 smithi019 ceph-mon[224043]: failed to encode map e577 with expected crc 2023-09-30T09:51:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:17 smithi019 ceph-mon[224043]: osdmap e577: 8 total, 8 up, 8 in 2023-09-30T09:51:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:17 smithi019 ceph-mon[224043]: failed to encode map e577 with expected crc 2023-09-30T09:51:18.299 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:18.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:18.404 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:18.434 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:18.753 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:18.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:18.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:18.867 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:18.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: osdmap e578: 8 total, 8 up, 8 in 2023-09-30T09:51:18.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.897 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: pgmap v1548: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 105 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12826/197673 objects degraded (6.488%); 1870/197673 objects misplaced (0.946%); 15 MiB/s, 245 objects/s recovering 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:18.898 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[216815]: osdmap e579: 8 total, 8 up, 8 in 2023-09-30T09:51:18.899 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.899 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.899 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.899 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: osdmap e578: 8 total, 8 up, 8 in 2023-09-30T09:51:18.899 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.899 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.899 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.899 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: failed to encode map e578 with expected crc 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: pgmap v1548: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 105 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12826/197673 objects degraded (6.488%); 1870/197673 objects misplaced (0.946%); 15 MiB/s, 245 objects/s recovering 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:18.900 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:18 smithi019 ceph-mon[224043]: osdmap e579: 8 total, 8 up, 8 in 2023-09-30T09:51:18.901 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: osdmap e578: 8 total, 8 up, 8 in 2023-09-30T09:51:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: failed to encode map e578 with expected crc 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: pgmap v1548: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 105 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12826/197673 objects degraded (6.488%); 1870/197673 objects misplaced (0.946%); 15 MiB/s, 245 objects/s recovering 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:19.023 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:18 smithi167 ceph-mon[192880]: osdmap e579: 8 total, 8 up, 8 in 2023-09-30T09:51:19.632 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:51:19.714 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:19.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:19.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:19.818 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:19.854 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:19.891 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:20.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:19 smithi167 ceph-mon[192880]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:19 smithi167 ceph-mon[192880]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:19 smithi167 ceph-mon[192880]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:19 smithi167 ceph-mon[192880]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 13194/200892 objects degraded (6.568%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:19 smithi167 ceph-mon[192880]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:19 smithi167 ceph-mon[192880]: osdmap e580: 8 total, 8 up, 8 in 2023-09-30T09:51:20.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:19 smithi167 ceph-mon[192880]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.030 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[224043]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[224043]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[224043]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[224043]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 13194/200892 objects degraded (6.568%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:20.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[224043]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[224043]: osdmap e580: 8 total, 8 up, 8 in 2023-09-30T09:51:20.031 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[224043]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.031 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:20.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[216815]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[216815]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[216815]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[216815]: failed to encode map e579 with expected crc 2023-09-30T09:51:20.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 13194/200892 objects degraded (6.568%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:20.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[216815]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[216815]: osdmap e580: 8 total, 8 up, 8 in 2023-09-30T09:51:20.032 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:19 smithi019 ceph-mon[216815]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:20.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:20.136 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:20.171 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:20.231 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:20.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:20.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:20.335 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:51:20.336 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:20.359 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:51:20.363 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:20.407 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:20.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:20.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:20.515 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:20.548 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:20.593 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:20.605 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:20.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:20.716 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:20.743 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:20.785 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:20.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:20.890 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:20.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: osdmap e581: 8 total, 8 up, 8 in 2023-09-30T09:51:20.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.915 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: pgmap v1552: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 103 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10554/194643 objects degraded (5.422%); 1451/194643 objects misplaced (0.745%); 20 MiB/s, 321 objects/s recovering 2023-09-30T09:51:20.916 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.916 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:20 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1238635323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:20.917 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:20.961 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:20.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: osdmap e581: 8 total, 8 up, 8 in 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: pgmap v1552: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 103 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10554/194643 objects degraded (5.422%); 1451/194643 objects misplaced (0.745%); 20 MiB/s, 321 objects/s recovering 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1238635323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: failed to encode map e580 with expected crc 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: osdmap e581: 8 total, 8 up, 8 in 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: pgmap v1552: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 103 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10554/194643 objects degraded (5.422%); 1451/194643 objects misplaced (0.745%); 20 MiB/s, 321 objects/s recovering 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: failed to encode map e581 with expected crc 2023-09-30T09:51:20.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:20 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1238635323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:21.068 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:21.094 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:21.136 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:21.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:21.241 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:21.267 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:21.311 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:21.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:21.416 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:21.443 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:21.486 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:21.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:21.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:21.591 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:21.623 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:21.972 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:21.973 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:21 smithi019 ceph-mon[216815]: failed to encode map e581 with expected crc 2023-09-30T09:51:21.974 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:21 smithi019 ceph-mon[216815]: failed to encode map e581 with expected crc 2023-09-30T09:51:21.974 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:21 smithi019 ceph-mon[216815]: failed to encode map e581 with expected crc 2023-09-30T09:51:21.974 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:21 smithi019 ceph-mon[224043]: failed to encode map e581 with expected crc 2023-09-30T09:51:21.974 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:21 smithi019 ceph-mon[224043]: failed to encode map e581 with expected crc 2023-09-30T09:51:21.974 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:21 smithi019 ceph-mon[224043]: failed to encode map e581 with expected crc 2023-09-30T09:51:21.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:21.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:22.021 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:21 smithi167 ceph-mon[192880]: failed to encode map e581 with expected crc 2023-09-30T09:51:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:21 smithi167 ceph-mon[192880]: failed to encode map e581 with expected crc 2023-09-30T09:51:22.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:21 smithi167 ceph-mon[192880]: failed to encode map e581 with expected crc 2023-09-30T09:51:22.076 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:22.102 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:22.144 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:22.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:22.249 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:22.276 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:22.320 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:22.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:22.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:22.427 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:22.465 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:22.723 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:22.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:22.828 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:22.860 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:22.917 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:22.919 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:22 smithi019 ceph-mon[224043]: pgmap v1553: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 103 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 10554/194643 objects degraded (5.422%); 1451/194643 objects misplaced (0.745%); 10 MiB/s, 164 objects/s recovering 2023-09-30T09:51:22.920 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:22 smithi019 ceph-mon[216815]: pgmap v1553: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 103 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 10554/194643 objects degraded (5.422%); 1451/194643 objects misplaced (0.745%); 10 MiB/s, 164 objects/s recovering 2023-09-30T09:51:22.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:22.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:23.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:22 smithi167 ceph-mon[192880]: pgmap v1553: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 103 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 10554/194643 objects degraded (5.422%); 1451/194643 objects misplaced (0.745%); 10 MiB/s, 164 objects/s recovering 2023-09-30T09:51:23.023 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:23.049 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:23.093 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:23.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:23.201 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:23.227 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:23.269 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:23.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:23.376 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:23.402 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:23.449 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:23.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:23.555 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:23.586 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:23.659 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:51:23.834 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:51:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:51:23] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.43.0" 2023-09-30T09:51:23.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3923772749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:51:23.862 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3923772749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:51:23.944 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:23.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:23.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:24.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3923772749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:51:24.048 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:24.074 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:24.118 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:24.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:24.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:24.224 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:24.255 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:24.731 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:51:24.731 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:51:24.731 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-30T09:51:24.732 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:51:24.732 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:51:24.732 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-30T09:51:24.946 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:24.947 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:24.947 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3923772749' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:51:24.947 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[216815]: failed to encode map e582 with expected crc 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[216815]: osdmap e582: 8 total, 8 up, 8 in 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 10554/194643 objects degraded (5.422%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[216815]: pgmap v1555: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 14598/194238 objects degraded (7.516%); 744/194238 objects misplaced (0.383%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3923772749' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[224043]: failed to encode map e582 with expected crc 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[224043]: osdmap e582: 8 total, 8 up, 8 in 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 10554/194643 objects degraded (5.422%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:24.948 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:24 smithi019 ceph-mon[224043]: pgmap v1555: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 14598/194238 objects degraded (7.516%); 744/194238 objects misplaced (0.383%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:51:24.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:24.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:24 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:24 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3923772749' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:51:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:24 smithi167 ceph-mon[192880]: failed to encode map e582 with expected crc 2023-09-30T09:51:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:24 smithi167 ceph-mon[192880]: osdmap e582: 8 total, 8 up, 8 in 2023-09-30T09:51:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 10554/194643 objects degraded (5.422%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:25.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:24 smithi167 ceph-mon[192880]: pgmap v1555: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 14598/194238 objects degraded (7.516%); 744/194238 objects misplaced (0.383%); 11 MiB/s, 180 objects/s recovering 2023-09-30T09:51:25.050 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:25.086 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:25.096 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:51:25.172 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:25.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:25.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:25.276 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:25.302 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:25.350 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:25.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:25.453 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:25.487 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:25.541 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:126957 8.11 8:8c2c7c44:::benchmark_data_smithi019_478679_object34509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 577.62414745900003, 2023-09-30T09:51:25.546 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8430575170000001, 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 126957 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:51:25.547 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914652+0000", 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914653+0000", 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2550000000000001e-06 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.548 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914659+0000", 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5400000000000001e-06 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914676+0000", 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6543999999999999e-05 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914685+0000", 2023-09-30T09:51:25.549 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5109999999999992e-06 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914770+0000", 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4579000000000005e-05 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.914933+0000", 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016312100000000001 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.550 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.915080+0000", 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000146699 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.917700+0000", 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026206659999999998 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:47.919771+0000", 2023-09-30T09:51:25.551 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002070985 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757615+0000", 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.837843213 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757648+0000", 2023-09-30T09:51:25.552 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.3256000000000001e-05 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:41:49.757709+0000", 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1148000000000005e-05 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.553 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136440 8.19 8:9e052cf0:::benchmark_data_smithi019_478679_object43992:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 544.11029431300005, 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8141369350000001, 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136440 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.554 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.555 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428505+0000", 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6399999999999998e-07 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428507+0000", 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0959999999999999e-06 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428513+0000", 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1639999999999998e-06 2023-09-30T09:51:25.556 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428519+0000", 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.541e-06 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429278+0000", 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00075965099999999997 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.557 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429285+0000", 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.618e-06 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.195903+0000", 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7666181060000001 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.196239+0000", 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033550600000000001 2023-09-30T09:51:25.558 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.196875+0000", 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000636519 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.199780+0000", 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002904883 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.559 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208312+0000", 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0085319590000000004 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242090+0000", 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.033777612999999998 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242261+0000", 2023-09-30T09:51:25.560 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000171507 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242642+0000", 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00038040799999999999 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136457 8.19 8:9a6b3ff1:::benchmark_data_smithi019_478679_object44009:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:51:25.561 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 544.11027342900002, 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.814734099, 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136457 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.562 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428526+0000", 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9700000000000002e-07 2023-09-30T09:51:25.563 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428528+0000", 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.905e-06 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428552+0000", 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.366e-05 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.564 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428558+0000", 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1160000000000004e-06 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429324+0000", 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00076669300000000004 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429333+0000", 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1120000000000002e-06 2023-09-30T09:51:25.565 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.199884+0000", 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7705505289999999 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.200283+0000", 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00039902299999999998 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.566 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.200894+0000", 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00061094000000000003 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208335+0000", 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0074407869999999999 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208590+0000", 2023-09-30T09:51:25.567 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025573000000000002 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242752+0000", 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.034161527999999997 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.242903+0000", 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000150569 2023-09-30T09:51:25.568 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243260+0000", 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035711 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:136497 8.19 8:9ab7e4b6:::benchmark_data_smithi019_478679_object44049:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e441)", 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:51:25.569 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 544.11023212299995, 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.815323341, 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 136497 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.570 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.571 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428567+0000", 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3499999999999996e-07 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428570+0000", 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.548e-06 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428577+0000", 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3159999999999999e-06 2023-09-30T09:51:25.572 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.428583+0000", 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.592e-06 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429357+0000", 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00077461499999999998 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.573 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:21.429365+0000", 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4390000000000003e-06 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.202754+0000", 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7733887419999999 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.203035+0000", 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00028132800000000001 2023-09-30T09:51:25.574 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4", 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.203627+0000", 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00059232499999999999 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.208358+0000", 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004730845 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.575 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.211841+0000", 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034829769999999999 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243353+0000", 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031512327 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243486+0000", 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013215899999999999 2023-09-30T09:51:25.576 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:42:23.243890+0000", 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00040459300000000002 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153596 8.6 8:63ca80c0:::benchmark_data_smithi019_478679_object61148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:51:25.577 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 503.533348359, 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2178367450000001, 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153596 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.578 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005451+0000", 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005452+0000", 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-06 2023-09-30T09:51:25.579 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005458+0000", 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0240000000000003e-06 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005476+0000", 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7638999999999998e-05 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.580 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005484+0000", 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6649999999999992e-06 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005563+0000", 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8406999999999997e-05 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.005575+0000", 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2533e-05 2023-09-30T09:51:25.581 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961360+0000", 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.955784462 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961374+0000", 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4756999999999999e-05 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.582 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197062+0000", 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2356875270000001 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197198+0000", 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013569999999999999 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197626+0000", 2023-09-30T09:51:25.583 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00042856900000000001 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.202382+0000", 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0047554479999999998 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.222890+0000", 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020507803000000002 2023-09-30T09:51:25.584 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.222938+0000", 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8897999999999998e-05 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223015+0000", 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6330999999999998e-05 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.585 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223287+0000", 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027260099999999997 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.35820.0:153617 8.6 8:671a25b8:::benchmark_data_smithi019_478679_object61169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 503.45945707599998, 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1478323110000002, 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.586 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.35820", 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 153617 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.587 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079342+0000", 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079343+0000", 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0410000000000001e-06 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079347+0000", 2023-09-30T09:51:25.588 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8829999999999999e-06 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079359+0000", 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1719000000000001e-05 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079364+0000", 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8889999999999999e-06 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.589 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079445+0000", 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0644999999999998e-05 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.079460+0000", 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5247e-05 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.590 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961403+0000", 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.88194313400000002 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:02.961413+0000", 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9450000000000005e-06 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.199966+0000", 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2385522090000001 2023-09-30T09:51:25.591 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.200059+0000", 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3688000000000002e-05 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 6,7", 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.200467+0000", 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000408093 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.592 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223502+0000", 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.023034289999999999 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.223531+0000", 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9584e-05 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.226902+0000", 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033704429999999999 2023-09-30T09:51:25.593 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.226956+0000", 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4129000000000003e-05 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.227174+0000", 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021837199999999999 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.594 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2292 9.8 9:1a1fa048:::benchmark_data_smithi019_560486_object2291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 420.496818238, 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.046112114, 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:51:25.595 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2292 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041981+0000", 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.596 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.041982+0000", 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.375e-06 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042038+0000", 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6124e-05 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.597 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042052+0000", 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3366e-05 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.042061+0000", 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9039999999999998e-06 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.043012+0000", 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00095105899999999998 2023-09-30T09:51:25.598 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.043037+0000", 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5621000000000001e-05 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046269+0000", 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.003231505 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.599 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046280+0000", 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1532000000000001e-05 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080018+0000", 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0337373249999999 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080180+0000", 2023-09-30T09:51:25.600 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000162268 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080377+0000", 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000197295 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080973+0000", 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000596292 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.601 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081788+0000", 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00081490800000000004 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087843+0000", 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0060543369999999999 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.602 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.087903+0000", 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9904000000000001e-05 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.088093+0000", 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000190299 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.603 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2304 9.5 9:a61d12ad:::benchmark_data_smithi019_560486_object2303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 420.48444223500002, 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0366920450000001, 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2304 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.604 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054357+0000", 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054358+0000", 2023-09-30T09:51:25.605 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.497e-06 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054413+0000", 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.486e-05 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054425+0000", 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.221e-05 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.606 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.054436+0000", 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0822000000000001e-05 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.055599+0000", 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001163182 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.607 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.055607+0000", 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2219999999999996e-06 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.039370+0000", 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98376339800000001 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.039387+0000", 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6816999999999998e-05 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.608 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081036+0000", 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0416488429999999 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081162+0000", 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012657 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:51:25.609 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081327+0000", 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000165043 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081830+0000", 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00050259000000000002 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082960+0000", 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001130147 2023-09-30T09:51:25.610 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090727+0000", 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0077666380000000002 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090794+0000", 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7787999999999995e-05 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.611 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.091049+0000", 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025441800000000002 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2309 9.8 9:1c660290:::benchmark_data_smithi019_560486_object2308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 420.46527392899998, 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0171991079999998, 2023-09-30T09:51:25.612 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2309 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.613 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073525+0000", 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073526+0000", 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-06 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.614 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073572+0000", 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5779999999999999e-05 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073585+0000", 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2870999999999999e-05 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073592+0000", 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5150000000000003e-06 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.615 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073661+0000", 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9664000000000003e-05 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073675+0000", 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.381e-05 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.616 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046310+0000", 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.972634836 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.046319+0000", 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.329e-06 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081008+0000", 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.034688308 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.617 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081125+0000", 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011733199999999999 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,5", 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081251+0000", 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012646599999999999 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.618 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081666+0000", 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00041515399999999999 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082565+0000", 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00089839100000000001 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090410+0000", 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.007844976 2023-09-30T09:51:25.619 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090481+0000", 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1354999999999997e-05 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090724+0000", 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024294 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.620 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2312 9.9 9:939ceb45:::benchmark_data_smithi019_560486_object2311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 420.45928620000001, 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0117532649999998, 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:51:25.621 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2312 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079513+0000", 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.622 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079514+0000", 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1200000000000002e-07 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079550+0000", 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6514000000000003e-05 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.623 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079562+0000", 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2275e-05 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079568+0000", 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0109999999999997e-06 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079619+0000", 2023-09-30T09:51:25.624 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0250999999999997e-05 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.079631+0000", 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2756e-05 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043823+0000", 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96419190499999996 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.625 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.043838+0000", 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4323e-05 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080176+0000", 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.036337939 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.626 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080316+0000", 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014040999999999999 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.080506+0000", 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019016399999999999 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.081229+0000", 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00072260300000000005 2023-09-30T09:51:25.627 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.082898+0000", 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001669366 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090790+0000", 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0078920009999999992 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.628 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.090859+0000", 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9210999999999999e-05 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.091266+0000", 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00040672400000000002 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.629 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101646 10.5 10:a3530dcc:::benchmark_data_smithi019_617837_object8246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e568)", 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.359483+0000", 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 45.179315660999997, 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1663677429999999, 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.630 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101646 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359483+0000", 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359483+0000", 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.631 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359485+0000", 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.751e-06 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359493+0000", 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5089999999999996e-06 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.632 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359507+0000", 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4337e-05 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359517+0000", 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.008e-05 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359567+0000", 2023-09-30T09:51:25.633 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9761e-05 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359755+0000", 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018860799999999999 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.359945+0000", 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018958199999999999 2023-09-30T09:51:25.634 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.362846+0000", 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002900695 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.368927+0000", 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0060816459999999996 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.635 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.391300+0000", 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022373113 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.391314+0000", 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3638000000000001e-05 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.438484+0000", 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.047169961 2023-09-30T09:51:25.636 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.438617+0000", 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013339100000000001 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525783+0000", 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.087165859999999998 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.637 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525851+0000", 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7811000000000005e-05 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101660 10.5 10:ae885e05:::benchmark_data_smithi019_617837_object8260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.472658+0000", 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 45.066141424999998, 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.05209475, 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.638 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101660 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.472658+0000", 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.639 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.472658+0000", 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.472659+0000", 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0899999999999999e-06 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.472664+0000", 2023-09-30T09:51:25.640 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8949999999999997e-06 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.472679+0000", 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5665e-05 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.472687+0000", 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5190000000000003e-06 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.641 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.472758+0000", 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0845000000000004e-05 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.472773+0000", 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5174000000000001e-05 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.642 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.438855+0000", 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96608240700000003 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.438991+0000", 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013546700000000001 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.439143+0000", 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015272199999999999 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.643 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.452829+0000", 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013685635999999999 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524497+0000", 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.071668397999999994 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.644 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524545+0000", 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7976000000000003e-05 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524615+0000", 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9369000000000006e-05 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524752+0000", 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000137587 2023-09-30T09:51:25.645 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101680 10.5 10:afcb67f2:::benchmark_data_smithi019_617837_object8280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.509048+0000", 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 45.029751249999997, 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0159291580000001, 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.646 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101680 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.509048+0000", 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.509048+0000", 2023-09-30T09:51:25.647 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.509049+0000", 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4300000000000002e-07 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.509053+0000", 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6720000000000003e-06 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.648 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.509065+0000", 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2254e-05 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.509074+0000", 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9320000000000002e-06 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.509137+0000", 2023-09-30T09:51:25.649 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2748000000000003e-05 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.509147+0000", 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5440000000000004e-06 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.442591+0000", 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.93344432600000005 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.650 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.442748+0000", 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000156467 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.442903+0000", 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000155501 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.452838+0000", 2023-09-30T09:51:25.651 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0099351040000000002 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524785+0000", 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.071947285999999999 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524821+0000", 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5991999999999997e-05 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.652 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524874+0000", 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2459e-05 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524977+0000", 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010302999999999999 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.653 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101689 10.8 10:1e24280e:::benchmark_data_smithi019_617837_object8289:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.530831+0000", 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 45.007968261000002, 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.00023745, 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101689 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.654 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530831+0000", 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530831+0000", 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.655 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530831+0000", 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7700000000000004e-07 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530835+0000", 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7730000000000001e-06 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530851+0000", 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5288999999999999e-05 2023-09-30T09:51:25.656 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530865+0000", 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4361e-05 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530938+0000", 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2731999999999994e-05 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.657 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530958+0000", 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0298000000000001e-05 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.455920+0000", 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.92496215299999995 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.456076+0000", 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015613700000000001 2023-09-30T09:51:25.658 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.456243+0000", 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016672200000000001 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.463256+0000", 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0070135090000000002 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.659 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.530824+0000", 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.067567664999999999 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.530869+0000", 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5238000000000003e-05 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.530941+0000", 2023-09-30T09:51:25.660 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1489999999999995e-05 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.531068+0000", 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012730599999999999 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101690 10.5 10:a20ee7ee:::benchmark_data_smithi019_617837_object8290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:51:25.661 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.530913+0000", 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 45.007885731000002, 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99434213400000004, 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101690 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.662 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530913+0000", 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530913+0000", 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530914+0000", 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.82e-07 2023-09-30T09:51:25.663 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530917+0000", 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6749999999999998e-06 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530924+0000", 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2389999999999996e-06 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.664 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530929+0000", 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6169999999999998e-06 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530979+0000", 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9313e-05 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.530990+0000", 2023-09-30T09:51:25.665 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0919e-05 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.446246+0000", 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.91525603700000002 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.446402+0000", 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015594099999999999 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.666 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.446576+0000", 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017433100000000001 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.452845+0000", 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0062690180000000003 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.667 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525016+0000", 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.072171468000000003 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525044+0000", 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7195e-05 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525100+0000", 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5928999999999999e-05 2023-09-30T09:51:25.668 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525255+0000", 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015587000000000001 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101696 10.5 10:a05b36af:::benchmark_data_smithi019_617837_object8296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.547780+0000", 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 44.991018738000001, 2023-09-30T09:51:25.669 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97768454800000004, 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101696 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.670 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.547780+0000", 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.547780+0000", 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.547781+0000", 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9999999999999996e-07 2023-09-30T09:51:25.671 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.547785+0000", 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9169999999999999e-06 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.547799+0000", 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3653e-05 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.672 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.547806+0000", 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0700000000000001e-06 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.547869+0000", 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3452000000000002e-05 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.547878+0000", 2023-09-30T09:51:25.673 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6479999999999992e-06 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.449785+0000", 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90190708900000005 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.449930+0000", 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000144878 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.674 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.450024+0000", 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4588999999999998e-05 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.524437+0000", 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.074412412999999997 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.675 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525283+0000", 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00084636399999999995 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525305+0000", 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1837000000000001e-05 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525355+0000", 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9758e-05 2023-09-30T09:51:25.676 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.525465+0000", 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011008 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:114192 10.a 10:577d93c2:::benchmark_data_smithi019_617837_object20792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e575)", 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:04.326242+0000", 2023-09-30T09:51:25.677 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 21.212556902999999, 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.34964364999999997, 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 114192 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326242+0000", 2023-09-30T09:51:25.678 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326242+0000", 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326244+0000", 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.057e-06 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.679 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326252+0000", 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0909999999999993e-06 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326266+0000", 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4022000000000001e-05 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326275+0000", 2023-09-30T09:51:25.680 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0559999999999994e-06 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.329525+0000", 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0032494519999999999 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.330435+0000", 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000910647 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.681 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.342672+0000", 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012236593 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.342738+0000", 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5621000000000001e-05 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:51:25.682 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.342827+0000", 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9174999999999999e-05 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.346555+0000", 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0037284750000000002 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.349427+0000", 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002872 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.683 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.673566+0000", 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32413839799999999 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.673629+0000", 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3324999999999994e-05 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.684 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.675886+0000", 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022567379999999999 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:114195 10.b 10:d2e4e41e:::benchmark_data_smithi019_617837_object20795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e575)", 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:04.326295+0000", 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 21.212503623, 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.34237197400000002, 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.685 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 114195 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326295+0000", 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.686 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326295+0000", 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326296+0000", 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4700000000000002e-07 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.687 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326299+0000", 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1420000000000002e-06 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326305+0000", 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9970000000000004e-06 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326310+0000", 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7960000000000003e-06 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.688 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.326371+0000", 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1328999999999995e-05 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.327045+0000", 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00067362200000000002 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.689 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.343112+0000", 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016067501000000001 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.343217+0000", 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000105287 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.343324+0000", 2023-09-30T09:51:25.690 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000106037 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.346912+0000", 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035885890000000001 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.346931+0000", 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8961e-05 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.691 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.666501+0000", 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31956994100000002 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.666566+0000", 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4751000000000004e-05 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.668667+0000", 2023-09-30T09:51:25.692 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0021015740000000002 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:114204 10.a 10:5922638e:::benchmark_data_smithi019_617837_object20804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e575)", 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:04.330688+0000", 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 21.208110967, 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.347448706, 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.693 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 114204 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.330688+0000", 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.694 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.330688+0000", 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.330689+0000", 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1699999999999998e-07 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.695 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.330691+0000", 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7030000000000002e-06 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.330699+0000", 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.79e-06 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.330705+0000", 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0700000000000003e-06 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.334130+0000", 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003425174 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.334862+0000", 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000731345 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.697 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.669871+0000", 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33500892500000001 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.669954+0000", 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3934e-05 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.670083+0000", 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000128966 2023-09-30T09:51:25.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.676006+0000", 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005922386 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.676023+0000", 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7354e-05 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.699 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.676069+0000", 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5615000000000003e-05 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.676124+0000", 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5506e-05 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.678137+0000", 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020124209999999999 2023-09-30T09:51:25.700 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:114216 10.a 10:5190b8e2:::benchmark_data_smithi019_617837_object20816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e575)", 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:04.351390+0000", 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 21.187408909999998, 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.350556284, 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:51:25.701 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 114216 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.351390+0000", 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.351390+0000", 2023-09-30T09:51:25.702 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.351391+0000", 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3499999999999996e-07 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.351394+0000", 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0900000000000001e-06 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.703 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.351402+0000", 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.371e-06 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.351410+0000", 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8599999999999993e-06 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.678177+0000", 2023-09-30T09:51:25.704 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32676666700000001 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.680570+0000", 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023931899999999999 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.691752+0000", 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011182196 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.705 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.691844+0000", 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1687000000000005e-05 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.691931+0000", 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7472000000000005e-05 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.706 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.695966+0000", 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040350580000000002 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.695980+0000", 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3859999999999999e-05 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.700488+0000", 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0045076589999999998 2023-09-30T09:51:25.707 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.700527+0000", 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9721000000000001e-05 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:04.701946+0000", 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0014189179999999999 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.708 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.709 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:51:25.709 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:51:25.709 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:51:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:25.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:25.810 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:25.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:25 smithi019 ceph-mon[216815]: failed to encode map e582 with expected crc 2023-09-30T09:51:25.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:25 smithi019 ceph-mon[224043]: failed to encode map e582 with expected crc 2023-09-30T09:51:25.836 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:25 smithi167 ceph-mon[192880]: failed to encode map e582 with expected crc 2023-09-30T09:51:25.839 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:25.885 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:25.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:25.989 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:26.023 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:26.073 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:26.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:26.177 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:26.204 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:26.247 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:26.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:26.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:26.350 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:26.380 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:26.432 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:26.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:26.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:26.536 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:26.566 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:26.657 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:26 smithi019 ceph-mon[216815]: pgmap v1556: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 14577/194076 objects degraded (7.511%); 744/194076 objects misplaced (0.383%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:51:26.852 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:26.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:26.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:26 smithi019 ceph-mon[224043]: pgmap v1556: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 14577/194076 objects degraded (7.511%); 744/194076 objects misplaced (0.383%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:51:26.921 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:51:26.956 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:26.980 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:26 smithi167 ceph-mon[192880]: pgmap v1556: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 14577/194076 objects degraded (7.511%); 744/194076 objects misplaced (0.383%); 8.6 MiB/s, 138 objects/s recovering 2023-09-30T09:51:26.983 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:27.026 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:27.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:27.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:27.086 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:51:27.129 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:27.167 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:27.237 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:27.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:27.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:27.341 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:27.375 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:27.381 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:27.460 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:27.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:27.564 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:27.590 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:27.632 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:27.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:27.736 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:27.760 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:27.801 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:27.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:27.832 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:27.904 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:27.937 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:27.994 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:27.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:28.099 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:28.125 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:28.169 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:28.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:28.272 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:28.305 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:28.925 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:28.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:28.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:29.029 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:51:29.029 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-30T09:51:29.029 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:51:29.030 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:29.061 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:29.212 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:29.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:29.315 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:29.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:29 smithi019 ceph-mon[216815]: pgmap v1557: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 14577/194076 objects degraded (7.511%); 12 MiB/s, 195 objects/s recovering 2023-09-30T09:51:29.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:29 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 14577/194076 objects degraded (7.511%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:29.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:29 smithi019 ceph-mon[216815]: failed to encode map e583 with expected crc 2023-09-30T09:51:29.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:29 smithi019 ceph-mon[216815]: osdmap e583: 8 total, 8 up, 8 in 2023-09-30T09:51:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:29 smithi019 ceph-mon[224043]: pgmap v1557: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 14577/194076 objects degraded (7.511%); 12 MiB/s, 195 objects/s recovering 2023-09-30T09:51:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:29 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 14577/194076 objects degraded (7.511%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:29 smithi019 ceph-mon[224043]: failed to encode map e583 with expected crc 2023-09-30T09:51:29.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:29 smithi019 ceph-mon[224043]: osdmap e583: 8 total, 8 up, 8 in 2023-09-30T09:51:29.359 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:29 smithi167 ceph-mon[192880]: pgmap v1557: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 106 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 14577/194076 objects degraded (7.511%); 12 MiB/s, 195 objects/s recovering 2023-09-30T09:51:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:29 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 14577/194076 objects degraded (7.511%), 13 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:29 smithi167 ceph-mon[192880]: failed to encode map e583 with expected crc 2023-09-30T09:51:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:29 smithi167 ceph-mon[192880]: osdmap e583: 8 total, 8 up, 8 in 2023-09-30T09:51:29.569 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:29.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:29.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:29.673 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:29.702 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:29.882 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:29.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:29.986 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:30.014 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:30.058 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:30.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:30.163 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:30.198 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:30.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:30 smithi167 ceph-mon[192880]: failed to encode map e583 with expected crc 2023-09-30T09:51:30.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:30 smithi167 ceph-mon[192880]: failed to encode map e583 with expected crc 2023-09-30T09:51:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:30 smithi167 ceph-mon[192880]: failed to encode map e584 with expected crc 2023-09-30T09:51:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:30 smithi167 ceph-mon[192880]: osdmap e584: 8 total, 8 up, 8 in 2023-09-30T09:51:30.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:30 smithi167 ceph-mon[192880]: failed to encode map e584 with expected crc 2023-09-30T09:51:30.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[216815]: failed to encode map e583 with expected crc 2023-09-30T09:51:30.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[216815]: failed to encode map e583 with expected crc 2023-09-30T09:51:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[216815]: failed to encode map e584 with expected crc 2023-09-30T09:51:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[216815]: osdmap e584: 8 total, 8 up, 8 in 2023-09-30T09:51:30.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[216815]: failed to encode map e584 with expected crc 2023-09-30T09:51:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[224043]: failed to encode map e583 with expected crc 2023-09-30T09:51:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[224043]: failed to encode map e583 with expected crc 2023-09-30T09:51:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[224043]: failed to encode map e584 with expected crc 2023-09-30T09:51:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[224043]: osdmap e584: 8 total, 8 up, 8 in 2023-09-30T09:51:30.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:30 smithi019 ceph-mon[224043]: failed to encode map e584 with expected crc 2023-09-30T09:51:30.513 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:30.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:30.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:30.618 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:30.645 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:30.689 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:30.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:30.746 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:51:30.746 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:51:30.796 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:30.830 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:31 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:51:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:31 smithi019 ceph-mon[216815]: failed to encode map e584 with expected crc 2023-09-30T09:51:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:31 smithi019 ceph-mon[216815]: pgmap v1560: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 14186/194076 objects degraded (7.310%); 7.5 MiB/s, 157 objects/s recovering 2023-09-30T09:51:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:31 smithi019 ceph-mon[216815]: failed to encode map e584 with expected crc 2023-09-30T09:51:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:31 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:51:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:31 smithi019 ceph-mon[224043]: failed to encode map e584 with expected crc 2023-09-30T09:51:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:31 smithi019 ceph-mon[224043]: pgmap v1560: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 14186/194076 objects degraded (7.310%); 7.5 MiB/s, 157 objects/s recovering 2023-09-30T09:51:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:31 smithi019 ceph-mon[224043]: failed to encode map e584 with expected crc 2023-09-30T09:51:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:31 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:51:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:31 smithi167 ceph-mon[192880]: failed to encode map e584 with expected crc 2023-09-30T09:51:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:31 smithi167 ceph-mon[192880]: pgmap v1560: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 14186/194076 objects degraded (7.310%); 7.5 MiB/s, 157 objects/s recovering 2023-09-30T09:51:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:31 smithi167 ceph-mon[192880]: failed to encode map e584 with expected crc 2023-09-30T09:51:31.642 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:31.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:31.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:31.746 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:31.773 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:31.818 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:31.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:31.923 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:31.950 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:31.992 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:32.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:32.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:32.069 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[224043]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.070 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[216815]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.070 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[216815]: osdmap e585: 8 total, 8 up, 8 in 2023-09-30T09:51:32.070 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[216815]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.070 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[216815]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.070 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2396378475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:32.074 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:51:32.101 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:32.131 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:32.173 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:32.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:32.278 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:32.304 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:32.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[224043]: osdmap e585: 8 total, 8 up, 8 in 2023-09-30T09:51:32.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[224043]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[224043]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2396378475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:32.348 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:32.352 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:32 smithi167 ceph-mon[192880]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.352 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:32 smithi167 ceph-mon[192880]: osdmap e585: 8 total, 8 up, 8 in 2023-09-30T09:51:32.352 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:32 smithi167 ceph-mon[192880]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.352 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:32 smithi167 ceph-mon[192880]: failed to encode map e585 with expected crc 2023-09-30T09:51:32.353 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2396378475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:32.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:32.366 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:51:32.454 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:32.481 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:32.524 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:32.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:32.629 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:32.656 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:32.699 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:32.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:32.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:32.805 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:32.831 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:32.875 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:32.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:32.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:32.982 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:33.014 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:33.257 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:33.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:33.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:33.291 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:33.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:51:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: pgmap v1562: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14186/194076 objects degraded (7.310%); 7.7 MiB/s, 163 objects/s recovering 2023-09-30T09:51:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:51:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2396378475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: failed to encode map e586 with expected crc 2023-09-30T09:51:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: failed to encode map e586 with expected crc 2023-09-30T09:51:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: osdmap e586: 8 total, 8 up, 8 in 2023-09-30T09:51:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: pgmap v1562: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14186/194076 objects degraded (7.310%); 7.7 MiB/s, 163 objects/s recovering 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2396378475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: failed to encode map e586 with expected crc 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: failed to encode map e586 with expected crc 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: osdmap e586: 8 total, 8 up, 8 in 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:33.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:33.363 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:33.387 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:51:33.387 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: pgmap v1562: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14186/194076 objects degraded (7.310%); 7.7 MiB/s, 163 objects/s recovering 2023-09-30T09:51:33.387 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:51:33.387 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2396378475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:33.387 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: failed to encode map e586 with expected crc 2023-09-30T09:51:33.387 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: failed to encode map e586 with expected crc 2023-09-30T09:51:33.388 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: osdmap e586: 8 total, 8 up, 8 in 2023-09-30T09:51:33.388 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:33.388 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:33.389 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:33.434 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:33.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:33.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:33.503 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:51:33.527 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:51:33.542 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:33.553 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:51:33.577 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:33.578 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:51:33.602 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:51:33.627 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:51:33.628 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:51:33.628 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:51:33.628 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:51:33.629 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:51:33.629 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:51:33.629 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:51:33.629 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:51:33.629 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:51:33.941 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:33.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:33.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:34.045 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:34.070 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:51:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:51:33] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-09-30T09:51:34.071 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:34 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:51:34.081 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:34.183 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:34.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:34.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:34.288 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:34.314 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:34.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:34 smithi019 ceph-mon[216815]: failed to encode map e586 with expected crc 2023-09-30T09:51:34.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:34 smithi019 ceph-mon[216815]: failed to encode map e586 with expected crc 2023-09-30T09:51:34.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:34 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 14186/194076 objects degraded (7.310%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:34 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:51:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:34 smithi019 ceph-mon[224043]: failed to encode map e586 with expected crc 2023-09-30T09:51:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:34 smithi019 ceph-mon[224043]: failed to encode map e586 with expected crc 2023-09-30T09:51:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:34 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 14186/194076 objects degraded (7.310%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:34.358 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:34.363 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:34 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:51:34.363 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:34 smithi167 ceph-mon[192880]: failed to encode map e586 with expected crc 2023-09-30T09:51:34.363 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:34 smithi167 ceph-mon[192880]: failed to encode map e586 with expected crc 2023-09-30T09:51:34.364 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:34 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 14186/194076 objects degraded (7.310%), 11 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:34.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:34.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:34.465 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:34.492 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:34.535 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:34.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:34.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:34.640 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:34.681 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:34.948 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:51:34.949 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:51:34.949 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-30T09:51:34.949 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-30T09:51:34.993 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:34.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:34.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:35.097 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:35.143 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:35.218 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:51:35.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:35 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:51:35.252 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:35 smithi019 ceph-mon[224043]: pgmap v1564: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 18034/193803 objects degraded (9.305%); 1412/193803 objects misplaced (0.729%); 6.4 MiB/s, 166 objects/s recovering 2023-09-30T09:51:35.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:35 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:51:35.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:35 smithi019 ceph-mon[216815]: pgmap v1564: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 18034/193803 objects degraded (9.305%); 1412/193803 objects misplaced (0.729%); 6.4 MiB/s, 166 objects/s recovering 2023-09-30T09:51:35.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:35 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:51:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:35 smithi167 ceph-mon[192880]: pgmap v1564: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 18034/193803 objects degraded (9.305%); 1412/193803 objects misplaced (0.729%); 6.4 MiB/s, 166 objects/s recovering 2023-09-30T09:51:35.864 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:35.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:35.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:35.969 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:36.018 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:36.089 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:36.157 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:36.177 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:36.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:36.288 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:36.343 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:36.504 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:36.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:36.585 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:51:36.585 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:51:36.585 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:51:36.585 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:51:36.585 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:51:36.608 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:36.653 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:37.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:37 smithi019 ceph-mon[216815]: pgmap v1565: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 18034/193803 objects degraded (9.305%); 1412/193803 objects misplaced (0.729%); 5.4 MiB/s, 112 objects/s recovering 2023-09-30T09:51:37.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:37 smithi019 ceph-mon[224043]: pgmap v1565: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 18034/193803 objects degraded (9.305%); 1412/193803 objects misplaced (0.729%); 5.4 MiB/s, 112 objects/s recovering 2023-09-30T09:51:37.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:37 smithi167 ceph-mon[192880]: pgmap v1565: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 18034/193803 objects degraded (9.305%); 1412/193803 objects misplaced (0.729%); 5.4 MiB/s, 112 objects/s recovering 2023-09-30T09:51:37.673 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:51:38.529 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:38.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:38.635 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:38.693 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:38.814 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:51:39.016 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:39.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:39.120 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:39.150 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:39.207 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:39.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:39.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:39.314 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:39.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:39 smithi019 ceph-mon[224043]: pgmap v1566: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 17979/193008 objects degraded (9.315%); 564/193008 objects misplaced (0.292%); 11 MiB/s, 199 objects/s recovering 2023-09-30T09:51:39.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:39 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 17979/193008 objects degraded (9.315%), 8 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:39.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:39 smithi019 ceph-mon[216815]: pgmap v1566: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 17979/193008 objects degraded (9.315%); 564/193008 objects misplaced (0.292%); 11 MiB/s, 199 objects/s recovering 2023-09-30T09:51:39.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:39 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 17979/193008 objects degraded (9.315%), 8 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:39.352 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:39.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:39 smithi167 ceph-mon[192880]: pgmap v1566: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 17979/193008 objects degraded (9.315%); 564/193008 objects misplaced (0.292%); 11 MiB/s, 199 objects/s recovering 2023-09-30T09:51:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:39 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 17979/193008 objects degraded (9.315%), 8 pgs degraded (PG_DEGRADED) 2023-09-30T09:51:39.771 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:39.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:39.792 INFO:teuthology.orchestra.run.smithi019.stdout:moved 10 / 357 (2.80112%) 2023-09-30T09:51:39.792 INFO:teuthology.orchestra.run.smithi019.stdout:avg 44.625 2023-09-30T09:51:39.792 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 9.2187 -> 6.70704 (expected baseline 6.24875) 2023-09-30T09:51:39.792 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.5 with 27 -> 37 pgs (0.605042 -> 0.829132 * mean) 2023-09-30T09:51:39.792 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.4 with 55 -> 53 pgs (1.23249 -> 1.18768 * mean) 2023-09-30T09:51:39.792 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:51:39.793 INFO:teuthology.orchestra.run.smithi019.stdout:oload 1000 2023-09-30T09:51:39.793 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 1 2023-09-30T09:51:39.793 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:51:39.793 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0255 2023-09-30T09:51:39.793 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.2546 2023-09-30T09:51:39.793 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 weight 0.6217 -> 0.8555 2023-09-30T09:51:39.795 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:51:39.877 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:39.918 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:40.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:51:39 smithi019 conmon[202899]: 2023-09-30T09:51:39.790+0000 7f1d921ed700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-30T09:51:40.234 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:40.297 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:40.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:40.403 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:40.440 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:40.944 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:40.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:41.048 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:41.087 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:41.176 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:41.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:41.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:41.186 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-30T09:51:41.187 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-30T09:51:41.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:41 smithi019 ceph-mon[216815]: from='client.46760 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:41.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:41 smithi019 ceph-mon[216815]: pgmap v1567: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 11774/191646 objects degraded (6.144%); 4532/191646 objects misplaced (2.365%); 11 MiB/s, 192 objects/s recovering 2023-09-30T09:51:41.273 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:41 smithi019 ceph-mon[224043]: from='client.46760 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:41.273 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:41 smithi019 ceph-mon[224043]: pgmap v1567: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 11774/191646 objects degraded (6.144%); 4532/191646 objects misplaced (2.365%); 11 MiB/s, 192 objects/s recovering 2023-09-30T09:51:41.281 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:41.311 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:41.369 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:41.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:41.373 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:41 smithi167 ceph-mon[192880]: from='client.46760 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:41.373 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:41 smithi167 ceph-mon[192880]: pgmap v1567: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 11774/191646 objects degraded (6.144%); 4532/191646 objects misplaced (2.365%); 11 MiB/s, 192 objects/s recovering 2023-09-30T09:51:41.474 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:41.503 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:41.550 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:41.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:41.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:41.657 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:41.685 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:41.731 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:41.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:41.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:41.837 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:41.868 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:41.937 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:41.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:41.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:42.047 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:42.111 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:42.387 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:42.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:42.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:42.491 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:42.528 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:42.559 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:42.612 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:42.641 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:42.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:42.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:42.747 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:42.784 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:42.860 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:42.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:42.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:42.972 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:43.001 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:43.049 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:43.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:43.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:43.126 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:51:43.154 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:43.185 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:43.238 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:43.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:43.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:43.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:43 smithi167 ceph-mon[192880]: pgmap v1568: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 11774/191646 objects degraded (6.144%); 4532/191646 objects misplaced (2.365%); 9.7 MiB/s, 171 objects/s recovering 2023-09-30T09:51:43.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1929712673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:51:43.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:43 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:51:43.315 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:43 smithi019 ceph-mon[224043]: pgmap v1568: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 11774/191646 objects degraded (6.144%); 4532/191646 objects misplaced (2.365%); 9.7 MiB/s, 171 objects/s recovering 2023-09-30T09:51:43.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1929712673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:51:43.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:43 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:51:43.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:43 smithi019 ceph-mon[216815]: pgmap v1568: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 110 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 11774/191646 objects degraded (6.144%); 4532/191646 objects misplaced (2.365%); 9.7 MiB/s, 171 objects/s recovering 2023-09-30T09:51:43.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1929712673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:51:43.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:43 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:51:43.346 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:43.395 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:44.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:51:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:51:43] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-09-30T09:51:44.315 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:44.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:44.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:44.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:44.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:51:44.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[224043]: failed to encode map e587 with expected crc 2023-09-30T09:51:44.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[224043]: osdmap e587: 8 total, 8 up, 8 in 2023-09-30T09:51:44.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[224043]: failed to encode map e587 with expected crc 2023-09-30T09:51:44.378 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 11774/191646 objects degraded (6.144%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:44.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:44.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:51:44.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[216815]: failed to encode map e587 with expected crc 2023-09-30T09:51:44.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[216815]: osdmap e587: 8 total, 8 up, 8 in 2023-09-30T09:51:44.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[216815]: failed to encode map e587 with expected crc 2023-09-30T09:51:44.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:44 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 11774/191646 objects degraded (6.144%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:44.423 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:44.477 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:44.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:44 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:44 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:51:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:44 smithi167 ceph-mon[192880]: failed to encode map e587 with expected crc 2023-09-30T09:51:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:44 smithi167 ceph-mon[192880]: osdmap e587: 8 total, 8 up, 8 in 2023-09-30T09:51:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:44 smithi167 ceph-mon[192880]: failed to encode map e587 with expected crc 2023-09-30T09:51:44.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:44 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 11774/191646 objects degraded (6.144%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:45.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:45 smithi019 ceph-mon[216815]: pgmap v1570: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 11665/188520 objects degraded (6.188%); 3898/188520 objects misplaced (2.068%); 11 MiB/s, 176 objects/s recovering 2023-09-30T09:51:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:45 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3202587872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:45 smithi019 ceph-mon[224043]: pgmap v1570: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 11665/188520 objects degraded (6.188%); 3898/188520 objects misplaced (2.068%); 11 MiB/s, 176 objects/s recovering 2023-09-30T09:51:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:45 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3202587872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:45.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:51:45.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:51:45.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:51:45.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:51:45.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:51:45.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:51:45.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:51:45.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:51:45.420 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:51:45.420 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:51:45.420 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:51:45.420 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:51:45.420 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:51:45.420 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:51:45.420 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:51:45.420 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:51:45.421 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:45.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:45.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:45 smithi167 ceph-mon[192880]: pgmap v1570: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 11665/188520 objects degraded (6.188%); 3898/188520 objects misplaced (2.068%); 11 MiB/s, 176 objects/s recovering 2023-09-30T09:51:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:45 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3202587872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:45.525 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:45.554 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:45.764 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:45.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:45.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:45.870 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:45.908 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:46.380 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:51:46.418 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:46.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:46.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:46.526 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:46.572 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:46.861 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:46.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:46.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:46.953 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:51:46.953 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:51:46.966 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:47.005 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:47.227 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:47.229 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:47 smithi019 ceph-mon[216815]: pgmap v1571: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 9637/187143 objects degraded (5.150%); 5333/187143 objects misplaced (2.850%); 15 MiB/s, 236 objects/s recovering 2023-09-30T09:51:47.229 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:47 smithi019 ceph-mon[224043]: pgmap v1571: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 9637/187143 objects degraded (5.150%); 5333/187143 objects misplaced (2.850%); 15 MiB/s, 236 objects/s recovering 2023-09-30T09:51:47.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:47.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:47.331 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:47.360 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:47.369 INFO:teuthology.orchestra.run.smithi019.stdout:moved 41 / 357 (11.4846%) 2023-09-30T09:51:47.369 INFO:teuthology.orchestra.run.smithi019.stdout:avg 44.625 2023-09-30T09:51:47.369 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 9.2187 -> 8.3806 (expected baseline 6.24875) 2023-09-30T09:51:47.369 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.5 with 27 -> 40 pgs (0.605042 -> 0.896359 * mean) 2023-09-30T09:51:47.369 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.4 with 55 -> 46 pgs (1.23249 -> 1.03081 * mean) 2023-09-30T09:51:47.369 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:51:47.369 INFO:teuthology.orchestra.run.smithi019.stdout:oload 110 2023-09-30T09:51:47.369 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 3 2023-09-30T09:51:47.370 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:51:47.370 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0251 2023-09-30T09:51:47.370 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0276 2023-09-30T09:51:47.370 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 weight 1.0000 -> 0.7048 2023-09-30T09:51:47.370 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 weight 1.0000 -> 0.7605 2023-09-30T09:51:47.370 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 weight 1.0000 -> 0.7769 2023-09-30T09:51:47.370 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 weight 0.6217 -> 0.8566 2023-09-30T09:51:47.373 INFO:teuthology.orchestra.run.smithi019.stderr:no change 2023-09-30T09:51:47.413 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:47.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:47.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:47.517 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:47.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:47 smithi167 ceph-mon[192880]: pgmap v1571: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 9637/187143 objects degraded (5.150%); 5333/187143 objects misplaced (2.850%); 15 MiB/s, 236 objects/s recovering 2023-09-30T09:51:47.552 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:47.584 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:51:47 smithi019 conmon[202899]: 2023-09-30T09:51:47.368+0000 7f1d921ed700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-30T09:51:47.607 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:47.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:47.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:47.713 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:47.747 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:47.800 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:47.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:47.908 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:47.939 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:48.112 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:48 smithi167 ceph-mon[192880]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:48.112 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:48.112 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:48.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:48 smithi019 ceph-mon[224043]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:48 smithi019 ceph-mon[216815]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:51:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:51:48.501 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:51:48.518 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:48.522 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:48.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:48.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:48.537 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:51:48.556 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-30T09:51:48.557 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-30T09:51:48.635 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:48.667 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:48.721 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:48.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:48.826 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:48.886 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:49.026 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:49.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:49.133 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:49.195 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:49.258 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:49.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:49.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:49.363 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:49.390 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:49 smithi167 ceph-mon[192880]: pgmap v1572: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 260 op/s; 9467/185178 objects degraded (5.112%); 5333/185178 objects misplaced (2.880%); 10 MiB/s, 162 objects/s recovering 2023-09-30T09:51:49.390 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:49 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9637/187143 objects degraded (5.150%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:49.392 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:49.443 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:49.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:49.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:49.547 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:49.576 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:49.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:49 smithi019 ceph-mon[224043]: pgmap v1572: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 260 op/s; 9467/185178 objects degraded (5.112%); 5333/185178 objects misplaced (2.880%); 10 MiB/s, 162 objects/s recovering 2023-09-30T09:51:49.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:49 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9637/187143 objects degraded (5.150%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:49.579 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:49 smithi019 ceph-mon[216815]: pgmap v1572: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 260 op/s; 9467/185178 objects degraded (5.112%); 5333/185178 objects misplaced (2.880%); 10 MiB/s, 162 objects/s recovering 2023-09-30T09:51:49.580 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:49 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9637/187143 objects degraded (5.150%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:49.629 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:49.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:49.736 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:49.764 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:49.811 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:49.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:49.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:49.921 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:49.953 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:50.012 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:50.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:50.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:50.117 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:50.149 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:50.276 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:50.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:50.318 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:50.375 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:50.381 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:50.420 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:50.471 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:50.552 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:50.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:50.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:50.657 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:50.686 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:50.737 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:50.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:50.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:50.842 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:50.878 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:51.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:51 smithi019 ceph-mon[216815]: pgmap v1573: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 9474/183213 objects degraded (5.171%); 4339/183213 objects misplaced (2.368%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:51:51.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:51 smithi019 ceph-mon[224043]: pgmap v1573: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 9474/183213 objects degraded (5.171%); 4339/183213 objects misplaced (2.368%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:51:51.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:51 smithi167 ceph-mon[192880]: pgmap v1573: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 9474/183213 objects degraded (5.171%); 4339/183213 objects misplaced (2.368%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:51:52.138 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:51:52.297 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:52.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:52.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:52.402 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:52.425 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2242055483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:52.425 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:52.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2242055483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:52.426 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:52.428 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2242055483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:52.428 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:51:52.430 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:52.488 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:52.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:52.594 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:52.627 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:52.678 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:52.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:52.785 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:52.826 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:52.880 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:52.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:52.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:52.987 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:53.040 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:53.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[216815]: pgmap v1574: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 9474/183213 objects degraded (5.171%); 4339/183213 objects misplaced (2.368%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:51:53.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:53.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:53.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[216815]: failed to encode map e588 with expected crc 2023-09-30T09:51:53.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[216815]: osdmap e588: 8 total, 8 up, 8 in 2023-09-30T09:51:53.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[216815]: failed to encode map e588 with expected crc 2023-09-30T09:51:53.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[224043]: pgmap v1574: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 9474/183213 objects degraded (5.171%); 4339/183213 objects misplaced (2.368%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:51:53.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:53.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:53.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[224043]: failed to encode map e588 with expected crc 2023-09-30T09:51:53.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[224043]: osdmap e588: 8 total, 8 up, 8 in 2023-09-30T09:51:53.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:53 smithi019 ceph-mon[224043]: failed to encode map e588 with expected crc 2023-09-30T09:51:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:53 smithi167 ceph-mon[192880]: pgmap v1574: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 9474/183213 objects degraded (5.171%); 4339/183213 objects misplaced (2.368%); 16 MiB/s, 249 objects/s recovering 2023-09-30T09:51:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:53 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:51:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:53 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:51:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:53 smithi167 ceph-mon[192880]: failed to encode map e588 with expected crc 2023-09-30T09:51:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:53 smithi167 ceph-mon[192880]: osdmap e588: 8 total, 8 up, 8 in 2023-09-30T09:51:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:53 smithi167 ceph-mon[192880]: failed to encode map e588 with expected crc 2023-09-30T09:51:53.665 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:53.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:53.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:53.771 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:53.802 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:53.854 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:53.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:53.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:53.907 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:51:53.907 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:51:53.935 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:51:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:51:53] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2023-09-30T09:51:53.959 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:53.988 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:54.039 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:54.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:54.122 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:54 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 9474/183213 objects degraded (5.171%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:54.146 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:54.184 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:54 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 9474/183213 objects degraded (5.171%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:54.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:54 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 9474/183213 objects degraded (5.171%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:54.186 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:55.065 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:55.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:55.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:55.168 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:55.202 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:55.274 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:55.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:55.295 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:51:55.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:55 smithi019 ceph-mon[224043]: pgmap v1576: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 260 op/s; 9389/180717 objects degraded (5.195%); 4256/180717 objects misplaced (2.355%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:51:55.325 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:55 smithi019 ceph-mon[216815]: pgmap v1576: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 260 op/s; 9389/180717 objects degraded (5.195%); 4256/180717 objects misplaced (2.355%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:51:55.380 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:55.412 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:55 smithi167 ceph-mon[192880]: pgmap v1576: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 111 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 260 op/s; 9389/180717 objects degraded (5.195%); 4256/180717 objects misplaced (2.355%); 12 MiB/s, 192 objects/s recovering 2023-09-30T09:51:55.413 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:55.460 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:55.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:55.566 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:55.594 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:55.641 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:55.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:55.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:55.698 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:51:55.747 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:51:55.774 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:55.820 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:51:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:55.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:51:55.926 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:51:55.964 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:56.045 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:51:56.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:56.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:51:56.124 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'46896','2':'50415','4':'51281','5':'56003'}"}]: dispatch 2023-09-30T09:51:56.130 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:moved 40 / 357 (11.2045%) 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:avg 44.625 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 9.2187 -> 8.03022 (expected baseline 6.24875) 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.5 with 27 -> 40 pgs (0.605042 -> 0.896359 * mean) 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.4 with 55 -> 46 pgs (1.23249 -> 1.03081 * mean) 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:oload 110 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 3 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0246 2023-09-30T09:51:56.133 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0271 2023-09-30T09:51:56.134 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 weight 1.0000 -> 0.7156 2023-09-30T09:51:56.134 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 weight 1.0000 -> 0.7693 2023-09-30T09:51:56.134 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 weight 1.0000 -> 0.7825 2023-09-30T09:51:56.134 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 weight 0.6217 -> 0.8545 2023-09-30T09:51:56.149 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:51:56.179 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:56.204 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:51:56.222 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:51:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:56.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:51:56.236 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:51:56.271 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:51:56.327 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:56.330 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:51:56.330 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:51:56.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'46896','2':'50415','4':'51281','5':'56003'}"}]: dispatch 2023-09-30T09:51:56.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'46896','2':'50415','4':'51281','5':'56003'}"}]: dispatch 2023-09-30T09:51:56.353 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:56.354 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:51:56.378 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:51:56.398 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:56.402 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:51:56.403 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:51:56.403 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:51:56.403 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:51:56.403 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:51:56.404 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:51:56.404 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:51:56.404 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:51:56.404 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:51:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:56.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:56.506 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:51:56.561 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:57.358 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-30T09:51:57.358 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-30T09:51:57.410 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:51:57.413 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[224043]: pgmap v1577: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 7966/179346 objects degraded (4.442%); 4138/179346 objects misplaced (2.307%); 15 MiB/s, 248 objects/s recovering 2023-09-30T09:51:57.413 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[224043]: from='client.36804 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:57.413 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'46896','2':'50415','4':'51281','5':'56003'}"}]': finished 2023-09-30T09:51:57.413 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[224043]: failed to encode map e589 with expected crc 2023-09-30T09:51:57.413 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[224043]: osdmap e589: 8 total, 8 up, 8 in 2023-09-30T09:51:57.413 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2254494867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:57.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:57.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:51:57.415 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[216815]: pgmap v1577: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 7966/179346 objects degraded (4.442%); 4138/179346 objects misplaced (2.307%); 15 MiB/s, 248 objects/s recovering 2023-09-30T09:51:57.415 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[216815]: from='client.36804 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:57.415 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'46896','2':'50415','4':'51281','5':'56003'}"}]': finished 2023-09-30T09:51:57.415 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[216815]: failed to encode map e589 with expected crc 2023-09-30T09:51:57.415 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[216815]: osdmap e589: 8 total, 8 up, 8 in 2023-09-30T09:51:57.415 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:57 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2254494867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:57.515 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:57.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:57 smithi167 ceph-mon[192880]: pgmap v1577: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 7966/179346 objects degraded (4.442%); 4138/179346 objects misplaced (2.307%); 15 MiB/s, 248 objects/s recovering 2023-09-30T09:51:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:57 smithi167 ceph-mon[192880]: from='client.36804 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:51:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'46896','2':'50415','4':'51281','5':'56003'}"}]': finished 2023-09-30T09:51:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:57 smithi167 ceph-mon[192880]: failed to encode map e589 with expected crc 2023-09-30T09:51:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:57 smithi167 ceph-mon[192880]: osdmap e589: 8 total, 8 up, 8 in 2023-09-30T09:51:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:57 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2254494867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:51:57.541 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:57.585 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:57.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:57.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:57.691 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:51:57.718 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:57.764 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:51:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:57.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:51:57.869 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:57.898 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: osdmap e590: 8 total, 8 up, 8 in 2023-09-30T09:51:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[224043]: osdmap e591: 8 total, 8 up, 8 in 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: osdmap e590: 8 total, 8 up, 8 in 2023-09-30T09:51:58.167 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.167 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.167 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.167 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.167 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.167 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.167 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:58 smithi019 ceph-mon[216815]: osdmap e591: 8 total, 8 up, 8 in 2023-09-30T09:51:58.477 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:58.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: osdmap e590: 8 total, 8 up, 8 in 2023-09-30T09:51:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: failed to encode map e590 with expected crc 2023-09-30T09:51:58.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:58 smithi167 ceph-mon[192880]: osdmap e591: 8 total, 8 up, 8 in 2023-09-30T09:51:58.587 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:58.617 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:58.678 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:58.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:58.783 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:51:58.814 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:58.891 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:51:58.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:58.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:51:58.999 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:59.029 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:59.141 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:59.162 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:51:59.164 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: pgmap v1580: 119 pgs: 1 activating, 6 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 106 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 3748/178188 objects degraded (2.103%); 4138/178188 objects misplaced (2.322%); 15 MiB/s, 259 objects/s recovering 2023-09-30T09:51:59.164 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.164 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.164 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.164 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.165 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.165 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 588'8115 2023-09-30T09:51:59.165 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.165 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.165 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.165 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.165 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: 10.0 starting backfill to osd.7 from (0'0,548'3773] MIN to 588'8010 2023-09-30T09:51:59.166 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 588'8157 2023-09-30T09:51:59.166 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 3748/178188 objects degraded (2.103%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:59.166 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: failed to encode map e592 with expected crc 2023-09-30T09:51:59.167 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:51:59 smithi167 ceph-mon[192880]: osdmap e592: 8 total, 8 up, 8 in 2023-09-30T09:51:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:59.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:51:59.269 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:51:59.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: pgmap v1580: 119 pgs: 1 activating, 6 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 106 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 3748/178188 objects degraded (2.103%); 4138/178188 objects misplaced (2.322%); 15 MiB/s, 259 objects/s recovering 2023-09-30T09:51:59.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 588'8115 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: 10.0 starting backfill to osd.7 from (0'0,548'3773] MIN to 588'8010 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 588'8157 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 3748/178188 objects degraded (2.103%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: failed to encode map e592 with expected crc 2023-09-30T09:51:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[224043]: osdmap e592: 8 total, 8 up, 8 in 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: pgmap v1580: 119 pgs: 1 activating, 6 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 106 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 3748/178188 objects degraded (2.103%); 4138/178188 objects misplaced (2.322%); 15 MiB/s, 259 objects/s recovering 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 588'8115 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e591 with expected crc 2023-09-30T09:51:59.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: 10.0 starting backfill to osd.7 from (0'0,548'3773] MIN to 588'8010 2023-09-30T09:51:59.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 588'8157 2023-09-30T09:51:59.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 3748/178188 objects degraded (2.103%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-30T09:51:59.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: failed to encode map e592 with expected crc 2023-09-30T09:51:59.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:51:59 smithi019 ceph-mon[216815]: osdmap e592: 8 total, 8 up, 8 in 2023-09-30T09:51:59.307 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:51:59.311 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:51:59.422 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:51:59.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:51:59.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:51:59.529 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:51:59.570 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:00.184 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:00.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:00.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:00.290 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:00.297 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:52:00.326 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:00.395 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:00.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:00.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:00.504 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:00.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:00 smithi167 ceph-mon[192880]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:00 smithi167 ceph-mon[192880]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:00 smithi167 ceph-mon[192880]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:00 smithi167 ceph-mon[192880]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:00 smithi167 ceph-mon[192880]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:00 smithi167 ceph-mon[192880]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:00 smithi167 ceph-mon[192880]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:00 smithi167 ceph-mon[192880]: osdmap e593: 8 total, 8 up, 8 in 2023-09-30T09:52:00.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[224043]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[224043]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[224043]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[224043]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[224043]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[224043]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[224043]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[224043]: osdmap e593: 8 total, 8 up, 8 in 2023-09-30T09:52:00.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[216815]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[216815]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[216815]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[216815]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[216815]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[216815]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[216815]: failed to encode map e592 with expected crc 2023-09-30T09:52:00.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:00 smithi019 ceph-mon[216815]: osdmap e593: 8 total, 8 up, 8 in 2023-09-30T09:52:00.534 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:00.587 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:00.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:00.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:00.697 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:00.749 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:00.830 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:52:00.831 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:52:00.831 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:52:00.831 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:52:00.831 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:52:00.856 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:00.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:00.961 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:00.988 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:01.037 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:01.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:01.142 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:01.171 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:01.224 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:01.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:01.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:01 smithi167 ceph-mon[192880]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:01 smithi167 ceph-mon[192880]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:01 smithi167 ceph-mon[192880]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:01 smithi167 ceph-mon[192880]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:01 smithi167 ceph-mon[192880]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:01 smithi167 ceph-mon[192880]: pgmap v1584: 119 pgs: 7 peering, 1 activating, 4 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 99 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 12430/176706 objects degraded (7.034%); 8673/176706 objects misplaced (4.908%); 14 MiB/s, 354 objects/s recovering 2023-09-30T09:52:01.330 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:01.357 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:01.413 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:01.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:01.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:01.519 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:01.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[216815]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[216815]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[216815]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[216815]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[216815]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[216815]: pgmap v1584: 119 pgs: 7 peering, 1 activating, 4 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 99 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 12430/176706 objects degraded (7.034%); 8673/176706 objects misplaced (4.908%); 14 MiB/s, 354 objects/s recovering 2023-09-30T09:52:01.564 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[224043]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.564 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[224043]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.564 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[224043]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.564 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[224043]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.564 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[224043]: failed to encode map e593 with expected crc 2023-09-30T09:52:01.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:01 smithi019 ceph-mon[224043]: pgmap v1584: 119 pgs: 7 peering, 1 activating, 4 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 99 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 12430/176706 objects degraded (7.034%); 8673/176706 objects misplaced (4.908%); 14 MiB/s, 354 objects/s recovering 2023-09-30T09:52:01.567 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:01.994 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:02.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:02.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:02.003 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:52:02.103 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:02.133 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:02.187 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:02.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:02.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:02.292 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:02.325 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:02.391 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:02.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:02.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:02.495 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:02.532 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:02.760 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:02.772 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:02.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:02.876 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:02.909 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:02.926 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:02.960 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:02.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:02.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:03.066 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:03.098 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:03.154 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:03.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:03.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:03.185 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:52:03.260 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:03.288 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:03.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:03 smithi019 ceph-mon[224043]: pgmap v1585: 119 pgs: 7 peering, 1 activating, 4 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 98 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 12430/176475 objects degraded (7.043%); 12415/176475 objects misplaced (7.035%); 11 MiB/s, 281 objects/s recovering 2023-09-30T09:52:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2845832501' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:03 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:03 smithi019 ceph-mon[216815]: pgmap v1585: 119 pgs: 7 peering, 1 activating, 4 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 98 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 12430/176475 objects degraded (7.043%); 12415/176475 objects misplaced (7.035%); 11 MiB/s, 281 objects/s recovering 2023-09-30T09:52:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2845832501' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:03 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:03.350 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:03.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:03.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:03.456 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:03.512 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:03 smithi167 ceph-mon[192880]: pgmap v1585: 119 pgs: 7 peering, 1 activating, 4 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 98 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 12430/176475 objects degraded (7.043%); 12415/176475 objects misplaced (7.035%); 11 MiB/s, 281 objects/s recovering 2023-09-30T09:52:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2845832501' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:03 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:03.586 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:03.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:03.690 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:03.723 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:03.776 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:03.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:03.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:03.882 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:03.915 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:03.977 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:03.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:03.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:04.010 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:52:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:52:03] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-09-30T09:52:04.086 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:04.123 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:04.185 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:04.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:04.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:04 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:04 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:52:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:04 smithi167 ceph-mon[192880]: failed to encode map e594 with expected crc 2023-09-30T09:52:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:04 smithi167 ceph-mon[192880]: osdmap e594: 8 total, 8 up, 8 in 2023-09-30T09:52:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:04 smithi167 ceph-mon[192880]: failed to encode map e594 with expected crc 2023-09-30T09:52:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:04 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 12430/176475 objects degraded (7.043%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:52:04.294 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:04.330 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:04.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:04.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:52:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[224043]: failed to encode map e594 with expected crc 2023-09-30T09:52:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[224043]: osdmap e594: 8 total, 8 up, 8 in 2023-09-30T09:52:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[224043]: failed to encode map e594 with expected crc 2023-09-30T09:52:04.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 12430/176475 objects degraded (7.043%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:52:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:52:04.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[216815]: failed to encode map e594 with expected crc 2023-09-30T09:52:04.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[216815]: osdmap e594: 8 total, 8 up, 8 in 2023-09-30T09:52:04.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[216815]: failed to encode map e594 with expected crc 2023-09-30T09:52:04.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:04 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 12430/176475 objects degraded (7.043%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:52:04.953 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:04.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:05.058 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:05.087 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout:moved 31 / 357 (8.68347%) 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout:avg 44.625 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 8.03022 -> 12.3485 (expected baseline 6.24875) 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.0 with 32 -> 24 pgs (0.717087 -> 0.537815 * mean) 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.1 with 54 -> 60 pgs (1.21008 -> 1.34454 * mean) 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout:oload 110 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 3 2023-09-30T09:52:05.176 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:52:05.177 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0243 2023-09-30T09:52:05.177 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0267 2023-09-30T09:52:05.177 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 weight 0.7156 -> 0.5193 2023-09-30T09:52:05.177 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 weight 0.7693 -> 0.5966 2023-09-30T09:52:05.177 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 weight 0.7825 -> 0.6085 2023-09-30T09:52:05.177 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 weight 0.8545 -> 1.0000 2023-09-30T09:52:05.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:05 smithi019 ceph-mon[216815]: pgmap v1587: 119 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 186 op/s; 15994/174894 objects degraded (9.145%); 13926/174894 objects misplaced (7.963%); 9.5 MiB/s, 265 objects/s recovering 2023-09-30T09:52:05.489 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'34033','2':'39097','4':'39879','5':'65536'}"}]: dispatch 2023-09-30T09:52:05.489 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:05 smithi019 ceph-mon[224043]: pgmap v1587: 119 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 186 op/s; 15994/174894 objects degraded (9.145%); 13926/174894 objects misplaced (7.963%); 9.5 MiB/s, 265 objects/s recovering 2023-09-30T09:52:05.490 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'34033','2':'39097','4':'39879','5':'65536'}"}]: dispatch 2023-09-30T09:52:05.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:05 smithi167 ceph-mon[192880]: pgmap v1587: 119 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 186 op/s; 15994/174894 objects degraded (9.145%); 13926/174894 objects misplaced (7.963%); 9.5 MiB/s, 265 objects/s recovering 2023-09-30T09:52:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'34033','2':'39097','4':'39879','5':'65536'}"}]: dispatch 2023-09-30T09:52:05.700 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:05.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:05.805 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:05.837 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:06.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:06 smithi167 ceph-mon[192880]: from='client.56611 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:52:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'34033','2':'39097','4':'39879','5':'65536'}"}]': finished 2023-09-30T09:52:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:06 smithi167 ceph-mon[192880]: failed to encode map e595 with expected crc 2023-09-30T09:52:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:06 smithi167 ceph-mon[192880]: osdmap e595: 8 total, 8 up, 8 in 2023-09-30T09:52:06.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:06 smithi167 ceph-mon[192880]: failed to encode map e595 with expected crc 2023-09-30T09:52:06.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[216815]: from='client.56611 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'34033','2':'39097','4':'39879','5':'65536'}"}]': finished 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[216815]: failed to encode map e595 with expected crc 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[216815]: osdmap e595: 8 total, 8 up, 8 in 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[216815]: failed to encode map e595 with expected crc 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[224043]: from='client.56611 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'34033','2':'39097','4':'39879','5':'65536'}"}]': finished 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[224043]: failed to encode map e595 with expected crc 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[224043]: osdmap e595: 8 total, 8 up, 8 in 2023-09-30T09:52:06.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:06 smithi019 ceph-mon[224043]: failed to encode map e595 with expected crc 2023-09-30T09:52:06.709 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:06.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:06.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:06.813 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:06.840 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:06.859 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-30T09:52:06.859 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-30T09:52:06.887 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:06.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:06.993 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:07.021 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:07.068 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:07.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:07.173 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:07.200 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:07.251 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:52:07.262 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:52:07.262 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:52:07.262 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:52:07.262 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:52:07.262 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.262 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:153642 8.f e456/448 8:f10f80bc:::benchmark_data_smithi019_478679_object61194:head v 456'8753, mlcod=384'5694)", 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:04.197104+0000", 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 543.06119944399995, 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0607071669999999, 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197104+0000", 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.263 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197104+0000", 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197105+0000", 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1149999999999999e-06 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.264 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197112+0000", 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.165e-06 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197123+0000", 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1051e-05 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.265 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.197132+0000", 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6230000000000003e-06 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.255956+0000", 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.058824032 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.266 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.255988+0000", 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1826000000000003e-05 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.257777+0000", 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001789037 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.257811+0000", 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4317999999999999e-05 2023-09-30T09:52:07.267 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:153648 8.1f e456/448 8:fd9e162a:::benchmark_data_smithi019_478679_object61200:head v 456'8785, mlcod=384'5695)", 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:04.246060+0000", 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 543.01224340800002, 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0126862059999999, 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.268 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.246060+0000", 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.246060+0000", 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.269 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.246061+0000", 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0999999999999997e-07 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.246067+0000", 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.195e-06 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.270 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.246077+0000", 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.645e-06 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:04.246084+0000", 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7649999999999996e-06 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.256604+0000", 2023-09-30T09:52:07.271 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0105202170000001 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.256618+0000", 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4314999999999999e-05 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.258708+0000", 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002089956 2023-09-30T09:52:07.272 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.258746+0000", 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8303000000000002e-05 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:153659 8.1f e457/448 8:fcf7805c:::benchmark_data_smithi019_478679_object61211:head v 457'8786, mlcod=384'5695)", 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:05.669073+0000", 2023-09-30T09:52:07.273 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 541.58923039000001, 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4879337750000001, 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.669073+0000", 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.274 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.669073+0000", 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.669074+0000", 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2900000000000002e-07 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.275 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.669080+0000", 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2450000000000003e-06 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.669096+0000", 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5916000000000001e-05 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:05.669104+0000", 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1820000000000006e-06 2023-09-30T09:52:07.276 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.155101+0000", 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4859970730000001 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.155128+0000", 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6472999999999999e-05 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.277 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.156969+0000", 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018412210000000001 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.157007+0000", 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7836000000000002e-05 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.278 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:153675 8.1f e459/448 8:fb569702:::benchmark_data_smithi019_478679_object61227:head v 459'8787, mlcod=384'5695)", 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:07.075339+0000", 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 540.18296428999997, 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99075435000000001, 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.279 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.075339+0000", 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.075339+0000", 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.280 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.075340+0000", 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0410000000000001e-06 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.075349+0000", 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4540000000000007e-06 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.075365+0000", 2023-09-30T09:52:07.281 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6008e-05 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.075378+0000", 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2896e-05 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.061483+0000", 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98610586700000002 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.282 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.061519+0000", 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5476999999999999e-05 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.066041+0000", 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0045215769999999997 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.283 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.066094+0000", 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3029999999999999e-05 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:153688 8.1f e459/448 8:ffa34a4f:::benchmark_data_smithi019_478679_object61240:head v 459'8788, mlcod=384'5695)", 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:07.097598+0000", 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 540.16070533899995, 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96862751199999997, 2023-09-30T09:52:07.284 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.097598+0000", 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.097598+0000", 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.285 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.097599+0000", 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1880000000000001e-06 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.097607+0000", 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3869999999999997e-06 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.286 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.097619+0000", 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2099000000000001e-05 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.097627+0000", 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1510000000000004e-06 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.062743+0000", 2023-09-30T09:52:07.287 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96511619100000001 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.062767+0000", 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.429e-05 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.066173+0000", 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034056939999999999 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.288 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.066226+0000", 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2512000000000001e-05 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.35820.0:153700 8.f e459/448 8:f1868564:::benchmark_data_smithi019_478679_object61252:head v 459'8754, mlcod=432'7988)", 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:43:07.128948+0000", 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 540.12935508999999, 2023-09-30T09:52:07.289 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.93722234000000004, 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.128948+0000", 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.128948+0000", 2023-09-30T09:52:07.290 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.128950+0000", 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.172e-06 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.128958+0000", 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6090000000000001e-06 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.291 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.128971+0000", 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2724999999999999e-05 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:07.128981+0000", 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6299999999999993e-06 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.292 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.062724+0000", 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.93374351700000002 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.062739+0000", 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4773000000000001e-05 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.066133+0000", 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003394364 2023-09-30T09:52:07.293 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:43:08.066171+0000", 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7549999999999998e-05 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2298 9.7 9:e131cc11:::benchmark_data_smithi019_560486_object2297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:52:07.294 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:26.045600+0000", 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 461.21270305199999, 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0400317670000001, 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2298 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.295 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045600+0000", 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045600+0000", 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045603+0000", 2023-09-30T09:52:07.296 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.571e-06 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045638+0000", 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4876000000000003e-05 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045650+0000", 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1997000000000001e-05 2023-09-30T09:52:07.297 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045658+0000", 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1340000000000004e-06 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045745+0000", 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6846000000000002e-05 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.298 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op must wait for map", 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.045754+0000", 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7250000000000003e-06 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.051361+0000", 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005607758 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.051373+0000", 2023-09-30T09:52:07.299 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1153e-05 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085465+0000", 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0340923209999999 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.085632+0000", 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000167386 2023-09-30T09:52:07.300 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101677 10.f 10:f6c8d05b:::benchmark_data_smithi019_617837_object8277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.505301+0000", 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 86.753002066999997, 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0377062750000001, 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.301 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.302 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.302 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.302 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101677 2023-09-30T09:52:07.302 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.302 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.302 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.302 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.302 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505301+0000", 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505301+0000", 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505303+0000", 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.855e-06 2023-09-30T09:52:07.303 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505311+0000", 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2969999999999998e-06 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505330+0000", 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9848999999999999e-05 2023-09-30T09:52:07.304 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505340+0000", 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4909999999999994e-06 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505430+0000", 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0117000000000005e-05 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.305 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505445+0000", 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5268e-05 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380620+0000", 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87517486700000002 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.306 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380634+0000", 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4201e-05 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.471995+0000", 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0913604770000001 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.472756+0000", 2023-09-30T09:52:07.307 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00076066000000000003 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.479103+0000", 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063471689999999997 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.479257+0000", 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000154597 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.308 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.479448+0000", 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019083199999999999 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.536950+0000", 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.057502191000000001 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.309 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.536993+0000", 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2487999999999997e-05 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.541678+0000", 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00468558 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.541722+0000", 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3269000000000002e-05 2023-09-30T09:52:07.310 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543008+0000", 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012860669999999999 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101679 10.f 10:f6ce7bfd:::benchmark_data_smithi019_617837_object8279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:52:07.311 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.508259+0000", 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 86.750044704000004, 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0366561970000001, 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101679 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.312 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508259+0000", 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508259+0000", 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.313 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508260+0000", 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1519999999999999e-06 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508266+0000", 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5500000000000002e-06 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508281+0000", 2023-09-30T09:52:07.314 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5359999999999999e-05 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508289+0000", 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.259e-06 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508347+0000", 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7509000000000001e-05 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.315 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508355+0000", 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5329999999999994e-06 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380662+0000", 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87230725399999998 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.316 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380671+0000", 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5939999999999994e-06 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.474961+0000", 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0942902880000001 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.475717+0000", 2023-09-30T09:52:07.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00075602500000000001 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.534287+0000", 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.058569953000000001 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.534371+0000", 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3350000000000007e-05 2023-09-30T09:52:07.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.534471+0000", 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000100073 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543024+0000", 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0085535560000000004 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543089+0000", 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4846999999999998e-05 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543967+0000", 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00087815400000000002 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.544119+0000", 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015164900000000001 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.544915+0000", 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00079609099999999996 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.321 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101692 10.f 10:fff5665a:::benchmark_data_smithi019_617837_object8292:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.537296+0000", 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 86.721007994999994, 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0086183219999998, 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101692 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.322 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537296+0000", 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537296+0000", 2023-09-30T09:52:07.323 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.325 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537297+0000", 2023-09-30T09:52:07.325 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3599999999999999e-06 2023-09-30T09:52:07.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537304+0000", 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6989999999999997e-06 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537322+0000", 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8817e-05 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537330+0000", 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8620000000000003e-06 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537420+0000", 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0041000000000001e-05 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537438+0000", 2023-09-30T09:52:07.327 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7496999999999999e-05 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380817+0000", 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.843379557 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380827+0000", 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0027000000000001e-05 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.475791+0000", 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0949634210000001 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.476372+0000", 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00058119900000000002 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.539164+0000", 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.062791945000000002 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.539256+0000", 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1797000000000002e-05 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.330 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.539346+0000", 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9776000000000002e-05 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543104+0000", 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003758094 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.544969+0000", 2023-09-30T09:52:07.331 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001865589 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.545017+0000", 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7542e-05 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.545055+0000", 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8742999999999998e-05 2023-09-30T09:52:07.332 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.545914+0000", 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00085835600000000001 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101699 10.f 10:fa38cb57:::benchmark_data_smithi019_617837_object8299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:52:07.333 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.503819+0000", 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.754484052999999, 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.078656871, 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101699 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.334 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503819+0000", 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503819+0000", 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.335 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503822+0000", 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.368e-06 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503837+0000", 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5077e-05 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503864+0000", 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6801999999999999e-05 2023-09-30T09:52:07.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503879+0000", 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5268e-05 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503992+0000", 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011311099999999999 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.504009+0000", 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6515e-05 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.545973+0000", 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.041964361 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.546650+0000", 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000676726 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.558255+0000", 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011605284 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.558332+0000", 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6892999999999997e-05 2023-09-30T09:52:07.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.558427+0000", 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5353999999999995e-05 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.568221+0000", 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0097933819999999998 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.568276+0000", 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5772000000000001e-05 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.580557+0000", 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012280307000000001 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.341 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.580634+0000", 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7471000000000004e-05 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.582476+0000", 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018421799999999999 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.342 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101748 10.f 10:feb50901:::benchmark_data_smithi019_617837_object8348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.594287+0000", 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.664016219000004, 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99615996500000004, 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101748 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.343 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594287+0000", 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594287+0000", 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594289+0000", 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9290000000000001e-06 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594296+0000", 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3729999999999998e-06 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594322+0000", 2023-09-30T09:52:07.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6910000000000002e-05 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594335+0000", 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2669e-05 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594420+0000", 2023-09-30T09:52:07.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4666000000000006e-05 2023-09-30T09:52:07.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594432+0000", 2023-09-30T09:52:07.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2429000000000001e-05 2023-09-30T09:52:07.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.546692+0000", 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95225960600000004 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.548960+0000", 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022686210000000002 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.349 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.565214+0000", 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016253648999999998 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.565301+0000", 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6386999999999998e-05 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.565390+0000", 2023-09-30T09:52:07.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9759999999999994e-05 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.574662+0000", 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0092722020000000002 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.574675+0000", 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2934e-05 2023-09-30T09:52:07.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.588018+0000", 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.01334277 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.588094+0000", 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5696000000000002e-05 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.590447+0000", 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023533640000000002 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101752 10.f 10:fe3db57c:::benchmark_data_smithi019_617837_object8352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.606755+0000", 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.651548164999994, 2023-09-30T09:52:07.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98578958500000002, 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101752 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606755+0000", 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606755+0000", 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606757+0000", 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1160000000000002e-06 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606766+0000", 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1380000000000007e-06 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606788+0000", 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2617e-05 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606803+0000", 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4627000000000001e-05 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606893+0000", 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0177000000000001e-05 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606908+0000", 2023-09-30T09:52:07.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4789e-05 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.549029+0000", 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94212122499999995 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.552141+0000", 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031117340000000001 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.358 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.571101+0000", 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018960685000000001 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.571183+0000", 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1176999999999999e-05 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.571305+0000", 2023-09-30T09:52:07.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012193600000000001 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.584504+0000", 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013199934 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.590561+0000", 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.006056797 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.590610+0000", 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8690999999999998e-05 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.590686+0000", 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5815999999999994e-05 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.592545+0000", 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001859126 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101772 10.f 10:fc2b4dbf:::benchmark_data_smithi019_617837_object8372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.642283+0000", 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.616020775999999, 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95144165599999997, 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.362 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101772 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642283+0000", 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642283+0000", 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642284+0000", 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6279999999999999e-06 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642290+0000", 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.981e-06 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642309+0000", 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8532000000000001e-05 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642319+0000", 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0508e-05 2023-09-30T09:52:07.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642385+0000", 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5679999999999995e-05 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642397+0000", 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1635e-05 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.552217+0000", 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90982037199999999 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.553472+0000", 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012552240000000001 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.577416+0000", 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.023943760000000001 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.577539+0000", 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000123406 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.577711+0000", 2023-09-30T09:52:07.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017168399999999999 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.587950+0000", 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010238939000000001 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.592604+0000", 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004653768 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.592632+0000", 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8572e-05 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.592688+0000", 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5447e-05 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.593724+0000", 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00103652 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101807 10.f 10:f1360924:::benchmark_data_smithi019_617837_object8407:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.699381+0000", 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.558922444000004, 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97619895899999998, 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.371 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101807 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699381+0000", 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699381+0000", 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699383+0000", 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3860000000000001e-06 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699392+0000", 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4109999999999996e-06 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699419+0000", 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7382e-05 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699431+0000", 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1914e-05 2023-09-30T09:52:07.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699514+0000", 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2646999999999997e-05 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699527+0000", 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.295e-05 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.594872+0000", 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89534542399999995 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.595403+0000", 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00053071599999999998 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.618246+0000", 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022842713000000001 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.618340+0000", 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4481000000000006e-05 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.618516+0000", 2023-09-30T09:52:07.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000175687 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.630704+0000", 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012188399000000001 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.638907+0000", 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0082030609999999993 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.673663+0000", 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.034755449000000001 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.673726+0000", 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3511999999999998e-05 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.675580+0000", 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018538269999999999 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101822 10.f 10:f4bbe328:::benchmark_data_smithi019_617837_object8422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.731256+0000", 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.527047143999994, 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95043909299999996, 2023-09-30T09:52:07.380 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101822 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731256+0000", 2023-09-30T09:52:07.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731256+0000", 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731258+0000", 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.742e-06 2023-09-30T09:52:07.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731269+0000", 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1353e-05 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731290+0000", 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0508000000000001e-05 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731300+0000", 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8369999999999992e-06 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731386+0000", 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6588000000000005e-05 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731404+0000", 2023-09-30T09:52:07.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7808999999999998e-05 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.595466+0000", 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86406184500000005 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.596751+0000", 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00128535 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.627388+0000", 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.030636587 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.627535+0000", 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000147374 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.627725+0000", 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018972800000000001 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.638916+0000", 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011191362999999999 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.680403+0000", 2023-09-30T09:52:07.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.041486519999999999 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.680446+0000", 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3257999999999998e-05 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.680488+0000", 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1591999999999998e-05 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681695+0000", 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012076389999999999 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101827 10.f 10:fcfbd7c6:::benchmark_data_smithi019_617837_object8427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.734887+0000", 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.523416749999996, 2023-09-30T09:52:07.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94782315299999997, 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101827 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.390 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734887+0000", 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734887+0000", 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734889+0000", 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.827e-06 2023-09-30T09:52:07.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734895+0000", 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.635e-06 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734917+0000", 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1786000000000001e-05 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734928+0000", 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0849999999999999e-05 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.735006+0000", 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7921000000000002e-05 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.735017+0000", 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1321000000000001e-05 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.596804+0000", 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86178649900000004 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.597425+0000", 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00062158600000000004 2023-09-30T09:52:07.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.635761+0000", 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.038336108000000001 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.635872+0000", 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000111202 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.636045+0000", 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017237099999999999 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.650267+0000", 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014222088000000001 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681746+0000", 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031478610999999997 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681815+0000", 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9295000000000007e-05 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681851+0000", 2023-09-30T09:52:07.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6275000000000001e-05 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.682710+0000", 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00085877800000000004 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.398 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101840 10.f 10:f3ea7b62:::benchmark_data_smithi019_617837_object8440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.753839+0000", 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.504464428000006, 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.92968448599999998, 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.399 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101840 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.753839+0000", 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.753839+0000", 2023-09-30T09:52:07.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.753841+0000", 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.579e-06 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.753847+0000", 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4980000000000003e-06 2023-09-30T09:52:07.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.753867+0000", 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9426e-05 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.753877+0000", 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.024e-05 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.753966+0000", 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9355000000000001e-05 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.753982+0000", 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5994e-05 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.403 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.597489+0000", 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.84350681199999999 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.598178+0000", 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00068865599999999995 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.642697+0000", 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.044519178999999999 2023-09-30T09:52:07.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.642785+0000", 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8012999999999999e-05 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.642886+0000", 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010149099999999999 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.661416+0000", 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018529243000000001 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681787+0000", 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020371469 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.406 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.682739+0000", 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00095163400000000003 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.682773+0000", 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4541e-05 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.683524+0000", 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00075035599999999998 2023-09-30T09:52:07.407 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127770 10.7 10:ea5acc8c:::benchmark_data_smithi019_617837_object34370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:56.740170+0000", 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 10.518133255, 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.661720278, 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.408 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127770 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740170+0000", 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.409 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740170+0000", 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740172+0000", 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4699999999999999e-06 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.410 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740179+0000", 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8120000000000002e-06 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740205+0000", 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6679999999999999e-05 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740213+0000", 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2670000000000006e-06 2023-09-30T09:52:07.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740304+0000", 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0814000000000002e-05 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740321+0000", 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6646e-05 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.412 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.141529+0000", 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.40120852600000001 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.141543+0000", 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3888e-05 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.327671+0000", 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1861271870000001 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.328554+0000", 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00088332799999999996 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.389968+0000", 2023-09-30T09:52:07.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.061414174000000002 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.390100+0000", 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013192299999999999 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4,6", 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.390266+0000", 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016573099999999999 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.415 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.396122+0000", 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058566570000000004 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.396244+0000", 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012189300000000001 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.396265+0000", 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0639999999999999e-05 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.399991+0000", 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0037262939999999998 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.400077+0000", 2023-09-30T09:52:07.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5991999999999999e-05 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.401891+0000", 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018133559999999999 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127773 10.7 10:e8a99f2e:::benchmark_data_smithi019_617837_object34373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-30T09:52:07.418 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:56.832885+0000", 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 10.42541892, 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6041828979999999, 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127773 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832885+0000", 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832885+0000", 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832886+0000", 2023-09-30T09:52:07.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0690000000000001e-06 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832890+0000", 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9190000000000001e-06 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832898+0000", 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5690000000000005e-06 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832904+0000", 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.772e-06 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832961+0000", 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7107000000000001e-05 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.422 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832969+0000", 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3939999999999996e-06 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.141571+0000", 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30860205000000002 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:52:07.423 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.141582+0000", 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0254999999999999e-05 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.393765+0000", 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2521827939999999 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.394470+0000", 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00070521500000000003 2023-09-30T09:52:07.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.410662+0000", 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016191855000000002 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.410797+0000", 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013578400000000001 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4,6", 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.410983+0000", 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000185649 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.431634+0000", 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020651051 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.435059+0000", 2023-09-30T09:52:07.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003424584 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.435089+0000", 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0303000000000001e-05 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.435121+0000", 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2403999999999997e-05 2023-09-30T09:52:07.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.435192+0000", 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1141000000000002e-05 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.437067+0000", 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001874983 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.428 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.429 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:52:07.429 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:52:07.429 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:52:07.429 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:07.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: pgmap v1589: 119 pgs: 1 active+remapped+backfilling, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 16074/172539 objects degraded (9.316%); 12578/172539 objects misplaced (7.290%); 13 MiB/s, 246 objects/s recovering 2023-09-30T09:52:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: osdmap e596: 8 total, 8 up, 8 in 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:07 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.531 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: pgmap v1589: 119 pgs: 1 active+remapped+backfilling, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 16074/172539 objects degraded (9.316%); 12578/172539 objects misplaced (7.290%); 13 MiB/s, 246 objects/s recovering 2023-09-30T09:52:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: osdmap e596: 8 total, 8 up, 8 in 2023-09-30T09:52:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: pgmap v1589: 119 pgs: 1 active+remapped+backfilling, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 16074/172539 objects degraded (9.316%); 12578/172539 objects misplaced (7.290%); 13 MiB/s, 246 objects/s recovering 2023-09-30T09:52:07.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: osdmap e596: 8 total, 8 up, 8 in 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:07 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:07.643 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:07.847 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:52:07.847 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:52:07.847 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:52:07.847 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:52:07.890 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:52:07.911 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:07.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:07.941 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:52:07.987 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:52:08.016 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:08.019 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:52:08.028 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:52:08.028 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:52:08.028 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:52:08.028 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:52:08.028 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:52:08.028 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:52:08.028 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:52:08.029 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:52:08.073 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:08.147 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:08.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:08.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:08.252 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:08.278 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:08.322 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:08.335 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:08 smithi167 ceph-mon[192880]: failed to encode map e596 with expected crc 2023-09-30T09:52:08.336 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:08 smithi167 ceph-mon[192880]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.336 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:08 smithi167 ceph-mon[192880]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.336 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:08 smithi167 ceph-mon[192880]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.336 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:08 smithi167 ceph-mon[192880]: osdmap e597: 8 total, 8 up, 8 in 2023-09-30T09:52:08.336 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:08 smithi167 ceph-mon[192880]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 594'8127 2023-09-30T09:52:08.336 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3388523117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:08.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:08.438 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:08.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[216815]: failed to encode map e596 with expected crc 2023-09-30T09:52:08.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[216815]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[216815]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[216815]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[216815]: osdmap e597: 8 total, 8 up, 8 in 2023-09-30T09:52:08.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[216815]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 594'8127 2023-09-30T09:52:08.510 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3388523117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:08.511 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[224043]: failed to encode map e596 with expected crc 2023-09-30T09:52:08.511 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[224043]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.511 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[224043]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.511 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[224043]: failed to encode map e597 with expected crc 2023-09-30T09:52:08.511 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[224043]: osdmap e597: 8 total, 8 up, 8 in 2023-09-30T09:52:08.511 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[224043]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 594'8127 2023-09-30T09:52:08.511 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3388523117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:08.515 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:08.982 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:08.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:08.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:09.088 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:09.167 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:09.311 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:52:09.312 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:09.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:09.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:09.425 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:09.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: failed to encode map e597 with expected crc 2023-09-30T09:52:09.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: failed to encode map e597 with expected crc 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: 10.e starting backfill to osd.5 from (541'1472,568'6397] MIN to 594'8161 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: pgmap v1592: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfilling, 1 unknown, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 108 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 12242/171696 objects degraded (7.130%); 11067/171696 objects misplaced (6.446%); 17 MiB/s, 276 objects/s recovering 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: osdmap e598: 8 total, 8 up, 8 in 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 12242/171696 objects degraded (7.130%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:09.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: failed to encode map e597 with expected crc 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: failed to encode map e597 with expected crc 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: 10.e starting backfill to osd.5 from (541'1472,568'6397] MIN to 594'8161 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: pgmap v1592: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfilling, 1 unknown, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 108 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 12242/171696 objects degraded (7.130%); 11067/171696 objects misplaced (6.446%); 17 MiB/s, 276 objects/s recovering 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: osdmap e598: 8 total, 8 up, 8 in 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.482 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 12242/171696 objects degraded (7.130%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:09.493 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:09.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: failed to encode map e597 with expected crc 2023-09-30T09:52:09.528 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: failed to encode map e597 with expected crc 2023-09-30T09:52:09.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: 10.e starting backfill to osd.5 from (541'1472,568'6397] MIN to 594'8161 2023-09-30T09:52:09.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: pgmap v1592: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfilling, 1 unknown, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 108 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 12242/171696 objects degraded (7.130%); 11067/171696 objects misplaced (6.446%); 17 MiB/s, 276 objects/s recovering 2023-09-30T09:52:09.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: osdmap e598: 8 total, 8 up, 8 in 2023-09-30T09:52:09.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: failed to encode map e598 with expected crc 2023-09-30T09:52:09.529 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 12242/171696 objects degraded (7.130%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:09.618 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:09.651 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:09.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:09.757 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:09.759 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:09.843 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:09.948 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:09.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:09.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:10.039 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:10.059 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:10.104 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:10.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: osdmap e599: 8 total, 8 up, 8 in 2023-09-30T09:52:10.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[216815]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: osdmap e599: 8 total, 8 up, 8 in 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.260 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.260 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.260 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.260 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.260 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.260 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:10 smithi019 ceph-mon[224043]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.439 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:10.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:10.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:10.502 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.502 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: osdmap e599: 8 total, 8 up, 8 in 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.503 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:10 smithi167 ceph-mon[192880]: failed to encode map e599 with expected crc 2023-09-30T09:52:10.547 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:10.599 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:10.720 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:10.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:10.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:10.830 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:10.891 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:11.076 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:11.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:11.183 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:11.258 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:11.311 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:11.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:11.415 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:11.443 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:11.492 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:11.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:11.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:11.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:11 smithi167 ceph-mon[192880]: pgmap v1595: 119 pgs: 2 active+recovering+undersized+remapped, 8 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 2 active+remapped+backfilling, 1 unknown, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 282 op/s; 11802/169146 objects degraded (6.977%); 15455/169146 objects misplaced (9.137%); 20 MiB/s, 420 objects/s recovering 2023-09-30T09:52:11.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:11 smithi019 ceph-mon[216815]: pgmap v1595: 119 pgs: 2 active+recovering+undersized+remapped, 8 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 2 active+remapped+backfilling, 1 unknown, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 282 op/s; 11802/169146 objects degraded (6.977%); 15455/169146 objects misplaced (9.137%); 20 MiB/s, 420 objects/s recovering 2023-09-30T09:52:11.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:11 smithi019 ceph-mon[224043]: pgmap v1595: 119 pgs: 2 active+recovering+undersized+remapped, 8 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 2 active+remapped+backfilling, 1 unknown, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 282 op/s; 11802/169146 objects degraded (6.977%); 15455/169146 objects misplaced (9.137%); 20 MiB/s, 420 objects/s recovering 2023-09-30T09:52:11.595 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:11.658 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:11.662 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:52:12.484 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:12.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:12.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:12.593 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:12.665 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:12.741 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:12.930 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:12.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:13.043 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:13.122 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:13.244 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:13.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:13.351 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:13.425 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:13.502 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:13.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:13.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:13 smithi167 ceph-mon[192880]: pgmap v1596: 119 pgs: 2 active+recovering+undersized+remapped, 8 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 11802/168669 objects degraded (6.997%); 14788/168669 objects misplaced (8.767%); 17 MiB/s, 340 objects/s recovering 2023-09-30T09:52:13.541 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:13 smithi019 ceph-mon[216815]: pgmap v1596: 119 pgs: 2 active+recovering+undersized+remapped, 8 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 11802/168669 objects degraded (6.997%); 14788/168669 objects misplaced (8.767%); 17 MiB/s, 340 objects/s recovering 2023-09-30T09:52:13.541 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:13 smithi019 ceph-mon[224043]: pgmap v1596: 119 pgs: 2 active+recovering+undersized+remapped, 8 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 11802/168669 objects degraded (6.997%); 14788/168669 objects misplaced (8.767%); 17 MiB/s, 340 objects/s recovering 2023-09-30T09:52:13.609 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:13.691 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:13.967 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:13.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:13.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:14.077 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:14.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:52:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:52:13] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-09-30T09:52:14.147 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:14.265 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:14.267 INFO:teuthology.orchestra.run.smithi019.stdout:moved 13 / 357 (3.64146%) 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:avg 44.625 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:stddev 12.3485 -> 14.6709 (expected baseline 6.24875) 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:min osd.0 with 24 -> 23 pgs (0.537815 -> 0.515406 * mean) 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:max osd.1 with 60 -> 63 pgs (1.34454 -> 1.41176 * mean) 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:oload 110 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:max_change 0.05 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:max_change_osds 4 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:average_utilization 0.0241 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:overload_utilization 0.0265 2023-09-30T09:52:14.268 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 weight 0.5193 -> 0.4693 2023-09-30T09:52:14.269 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 weight 0.5966 -> 0.5466 2023-09-30T09:52:14.269 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 weight 0.6085 -> 0.5585 2023-09-30T09:52:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:14.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:14.299 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:52:14.380 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:14.528 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:14.576 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:14.577 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:14 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'30757','2':'35821','4':'36603'}"}]: dispatch 2023-09-30T09:52:14.577 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 11802/168669 objects degraded (6.997%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:14.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:14 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1814722436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:14.578 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:14 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:14.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:14.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:14.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:14 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'30757','2':'35821','4':'36603'}"}]: dispatch 2023-09-30T09:52:14.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 11802/168669 objects degraded (6.997%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:14.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:14 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1814722436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:14.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:14 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:14 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'30757','2':'35821','4':'36603'}"}]: dispatch 2023-09-30T09:52:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 11802/168669 objects degraded (6.997%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:14 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1814722436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:14.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:14 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:14.680 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:14.729 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:14.813 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:14.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:14.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:14.920 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:14.951 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:15.005 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:15.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:15.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:15.109 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:15.140 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:15.194 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:15.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:15.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:15.300 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:15.338 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:15.397 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:15.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:15.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:15.501 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:15.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: from='client.46811 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:52:15.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: pgmap v1597: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 3 peering, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 13576/164028 objects degraded (8.277%); 13929/164028 objects misplaced (8.492%); 22 MiB/s, 410 objects/s recovering 2023-09-30T09:52:15.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:52:15.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'30757','2':'35821','4':'36603'}"}]': finished 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: osdmap e600: 8 total, 8 up, 8 in 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[216815]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: from='client.46811 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: pgmap v1597: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 3 peering, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 13576/164028 objects degraded (8.277%); 13929/164028 objects misplaced (8.492%); 22 MiB/s, 410 objects/s recovering 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'30757','2':'35821','4':'36603'}"}]': finished 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: osdmap e600: 8 total, 8 up, 8 in 2023-09-30T09:52:15.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:15 smithi019 ceph-mon[224043]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.538 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:15.685 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: from='client.46811 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-30T09:52:15.685 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: pgmap v1597: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 3 peering, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 13576/164028 objects degraded (8.277%); 13929/164028 objects misplaced (8.492%); 22 MiB/s, 410 objects/s recovering 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'30757','2':'35821','4':'36603'}"}]': finished 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: osdmap e600: 8 total, 8 up, 8 in 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.686 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:15.687 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:15 smithi167 ceph-mon[192880]: failed to encode map e600 with expected crc 2023-09-30T09:52:16.210 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:52:16.211 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:52:16.355 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:16.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:16.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:16.462 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:16.494 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[224043]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.494 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[224043]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.495 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[224043]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.495 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[224043]: osdmap e601: 8 total, 8 up, 8 in 2023-09-30T09:52:16.495 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[224043]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.495 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[224043]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.495 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[224043]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.496 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[224043]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[216815]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[216815]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[216815]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[216815]: osdmap e601: 8 total, 8 up, 8 in 2023-09-30T09:52:16.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[216815]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[216815]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.497 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[216815]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.497 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:16 smithi019 ceph-mon[216815]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.501 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:16 smithi167 ceph-mon[192880]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:16 smithi167 ceph-mon[192880]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:16 smithi167 ceph-mon[192880]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:16 smithi167 ceph-mon[192880]: osdmap e601: 8 total, 8 up, 8 in 2023-09-30T09:52:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:16 smithi167 ceph-mon[192880]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:16 smithi167 ceph-mon[192880]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:16 smithi167 ceph-mon[192880]: failed to encode map e601 with expected crc 2023-09-30T09:52:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:16 smithi167 ceph-mon[192880]: failed to encode map e601 with expected crc 2023-09-30T09:52:17.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: pgmap v1600: 119 pgs: 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 7756/159378 objects degraded (4.866%); 12804/159378 objects misplaced (8.034%); 17 MiB/s, 322 objects/s recovering 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: osdmap e602: 8 total, 8 up, 8 in 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: 10.3 starting backfill to osd.0 from (0'0,0'0] MAX to 599'8477 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[216815]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: pgmap v1600: 119 pgs: 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 7756/159378 objects degraded (4.866%); 12804/159378 objects misplaced (8.034%); 17 MiB/s, 322 objects/s recovering 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: osdmap e602: 8 total, 8 up, 8 in 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: 10.3 starting backfill to osd.0 from (0'0,0'0] MAX to 599'8477 2023-09-30T09:52:17.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:17 smithi019 ceph-mon[224043]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: pgmap v1600: 119 pgs: 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 7756/159378 objects degraded (4.866%); 12804/159378 objects misplaced (8.034%); 17 MiB/s, 322 objects/s recovering 2023-09-30T09:52:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:52:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: osdmap e602: 8 total, 8 up, 8 in 2023-09-30T09:52:17.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: 10.3 starting backfill to osd.0 from (0'0,0'0] MAX to 599'8477 2023-09-30T09:52:17.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:17 smithi167 ceph-mon[192880]: failed to encode map e602 with expected crc 2023-09-30T09:52:17.952 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:17.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:17.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:18.066 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:18.103 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:18.131 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:52:18.155 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:18.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:18.261 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:18.290 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:18.353 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:18.359 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:18 smithi167 ceph-mon[192880]: osdmap e603: 8 total, 8 up, 8 in 2023-09-30T09:52:18.359 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:18 smithi167 ceph-mon[192880]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.359 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:18 smithi167 ceph-mon[192880]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.359 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:18 smithi167 ceph-mon[192880]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.359 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:18 smithi167 ceph-mon[192880]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.360 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:18.360 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:18.360 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:52:18.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:18.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:18.461 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:18.535 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:52:18.538 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:18.566 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:52:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[224043]: osdmap e603: 8 total, 8 up, 8 in 2023-09-30T09:52:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[224043]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[224043]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[224043]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[224043]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:18.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:52:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[216815]: osdmap e603: 8 total, 8 up, 8 in 2023-09-30T09:52:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[216815]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[216815]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[216815]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[216815]: failed to encode map e603 with expected crc 2023-09-30T09:52:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:18.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:52:18.595 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:52:18.630 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:52:18.659 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:18.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:18.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:18.679 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:52:18.746 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:52:18.771 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:18.785 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:52:18.802 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:18.834 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:52:18.847 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:52:18.847 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:52:18.847 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:52:18.847 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:52:18.848 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:52:18.848 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:52:18.848 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:52:18.848 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:52:18.862 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:18.974 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:18.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:19.085 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:19.140 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:19.435 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:19.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:19.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:19.547 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: failed to encode map e603 with expected crc 2023-09-30T09:52:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: failed to encode map e603 with expected crc 2023-09-30T09:52:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: pgmap v1603: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 7 peering, 2 active+remapped+backfilling, 3 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 5862/157509 objects degraded (3.722%); 13795/157509 objects misplaced (8.758%); 13 MiB/s, 306 objects/s recovering 2023-09-30T09:52:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: osdmap e604: 8 total, 8 up, 8 in 2023-09-30T09:52:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2377682940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:19.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7756/159378 objects degraded (4.866%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: failed to encode map e603 with expected crc 2023-09-30T09:52:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: failed to encode map e603 with expected crc 2023-09-30T09:52:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: pgmap v1603: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 7 peering, 2 active+remapped+backfilling, 3 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 5862/157509 objects degraded (3.722%); 13795/157509 objects misplaced (8.758%); 13 MiB/s, 306 objects/s recovering 2023-09-30T09:52:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: osdmap e604: 8 total, 8 up, 8 in 2023-09-30T09:52:19.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2377682940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:19.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7756/159378 objects degraded (4.866%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: failed to encode map e603 with expected crc 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: failed to encode map e603 with expected crc 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: pgmap v1603: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 7 peering, 2 active+remapped+backfilling, 3 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 5862/157509 objects degraded (3.722%); 13795/157509 objects misplaced (8.758%); 13 MiB/s, 306 objects/s recovering 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: osdmap e604: 8 total, 8 up, 8 in 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: failed to encode map e604 with expected crc 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2377682940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:19.624 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7756/159378 objects degraded (4.866%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:52:19.632 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:19.756 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:19.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:19.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:19.866 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:19.934 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:19.989 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:52:20.649 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:20.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:20.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:20.723 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:20 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:52:20.756 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:20.828 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:20.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:20 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:52:20.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:20 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:52:20.951 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:20.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:21.066 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:21.183 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:21.415 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:21.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:21.521 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:21.582 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:21.649 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:52:21.649 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:52:21.650 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:52:21.650 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:52:21.650 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:52:21.685 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:21 smithi167 ceph-mon[192880]: pgmap v1605: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 385 op/s; 12689/154392 objects degraded (8.219%); 10842/154392 objects misplaced (7.022%); 24 MiB/s, 2 keys/s, 467 objects/s recovering 2023-09-30T09:52:21.686 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:21.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:21.702 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:21.793 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:21.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:21 smithi019 ceph-mon[216815]: pgmap v1605: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 385 op/s; 12689/154392 objects degraded (8.219%); 10842/154392 objects misplaced (7.022%); 24 MiB/s, 2 keys/s, 467 objects/s recovering 2023-09-30T09:52:21.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:21 smithi019 ceph-mon[224043]: pgmap v1605: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 385 op/s; 12689/154392 objects degraded (8.219%); 10842/154392 objects misplaced (7.022%); 24 MiB/s, 2 keys/s, 467 objects/s recovering 2023-09-30T09:52:21.903 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:22.082 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:22.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:22.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:22.189 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:22.247 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:22.766 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:22.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:22.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:22.871 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:22.917 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:22.951 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:52:23.052 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:52:23.052 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:52:23.053 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-30T09:52:23.053 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:52:23.053 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:52:23.053 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-30T09:52:23.474 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:52:23.661 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:23.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:23.679 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:23 smithi019 ceph-mon[224043]: pgmap v1606: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 12689/154002 objects degraded (8.240%); 9389/154002 objects misplaced (6.097%); 25 MiB/s, 1 keys/s, 475 objects/s recovering 2023-09-30T09:52:23.679 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:23 smithi019 ceph-mon[216815]: pgmap v1606: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 12689/154002 objects degraded (8.240%); 9389/154002 objects misplaced (6.097%); 25 MiB/s, 1 keys/s, 475 objects/s recovering 2023-09-30T09:52:23.771 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:23.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:23 smithi167 ceph-mon[192880]: pgmap v1606: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 12689/154002 objects degraded (8.240%); 9389/154002 objects misplaced (6.097%); 25 MiB/s, 1 keys/s, 475 objects/s recovering 2023-09-30T09:52:23.780 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:23.857 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:23.892 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:23.985 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:23.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:24.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:52:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:52:23] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-30T09:52:24.091 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:24.178 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:24.418 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 12689/154002 objects degraded (8.240%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:24.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 12689/154002 objects degraded (8.240%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:24.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 12689/154002 objects degraded (8.240%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:24.683 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:24.688 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:24.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:24.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:24.790 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:24.885 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:25.097 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:25.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:25.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:25.206 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:25.299 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:25.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:25 smithi167 ceph-mon[192880]: pgmap v1607: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 12521/151494 objects degraded (8.265%); 8939/151494 objects misplaced (5.901%); 25 MiB/s, 1 keys/s, 450 objects/s recovering 2023-09-30T09:52:25.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:25 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2217559577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:52:25.442 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:25 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:52:25.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:25 smithi019 ceph-mon[216815]: pgmap v1607: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 12521/151494 objects degraded (8.265%); 8939/151494 objects misplaced (5.901%); 25 MiB/s, 1 keys/s, 450 objects/s recovering 2023-09-30T09:52:25.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:25 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2217559577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:52:25.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:25 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:52:25.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:25 smithi019 ceph-mon[224043]: pgmap v1607: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 12521/151494 objects degraded (8.265%); 8939/151494 objects misplaced (5.901%); 25 MiB/s, 1 keys/s, 450 objects/s recovering 2023-09-30T09:52:25.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:25 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2217559577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:52:25.448 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:25 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:52:25.674 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:25.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:25.779 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:25.837 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:25.916 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:52:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:26 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:26 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:52:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:26 smithi167 ceph-mon[192880]: osdmap e605: 8 total, 8 up, 8 in 2023-09-30T09:52:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:26 smithi167 ceph-mon[192880]: failed to encode map e605 with expected crc 2023-09-30T09:52:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:26 smithi167 ceph-mon[192880]: failed to encode map e605 with expected crc 2023-09-30T09:52:26.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:26 smithi167 ceph-mon[192880]: pgmap v1609: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 12311/145137 objects degraded (8.482%); 7814/145137 objects misplaced (5.384%); 26 MiB/s, 1 keys/s, 464 objects/s recovering 2023-09-30T09:52:26.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:52:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[216815]: osdmap e605: 8 total, 8 up, 8 in 2023-09-30T09:52:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[216815]: failed to encode map e605 with expected crc 2023-09-30T09:52:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[216815]: failed to encode map e605 with expected crc 2023-09-30T09:52:26.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[216815]: pgmap v1609: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 12311/145137 objects degraded (8.482%); 7814/145137 objects misplaced (5.384%); 26 MiB/s, 1 keys/s, 464 objects/s recovering 2023-09-30T09:52:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:26.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:52:26.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[224043]: osdmap e605: 8 total, 8 up, 8 in 2023-09-30T09:52:26.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[224043]: failed to encode map e605 with expected crc 2023-09-30T09:52:26.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[224043]: failed to encode map e605 with expected crc 2023-09-30T09:52:26.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:26 smithi019 ceph-mon[224043]: pgmap v1609: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 12311/145137 objects degraded (8.482%); 7814/145137 objects misplaced (5.384%); 26 MiB/s, 1 keys/s, 464 objects/s recovering 2023-09-30T09:52:27.354 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:52:27.354 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:153578 8.9 8:90d1d7bb:::benchmark_data_smithi019_478679_object61130:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:02.966549+0000", 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 564.38322590200005, 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0650080519999996, 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:52:27.355 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 153578 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966549+0000", 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966549+0000", 2023-09-30T09:52:27.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966548+0000", 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966554+0000", 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9290000000000003e-06 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966567+0000", 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2621e-05 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966575+0000", 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5099999999999998e-06 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967360+0000", 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00078461500000000001 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967368+0000", 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.1240000000000005e-06 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.637823+0000", 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6704550610000002 2023-09-30T09:52:27.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for missing object", 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.640292+0000", 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024689769999999998 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.029951+0000", 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.389658828 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031557+0000", 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016064709999999999 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:153601 8.9 8:96e6158f:::benchmark_data_smithi019_478679_object61153:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:02.966582+0000", 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 564.38319340199996, 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1046012760000004, 2023-09-30T09:52:27.361 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.362 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.362 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.362 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:52:27.362 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:52:27.362 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 153601 2023-09-30T09:52:27.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.362 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966582+0000", 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966582+0000", 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966581+0000", 2023-09-30T09:52:27.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966584+0000", 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9730000000000002e-06 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966591+0000", 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5799999999999997e-06 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966595+0000", 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3209999999999998e-06 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967390+0000", 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00079515100000000002 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967396+0000", 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3090000000000003e-06 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.040459+0000", 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0730634190000004 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.042937+0000", 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024781439999999998 2023-09-30T09:52:27.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,6", 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.047464+0000", 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045271729999999998 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.064276+0000", 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016811621999999998 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.064915+0000", 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00063849399999999998 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.065536+0000", 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00062178699999999995 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.066784+0000", 2023-09-30T09:52:27.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012479609999999999 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.071183+0000", 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0043987760000000001 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.35820.0:153614 8.9 8:92103489:::benchmark_data_smithi019_478679_object61166:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2023-09-30T09:52:27.369 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:02.966600+0000", 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 564.38317535500005, 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1120596819999999, 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.35820", 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4047810267", 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 153614 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966600+0000", 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966600+0000", 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966599+0000", 2023-09-30T09:52:27.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966602+0000", 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8600000000000001e-06 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966609+0000", 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3319999999999997e-06 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.966613+0000", 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0640000000000004e-06 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967413+0000", 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00080035700000000002 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:02.967420+0000", 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4740000000000002e-06 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.053697+0000", 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0862776460000001 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.056675+0000", 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029778230000000001 2023-09-30T09:52:27.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,6", 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.060861+0000", 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0041855140000000004 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.071889+0000", 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011028911000000001 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.072098+0000", 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000208333 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.072683+0000", 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000585495 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.073852+0000", 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011684009999999999 2023-09-30T09:52:27.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.078660+0000", 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048078820000000003 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGRecoveryDeleteReply(8.9 e457,455 8:90d1d7bb:::benchmark_data_smithi019_478679_object61130:head,453'7128)", 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:05.642513+0000", 2023-09-30T09:52:27.377 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 561.70726183199997, 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3871478769999999, 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642513+0000", 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642513+0000", 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642513+0000", 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642518+0000", 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7180000000000004e-06 2023-09-30T09:52:27.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642530+0000", 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2391e-05 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.642537+0000", 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2490000000000004e-06 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.019708+0000", 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.377170352 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.029661+0000", 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0099537389999999996 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.381 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35820.0:153603 8.19 e457/455 8:9d676f8e:::benchmark_data_smithi019_478679_object61155:head v 457'7206, mlcod=0'0)", 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:05.654142+0000", 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 561.69563373100004, 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3773550720000001, 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654142+0000", 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654142+0000", 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654141+0000", 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654148+0000", 2023-09-30T09:52:27.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1420000000000004e-06 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654162+0000", 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3559e-05 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.654168+0000", 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0909999999999998e-06 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.019529+0000", 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.365361477 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.019550+0000", 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0965e-05 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031419+0000", 2023-09-30T09:52:27.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011869104 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031497+0000", 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7324999999999996e-05 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.386 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35820.0:153650 8.19 e457/455 8:9a1a13f4:::benchmark_data_smithi019_478679_object61202:head v 457'7207, mlcod=0'0)", 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:05.655233+0000", 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 561.69454244799999, 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.37629468, 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655233+0000", 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655233+0000", 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655232+0000", 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655238+0000", 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5890000000000002e-06 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655249+0000", 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1357e-05 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.655255+0000", 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3460000000000001e-06 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.021117+0000", 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3658622629999999 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.021139+0000", 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2002e-05 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031499+0000", 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010360596999999999 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031527+0000", 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8055000000000002e-05 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.391 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.35820.0:153653 8.19 e457/455 8:9f29c27f:::benchmark_data_smithi019_478679_object61205:head v 457'7208, mlcod=0'0)", 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:43:05.656934+0000", 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 561.69284147200005, 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3746209979999999, 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656934+0000", 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656934+0000", 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656933+0000", 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656939+0000", 2023-09-30T09:52:27.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6119999999999998e-06 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656951+0000", 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2422000000000001e-05 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:05.656957+0000", 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2290000000000002e-06 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.022470+0000", 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3655130689999999 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.022489+0000", 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9623000000000002e-05 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031533+0000", 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0090435029999999996 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:43:07.031555+0000", 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2065e-05 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.396 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101654 10.e 10:7e27f7a0:::benchmark_data_smithi019_617837_object8254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.458430+0000", 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 106.89134556099999, 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0722781880000001, 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101654 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458430+0000", 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458430+0000", 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458429+0000", 2023-09-30T09:52:27.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458434+0000", 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6629999999999998e-06 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458451+0000", 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.643e-05 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458456+0000", 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2240000000000003e-06 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458512+0000", 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5869999999999999e-05 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458523+0000", 2023-09-30T09:52:27.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0805000000000001e-05 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.457801+0000", 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99927842300000003 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.457958+0000", 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015693099999999999 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.458122+0000", 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016425700000000001 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.472646+0000", 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01452355 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.473651+0000", 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001005288 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.529695+0000", 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.056044163000000001 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.529758+0000", 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3034000000000007e-05 2023-09-30T09:52:27.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.530708+0000", 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00094970799999999995 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101664 10.e 10:7d1aed30:::benchmark_data_smithi019_617837_object8264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.484162+0000", 2023-09-30T09:52:27.404 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 106.865612801, 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.057487034, 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101664 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484162+0000", 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484162+0000", 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484161+0000", 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:27.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484167+0000", 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7599999999999999e-06 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484180+0000", 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2488e-05 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484185+0000", 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1989999999999997e-06 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484244+0000", 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8811000000000003e-05 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484254+0000", 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9310000000000003e-06 2023-09-30T09:52:27.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462696+0000", 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97844258799999995 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462826+0000", 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000130183 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:27.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462966+0000", 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013949200000000001 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.472671+0000", 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097053090000000005 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.473685+0000", 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010137290000000001 2023-09-30T09:52:27.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.540844+0000", 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.067158849000000007 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.540902+0000", 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8048999999999999e-05 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.541649+0000", 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00074773600000000004 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127781 10.4 10:25ddb007:::benchmark_data_smithi019_617837_object34381:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:57.175490+0000", 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 30.174285494999999, 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1234206959999999, 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.412 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127781 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175490+0000", 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175490+0000", 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175488+0000", 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175495+0000", 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8619999999999997e-06 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175508+0000", 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2673e-05 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175515+0000", 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2049999999999996e-06 2023-09-30T09:52:27.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175913+0000", 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00039778900000000001 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175926+0000", 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3489000000000001e-05 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.279219+0000", 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1032926949999999 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.281411+0000", 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021921359999999999 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,3", 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.281685+0000", 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027379999999999999 2023-09-30T09:52:27.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.292203+0000", 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010517581999999999 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.293803+0000", 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016001679999999999 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298752+0000", 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049493999999999996 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298803+0000", 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.083e-05 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298843+0000", 2023-09-30T09:52:27.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0145000000000003e-05 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298910+0000", 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7354000000000006e-05 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127783 10.e 10:75bd171f:::benchmark_data_smithi019_617837_object34383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:52:27.420 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:57.262645+0000", 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 30.087130102, 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92964328500000004, 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127783 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262645+0000", 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262645+0000", 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262644+0000", 2023-09-30T09:52:27.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262651+0000", 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5349999999999999e-06 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262667+0000", 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5947000000000002e-05 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262681+0000", 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3978999999999999e-05 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262734+0000", 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2907e-05 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262741+0000", 2023-09-30T09:52:27.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7719999999999997e-06 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.186184+0000", 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92344351300000005 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.186337+0000", 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015267500000000001 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.186484+0000", 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014657599999999999 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.189850+0000", 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033660320000000001 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.191036+0000", 2023-09-30T09:52:27.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011866800000000001 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.192173+0000", 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011366900000000001 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.192234+0000", 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1278999999999994e-05 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.192288+0000", 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4039999999999998e-05 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127784 10.4 10:218c61d4:::benchmark_data_smithi019_617837_object34384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:57.336285+0000", 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 30.013490244, 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.962762848, 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.428 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127784 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336285+0000", 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336285+0000", 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336284+0000", 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336291+0000", 2023-09-30T09:52:27.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9399999999999996e-06 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336303+0000", 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2449999999999999e-05 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336308+0000", 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0900000000000004e-06 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336368+0000", 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9815000000000002e-05 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336383+0000", 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5102e-05 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.289022+0000", 2023-09-30T09:52:27.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95263867800000002 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.289175+0000", 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015364900000000001 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,3", 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.289322+0000", 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014690699999999999 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.292213+0000", 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002890882 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.296819+0000", 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004605441 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298780+0000", 2023-09-30T09:52:27.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019614049999999998 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298940+0000", 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016015800000000001 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298971+0000", 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0460999999999999e-05 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.299048+0000", 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7195999999999997e-05 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129750 10.d 10:b52c4f95:::benchmark_data_smithi019_617837_object36350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e595)", 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:05.215988+0000", 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 22.133786799999999, 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3956790290000001, 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.436 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129750 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.215988+0000", 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.215988+0000", 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.215987+0000", 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.215995+0000", 2023-09-30T09:52:27.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2190000000000004e-06 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.216012+0000", 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7005999999999999e-05 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.216019+0000", 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6830000000000002e-06 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.216073+0000", 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3575999999999997e-05 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.216080+0000", 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4329999999999998e-06 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.213695+0000", 2023-09-30T09:52:27.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99761513400000001 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.213707+0000", 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1734e-05 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.510717+0000", 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.297010123 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.510818+0000", 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100617 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.510914+0000", 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6471999999999994e-05 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.519218+0000", 2023-09-30T09:52:27.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0083036840000000004 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532043+0000", 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012824797000000001 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.609411+0000", 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.07736809 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.609473+0000", 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2677999999999999e-05 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.611667+0000", 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021942200000000002 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129758 10.e 10:7ba26817:::benchmark_data_smithi019_617837_object36358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.493951+0000", 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.855823772000001, 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.85264479900000001, 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129758 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.445 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493951+0000", 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493951+0000", 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493950+0000", 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493956+0000", 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4110000000000004e-06 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493969+0000", 2023-09-30T09:52:27.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3861e-05 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493975+0000", 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7579999999999998e-06 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.494043+0000", 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7724000000000004e-05 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.494051+0000", 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2160000000000006e-06 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.336428+0000", 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.84237695599999995 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.336555+0000", 2023-09-30T09:52:27.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012707500000000001 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5,6", 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.336652+0000", 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6813999999999998e-05 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339699+0000", 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030473219999999999 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339768+0000", 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8276999999999997e-05 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339779+0000", 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0873000000000001e-05 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346487+0000", 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0067086469999999999 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346522+0000", 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5327000000000002e-05 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346596+0000", 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3813999999999995e-05 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.452 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129760 10.d 10:b92f266b:::benchmark_data_smithi019_617837_object36360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.503079+0000", 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.846695930999999, 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1105047159999999, 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.453 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129760 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503079+0000", 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503079+0000", 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503078+0000", 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503083+0000", 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1549999999999998e-06 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503096+0000", 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3030000000000001e-05 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503102+0000", 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1839999999999998e-06 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503147+0000", 2023-09-30T09:52:27.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.549e-05 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503156+0000", 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5199999999999997e-06 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.513645+0000", 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.010489609 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.513875+0000", 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00022947700000000001 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.514017+0000", 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014177600000000001 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.526800+0000", 2023-09-30T09:52:27.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012783216 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532094+0000", 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005293865 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.611739+0000", 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.079645117000000001 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.611805+0000", 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6489999999999995e-05 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.613584+0000", 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017788280000000001 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129774 10.d 10:b44bed1b:::benchmark_data_smithi019_617837_object36374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.525695+0000", 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.824080577, 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.089412802, 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129774 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525695+0000", 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525695+0000", 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525693+0000", 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525713+0000", 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9320000000000001e-05 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525726+0000", 2023-09-30T09:52:27.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3645000000000001e-05 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525733+0000", 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8419999999999999e-06 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525799+0000", 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6038999999999996e-05 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525810+0000", 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1438e-05 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.516754+0000", 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99094344400000001 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.516870+0000", 2023-09-30T09:52:27.465 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0001158 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.516996+0000", 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012621099999999999 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.529302+0000", 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012305732999999999 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.466 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532146+0000", 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028442839999999999 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.613698+0000", 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.081551636999999996 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.467 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.613755+0000", 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7737e-05 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.615107+0000", 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00135217 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.468 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129775 10.e 10:77c928e6:::benchmark_data_smithi019_617837_object36375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.528831+0000", 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.820944636, 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82096436800000006, 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129775 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.469 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528831+0000", 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528831+0000", 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.470 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528829+0000", 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528835+0000", 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9660000000000001e-06 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528853+0000", 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7550999999999998e-05 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.471 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528860+0000", 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6669999999999999e-06 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528925+0000", 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5370999999999995e-05 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.472 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528937+0000", 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2218000000000001e-05 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339805+0000", 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81086787900000001 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339934+0000", 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012879300000000001 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.473 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5,6", 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.340067+0000", 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013331899999999999 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.342961+0000", 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028942180000000001 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.474 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.345879+0000", 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029180859999999999 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.345952+0000", 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2434000000000005e-05 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349615+0000", 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003662733 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.475 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349674+0000", 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9312999999999999e-05 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349795+0000", 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012102700000000001 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.476 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129782 10.e 10:7bfbc37c:::benchmark_data_smithi019_617837_object36382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.537734+0000", 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.812041639, 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81226853300000001, 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129782 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.477 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537734+0000", 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537734+0000", 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.478 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537732+0000", 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537738+0000", 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1369999999999999e-06 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537752+0000", 2023-09-30T09:52:27.479 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4039000000000001e-05 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537759+0000", 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2829999999999998e-06 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537814+0000", 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4897e-05 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.480 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537822+0000", 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2439999999999993e-06 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.342981+0000", 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.80515920500000004 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.343129+0000", 2023-09-30T09:52:27.481 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000147885 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5,6", 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.343267+0000", 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000137775 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.345989+0000", 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027219599999999998 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.482 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346026+0000", 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7230000000000001e-05 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346034+0000", 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9869999999999999e-06 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349847+0000", 2023-09-30T09:52:27.483 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0038129589999999999 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349899+0000", 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1651999999999999e-05 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.350002+0000", 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103626 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.484 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129787 10.d 10:b2aeaf7e:::benchmark_data_smithi019_617837_object36387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.544982+0000", 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.804792913, 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0717242520000001, 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129787 2023-09-30T09:52:27.485 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.544982+0000", 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.544982+0000", 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.486 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.544981+0000", 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.544987+0000", 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5820000000000001e-06 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.545000+0000", 2023-09-30T09:52:27.487 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2938999999999999e-05 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.545005+0000", 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0379999999999999e-06 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.545065+0000", 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9932999999999997e-05 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.488 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.545076+0000", 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1488000000000001e-05 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.519270+0000", 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97419379699999997 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.519425+0000", 2023-09-30T09:52:27.489 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015525099999999999 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.519549+0000", 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012434600000000001 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.529308+0000", 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097589800000000004 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.490 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532180+0000", 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028711100000000001 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.615153+0000", 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.082973619999999998 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.615192+0000", 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9079000000000003e-05 2023-09-30T09:52:27.491 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.616707+0000", 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015143489999999999 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129794 10.d 10:b60cc818:::benchmark_data_smithi019_617837_object36394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.566027+0000", 2023-09-30T09:52:27.492 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 20.783747981000001, 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0527564389999999, 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129794 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:27.493 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566027+0000", 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566027+0000", 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566026+0000", 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:27.494 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566032+0000", 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0100000000000001e-06 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566045+0000", 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2456999999999999e-05 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.495 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566050+0000", 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9529999999999998e-06 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566098+0000", 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8383000000000001e-05 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:27.496 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566105+0000", 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5290000000000003e-06 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.521542+0000", 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95543642299999998 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.521665+0000", 2023-09-30T09:52:27.497 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012298800000000001 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.521787+0000", 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000121971 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.529314+0000", 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075275400000000001 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.498 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.499 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532211+0000", 2023-09-30T09:52:27.499 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028969 2023-09-30T09:52:27.499 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.499 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.499 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:27.499 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.616756+0000", 2023-09-30T09:52:27.499 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.084544362999999997 2023-09-30T09:52:27.499 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.616805+0000", 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8987999999999999e-05 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.618784+0000", 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019790889999999998 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.500 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.501 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:27.501 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:27.501 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:52:27.578 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:27.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:27.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:27.685 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:27.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[216815]: failed to encode map e605 with expected crc 2023-09-30T09:52:27.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[216815]: failed to encode map e605 with expected crc 2023-09-30T09:52:27.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[216815]: osdmap e606: 8 total, 8 up, 8 in 2023-09-30T09:52:27.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[216815]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[216815]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[216815]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[216815]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.713 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[224043]: failed to encode map e605 with expected crc 2023-09-30T09:52:27.714 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[224043]: failed to encode map e605 with expected crc 2023-09-30T09:52:27.714 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[224043]: osdmap e606: 8 total, 8 up, 8 in 2023-09-30T09:52:27.714 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[224043]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.714 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[224043]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.714 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[224043]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.714 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:27 smithi019 ceph-mon[224043]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.716 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:27 smithi167 ceph-mon[192880]: failed to encode map e605 with expected crc 2023-09-30T09:52:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:27 smithi167 ceph-mon[192880]: failed to encode map e605 with expected crc 2023-09-30T09:52:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:27 smithi167 ceph-mon[192880]: osdmap e606: 8 total, 8 up, 8 in 2023-09-30T09:52:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:27 smithi167 ceph-mon[192880]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:27 smithi167 ceph-mon[192880]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:27 smithi167 ceph-mon[192880]: failed to encode map e606 with expected crc 2023-09-30T09:52:27.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:27 smithi167 ceph-mon[192880]: failed to encode map e606 with expected crc 2023-09-30T09:52:28.184 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:52:28.184 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "" 2023-09-30T09:52:28.184 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:52:28.403 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:28.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:28.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:28.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[224043]: failed to encode map e606 with expected crc 2023-09-30T09:52:28.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[224043]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[224043]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[224043]: osdmap e607: 8 total, 8 up, 8 in 2023-09-30T09:52:28.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[224043]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[224043]: pgmap v1612: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 12143/142530 objects degraded (8.520%); 7400/142530 objects misplaced (5.192%); 19 MiB/s, 309 objects/s recovering 2023-09-30T09:52:28.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:28.594 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:28.744 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:28.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:28 smithi167 ceph-mon[192880]: failed to encode map e606 with expected crc 2023-09-30T09:52:28.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:28 smithi167 ceph-mon[192880]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:28 smithi167 ceph-mon[192880]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.747 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:28 smithi167 ceph-mon[192880]: osdmap e607: 8 total, 8 up, 8 in 2023-09-30T09:52:28.748 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:28 smithi167 ceph-mon[192880]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.748 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:28 smithi167 ceph-mon[192880]: pgmap v1612: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 12143/142530 objects degraded (8.520%); 7400/142530 objects misplaced (5.192%); 19 MiB/s, 309 objects/s recovering 2023-09-30T09:52:28.749 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[216815]: failed to encode map e606 with expected crc 2023-09-30T09:52:28.749 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[216815]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.749 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[216815]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.749 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[216815]: osdmap e607: 8 total, 8 up, 8 in 2023-09-30T09:52:28.749 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[216815]: failed to encode map e607 with expected crc 2023-09-30T09:52:28.749 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:28 smithi019 ceph-mon[216815]: pgmap v1612: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 12143/142530 objects degraded (8.520%); 7400/142530 objects misplaced (5.192%); 19 MiB/s, 309 objects/s recovering 2023-09-30T09:52:28.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:28.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:28.855 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:28.881 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:29.008 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:52:29.027 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:52:29.055 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:52:29.095 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:52:29.138 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:52:29.160 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:52:29.186 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:52:29.239 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:52:29.289 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:52:29.311 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:52:29.311 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:52:29.311 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:52:29.312 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:52:29.312 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:52:29.312 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:52:29.312 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:52:29.312 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:52:29.458 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:29.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:29.570 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:29.651 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:29.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[216815]: failed to encode map e607 with expected crc 2023-09-30T09:52:29.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 12311/145137 objects degraded (8.482%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:29.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2596578555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:29.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.669 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[224043]: failed to encode map e607 with expected crc 2023-09-30T09:52:29.669 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.669 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.669 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 12311/145137 objects degraded (8.482%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:29.669 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2596578555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:29.669 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.669 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.779 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:29 smithi167 ceph-mon[192880]: failed to encode map e607 with expected crc 2023-09-30T09:52:29.780 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:29 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.780 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:29 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.780 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:29 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 12311/145137 objects degraded (8.482%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:29.780 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:29 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2596578555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:29.780 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:29 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.780 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:29 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:29.780 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:29.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:29.857 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:29.892 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:29.922 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:30.394 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:30.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:30.502 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:30.572 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:30.578 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:52:30.724 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:30.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:30.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:30 smithi019 ceph-mon[224043]: pgmap v1613: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 11972/137247 objects degraded (8.723%); 6863/137247 objects misplaced (5.000%); 16 MiB/s, 251 objects/s recovering 2023-09-30T09:52:30.741 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:30 smithi019 ceph-mon[216815]: pgmap v1613: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 11972/137247 objects degraded (8.723%); 6863/137247 objects misplaced (5.000%); 16 MiB/s, 251 objects/s recovering 2023-09-30T09:52:30.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:30 smithi167 ceph-mon[192880]: pgmap v1613: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 11972/137247 objects degraded (8.723%); 6863/137247 objects misplaced (5.000%); 16 MiB/s, 251 objects/s recovering 2023-09-30T09:52:30.841 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:30.875 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:30.932 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:30.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:30.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:31.039 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:31.141 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:31.150 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:31.294 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:31.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:31.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:31.418 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:31.501 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:31.621 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:31.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:31.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:31.736 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:31.788 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:31.962 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:31.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:31.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:32.069 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:32.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:32.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:31 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:32.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:32.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:31 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:32.123 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:32.225 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:32.233 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:32.233 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:31 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:32.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:32.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:32.273 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:52:32.273 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:52:32.336 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:32.364 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:32.408 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:32.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:32.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:32.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:32.538 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:52:32.613 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:32.710 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:32.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:32.814 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:32.840 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:32.880 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:32.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:32.984 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:33.067 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:33.092 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:33.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:33 smithi167 ceph-mon[192880]: pgmap v1614: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 11972/137247 objects degraded (8.723%); 6863/137247 objects misplaced (5.000%); 8.7 MiB/s, 138 objects/s recovering 2023-09-30T09:52:33.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:33 smithi019 ceph-mon[224043]: pgmap v1614: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 11972/137247 objects degraded (8.723%); 6863/137247 objects misplaced (5.000%); 8.7 MiB/s, 138 objects/s recovering 2023-09-30T09:52:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:33 smithi019 ceph-mon[216815]: pgmap v1614: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 11972/137247 objects degraded (8.723%); 6863/137247 objects misplaced (5.000%); 8.7 MiB/s, 138 objects/s recovering 2023-09-30T09:52:33.350 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:33.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:33.456 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:52:33.465 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:33.526 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:33.872 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:33.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:33.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:33.999 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:34.028 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:52:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:52:33] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-09-30T09:52:34.071 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:34.157 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:34.284 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:34 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:34.284 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:34 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:34.284 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:34 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 11972/137247 objects degraded (8.723%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:34.285 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:34 smithi019 ceph-mon[216815]: osdmap e608: 8 total, 8 up, 8 in 2023-09-30T09:52:34.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:34 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:34.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:34 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:34.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:34 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 11972/137247 objects degraded (8.723%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:34.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:34 smithi019 ceph-mon[224043]: osdmap e608: 8 total, 8 up, 8 in 2023-09-30T09:52:34.433 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:34 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:34.434 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:34 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:34.434 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:34 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 11972/137247 objects degraded (8.723%), 6 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:34.434 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:34 smithi167 ceph-mon[192880]: osdmap e608: 8 total, 8 up, 8 in 2023-09-30T09:52:34.459 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:52:34.459 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:52:34.460 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-09-30T09:52:34.460 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:52:34.597 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:34.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:34.700 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:34.733 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:34.910 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:52:35.055 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:35.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:35.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:35.169 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:35.211 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:35.324 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:35.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: pgmap v1616: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 11808/133143 objects degraded (8.869%); 6653/133143 objects misplaced (4.997%); 10 MiB/s, 223 objects/s recovering 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1275617729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: failed to encode map e609 with expected crc 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[216815]: osdmap e609: 8 total, 8 up, 8 in 2023-09-30T09:52:35.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: pgmap v1616: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 11808/133143 objects degraded (8.869%); 6653/133143 objects misplaced (4.997%); 10 MiB/s, 223 objects/s recovering 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1275617729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: failed to encode map e609 with expected crc 2023-09-30T09:52:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:35 smithi019 ceph-mon[224043]: osdmap e609: 8 total, 8 up, 8 in 2023-09-30T09:52:35.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:35.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:35.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.338 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: failed to encode map e608 with expected crc 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: pgmap v1616: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 11808/133143 objects degraded (8.869%); 6653/133143 objects misplaced (4.997%); 10 MiB/s, 223 objects/s recovering 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1275617729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: failed to encode map e609 with expected crc 2023-09-30T09:52:35.339 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:35 smithi167 ceph-mon[192880]: osdmap e609: 8 total, 8 up, 8 in 2023-09-30T09:52:35.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:35.500 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:35.812 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:35.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:35.916 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:35.968 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:36.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:36 smithi019 ceph-mon[216815]: failed to encode map e609 with expected crc 2023-09-30T09:52:36.068 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:36 smithi019 ceph-mon[216815]: failed to encode map e609 with expected crc 2023-09-30T09:52:36.329 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:36.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:36 smithi019 ceph-mon[224043]: failed to encode map e609 with expected crc 2023-09-30T09:52:36.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:36 smithi019 ceph-mon[224043]: failed to encode map e609 with expected crc 2023-09-30T09:52:36.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:36.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:36.441 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:36.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:36 smithi167 ceph-mon[192880]: failed to encode map e609 with expected crc 2023-09-30T09:52:36.527 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:36 smithi167 ceph-mon[192880]: failed to encode map e609 with expected crc 2023-09-30T09:52:36.533 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:36.657 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:36.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:36.764 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:36.830 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:36.936 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:36.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:36.989 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:52:37.017 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:37.037 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:52:37.044 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:37.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:37 smithi019 ceph-mon[216815]: pgmap v1618: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7676/128709 objects degraded (5.964%); 7865/128709 objects misplaced (6.111%); 7.1 MiB/s, 176 objects/s recovering 2023-09-30T09:52:37.067 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1697005045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:37.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:37 smithi019 ceph-mon[224043]: pgmap v1618: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7676/128709 objects degraded (5.964%); 7865/128709 objects misplaced (6.111%); 7.1 MiB/s, 176 objects/s recovering 2023-09-30T09:52:37.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1697005045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:37.087 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:52:37.098 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:37 smithi167 ceph-mon[192880]: pgmap v1618: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7676/128709 objects degraded (5.964%); 7865/128709 objects misplaced (6.111%); 7.1 MiB/s, 176 objects/s recovering 2023-09-30T09:52:37.098 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1697005045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:37.103 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:37.127 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:52:37.172 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:52:37.216 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:52:37.257 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:52:37.261 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:37.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:37.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:37.294 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:52:37.295 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:52:37.296 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:52:37.296 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:52:37.296 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:52:37.296 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:52:37.296 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:52:37.296 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:52:37.296 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:52:37.375 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:37.437 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:37.698 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:37.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:37.803 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:37.850 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:38.107 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:52:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:52:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:52:38.107 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:52:38.107 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:52:38.160 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:38.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:38.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:38.268 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:38.301 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:38.416 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:38.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:38.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:38.449 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:52:38.521 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:38.606 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:39.083 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:39 smithi019 ceph-mon[216815]: pgmap v1619: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 7492/126705 objects degraded (5.913%); 7865/126705 objects misplaced (6.207%); 6.4 MiB/s, 165 objects/s recovering 2023-09-30T09:52:39.083 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:39 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7492/126705 objects degraded (5.913%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:39.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:39 smithi019 ceph-mon[224043]: pgmap v1619: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 7492/126705 objects degraded (5.913%); 7865/126705 objects misplaced (6.207%); 6.4 MiB/s, 165 objects/s recovering 2023-09-30T09:52:39.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:39 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7492/126705 objects degraded (5.913%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:39.495 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:39.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:39.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:39 smithi167 ceph-mon[192880]: pgmap v1619: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 7492/126705 objects degraded (5.913%); 7865/126705 objects misplaced (6.207%); 6.4 MiB/s, 165 objects/s recovering 2023-09-30T09:52:39.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:39 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7492/126705 objects degraded (5.913%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:39.601 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:39.649 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:39.693 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:39.820 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:39.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:39.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:39.931 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:39.983 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:40.346 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:40.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:40.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:40.384 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:52:40.463 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:40.570 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:40.659 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:52:40.674 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:52:40.725 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:40.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:40.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:40.832 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:40.909 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:41.089 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:41.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:41.185 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:41.214 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:41.299 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:41.440 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:41.449 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:41 smithi167 ceph-mon[192880]: pgmap v1620: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 7311/120159 objects degraded (6.084%); 6269/120159 objects misplaced (5.217%); 16 MiB/s, 316 objects/s recovering 2023-09-30T09:52:41.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:41.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:41.459 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:52:41.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:41 smithi019 ceph-mon[216815]: pgmap v1620: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 7311/120159 objects degraded (6.084%); 6269/120159 objects misplaced (5.217%); 16 MiB/s, 316 objects/s recovering 2023-09-30T09:52:41.542 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:41 smithi019 ceph-mon[224043]: pgmap v1620: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 7311/120159 objects degraded (6.084%); 6269/120159 objects misplaced (5.217%); 16 MiB/s, 316 objects/s recovering 2023-09-30T09:52:41.550 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:41.619 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:42.046 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:42.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:42.153 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:42.182 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:42.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:42 smithi167 ceph-mon[192880]: from='client.46844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:52:42.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:42 smithi019 ceph-mon[216815]: from='client.46844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:52:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:42 smithi019 ceph-mon[224043]: from='client.46844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:52:42.850 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1620,"stamp":"2023-09-30T09:52:40.030310+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2627812016,"num_objects":40053,"num_object_clones":0,"num_object_copies":120159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7311,"num_objects_misplaced":6269,"num_objects_unfound":0,"num_objects_dirty":40053,"num_whiteouts":0,"num_read":5989,"num_read_kb":13222,"num_write":147682,"num_write_kb":5990171,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35451,"num_bytes_recovered":2127759090,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97204,"ondisk_log_size":97204,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":414,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15175856,"kb_used_data":9616488,"kb_used_omap":30,"kb_used_meta":5558817,"kb_avail":734785360,"statfs":{"total":767960285184,"available":752420208640,"internally_reserved":0,"allocated":9847283712,"data_stored":9825086337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":5692228962},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,6,5,2,11,2,1,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-373293056,"num_objects":-5696,"num_object_clones":0,"num_object_copies":-17088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-4661,"num_objects_misplaced":-594,"num_objects_unfound":0,"num_objects_dirty":-5696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5696,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2534,"num_bytes_recovered":133103616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002262"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":1064,"reported_epoch":608,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:52:33.889559+0000","last_change":"2023-09-30T09:52:16.331889+0000","last_active":"2023-09-30T09:52:33.889559+0000","last_peered":"2023-09-30T09:52:33.889559+0000","last_clean":"2023-09-30T09:52:09.251646+0000","last_became_active":"2023-09-30T09:52:16.331131+0000","last_became_peered":"2023-09-30T09:52:16.331131+0000","last_unstale":"2023-09-30T09:52:33.889559+0000","last_undegraded":"2023-09-30T09:52:16.322829+0000","last_fullsized":"2023-09-30T09:52:16.322421+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:47.387781+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":646,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895402+0000","last_change":"2023-09-30T09:50:41.402505+0000","last_active":"2023-09-30T09:52:33.895402+0000","last_peered":"2023-09-30T09:52:33.895402+0000","last_clean":"2023-09-30T09:52:33.895402+0000","last_became_active":"2023-09-30T09:50:41.402099+0000","last_became_peered":"2023-09-30T09:50:41.402099+0000","last_unstale":"2023-09-30T09:52:33.895402+0000","last_undegraded":"2023-09-30T09:52:33.895402+0000","last_fullsized":"2023-09-30T09:52:33.895402+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:45.502519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":523,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.900408+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:52:33.900408+0000","last_peered":"2023-09-30T09:52:33.900408+0000","last_clean":"2023-09-30T09:52:33.900408+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:52:33.900408+0000","last_undegraded":"2023-09-30T09:52:33.900408+0000","last_fullsized":"2023-09-30T09:52:33.900408+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":1054,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.896022+0000","last_change":"2023-09-30T09:52:09.288155+0000","last_active":"2023-09-30T09:52:33.896022+0000","last_peered":"2023-09-30T09:52:33.896022+0000","last_clean":"2023-09-30T09:52:33.896022+0000","last_became_active":"2023-09-30T09:52:09.287827+0000","last_became_peered":"2023-09-30T09:52:09.287827+0000","last_unstale":"2023-09-30T09:52:33.896022+0000","last_undegraded":"2023-09-30T09:52:33.896022+0000","last_fullsized":"2023-09-30T09:52:33.896022+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:04.226064+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":184,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:34.912631+0000","last_change":"2023-09-30T09:51:57.147377+0000","last_active":"2023-09-30T09:52:34.912631+0000","last_peered":"2023-09-30T09:52:34.912631+0000","last_clean":"2023-09-30T09:52:34.912631+0000","last_became_active":"2023-09-30T09:51:57.146997+0000","last_became_peered":"2023-09-30T09:51:57.146997+0000","last_unstale":"2023-09-30T09:52:34.912631+0000","last_undegraded":"2023-09-30T09:52:34.912631+0000","last_fullsized":"2023-09-30T09:52:34.912631+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:01.086217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":323,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.911062+0000","last_change":"2023-09-30T09:52:06.288110+0000","last_active":"2023-09-30T09:52:33.911062+0000","last_peered":"2023-09-30T09:52:33.911062+0000","last_clean":"2023-09-30T09:52:33.911062+0000","last_became_active":"2023-09-30T09:52:06.287784+0000","last_became_peered":"2023-09-30T09:52:06.287784+0000","last_unstale":"2023-09-30T09:52:33.911062+0000","last_undegraded":"2023-09-30T09:52:33.911062+0000","last_fullsized":"2023-09-30T09:52:33.911062+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:26:56.675676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":619,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906142+0000","last_change":"2023-09-30T09:52:09.270404+0000","last_active":"2023-09-30T09:52:33.906142+0000","last_peered":"2023-09-30T09:52:33.906142+0000","last_clean":"2023-09-30T09:52:33.906142+0000","last_became_active":"2023-09-30T09:52:09.269933+0000","last_became_peered":"2023-09-30T09:52:09.269933+0000","last_unstale":"2023-09-30T09:52:33.906142+0000","last_undegraded":"2023-09-30T09:52:33.906142+0000","last_fullsized":"2023-09-30T09:52:33.906142+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:26.548074+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":30,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.899686+0000","last_change":"2023-09-30T09:51:57.155291+0000","last_active":"2023-09-30T09:52:33.899686+0000","last_peered":"2023-09-30T09:52:33.899686+0000","last_clean":"2023-09-30T09:52:33.899686+0000","last_became_active":"2023-09-30T09:51:57.154983+0000","last_became_peered":"2023-09-30T09:51:57.154983+0000","last_unstale":"2023-09-30T09:52:33.899686+0000","last_undegraded":"2023-09-30T09:52:33.899686+0000","last_fullsized":"2023-09-30T09:52:33.899686+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:44:16.352650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.907289+0000","last_change":"2023-09-30T09:49:31.310014+0000","last_active":"2023-09-30T09:52:33.907289+0000","last_peered":"2023-09-30T09:52:33.907289+0000","last_clean":"2023-09-30T09:52:33.907289+0000","last_became_active":"2023-09-30T09:49:31.309121+0000","last_became_peered":"2023-09-30T09:49:31.309121+0000","last_unstale":"2023-09-30T09:52:33.907289+0000","last_undegraded":"2023-09-30T09:52:33.907289+0000","last_fullsized":"2023-09-30T09:52:33.907289+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:46.598805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":1010,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906272+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:52:33.906272+0000","last_peered":"2023-09-30T09:52:33.906272+0000","last_clean":"2023-09-30T09:52:33.906272+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:52:33.906272+0000","last_undegraded":"2023-09-30T09:52:33.906272+0000","last_fullsized":"2023-09-30T09:52:33.906272+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":334,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.903036+0000","last_change":"2023-09-30T09:52:06.288453+0000","last_active":"2023-09-30T09:52:33.903036+0000","last_peered":"2023-09-30T09:52:33.903036+0000","last_clean":"2023-09-30T09:52:33.903036+0000","last_became_active":"2023-09-30T09:52:06.287939+0000","last_became_peered":"2023-09-30T09:52:06.287939+0000","last_unstale":"2023-09-30T09:52:33.903036+0000","last_undegraded":"2023-09-30T09:52:33.903036+0000","last_fullsized":"2023-09-30T09:52:33.903036+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:13:58.390726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":208,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.886145+0000","last_change":"2023-09-30T09:49:31.295632+0000","last_active":"2023-09-30T09:52:33.886145+0000","last_peered":"2023-09-30T09:52:33.886145+0000","last_clean":"2023-09-30T09:52:33.886145+0000","last_became_active":"2023-09-30T09:49:31.295269+0000","last_became_peered":"2023-09-30T09:49:31.295269+0000","last_unstale":"2023-09-30T09:52:33.886145+0000","last_undegraded":"2023-09-30T09:52:33.886145+0000","last_fullsized":"2023-09-30T09:52:33.886145+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:32.754585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":610,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.900465+0000","last_change":"2023-09-30T09:51:59.874993+0000","last_active":"2023-09-30T09:52:33.900465+0000","last_peered":"2023-09-30T09:52:33.900465+0000","last_clean":"2023-09-30T09:52:33.900465+0000","last_became_active":"2023-09-30T09:51:59.874470+0000","last_became_peered":"2023-09-30T09:51:59.874470+0000","last_unstale":"2023-09-30T09:52:33.900465+0000","last_undegraded":"2023-09-30T09:52:33.900465+0000","last_fullsized":"2023-09-30T09:52:33.900465+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:44.626194+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":29,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.890814+0000","last_change":"2023-09-30T09:51:57.160642+0000","last_active":"2023-09-30T09:52:33.890814+0000","last_peered":"2023-09-30T09:52:33.890814+0000","last_clean":"2023-09-30T09:52:33.890814+0000","last_became_active":"2023-09-30T09:51:57.160357+0000","last_became_peered":"2023-09-30T09:51:57.160357+0000","last_unstale":"2023-09-30T09:52:33.890814+0000","last_undegraded":"2023-09-30T09:52:33.890814+0000","last_fullsized":"2023-09-30T09:52:33.890814+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:40.417904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":585,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906265+0000","last_change":"2023-09-30T09:49:31.309241+0000","last_active":"2023-09-30T09:52:33.906265+0000","last_peered":"2023-09-30T09:52:33.906265+0000","last_clean":"2023-09-30T09:52:33.906265+0000","last_became_active":"2023-09-30T09:49:31.308703+0000","last_became_peered":"2023-09-30T09:49:31.308703+0000","last_unstale":"2023-09-30T09:52:33.906265+0000","last_undegraded":"2023-09-30T09:52:33.906265+0000","last_fullsized":"2023-09-30T09:52:33.906265+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:06.877644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":798,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.902787+0000","last_change":"2023-09-30T09:52:09.265362+0000","last_active":"2023-09-30T09:52:33.902787+0000","last_peered":"2023-09-30T09:52:33.902787+0000","last_clean":"2023-09-30T09:52:33.902787+0000","last_became_active":"2023-09-30T09:52:09.265043+0000","last_became_peered":"2023-09-30T09:52:09.265043+0000","last_unstale":"2023-09-30T09:52:33.902787+0000","last_undegraded":"2023-09-30T09:52:33.902787+0000","last_fullsized":"2023-09-30T09:52:33.902787+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:39.738075+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":323,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.908373+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:52:33.908373+0000","last_peered":"2023-09-30T09:52:33.908373+0000","last_clean":"2023-09-30T09:52:33.908373+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:52:33.908373+0000","last_undegraded":"2023-09-30T09:52:33.908373+0000","last_fullsized":"2023-09-30T09:52:33.908373+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":50,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.900864+0000","last_change":"2023-09-30T09:50:41.399925+0000","last_active":"2023-09-30T09:52:33.900864+0000","last_peered":"2023-09-30T09:52:33.900864+0000","last_clean":"2023-09-30T09:52:33.900864+0000","last_became_active":"2023-09-30T09:50:41.399642+0000","last_became_peered":"2023-09-30T09:50:41.399642+0000","last_unstale":"2023-09-30T09:52:33.900864+0000","last_undegraded":"2023-09-30T09:52:33.900864+0000","last_fullsized":"2023-09-30T09:52:33.900864+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:45:44.698050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":667,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:34.925844+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:52:34.925844+0000","last_peered":"2023-09-30T09:52:34.925844+0000","last_clean":"2023-09-30T09:52:34.925844+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:52:34.925844+0000","last_undegraded":"2023-09-30T09:52:34.925844+0000","last_fullsized":"2023-09-30T09:52:34.925844+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":382,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.909111+0000","last_change":"2023-09-30T09:52:15.347512+0000","last_active":"2023-09-30T09:52:33.909111+0000","last_peered":"2023-09-30T09:52:33.909111+0000","last_clean":"2023-09-30T09:52:33.909111+0000","last_became_active":"2023-09-30T09:52:15.339136+0000","last_became_peered":"2023-09-30T09:52:15.339136+0000","last_unstale":"2023-09-30T09:52:33.909111+0000","last_undegraded":"2023-09-30T09:52:33.909111+0000","last_fullsized":"2023-09-30T09:52:33.909111+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:48.631308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9444,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":650,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895695+0000","last_change":"2023-09-30T09:49:31.304624+0000","last_active":"2023-09-30T09:52:33.895695+0000","last_peered":"2023-09-30T09:52:33.895695+0000","last_clean":"2023-09-30T09:52:33.895695+0000","last_became_active":"2023-09-30T09:49:31.304106+0000","last_became_peered":"2023-09-30T09:49:31.304106+0000","last_unstale":"2023-09-30T09:52:33.895695+0000","last_undegraded":"2023-09-30T09:52:33.895695+0000","last_fullsized":"2023-09-30T09:52:33.895695+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:42.343641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":889,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.892702+0000","last_change":"2023-09-30T09:51:59.873231+0000","last_active":"2023-09-30T09:52:33.892702+0000","last_peered":"2023-09-30T09:52:33.892702+0000","last_clean":"2023-09-30T09:52:33.892702+0000","last_became_active":"2023-09-30T09:51:59.873003+0000","last_became_peered":"2023-09-30T09:51:59.873003+0000","last_unstale":"2023-09-30T09:52:33.892702+0000","last_undegraded":"2023-09-30T09:52:33.892702+0000","last_fullsized":"2023-09-30T09:52:33.892702+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:04.437753+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":307,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.909869+0000","last_change":"2023-09-30T09:52:15.360214+0000","last_active":"2023-09-30T09:52:33.909869+0000","last_peered":"2023-09-30T09:52:33.909869+0000","last_clean":"2023-09-30T09:52:33.909869+0000","last_became_active":"2023-09-30T09:52:15.339270+0000","last_became_peered":"2023-09-30T09:52:15.339270+0000","last_unstale":"2023-09-30T09:52:33.909869+0000","last_undegraded":"2023-09-30T09:52:33.909869+0000","last_fullsized":"2023-09-30T09:52:33.909869+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:37:34.556170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":586,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.909285+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:52:33.909285+0000","last_peered":"2023-09-30T09:52:33.909285+0000","last_clean":"2023-09-30T09:52:33.909285+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:52:33.909285+0000","last_undegraded":"2023-09-30T09:52:33.909285+0000","last_fullsized":"2023-09-30T09:52:33.909285+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":970,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889243+0000","last_change":"2023-09-30T09:49:34.332543+0000","last_active":"2023-09-30T09:52:33.889243+0000","last_peered":"2023-09-30T09:52:33.889243+0000","last_clean":"2023-09-30T09:52:33.889243+0000","last_became_active":"2023-09-30T09:49:34.332215+0000","last_became_peered":"2023-09-30T09:49:34.332215+0000","last_unstale":"2023-09-30T09:52:33.889243+0000","last_undegraded":"2023-09-30T09:52:33.889243+0000","last_fullsized":"2023-09-30T09:52:33.889243+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.827671+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":362,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.911331+0000","last_change":"2023-09-30T09:52:06.287263+0000","last_active":"2023-09-30T09:52:33.911331+0000","last_peered":"2023-09-30T09:52:33.911331+0000","last_clean":"2023-09-30T09:52:33.911331+0000","last_became_active":"2023-09-30T09:52:06.286870+0000","last_became_peered":"2023-09-30T09:52:06.286870+0000","last_unstale":"2023-09-30T09:52:33.911331+0000","last_undegraded":"2023-09-30T09:52:33.911331+0000","last_fullsized":"2023-09-30T09:52:33.911331+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:01:20.147214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":29,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889670+0000","last_change":"2023-09-30T09:51:57.157155+0000","last_active":"2023-09-30T09:52:33.889670+0000","last_peered":"2023-09-30T09:52:33.889670+0000","last_clean":"2023-09-30T09:52:33.889670+0000","last_became_active":"2023-09-30T09:51:57.156843+0000","last_became_peered":"2023-09-30T09:51:57.156843+0000","last_unstale":"2023-09-30T09:52:33.889670+0000","last_undegraded":"2023-09-30T09:52:33.889670+0000","last_fullsized":"2023-09-30T09:52:33.889670+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:12:00.333473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":1027,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.887677+0000","last_change":"2023-09-30T09:51:59.874841+0000","last_active":"2023-09-30T09:52:33.887677+0000","last_peered":"2023-09-30T09:52:33.887677+0000","last_clean":"2023-09-30T09:52:33.887677+0000","last_became_active":"2023-09-30T09:51:59.874418+0000","last_became_peered":"2023-09-30T09:51:59.874418+0000","last_unstale":"2023-09-30T09:52:33.887677+0000","last_undegraded":"2023-09-30T09:52:33.887677+0000","last_fullsized":"2023-09-30T09:52:33.887677+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:16.097188+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":313,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906393+0000","last_change":"2023-09-30T09:49:31.309022+0000","last_active":"2023-09-30T09:52:33.906393+0000","last_peered":"2023-09-30T09:52:33.906393+0000","last_clean":"2023-09-30T09:52:33.906393+0000","last_became_active":"2023-09-30T09:49:31.308236+0000","last_became_peered":"2023-09-30T09:49:31.308236+0000","last_unstale":"2023-09-30T09:52:33.906393+0000","last_undegraded":"2023-09-30T09:52:33.906393+0000","last_fullsized":"2023-09-30T09:52:33.906393+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:39.779954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":367,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.911251+0000","last_change":"2023-09-30T09:52:06.288331+0000","last_active":"2023-09-30T09:52:33.911251+0000","last_peered":"2023-09-30T09:52:33.911251+0000","last_clean":"2023-09-30T09:52:33.911251+0000","last_became_active":"2023-09-30T09:52:06.288031+0000","last_became_peered":"2023-09-30T09:52:06.288031+0000","last_unstale":"2023-09-30T09:52:33.911251+0000","last_undegraded":"2023-09-30T09:52:33.911251+0000","last_fullsized":"2023-09-30T09:52:33.911251+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:47:56.726506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":1059,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.892869+0000","last_change":"2023-09-30T09:49:34.332561+0000","last_active":"2023-09-30T09:52:33.892869+0000","last_peered":"2023-09-30T09:52:33.892869+0000","last_clean":"2023-09-30T09:52:33.892869+0000","last_became_active":"2023-09-30T09:49:34.332170+0000","last_became_peered":"2023-09-30T09:49:34.332170+0000","last_unstale":"2023-09-30T09:52:33.892869+0000","last_undegraded":"2023-09-30T09:52:33.892869+0000","last_fullsized":"2023-09-30T09:52:33.892869+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:22.548068+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":34,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889343+0000","last_change":"2023-09-30T09:52:06.314076+0000","last_active":"2023-09-30T09:52:33.889343+0000","last_peered":"2023-09-30T09:52:33.889343+0000","last_clean":"2023-09-30T09:52:33.889343+0000","last_became_active":"2023-09-30T09:52:06.286706+0000","last_became_peered":"2023-09-30T09:52:06.286706+0000","last_unstale":"2023-09-30T09:52:33.889343+0000","last_undegraded":"2023-09-30T09:52:33.889343+0000","last_fullsized":"2023-09-30T09:52:33.889343+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:35.387500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":176,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.894813+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:52:33.894813+0000","last_peered":"2023-09-30T09:52:33.894813+0000","last_clean":"2023-09-30T09:52:33.894813+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:52:33.894813+0000","last_undegraded":"2023-09-30T09:52:33.894813+0000","last_fullsized":"2023-09-30T09:52:33.894813+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"609'8868","reported_seq":8954,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:37.420890+0000","last_change":"2023-09-30T09:48:40.945814+0000","last_active":"2023-09-30T09:52:37.420890+0000","last_peered":"2023-09-30T09:52:37.420890+0000","last_clean":"2023-09-30T09:52:37.420890+0000","last_became_active":"2023-09-30T09:48:40.945431+0000","last_became_peered":"2023-09-30T09:48:40.945431+0000","last_unstale":"2023-09-30T09:52:37.420890+0000","last_undegraded":"2023-09-30T09:52:37.420890+0000","last_fullsized":"2023-09-30T09:52:37.420890+0000","mapping_epoch":536,"log_start":"548'3512","ondisk_log_start":"548'3512","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5356,"log_dups_size":0,"ondisk_log_size":5356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:31:32.281640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167116800,"num_objects":2550,"num_object_clones":0,"num_object_copies":7650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8868,"num_write_kb":365376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1005,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906367+0000","last_change":"2023-09-30T09:52:09.263017+0000","last_active":"2023-09-30T09:52:33.906367+0000","last_peered":"2023-09-30T09:52:33.906367+0000","last_clean":"2023-09-30T09:52:33.906367+0000","last_became_active":"2023-09-30T09:52:09.261512+0000","last_became_peered":"2023-09-30T09:52:09.261512+0000","last_unstale":"2023-09-30T09:52:33.906367+0000","last_undegraded":"2023-09-30T09:52:33.906367+0000","last_fullsized":"2023-09-30T09:52:33.906367+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:13.359576+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":299,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.908824+0000","last_change":"2023-09-30T09:51:57.189281+0000","last_active":"2023-09-30T09:52:33.908824+0000","last_peered":"2023-09-30T09:52:33.908824+0000","last_clean":"2023-09-30T09:52:33.908824+0000","last_became_active":"2023-09-30T09:51:57.165927+0000","last_became_peered":"2023-09-30T09:51:57.165927+0000","last_unstale":"2023-09-30T09:52:33.908824+0000","last_undegraded":"2023-09-30T09:52:33.908824+0000","last_fullsized":"2023-09-30T09:52:33.908824+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:30.207994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":29,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889166+0000","last_change":"2023-09-30T09:51:57.167306+0000","last_active":"2023-09-30T09:52:33.889166+0000","last_peered":"2023-09-30T09:52:33.889166+0000","last_clean":"2023-09-30T09:52:33.889166+0000","last_became_active":"2023-09-30T09:51:57.167040+0000","last_became_peered":"2023-09-30T09:51:57.167040+0000","last_unstale":"2023-09-30T09:52:33.889166+0000","last_undegraded":"2023-09-30T09:52:33.889166+0000","last_fullsized":"2023-09-30T09:52:33.889166+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:27:26.793530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"609'9304","reported_seq":18041,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:39.417897+0000","last_change":"2023-09-30T09:52:34.978315+0000","last_active":"2023-09-30T09:52:39.417897+0000","last_peered":"2023-09-30T09:52:39.417897+0000","last_clean":"2023-09-30T09:52:39.417897+0000","last_became_active":"2023-09-30T09:52:34.912855+0000","last_became_peered":"2023-09-30T09:52:34.912855+0000","last_unstale":"2023-09-30T09:52:39.417897+0000","last_undegraded":"2023-09-30T09:52:39.417897+0000","last_fullsized":"2023-09-30T09:52:39.417897+0000","mapping_epoch":608,"log_start":"548'3447","ondisk_log_start":"548'3447","created":536,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5857,"log_dups_size":0,"ondisk_log_size":5857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:19.028693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164757504,"num_objects":2514,"num_object_clones":0,"num_object_copies":7542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9304,"num_write_kb":378176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4301,"num_bytes_recovered":248774656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":1041,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.891646+0000","last_change":"2023-09-30T09:49:34.340125+0000","last_active":"2023-09-30T09:52:33.891646+0000","last_peered":"2023-09-30T09:52:33.891646+0000","last_clean":"2023-09-30T09:52:33.891646+0000","last_became_active":"2023-09-30T09:49:34.339638+0000","last_became_peered":"2023-09-30T09:49:34.339638+0000","last_unstale":"2023-09-30T09:52:33.891646+0000","last_undegraded":"2023-09-30T09:52:33.891646+0000","last_fullsized":"2023-09-30T09:52:33.891646+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:43:43.258960+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":269,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.911123+0000","last_change":"2023-09-30T09:51:57.183013+0000","last_active":"2023-09-30T09:52:33.911123+0000","last_peered":"2023-09-30T09:52:33.911123+0000","last_clean":"2023-09-30T09:52:33.911123+0000","last_became_active":"2023-09-30T09:51:57.161490+0000","last_became_peered":"2023-09-30T09:51:57.161490+0000","last_unstale":"2023-09-30T09:52:33.911123+0000","last_undegraded":"2023-09-30T09:52:33.911123+0000","last_fullsized":"2023-09-30T09:52:33.911123+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:14:47.256298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":669,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.898244+0000","last_change":"2023-09-30T09:50:26.903683+0000","last_active":"2023-09-30T09:52:33.898244+0000","last_peered":"2023-09-30T09:52:33.898244+0000","last_clean":"2023-09-30T09:52:33.898244+0000","last_became_active":"2023-09-30T09:50:26.903262+0000","last_became_peered":"2023-09-30T09:50:26.903262+0000","last_unstale":"2023-09-30T09:52:33.898244+0000","last_undegraded":"2023-09-30T09:52:33.898244+0000","last_fullsized":"2023-09-30T09:52:33.898244+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:40.724403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"609'9392","reported_seq":10351,"reported_epoch":609,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:52:39.192761+0000","last_change":"2023-09-30T09:52:07.362883+0000","last_active":"2023-09-30T09:52:39.192761+0000","last_peered":"2023-09-30T09:52:39.192761+0000","last_clean":"2023-09-30T09:51:55.533937+0000","last_became_active":"2023-09-30T09:51:58.262846+0000","last_became_peered":"2023-09-30T09:51:58.262846+0000","last_unstale":"2023-09-30T09:52:39.192761+0000","last_undegraded":"2023-09-30T09:52:39.192761+0000","last_fullsized":"2023-09-30T09:52:39.192761+0000","mapping_epoch":590,"log_start":"547'3339","ondisk_log_start":"547'3339","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6053,"log_dups_size":0,"ondisk_log_size":6053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:19:17.909153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165806080,"num_objects":2530,"num_object_clones":0,"num_object_copies":7590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2230,"num_objects_unfound":0,"num_objects_dirty":2530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9392,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":430,"num_bytes_recovered":28180480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":2530}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":691,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.901788+0000","last_change":"2023-09-30T09:52:18.342336+0000","last_active":"2023-09-30T09:52:33.901788+0000","last_peered":"2023-09-30T09:52:33.901788+0000","last_clean":"2023-09-30T09:52:33.901788+0000","last_became_active":"2023-09-30T09:52:18.341952+0000","last_became_peered":"2023-09-30T09:52:18.341952+0000","last_unstale":"2023-09-30T09:52:33.901788+0000","last_undegraded":"2023-09-30T09:52:33.901788+0000","last_fullsized":"2023-09-30T09:52:33.901788+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:16.116795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:16.116795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:01:10.275488+0000","scrub_duration":0.045979904000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":378,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906093+0000","last_change":"2023-09-30T09:51:57.192521+0000","last_active":"2023-09-30T09:52:33.906093+0000","last_peered":"2023-09-30T09:52:33.906093+0000","last_clean":"2023-09-30T09:52:33.906093+0000","last_became_active":"2023-09-30T09:51:57.170876+0000","last_became_peered":"2023-09-30T09:51:57.170876+0000","last_unstale":"2023-09-30T09:52:33.906093+0000","last_undegraded":"2023-09-30T09:52:33.906093+0000","last_fullsized":"2023-09-30T09:52:33.906093+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:44:49.877225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":494,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.892738+0000","last_change":"2023-09-30T09:50:13.889514+0000","last_active":"2023-09-30T09:52:33.892738+0000","last_peered":"2023-09-30T09:52:33.892738+0000","last_clean":"2023-09-30T09:52:33.892738+0000","last_became_active":"2023-09-30T09:50:13.889033+0000","last_became_peered":"2023-09-30T09:50:13.889033+0000","last_unstale":"2023-09-30T09:52:33.892738+0000","last_undegraded":"2023-09-30T09:52:33.892738+0000","last_fullsized":"2023-09-30T09:52:33.892738+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:19.671881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"609'9161","reported_seq":9991,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:39.085672+0000","last_change":"2023-09-30T09:52:18.372502+0000","last_active":"2023-09-30T09:52:39.085672+0000","last_peered":"2023-09-30T09:52:39.085672+0000","last_clean":"2023-09-30T09:52:39.085672+0000","last_became_active":"2023-09-30T09:52:18.341874+0000","last_became_peered":"2023-09-30T09:52:18.341874+0000","last_unstale":"2023-09-30T09:52:39.085672+0000","last_undegraded":"2023-09-30T09:52:39.085672+0000","last_fullsized":"2023-09-30T09:52:39.085672+0000","mapping_epoch":603,"log_start":"547'3115","ondisk_log_start":"547'3115","created":536,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6046,"log_dups_size":0,"ondisk_log_size":6046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:24:44.855270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162201600,"num_objects":2475,"num_object_clones":0,"num_object_copies":7425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9161,"num_write_kb":372352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":359,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":587,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.911016+0000","last_change":"2023-09-30T09:51:59.873050+0000","last_active":"2023-09-30T09:52:33.911016+0000","last_peered":"2023-09-30T09:52:33.911016+0000","last_clean":"2023-09-30T09:52:33.911016+0000","last_became_active":"2023-09-30T09:51:59.872660+0000","last_became_peered":"2023-09-30T09:51:59.872660+0000","last_unstale":"2023-09-30T09:52:33.911016+0000","last_undegraded":"2023-09-30T09:52:33.911016+0000","last_fullsized":"2023-09-30T09:52:33.911016+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:47:10.945792+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":564,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889974+0000","last_change":"2023-09-30T09:49:31.301460+0000","last_active":"2023-09-30T09:52:33.889974+0000","last_peered":"2023-09-30T09:52:33.889974+0000","last_clean":"2023-09-30T09:52:33.889974+0000","last_became_active":"2023-09-30T09:49:31.301114+0000","last_became_peered":"2023-09-30T09:49:31.301114+0000","last_unstale":"2023-09-30T09:52:33.889974+0000","last_undegraded":"2023-09-30T09:52:33.889974+0000","last_fullsized":"2023-09-30T09:52:33.889974+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:42:01.066974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":211,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.907753+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:52:33.907753+0000","last_peered":"2023-09-30T09:52:33.907753+0000","last_clean":"2023-09-30T09:52:33.907753+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:52:33.907753+0000","last_undegraded":"2023-09-30T09:52:33.907753+0000","last_fullsized":"2023-09-30T09:52:33.907753+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"609'9305","reported_seq":9385,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:39.097586+0000","last_change":"2023-09-30T09:48:40.937693+0000","last_active":"2023-09-30T09:52:39.097586+0000","last_peered":"2023-09-30T09:52:39.097586+0000","last_clean":"2023-09-30T09:52:39.097586+0000","last_became_active":"2023-09-30T09:48:40.937364+0000","last_became_peered":"2023-09-30T09:48:40.937364+0000","last_unstale":"2023-09-30T09:52:39.097586+0000","last_undegraded":"2023-09-30T09:52:39.097586+0000","last_fullsized":"2023-09-30T09:52:39.097586+0000","mapping_epoch":536,"log_start":"547'3345","ondisk_log_start":"547'3345","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5960,"log_dups_size":0,"ondisk_log_size":5960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:58.223575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9305,"num_write_kb":376768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1151,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.910335+0000","last_change":"2023-09-30T09:52:09.259618+0000","last_active":"2023-09-30T09:52:33.910335+0000","last_peered":"2023-09-30T09:52:33.910335+0000","last_clean":"2023-09-30T09:52:33.910335+0000","last_became_active":"2023-09-30T09:52:09.259274+0000","last_became_peered":"2023-09-30T09:52:09.259274+0000","last_unstale":"2023-09-30T09:52:33.910335+0000","last_undegraded":"2023-09-30T09:52:33.910335+0000","last_fullsized":"2023-09-30T09:52:33.910335+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:41:35.328752+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":104,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895816+0000","last_change":"2023-09-30T09:51:57.173028+0000","last_active":"2023-09-30T09:52:33.895816+0000","last_peered":"2023-09-30T09:52:33.895816+0000","last_clean":"2023-09-30T09:52:33.895816+0000","last_became_active":"2023-09-30T09:51:57.152483+0000","last_became_peered":"2023-09-30T09:51:57.152483+0000","last_unstale":"2023-09-30T09:52:33.895816+0000","last_undegraded":"2023-09-30T09:52:33.895816+0000","last_fullsized":"2023-09-30T09:52:33.895816+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:11.989575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":626,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895808+0000","last_change":"2023-09-30T09:49:31.300551+0000","last_active":"2023-09-30T09:52:33.895808+0000","last_peered":"2023-09-30T09:52:33.895808+0000","last_clean":"2023-09-30T09:52:33.895808+0000","last_became_active":"2023-09-30T09:49:31.300231+0000","last_became_peered":"2023-09-30T09:49:31.300231+0000","last_unstale":"2023-09-30T09:52:33.895808+0000","last_undegraded":"2023-09-30T09:52:33.895808+0000","last_fullsized":"2023-09-30T09:52:33.895808+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:45.832564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"609'9314","reported_seq":11280,"reported_epoch":609,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:52:39.487704+0000","last_change":"2023-09-30T09:52:16.579692+0000","last_active":"2023-09-30T09:52:39.487704+0000","last_peered":"2023-09-30T09:52:39.487704+0000","last_clean":"2023-09-30T09:51:55.471595+0000","last_became_active":"2023-09-30T09:52:16.492103+0000","last_became_peered":"2023-09-30T09:52:16.492103+0000","last_unstale":"2023-09-30T09:52:39.487704+0000","last_undegraded":"2023-09-30T09:52:16.484021+0000","last_fullsized":"2023-09-30T09:52:39.487704+0000","mapping_epoch":601,"log_start":"548'3508","ondisk_log_start":"548'3508","created":536,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:38.592310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162529280,"num_objects":2480,"num_object_clones":0,"num_object_copies":7440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2480,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9314,"num_write_kb":377408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":916,"num_bytes_recovered":59965440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,4],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2479},{"shards":"3,4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":984,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.899837+0000","last_change":"2023-09-30T09:52:09.249933+0000","last_active":"2023-09-30T09:52:33.899837+0000","last_peered":"2023-09-30T09:52:33.899837+0000","last_clean":"2023-09-30T09:52:33.899837+0000","last_became_active":"2023-09-30T09:52:09.249567+0000","last_became_peered":"2023-09-30T09:52:09.249567+0000","last_unstale":"2023-09-30T09:52:33.899837+0000","last_undegraded":"2023-09-30T09:52:33.899837+0000","last_fullsized":"2023-09-30T09:52:33.899837+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:52:00.664301+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":451,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895194+0000","last_change":"2023-09-30T09:50:13.887314+0000","last_active":"2023-09-30T09:52:33.895194+0000","last_peered":"2023-09-30T09:52:33.895194+0000","last_clean":"2023-09-30T09:52:33.895194+0000","last_became_active":"2023-09-30T09:50:13.887032+0000","last_became_peered":"2023-09-30T09:50:13.887032+0000","last_unstale":"2023-09-30T09:52:33.895194+0000","last_undegraded":"2023-09-30T09:52:33.895194+0000","last_fullsized":"2023-09-30T09:52:33.895194+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:27:26.626380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":18,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889446+0000","last_change":"2023-09-30T09:52:15.326810+0000","last_active":"2023-09-30T09:52:33.889446+0000","last_peered":"2023-09-30T09:52:33.889446+0000","last_clean":"2023-09-30T09:52:33.889446+0000","last_became_active":"2023-09-30T09:52:15.326507+0000","last_became_peered":"2023-09-30T09:52:15.326507+0000","last_unstale":"2023-09-30T09:52:33.889446+0000","last_undegraded":"2023-09-30T09:52:33.889446+0000","last_fullsized":"2023-09-30T09:52:33.889446+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:51:01.576127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.0","version":"609'9242","reported_seq":21316,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:38.979841+0000","last_change":"2023-09-30T09:52:28.113587+0000","last_active":"2023-09-30T09:52:38.979841+0000","last_peered":"2023-09-30T09:52:38.979841+0000","last_clean":"2023-09-30T09:52:38.979841+0000","last_became_active":"2023-09-30T09:52:28.047541+0000","last_became_peered":"2023-09-30T09:52:28.047541+0000","last_unstale":"2023-09-30T09:52:38.979841+0000","last_undegraded":"2023-09-30T09:52:38.979841+0000","last_fullsized":"2023-09-30T09:52:38.979841+0000","mapping_epoch":606,"log_start":"554'4000","ondisk_log_start":"554'4000","created":536,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5242,"log_dups_size":0,"ondisk_log_size":5242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:02:13.553694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168034304,"num_objects":2564,"num_object_clones":0,"num_object_copies":7692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9242,"num_write_kb":377792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5978,"num_bytes_recovered":391774208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":976,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.891038+0000","last_change":"2023-09-30T09:52:18.345744+0000","last_active":"2023-09-30T09:52:33.891038+0000","last_peered":"2023-09-30T09:52:33.891038+0000","last_clean":"2023-09-30T09:52:33.891038+0000","last_became_active":"2023-09-30T09:52:18.345422+0000","last_became_peered":"2023-09-30T09:52:18.345422+0000","last_unstale":"2023-09-30T09:52:33.891038+0000","last_undegraded":"2023-09-30T09:52:33.891038+0000","last_fullsized":"2023-09-30T09:52:33.891038+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:49:09.924646+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":610,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.900619+0000","last_change":"2023-09-30T09:52:06.274034+0000","last_active":"2023-09-30T09:52:33.900619+0000","last_peered":"2023-09-30T09:52:33.900619+0000","last_clean":"2023-09-30T09:52:33.900619+0000","last_became_active":"2023-09-30T09:52:06.273748+0000","last_became_peered":"2023-09-30T09:52:06.273748+0000","last_unstale":"2023-09-30T09:52:33.900619+0000","last_undegraded":"2023-09-30T09:52:33.900619+0000","last_fullsized":"2023-09-30T09:52:33.900619+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:22:02.105157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":640,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.905818+0000","last_change":"2023-09-30T09:49:31.311818+0000","last_active":"2023-09-30T09:52:33.905818+0000","last_peered":"2023-09-30T09:52:33.905818+0000","last_clean":"2023-09-30T09:52:33.905818+0000","last_became_active":"2023-09-30T09:49:31.311481+0000","last_became_peered":"2023-09-30T09:49:31.311481+0000","last_unstale":"2023-09-30T09:52:33.905818+0000","last_undegraded":"2023-09-30T09:52:33.905818+0000","last_fullsized":"2023-09-30T09:52:33.905818+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:54.941072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"609'9205","reported_seq":12743,"reported_epoch":609,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:52:39.007233+0000","last_change":"2023-09-30T09:52:16.758569+0000","last_active":"2023-09-30T09:52:39.007233+0000","last_peered":"2023-09-30T09:52:39.007233+0000","last_clean":"2023-09-30T09:52:04.984831+0000","last_became_active":"2023-09-30T09:52:07.326751+0000","last_became_peered":"2023-09-30T09:52:07.326751+0000","last_unstale":"2023-09-30T09:52:39.007233+0000","last_undegraded":"2023-09-30T09:52:39.007233+0000","last_fullsized":"2023-09-30T09:52:39.007233+0000","mapping_epoch":596,"log_start":"548'3576","ondisk_log_start":"548'3576","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5629,"log_dups_size":0,"ondisk_log_size":5629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:12:07.668051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161546240,"num_objects":2465,"num_object_clones":0,"num_object_copies":7395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1181,"num_objects_unfound":0,"num_objects_dirty":2465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9205,"num_write_kb":373440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1720,"num_bytes_recovered":112721920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,4],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":2465}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":807,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.900784+0000","last_change":"2023-09-30T09:50:44.462926+0000","last_active":"2023-09-30T09:52:33.900784+0000","last_peered":"2023-09-30T09:52:33.900784+0000","last_clean":"2023-09-30T09:52:33.900784+0000","last_became_active":"2023-09-30T09:50:44.462573+0000","last_became_peered":"2023-09-30T09:50:44.462573+0000","last_unstale":"2023-09-30T09:52:33.900784+0000","last_undegraded":"2023-09-30T09:52:33.900784+0000","last_fullsized":"2023-09-30T09:52:33.900784+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:28.805205+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":176,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.886074+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:52:33.886074+0000","last_peered":"2023-09-30T09:52:33.886074+0000","last_clean":"2023-09-30T09:52:33.886074+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:52:33.886074+0000","last_undegraded":"2023-09-30T09:52:33.886074+0000","last_fullsized":"2023-09-30T09:52:33.886074+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":293,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.897424+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:52:33.897424+0000","last_peered":"2023-09-30T09:52:33.897424+0000","last_clean":"2023-09-30T09:52:33.897424+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:52:33.897424+0000","last_undegraded":"2023-09-30T09:52:33.897424+0000","last_fullsized":"2023-09-30T09:52:33.897424+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"609'9223","reported_seq":9304,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:38.973314+0000","last_change":"2023-09-30T09:48:40.942045+0000","last_active":"2023-09-30T09:52:38.973314+0000","last_peered":"2023-09-30T09:52:38.973314+0000","last_clean":"2023-09-30T09:52:38.973314+0000","last_became_active":"2023-09-30T09:48:40.941786+0000","last_became_peered":"2023-09-30T09:48:40.941786+0000","last_unstale":"2023-09-30T09:52:38.973314+0000","last_undegraded":"2023-09-30T09:52:38.973314+0000","last_fullsized":"2023-09-30T09:52:38.973314+0000","mapping_epoch":536,"log_start":"548'3632","ondisk_log_start":"548'3632","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5591,"log_dups_size":0,"ondisk_log_size":5591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:03:11.354244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168361984,"num_objects":2569,"num_object_clones":0,"num_object_copies":7707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9223,"num_write_kb":377344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":849,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.908479+0000","last_change":"2023-09-30T09:50:13.903010+0000","last_active":"2023-09-30T09:52:33.908479+0000","last_peered":"2023-09-30T09:52:33.908479+0000","last_clean":"2023-09-30T09:52:33.908479+0000","last_became_active":"2023-09-30T09:50:13.902355+0000","last_became_peered":"2023-09-30T09:50:13.902355+0000","last_unstale":"2023-09-30T09:52:33.908479+0000","last_undegraded":"2023-09-30T09:52:33.908479+0000","last_fullsized":"2023-09-30T09:52:33.908479+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:35.240830+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":842,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.907637+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:52:33.907637+0000","last_peered":"2023-09-30T09:52:33.907637+0000","last_clean":"2023-09-30T09:52:33.907637+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:52:33.907637+0000","last_undegraded":"2023-09-30T09:52:33.907637+0000","last_fullsized":"2023-09-30T09:52:33.907637+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":557,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889748+0000","last_change":"2023-09-30T09:50:41.396047+0000","last_active":"2023-09-30T09:52:33.889748+0000","last_peered":"2023-09-30T09:52:33.889748+0000","last_clean":"2023-09-30T09:52:33.889748+0000","last_became_active":"2023-09-30T09:50:41.395627+0000","last_became_peered":"2023-09-30T09:50:41.395627+0000","last_unstale":"2023-09-30T09:52:33.889748+0000","last_undegraded":"2023-09-30T09:52:33.889748+0000","last_fullsized":"2023-09-30T09:52:33.889748+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:16.050977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":49,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889393+0000","last_change":"2023-09-30T09:52:15.338190+0000","last_active":"2023-09-30T09:52:33.889393+0000","last_peered":"2023-09-30T09:52:33.889393+0000","last_clean":"2023-09-30T09:52:33.889393+0000","last_became_active":"2023-09-30T09:52:15.337907+0000","last_became_peered":"2023-09-30T09:52:15.337907+0000","last_unstale":"2023-09-30T09:52:33.889393+0000","last_undegraded":"2023-09-30T09:52:33.889393+0000","last_fullsized":"2023-09-30T09:52:33.889393+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:20:09.336859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"609'9386","reported_seq":18495,"reported_epoch":609,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:52:39.470134+0000","last_change":"2023-09-30T09:52:33.171256+0000","last_active":"2023-09-30T09:52:39.470134+0000","last_peered":"2023-09-30T09:52:39.470134+0000","last_clean":"2023-09-30T09:51:56.097639+0000","last_became_active":"2023-09-30T09:51:58.327478+0000","last_became_peered":"2023-09-30T09:51:58.327478+0000","last_unstale":"2023-09-30T09:52:39.470134+0000","last_undegraded":"2023-09-30T09:52:39.470134+0000","last_fullsized":"2023-09-30T09:51:58.140261+0000","mapping_epoch":590,"log_start":"548'3588","ondisk_log_start":"548'3588","created":536,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:41:17.448598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840000,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1448,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9386,"num_write_kb":380352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4495,"num_bytes_recovered":294584320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":1448},{"shards":"5,6","objects":1052}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":1110,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.892939+0000","last_change":"2023-09-30T09:52:09.283412+0000","last_active":"2023-09-30T09:52:33.892939+0000","last_peered":"2023-09-30T09:52:33.892939+0000","last_clean":"2023-09-30T09:52:33.892939+0000","last_became_active":"2023-09-30T09:52:09.283074+0000","last_became_peered":"2023-09-30T09:52:09.283074+0000","last_unstale":"2023-09-30T09:52:33.892939+0000","last_undegraded":"2023-09-30T09:52:33.892939+0000","last_fullsized":"2023-09-30T09:52:33.892939+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:33:20.204709+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":406,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.892659+0000","last_change":"2023-09-30T09:50:13.889332+0000","last_active":"2023-09-30T09:52:33.892659+0000","last_peered":"2023-09-30T09:52:33.892659+0000","last_clean":"2023-09-30T09:52:33.892659+0000","last_became_active":"2023-09-30T09:50:13.888518+0000","last_became_peered":"2023-09-30T09:50:13.888518+0000","last_unstale":"2023-09-30T09:52:33.892659+0000","last_undegraded":"2023-09-30T09:52:33.892659+0000","last_fullsized":"2023-09-30T09:52:33.892659+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:40:20.465620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":219,"num_read_kb":2265,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":657,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895737+0000","last_change":"2023-09-30T09:52:15.328572+0000","last_active":"2023-09-30T09:52:33.895737+0000","last_peered":"2023-09-30T09:52:33.895737+0000","last_clean":"2023-09-30T09:52:33.895737+0000","last_became_active":"2023-09-30T09:52:15.328181+0000","last_became_peered":"2023-09-30T09:52:15.328181+0000","last_unstale":"2023-09-30T09:52:33.895737+0000","last_undegraded":"2023-09-30T09:52:33.895737+0000","last_fullsized":"2023-09-30T09:52:33.895737+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:08:51.469327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":277,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.890013+0000","last_change":"2023-09-30T09:52:15.332419+0000","last_active":"2023-09-30T09:52:33.890013+0000","last_peered":"2023-09-30T09:52:33.890013+0000","last_clean":"2023-09-30T09:52:33.890013+0000","last_became_active":"2023-09-30T09:52:15.332098+0000","last_became_peered":"2023-09-30T09:52:15.332098+0000","last_unstale":"2023-09-30T09:52:33.890013+0000","last_undegraded":"2023-09-30T09:52:33.890013+0000","last_fullsized":"2023-09-30T09:52:33.890013+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:53:34.176892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"609'9128","reported_seq":10911,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:39.473520+0000","last_change":"2023-09-30T09:52:16.376606+0000","last_active":"2023-09-30T09:52:39.473520+0000","last_peered":"2023-09-30T09:52:39.473520+0000","last_clean":"2023-09-30T09:52:39.473520+0000","last_became_active":"2023-09-30T09:52:16.330391+0000","last_became_peered":"2023-09-30T09:52:16.330391+0000","last_unstale":"2023-09-30T09:52:39.473520+0000","last_undegraded":"2023-09-30T09:52:39.473520+0000","last_fullsized":"2023-09-30T09:52:39.473520+0000","mapping_epoch":601,"log_start":"547'3264","ondisk_log_start":"547'3264","created":536,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:02:19.237518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160956416,"num_objects":2456,"num_object_clones":0,"num_object_copies":7368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9128,"num_write_kb":370688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":1068,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906273+0000","last_change":"2023-09-30T09:52:09.253155+0000","last_active":"2023-09-30T09:52:33.906273+0000","last_peered":"2023-09-30T09:52:33.906273+0000","last_clean":"2023-09-30T09:52:33.906273+0000","last_became_active":"2023-09-30T09:52:09.252824+0000","last_became_peered":"2023-09-30T09:52:09.252824+0000","last_unstale":"2023-09-30T09:52:33.906273+0000","last_undegraded":"2023-09-30T09:52:33.906273+0000","last_fullsized":"2023-09-30T09:52:33.906273+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:50.451301+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":523,"reported_epoch":608,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:52:33.905950+0000","last_change":"2023-09-30T09:52:16.329223+0000","last_active":"2023-09-30T09:52:33.905950+0000","last_peered":"2023-09-30T09:52:33.905950+0000","last_clean":"2023-09-30T09:23:03.976898+0000","last_became_active":"2023-09-30T09:52:16.328685+0000","last_became_peered":"2023-09-30T09:52:16.328685+0000","last_unstale":"2023-09-30T09:52:33.905950+0000","last_undegraded":"2023-09-30T09:52:16.323100+0000","last_fullsized":"2023-09-30T09:52:16.321702+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:30.531480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":50,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.899840+0000","last_change":"2023-09-30T09:50:41.394348+0000","last_active":"2023-09-30T09:52:33.899840+0000","last_peered":"2023-09-30T09:52:33.899840+0000","last_clean":"2023-09-30T09:52:33.899840+0000","last_became_active":"2023-09-30T09:50:41.394017+0000","last_became_peered":"2023-09-30T09:50:41.394017+0000","last_unstale":"2023-09-30T09:52:33.899840+0000","last_undegraded":"2023-09-30T09:52:33.899840+0000","last_fullsized":"2023-09-30T09:52:33.899840+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:37:30.133826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":468,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.902961+0000","last_change":"2023-09-30T09:51:57.162010+0000","last_active":"2023-09-30T09:52:33.902961+0000","last_peered":"2023-09-30T09:52:33.902961+0000","last_clean":"2023-09-30T09:52:33.902961+0000","last_became_active":"2023-09-30T09:51:57.161672+0000","last_became_peered":"2023-09-30T09:51:57.161672+0000","last_unstale":"2023-09-30T09:52:33.902961+0000","last_undegraded":"2023-09-30T09:52:33.902961+0000","last_fullsized":"2023-09-30T09:52:33.902961+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:28.744549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"609'8875","reported_seq":15411,"reported_epoch":609,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-30T09:52:37.502270+0000","last_change":"2023-09-30T09:52:06.380535+0000","last_active":"2023-09-30T09:52:37.502270+0000","last_peered":"2023-09-30T09:52:37.502270+0000","last_clean":"2023-09-30T09:49:30.256457+0000","last_became_active":"2023-09-30T09:50:14.978894+0000","last_became_peered":"2023-09-30T09:50:14.978894+0000","last_unstale":"2023-09-30T09:52:37.502270+0000","last_undegraded":"2023-09-30T09:52:06.379384+0000","last_fullsized":"2023-09-30T09:52:37.502270+0000","mapping_epoch":560,"log_start":"548'3604","ondisk_log_start":"548'3604","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5271,"log_dups_size":0,"ondisk_log_size":5271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:20:48.615096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159711232,"num_objects":2437,"num_object_clones":0,"num_object_copies":7311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":408,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8875,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3219,"num_bytes_recovered":210960384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":408},{"shards":"0,1,6","objects":2029}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1192,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.896192+0000","last_change":"2023-09-30T09:50:42.387920+0000","last_active":"2023-09-30T09:52:33.896192+0000","last_peered":"2023-09-30T09:52:33.896192+0000","last_clean":"2023-09-30T09:52:33.896192+0000","last_became_active":"2023-09-30T09:50:42.387121+0000","last_became_peered":"2023-09-30T09:50:42.387121+0000","last_unstale":"2023-09-30T09:52:33.896192+0000","last_undegraded":"2023-09-30T09:52:33.896192+0000","last_fullsized":"2023-09-30T09:52:33.896192+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:34:05.192090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1031,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906211+0000","last_change":"2023-09-30T09:51:59.878404+0000","last_active":"2023-09-30T09:52:33.906211+0000","last_peered":"2023-09-30T09:52:33.906211+0000","last_clean":"2023-09-30T09:52:33.906211+0000","last_became_active":"2023-09-30T09:51:59.878042+0000","last_became_peered":"2023-09-30T09:51:59.878042+0000","last_unstale":"2023-09-30T09:52:33.906211+0000","last_undegraded":"2023-09-30T09:52:33.906211+0000","last_fullsized":"2023-09-30T09:52:33.906211+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:53:00.431808+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889472+0000","last_change":"2023-09-30T09:51:57.166214+0000","last_active":"2023-09-30T09:52:33.889472+0000","last_peered":"2023-09-30T09:52:33.889472+0000","last_clean":"2023-09-30T09:52:33.889472+0000","last_became_active":"2023-09-30T09:51:57.165926+0000","last_became_peered":"2023-09-30T09:51:57.165926+0000","last_unstale":"2023-09-30T09:52:33.889472+0000","last_undegraded":"2023-09-30T09:52:33.889472+0000","last_fullsized":"2023-09-30T09:52:33.889472+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:19:29.596549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":466,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906178+0000","last_change":"2023-09-30T09:52:15.339945+0000","last_active":"2023-09-30T09:52:33.906178+0000","last_peered":"2023-09-30T09:52:33.906178+0000","last_clean":"2023-09-30T09:52:33.906178+0000","last_became_active":"2023-09-30T09:52:15.339624+0000","last_became_peered":"2023-09-30T09:52:15.339624+0000","last_unstale":"2023-09-30T09:52:33.906178+0000","last_undegraded":"2023-09-30T09:52:33.906178+0000","last_fullsized":"2023-09-30T09:52:33.906178+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:03.885650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"609'8945","reported_seq":9774,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:37.512965+0000","last_change":"2023-09-30T09:51:59.899383+0000","last_active":"2023-09-30T09:52:37.512965+0000","last_peered":"2023-09-30T09:52:37.512965+0000","last_clean":"2023-09-30T09:52:37.512965+0000","last_became_active":"2023-09-30T09:51:59.876329+0000","last_became_peered":"2023-09-30T09:51:59.876329+0000","last_unstale":"2023-09-30T09:52:37.512965+0000","last_undegraded":"2023-09-30T09:52:37.512965+0000","last_fullsized":"2023-09-30T09:52:37.512965+0000","mapping_epoch":592,"log_start":"548'3602","ondisk_log_start":"548'3602","created":536,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5343,"log_dups_size":0,"ondisk_log_size":5343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:49:07.332301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164954112,"num_objects":2517,"num_object_clones":0,"num_object_copies":7551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8945,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":360,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":750,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.897876+0000","last_change":"2023-09-30T09:52:09.254399+0000","last_active":"2023-09-30T09:52:33.897876+0000","last_peered":"2023-09-30T09:52:33.897876+0000","last_clean":"2023-09-30T09:52:33.897876+0000","last_became_active":"2023-09-30T09:52:09.253999+0000","last_became_peered":"2023-09-30T09:52:09.253999+0000","last_unstale":"2023-09-30T09:52:33.897876+0000","last_undegraded":"2023-09-30T09:52:33.897876+0000","last_fullsized":"2023-09-30T09:52:33.897876+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:39.230227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.230227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:12.791078+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":1017,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906122+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:52:33.906122+0000","last_peered":"2023-09-30T09:52:33.906122+0000","last_clean":"2023-09-30T09:52:33.906122+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:52:33.906122+0000","last_undegraded":"2023-09-30T09:52:33.906122+0000","last_fullsized":"2023-09-30T09:52:33.906122+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":232,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906177+0000","last_change":"2023-09-30T09:51:57.155491+0000","last_active":"2023-09-30T09:52:33.906177+0000","last_peered":"2023-09-30T09:52:33.906177+0000","last_clean":"2023-09-30T09:52:33.906177+0000","last_became_active":"2023-09-30T09:51:57.153846+0000","last_became_peered":"2023-09-30T09:51:57.153846+0000","last_unstale":"2023-09-30T09:52:33.906177+0000","last_undegraded":"2023-09-30T09:52:33.906177+0000","last_fullsized":"2023-09-30T09:52:33.906177+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:00.681929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":112,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.910855+0000","last_change":"2023-09-30T09:52:06.219826+0000","last_active":"2023-09-30T09:52:33.910855+0000","last_peered":"2023-09-30T09:52:33.910855+0000","last_clean":"2023-09-30T09:52:33.910855+0000","last_became_active":"2023-09-30T09:52:06.219645+0000","last_became_peered":"2023-09-30T09:52:06.219645+0000","last_unstale":"2023-09-30T09:52:33.910855+0000","last_undegraded":"2023-09-30T09:52:33.910855+0000","last_fullsized":"2023-09-30T09:52:33.910855+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:08.551862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.d","version":"609'9125","reported_seq":16515,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:52:39.167978+0000","last_change":"2023-09-30T09:52:15.367163+0000","last_active":"2023-09-30T09:52:39.167978+0000","last_peered":"2023-09-30T09:52:39.167978+0000","last_clean":"2023-09-30T09:49:30.256680+0000","last_became_active":"2023-09-30T09:52:07.487724+0000","last_became_peered":"2023-09-30T09:52:07.487724+0000","last_unstale":"2023-09-30T09:52:39.167978+0000","last_undegraded":"2023-09-30T09:52:15.364556+0000","last_fullsized":"2023-09-30T09:52:07.219110+0000","mapping_epoch":596,"log_start":"547'3307","ondisk_log_start":"547'3307","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5818,"log_dups_size":0,"ondisk_log_size":5818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:07:48.953677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840024,"num_objects":2501,"num_object_clones":0,"num_object_copies":7503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2501,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9125,"num_write_kb":371969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3637,"num_bytes_recovered":238354432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":48},{"shards":"0,4,6","objects":2127},{"shards":"4,6","objects":326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":814,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.905456+0000","last_change":"2023-09-30T09:52:09.278067+0000","last_active":"2023-09-30T09:52:33.905456+0000","last_peered":"2023-09-30T09:52:33.905456+0000","last_clean":"2023-09-30T09:52:33.905456+0000","last_became_active":"2023-09-30T09:52:09.277739+0000","last_became_peered":"2023-09-30T09:52:09.277739+0000","last_unstale":"2023-09-30T09:52:33.905456+0000","last_undegraded":"2023-09-30T09:52:33.905456+0000","last_fullsized":"2023-09-30T09:52:33.905456+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:42.634023+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1207,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.908468+0000","last_change":"2023-09-30T09:52:09.264946+0000","last_active":"2023-09-30T09:52:33.908468+0000","last_peered":"2023-09-30T09:52:33.908468+0000","last_clean":"2023-09-30T09:52:33.908468+0000","last_became_active":"2023-09-30T09:52:09.262504+0000","last_became_peered":"2023-09-30T09:52:09.262504+0000","last_unstale":"2023-09-30T09:52:33.908468+0000","last_undegraded":"2023-09-30T09:52:33.908468+0000","last_fullsized":"2023-09-30T09:52:33.908468+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:10:13.204007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":64,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889809+0000","last_change":"2023-09-30T09:52:06.230958+0000","last_active":"2023-09-30T09:52:33.889809+0000","last_peered":"2023-09-30T09:52:33.889809+0000","last_clean":"2023-09-30T09:52:33.889809+0000","last_became_active":"2023-09-30T09:52:06.230575+0000","last_became_peered":"2023-09-30T09:52:06.230575+0000","last_unstale":"2023-09-30T09:52:33.889809+0000","last_undegraded":"2023-09-30T09:52:33.889809+0000","last_fullsized":"2023-09-30T09:52:33.889809+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:42:57.084463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":224,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.908069+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:52:33.908069+0000","last_peered":"2023-09-30T09:52:33.908069+0000","last_clean":"2023-09-30T09:52:33.908069+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:52:33.908069+0000","last_undegraded":"2023-09-30T09:52:33.908069+0000","last_fullsized":"2023-09-30T09:52:33.908069+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"609'9263","reported_seq":19636,"reported_epoch":609,"state":"active+remapped+backfilling","last_fresh":"2023-09-30T09:52:39.192170+0000","last_change":"2023-09-30T09:52:33.155740+0000","last_active":"2023-09-30T09:52:39.192170+0000","last_peered":"2023-09-30T09:52:39.192170+0000","last_clean":"2023-09-30T09:52:05.046523+0000","last_became_active":"2023-09-30T09:52:07.321927+0000","last_became_peered":"2023-09-30T09:52:07.321927+0000","last_unstale":"2023-09-30T09:52:39.192170+0000","last_undegraded":"2023-09-30T09:52:39.192170+0000","last_fullsized":"2023-09-30T09:52:39.192170+0000","mapping_epoch":596,"log_start":"547'3289","ondisk_log_start":"547'3289","created":536,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:45:09.841252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162070528,"num_objects":2473,"num_object_clones":0,"num_object_copies":7419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1410,"num_objects_unfound":0,"num_objects_dirty":2473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9263,"num_write_kb":375552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5119,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2473}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1223,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889533+0000","last_change":"2023-09-30T09:52:18.344243+0000","last_active":"2023-09-30T09:52:33.889533+0000","last_peered":"2023-09-30T09:52:33.889533+0000","last_clean":"2023-09-30T09:52:33.889533+0000","last_became_active":"2023-09-30T09:52:18.343870+0000","last_became_peered":"2023-09-30T09:52:18.343870+0000","last_unstale":"2023-09-30T09:52:33.889533+0000","last_undegraded":"2023-09-30T09:52:33.889533+0000","last_fullsized":"2023-09-30T09:52:33.889533+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:09:04.148163+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":640,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.908617+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:52:33.908617+0000","last_peered":"2023-09-30T09:52:33.908617+0000","last_clean":"2023-09-30T09:52:33.908617+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:52:33.908617+0000","last_undegraded":"2023-09-30T09:52:33.908617+0000","last_fullsized":"2023-09-30T09:52:33.908617+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1336,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.905735+0000","last_change":"2023-09-30T09:50:13.858344+0000","last_active":"2023-09-30T09:52:33.905735+0000","last_peered":"2023-09-30T09:52:33.905735+0000","last_clean":"2023-09-30T09:52:33.905735+0000","last_became_active":"2023-09-30T09:50:13.857963+0000","last_became_peered":"2023-09-30T09:50:13.857963+0000","last_unstale":"2023-09-30T09:52:33.905735+0000","last_undegraded":"2023-09-30T09:52:33.905735+0000","last_fullsized":"2023-09-30T09:52:33.905735+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:21.850425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":654,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.905978+0000","last_change":"2023-09-30T09:51:57.161323+0000","last_active":"2023-09-30T09:52:33.905978+0000","last_peered":"2023-09-30T09:52:33.905978+0000","last_clean":"2023-09-30T09:52:33.905978+0000","last_became_active":"2023-09-30T09:51:57.160615+0000","last_became_peered":"2023-09-30T09:51:57.160615+0000","last_unstale":"2023-09-30T09:52:33.905978+0000","last_undegraded":"2023-09-30T09:52:33.905978+0000","last_fullsized":"2023-09-30T09:52:33.905978+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:53:15.253650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"609'9035","reported_seq":17025,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-30T09:52:37.089623+0000","last_change":"2023-09-30T09:51:29.884588+0000","last_active":"2023-09-30T09:52:37.089623+0000","last_peered":"2023-09-30T09:52:37.089623+0000","last_clean":"2023-09-30T09:52:37.089623+0000","last_became_active":"2023-09-30T09:51:29.869941+0000","last_became_peered":"2023-09-30T09:51:29.869941+0000","last_unstale":"2023-09-30T09:52:37.089623+0000","last_undegraded":"2023-09-30T09:52:37.089623+0000","last_fullsized":"2023-09-30T09:52:37.089623+0000","mapping_epoch":583,"log_start":"548'3461","ondisk_log_start":"548'3461","created":536,"last_epoch_clean":584,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5574,"log_dups_size":0,"ondisk_log_size":5574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:25:01.026547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165478400,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9035,"num_write_kb":369920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3740,"num_bytes_recovered":219152384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":1036,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.891446+0000","last_change":"2023-09-30T09:51:59.873401+0000","last_active":"2023-09-30T09:52:33.891446+0000","last_peered":"2023-09-30T09:52:33.891446+0000","last_clean":"2023-09-30T09:52:33.891446+0000","last_became_active":"2023-09-30T09:51:59.873066+0000","last_became_peered":"2023-09-30T09:51:59.873066+0000","last_unstale":"2023-09-30T09:52:33.891446+0000","last_undegraded":"2023-09-30T09:52:33.891446+0000","last_fullsized":"2023-09-30T09:52:33.891446+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:48:11.565720+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":914,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.908748+0000","last_change":"2023-09-30T09:50:41.393850+0000","last_active":"2023-09-30T09:52:33.908748+0000","last_peered":"2023-09-30T09:52:33.908748+0000","last_clean":"2023-09-30T09:52:33.908748+0000","last_became_active":"2023-09-30T09:50:41.393481+0000","last_became_peered":"2023-09-30T09:50:41.393481+0000","last_unstale":"2023-09-30T09:52:33.908748+0000","last_undegraded":"2023-09-30T09:52:33.908748+0000","last_fullsized":"2023-09-30T09:52:33.908748+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:47:48.909491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":189,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.905915+0000","last_change":"2023-09-30T09:51:57.187258+0000","last_active":"2023-09-30T09:52:33.905915+0000","last_peered":"2023-09-30T09:52:33.905915+0000","last_clean":"2023-09-30T09:52:33.905915+0000","last_became_active":"2023-09-30T09:51:57.158336+0000","last_became_peered":"2023-09-30T09:51:57.158336+0000","last_unstale":"2023-09-30T09:52:33.905915+0000","last_undegraded":"2023-09-30T09:52:33.905915+0000","last_fullsized":"2023-09-30T09:52:33.905915+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:10:59.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":587,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.892862+0000","last_change":"2023-09-30T09:52:06.382180+0000","last_active":"2023-09-30T09:52:33.892862+0000","last_peered":"2023-09-30T09:52:33.892862+0000","last_clean":"2023-09-30T09:52:33.892862+0000","last_became_active":"2023-09-30T09:52:06.381759+0000","last_became_peered":"2023-09-30T09:52:06.381759+0000","last_unstale":"2023-09-30T09:52:33.892862+0000","last_undegraded":"2023-09-30T09:52:33.892862+0000","last_fullsized":"2023-09-30T09:52:33.892862+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:35:44.180146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":968,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.905882+0000","last_change":"2023-09-30T09:51:59.878763+0000","last_active":"2023-09-30T09:52:33.905882+0000","last_peered":"2023-09-30T09:52:33.905882+0000","last_clean":"2023-09-30T09:52:33.905882+0000","last_became_active":"2023-09-30T09:51:59.878357+0000","last_became_peered":"2023-09-30T09:51:59.878357+0000","last_unstale":"2023-09-30T09:52:33.905882+0000","last_undegraded":"2023-09-30T09:52:33.905882+0000","last_fullsized":"2023-09-30T09:52:33.905882+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:31:08.208858+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":360,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895777+0000","last_change":"2023-09-30T09:51:57.164777+0000","last_active":"2023-09-30T09:52:33.895777+0000","last_peered":"2023-09-30T09:52:33.895777+0000","last_clean":"2023-09-30T09:52:33.895777+0000","last_became_active":"2023-09-30T09:51:57.164468+0000","last_became_peered":"2023-09-30T09:51:57.164468+0000","last_unstale":"2023-09-30T09:52:33.895777+0000","last_undegraded":"2023-09-30T09:52:33.895777+0000","last_fullsized":"2023-09-30T09:52:33.895777+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:14.012018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":29,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895601+0000","last_change":"2023-09-30T09:51:57.164689+0000","last_active":"2023-09-30T09:52:33.895601+0000","last_peered":"2023-09-30T09:52:33.895601+0000","last_clean":"2023-09-30T09:52:33.895601+0000","last_became_active":"2023-09-30T09:51:57.164384+0000","last_became_peered":"2023-09-30T09:51:57.164384+0000","last_unstale":"2023-09-30T09:52:33.895601+0000","last_undegraded":"2023-09-30T09:52:33.895601+0000","last_fullsized":"2023-09-30T09:52:33.895601+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:47:39.321245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":755,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.911186+0000","last_change":"2023-09-30T09:50:13.849267+0000","last_active":"2023-09-30T09:52:33.911186+0000","last_peered":"2023-09-30T09:52:33.911186+0000","last_clean":"2023-09-30T09:52:33.911186+0000","last_became_active":"2023-09-30T09:50:13.848637+0000","last_became_peered":"2023-09-30T09:50:13.848637+0000","last_unstale":"2023-09-30T09:52:33.911186+0000","last_undegraded":"2023-09-30T09:52:33.911186+0000","last_fullsized":"2023-09-30T09:52:33.911186+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:24.775922+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":537,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889931+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:52:33.889931+0000","last_peered":"2023-09-30T09:52:33.889931+0000","last_clean":"2023-09-30T09:52:33.889931+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:52:33.889931+0000","last_undegraded":"2023-09-30T09:52:33.889931+0000","last_fullsized":"2023-09-30T09:52:33.889931+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.911282+0000","last_change":"2023-09-30T09:52:06.292310+0000","last_active":"2023-09-30T09:52:33.911282+0000","last_peered":"2023-09-30T09:52:33.911282+0000","last_clean":"2023-09-30T09:52:33.911282+0000","last_became_active":"2023-09-30T09:52:06.291977+0000","last_became_peered":"2023-09-30T09:52:06.291977+0000","last_unstale":"2023-09-30T09:52:33.911282+0000","last_undegraded":"2023-09-30T09:52:33.911282+0000","last_fullsized":"2023-09-30T09:52:33.911282+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:00.274025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":507,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.889886+0000","last_change":"2023-09-30T09:49:31.313229+0000","last_active":"2023-09-30T09:52:33.889886+0000","last_peered":"2023-09-30T09:52:33.889886+0000","last_clean":"2023-09-30T09:52:33.889886+0000","last_became_active":"2023-09-30T09:49:31.301137+0000","last_became_peered":"2023-09-30T09:49:31.301137+0000","last_unstale":"2023-09-30T09:52:33.889886+0000","last_undegraded":"2023-09-30T09:52:33.889886+0000","last_fullsized":"2023-09-30T09:52:33.889886+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:57.362379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":212,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.906606+0000","last_change":"2023-09-30T09:52:06.388361+0000","last_active":"2023-09-30T09:52:33.906606+0000","last_peered":"2023-09-30T09:52:33.906606+0000","last_clean":"2023-09-30T09:52:33.906606+0000","last_became_active":"2023-09-30T09:52:06.382251+0000","last_became_peered":"2023-09-30T09:52:06.382251+0000","last_unstale":"2023-09-30T09:52:33.906606+0000","last_undegraded":"2023-09-30T09:52:33.906606+0000","last_fullsized":"2023-09-30T09:52:33.906606+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:31.835279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":739,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.908909+0000","last_change":"2023-09-30T09:51:33.641471+0000","last_active":"2023-09-30T09:52:33.908909+0000","last_peered":"2023-09-30T09:52:33.908909+0000","last_clean":"2023-09-30T09:52:33.908909+0000","last_became_active":"2023-09-30T09:50:29.883457+0000","last_became_peered":"2023-09-30T09:50:29.883457+0000","last_unstale":"2023-09-30T09:52:33.908909+0000","last_undegraded":"2023-09-30T09:52:33.908909+0000","last_fullsized":"2023-09-30T09:52:33.908909+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:14.071789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:14.071789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":81,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.887455+0000","last_change":"2023-09-30T09:52:06.286929+0000","last_active":"2023-09-30T09:52:33.887455+0000","last_peered":"2023-09-30T09:52:33.887455+0000","last_clean":"2023-09-30T09:52:33.887455+0000","last_became_active":"2023-09-30T09:52:06.286569+0000","last_became_peered":"2023-09-30T09:52:06.286569+0000","last_unstale":"2023-09-30T09:52:33.887455+0000","last_undegraded":"2023-09-30T09:52:33.887455+0000","last_fullsized":"2023-09-30T09:52:33.887455+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:29.837107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":44,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.900695+0000","last_change":"2023-09-30T09:50:41.404832+0000","last_active":"2023-09-30T09:52:33.900695+0000","last_peered":"2023-09-30T09:52:33.900695+0000","last_clean":"2023-09-30T09:52:33.900695+0000","last_became_active":"2023-09-30T09:50:41.404445+0000","last_became_peered":"2023-09-30T09:50:41.404445+0000","last_unstale":"2023-09-30T09:52:33.900695+0000","last_undegraded":"2023-09-30T09:52:33.900695+0000","last_fullsized":"2023-09-30T09:52:33.900695+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:27.041434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":771,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.901052+0000","last_change":"2023-09-30T09:52:09.241112+0000","last_active":"2023-09-30T09:52:33.901052+0000","last_peered":"2023-09-30T09:52:33.901052+0000","last_clean":"2023-09-30T09:52:33.901052+0000","last_became_active":"2023-09-30T09:52:09.240703+0000","last_became_peered":"2023-09-30T09:52:09.240703+0000","last_unstale":"2023-09-30T09:52:33.901052+0000","last_undegraded":"2023-09-30T09:52:33.901052+0000","last_fullsized":"2023-09-30T09:52:33.901052+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:40:40.400090+0000","scrub_duration":0.004695744,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":308,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.902881+0000","last_change":"2023-09-30T09:52:06.308608+0000","last_active":"2023-09-30T09:52:33.902881+0000","last_peered":"2023-09-30T09:52:33.902881+0000","last_clean":"2023-09-30T09:52:33.902881+0000","last_became_active":"2023-09-30T09:52:06.287432+0000","last_became_peered":"2023-09-30T09:52:06.287432+0000","last_unstale":"2023-09-30T09:52:33.902881+0000","last_undegraded":"2023-09-30T09:52:33.902881+0000","last_fullsized":"2023-09-30T09:52:33.902881+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:30:09.888951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":653,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-30T09:52:33.895238+0000","last_change":"2023-09-30T09:50:26.905427+0000","last_active":"2023-09-30T09:52:33.895238+0000","last_peered":"2023-09-30T09:52:33.895238+0000","last_clean":"2023-09-30T09:52:33.895238+0000","last_became_active":"2023-09-30T09:50:26.905100+0000","last_became_peered":"2023-09-30T09:50:26.905100+0000","last_unstale":"2023-09-30T09:52:33.895238+0000","last_undegraded":"2023-09-30T09:52:33.895238+0000","last_fullsized":"2023-09-30T09:52:33.895238+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:38.036208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2623012888,"num_objects":40025,"num_object_clones":0,"num_object_copies":120075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7300,"num_objects_misplaced":6269,"num_objects_unfound":0,"num_objects_dirty":40025,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146771,"num_write_kb":5977409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35098,"num_bytes_recovered":2114453504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9709629440,"data_stored":9709617224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91182,"ondisk_log_size":91182,"up":48,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":13644,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":599313,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":709030,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234491,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1435200,"kb_used_data":821048,"kb_used_omap":0,"kb_used_meta":614144,"kb_avail":92309952,"statfs":{"total":95995035648,"available":94525390848,"internally_reserved":0,"allocated":840753152,"data_stored":838733051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":628883456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.5209999999999999,"15min":5.2839999999999998},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.22900000000000001},"max":{"1min":14.641999999999999,"5min":45.895000000000003,"15min":215.416},"last":100.176},{"interface":"front","average":{"1min":0.80800000000000005,"5min":2.661,"15min":5.3259999999999996},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.1799999999999997,"5min":46.451000000000001,"15min":215.041},"last":99.890000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:52:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.823,"15min":3.77},"min":{"1min":0.29299999999999998,"5min":0.26700000000000002,"15min":0.25},"max":{"1min":6.2809999999999997,"5min":10.749000000000001,"15min":157.578},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.946,"15min":3.7890000000000001},"min":{"1min":0.32700000000000001,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":4.4260000000000002,"5min":12.013999999999999,"15min":157.589},"last":0.311}]},{"osd":2,"last update":"Sat Sep 30 09:51:47 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":4.9210000000000003,"15min":4.5789999999999997},"min":{"1min":0.28699999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.5289999999999999,"5min":187.74600000000001,"15min":187.74600000000001},"last":0.374},{"interface":"front","average":{"1min":2.516,"5min":4.9900000000000002,"15min":4.5830000000000002},"min":{"1min":0.34300000000000003,"5min":0.33500000000000002,"15min":0.28399999999999997},"max":{"1min":10.949,"5min":187.583,"15min":187.583},"last":0.64100000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:52:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.867,"15min":4.9790000000000001},"min":{"1min":0.34799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.3520000000000003,"5min":13.303000000000001,"15min":248.691},"last":102.139},{"interface":"front","average":{"1min":1.0629999999999999,"5min":2.2730000000000001,"15min":5.2169999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.21,"5min":13.257,"15min":249.36500000000001},"last":99.713999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:51:47 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":3.9380000000000002,"15min":2.5899999999999999},"min":{"1min":0.29099999999999998,"5min":0.27200000000000002,"15min":0.23799999999999999},"max":{"1min":3.2360000000000002,"5min":238.511,"15min":238.511},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.3620000000000001,"5min":4.2140000000000004,"15min":2.6179999999999999},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.252},"max":{"1min":11,"5min":238.387,"15min":238.387},"last":0.68300000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:52:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":4.165,"15min":7.9400000000000004},"min":{"1min":0.39000000000000001,"5min":0.32700000000000001,"15min":0.28399999999999997},"max":{"1min":9.3659999999999997,"5min":187.374,"15min":197.75700000000001},"last":0.71199999999999997},{"interface":"front","average":{"1min":2.0590000000000002,"5min":3.8500000000000001,"15min":7.8220000000000001},"min":{"1min":0.32400000000000001,"5min":0.309,"15min":0.309},"max":{"1min":11.863,"5min":187.416,"15min":197.63300000000001},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:51:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.5640000000000001,"15min":2.234},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":3.198,"5min":7.6479999999999997,"15min":206.244},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.3250000000000002,"5min":1.613,"15min":2.2309999999999999},"min":{"1min":0.32700000000000001,"5min":0.32200000000000001,"15min":0.29599999999999999},"max":{"1min":11.121,"5min":12.058,"15min":202.78800000000001},"last":0.84899999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496129,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1737272,"kb_used_data":1090992,"kb_used_omap":0,"kb_used_meta":646144,"kb_avail":92007880,"statfs":{"total":95995035648,"available":94216069120,"internally_reserved":0,"allocated":1117175808,"data_stored":1115163252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":661651456},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":5.9130000000000003,"15min":6.5599999999999996},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.3609999999999998,"5min":192.511,"15min":192.511},"last":1.095},{"interface":"front","average":{"1min":0.87,"5min":6.2089999999999996,"15min":6.6340000000000003},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.246},"max":{"1min":4.3630000000000004,"5min":192.346,"15min":192.346},"last":1.2190000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:52:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":3.073,"15min":2.2989999999999999},"min":{"1min":0.27000000000000002,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":4.2430000000000003,"5min":103.038,"15min":103.038},"last":1.054},{"interface":"front","average":{"1min":2.367,"5min":3.286,"15min":2.492},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":25.446999999999999,"5min":103.05,"15min":103.05},"last":1.0720000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:52:01 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":3.6840000000000002,"15min":4.4930000000000003},"min":{"1min":0.29799999999999999,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":6.2030000000000003,"5min":109.88500000000001,"15min":207.02500000000001},"last":1.2629999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":3.258,"15min":4.4029999999999996},"min":{"1min":0.32900000000000001,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":4.3010000000000002,"5min":105.81699999999999,"15min":207.114},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":2.706,"15min":2.7869999999999999},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":6.3330000000000002,"5min":106.402,"15min":107.11799999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.359,"5min":3.177,"15min":2.9390000000000001},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.31900000000000001},"max":{"1min":4.4290000000000003,"5min":106.34699999999999,"15min":106.982},"last":2.9969999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":2.286,"15min":1.6739999999999999},"min":{"1min":0.28899999999999998,"5min":0.28100000000000003,"15min":0.24399999999999999},"max":{"1min":6.976,"5min":30.568000000000001,"15min":63.798000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.268,"5min":1.919,"15min":1.532},"min":{"1min":0.32200000000000001,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":7.5330000000000004,"5min":29.27,"15min":63.133000000000003},"last":0.378}]},{"osd":5,"last update":"Sat Sep 30 09:51:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.899,"15min":1.9399999999999999},"min":{"1min":0.37,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":7.4630000000000001,"5min":19.212,"15min":19.212},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.089,"5min":1.667,"15min":2.1909999999999998},"min":{"1min":0.42399999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.1609999999999996,"5min":12.731,"15min":16.443000000000001},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:52:01 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.0089999999999999,"15min":1.792},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":39.868000000000002,"5min":39.868000000000002,"15min":138.191},"last":0.442},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.7030000000000001,"15min":1.619},"min":{"1min":0.38800000000000001,"5min":0.36899999999999999,"15min":0.27500000000000002},"max":{"1min":4.1189999999999998,"5min":12.625,"15min":140.33500000000001},"last":0.436}]}]},{"osd":1,"up_from":301,"seq":1292785156326,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1364444,"kb_used_data":628816,"kb_used_omap":9,"kb_used_meta":735606,"kb_avail":92380708,"statfs":{"total":95995035648,"available":94597844992,"internally_reserved":0,"allocated":643907584,"data_stored":640432996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":753260562},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:06 2023","interfaces":[{"interface":"back","average":{"1min":22.524000000000001,"5min":10.657,"15min":5.9180000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.17100000000000001},"max":{"1min":151.256,"5min":191.62700000000001,"15min":191.62700000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":22.704999999999998,"5min":10.834,"15min":5.7309999999999999},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":150.733,"5min":191.58799999999999,"15min":191.58799999999999},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:52:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":5.1760000000000002,"15min":3.3620000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.184},"max":{"1min":9.7040000000000006,"5min":189.14500000000001,"15min":189.14500000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":5.0640000000000001,"15min":3.4580000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.20999999999999999},"max":{"1min":4.1639999999999997,"5min":189.333,"15min":189.333},"last":0.48199999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:52:06 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":3.3260000000000001,"15min":4.5529999999999999},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.23400000000000001},"max":{"1min":31.151,"5min":48.732999999999997,"15min":199.41},"last":0.308},{"interface":"front","average":{"1min":3.2040000000000002,"5min":2.5840000000000001,"15min":4.2560000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":31.727,"5min":48.576999999999998,"15min":199.58099999999999},"last":0.375}]},{"osd":4,"last update":"Sat Sep 30 09:52:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.069,"15min":2.1190000000000002},"min":{"1min":0.33800000000000002,"5min":0.29799999999999999,"15min":0.26800000000000002},"max":{"1min":2.8159999999999998,"5min":48.604999999999997,"15min":142.77099999999999},"last":0.871},{"interface":"front","average":{"1min":1.222,"5min":2.194,"15min":2.5739999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":6.234,"5min":22.373000000000001,"15min":144.898},"last":0.95399999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:51:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":2.4870000000000001,"15min":2.1469999999999998},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.3360000000000003,"5min":22.259,"15min":22.259},"last":2.4780000000000002},{"interface":"front","average":{"1min":1.097,"5min":1.752,"15min":1.4399999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.35599999999999998},"max":{"1min":5.6159999999999997,"5min":12.41,"15min":15.250999999999999},"last":2.4910000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:52:06 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":1.8420000000000001,"15min":2.3420000000000001},"min":{"1min":0.377,"5min":0.33700000000000002,"15min":0.29899999999999999},"max":{"1min":23.521000000000001,"5min":23.521000000000001,"15min":96.719999999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.8,"5min":1.962,"15min":2.4140000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":23.469999999999999,"5min":23.469999999999999,"15min":96.356999999999999},"last":0.63500000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:52:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.5029999999999999,"15min":1.663},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.28299999999999997},"max":{"1min":9.8390000000000004,"5min":9.8390000000000004,"15min":15.587},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.595,"15min":1.6699999999999999},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":9.8109999999999999,"5min":9.8109999999999999,"15min":15.226000000000001},"last":3.262}]}]},{"osd":0,"up_from":94,"seq":403726926225,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2632344,"kb_used_data":1796176,"kb_used_omap":5,"kb_used_meta":836154,"kb_avail":91112808,"statfs":{"total":95995035648,"available":93299515392,"internally_reserved":0,"allocated":1839284224,"data_stored":1835586655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":856221949},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:52:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.3839999999999999,"15min":1.621},"min":{"1min":0.20799999999999999,"5min":0.19700000000000001,"15min":0.191},"max":{"1min":8.0269999999999992,"5min":13.308999999999999,"15min":57.482999999999997},"last":1.02},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.794,"15min":2.0329999999999999},"min":{"1min":0.28000000000000003,"5min":0.22500000000000001,"15min":0.19800000000000001},"max":{"1min":2.258,"5min":23.503,"15min":57.140999999999998},"last":1.226}]},{"osd":2,"last update":"Sat Sep 30 09:51:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":4.6420000000000003,"15min":4.1200000000000001},"min":{"1min":0.24399999999999999,"5min":0.24299999999999999,"15min":0.20799999999999999},"max":{"1min":7.4299999999999997,"5min":173.185,"15min":173.185},"last":1.2729999999999999},{"interface":"front","average":{"1min":0.88800000000000001,"5min":4.367,"15min":4.1580000000000004},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":3.754,"5min":173.22499999999999,"15min":173.22499999999999},"last":1.304}]},{"osd":3,"last update":"Sat Sep 30 09:52:20 2023","interfaces":[{"interface":"back","average":{"1min":3.8599999999999999,"5min":2.7490000000000001,"15min":3.496},"min":{"1min":0.37,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":53.192999999999998,"5min":53.192999999999998,"15min":125.562},"last":0.27600000000000002},{"interface":"front","average":{"1min":3.6619999999999999,"5min":2.4140000000000001,"15min":2.9670000000000001},"min":{"1min":0.371,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":53.207000000000001,"5min":53.207000000000001,"15min":117.295},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:52:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":2.8279999999999998,"15min":2.4009999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3149999999999995,"5min":67.945999999999998,"15min":72.403999999999996},"last":1.77},{"interface":"front","average":{"1min":1.363,"5min":2.835,"15min":2.484},"min":{"1min":0.36199999999999999,"5min":0.35999999999999999,"15min":0.29699999999999999},"max":{"1min":10.215,"5min":67.792000000000002,"15min":72.162999999999997},"last":3.3999999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":3.0209999999999999,"15min":1.917},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":7.4050000000000002,"5min":71.376000000000005,"15min":71.376000000000005},"last":1.95},{"interface":"front","average":{"1min":1.125,"5min":2.5939999999999999,"15min":1.5900000000000001},"min":{"1min":0.41699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.6040000000000001,"5min":71.706000000000003,"15min":71.706000000000003},"last":1.3520000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":4.0880000000000001,"15min":2.4889999999999999},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.33200000000000002},"max":{"1min":2.1259999999999999,"5min":240.946,"15min":240.946},"last":1.7869999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":4.5540000000000003,"15min":2.9399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.34999999999999998},"max":{"1min":3.8479999999999999,"5min":240.977,"15min":240.977},"last":1.7989999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:52:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":3.359,"15min":2.1960000000000002},"min":{"1min":0.43099999999999999,"5min":0.41499999999999998,"15min":0.36099999999999999},"max":{"1min":9.1460000000000008,"5min":146.828,"15min":146.828},"last":4.9219999999999997},{"interface":"front","average":{"1min":0.93000000000000005,"5min":3.843,"15min":2.694},"min":{"1min":0.436,"5min":0.41599999999999998,"15min":0.33100000000000002},"max":{"1min":3.8889999999999998,"5min":146.876,"15min":146.876},"last":4.0670000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599164,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2484648,"kb_used_data":1565728,"kb_used_omap":5,"kb_used_meta":918842,"kb_avail":91260504,"statfs":{"total":95995035648,"available":93450756096,"internally_reserved":0,"allocated":1603305472,"data_stored":1599884010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":940894861},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":4.6230000000000002,"15min":3.9670000000000001},"min":{"1min":0.20799999999999999,"5min":0.188,"15min":0.188},"max":{"1min":28.606000000000002,"5min":202.40899999999999,"15min":202.40899999999999},"last":5.0570000000000004},{"interface":"front","average":{"1min":2.5139999999999998,"5min":5.0190000000000001,"15min":4.2149999999999999},"min":{"1min":0.23000000000000001,"5min":0.222,"15min":0.217},"max":{"1min":28.582000000000001,"5min":202.38800000000001,"15min":202.38800000000001},"last":0.81000000000000005}]},{"osd":1,"last update":"Sat Sep 30 09:52:03 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":4.5529999999999999,"15min":3.698},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":8.1310000000000002,"5min":202.369,"15min":202.369},"last":6.3760000000000003},{"interface":"front","average":{"1min":0.88500000000000001,"5min":4.4859999999999998,"15min":3.7519999999999998},"min":{"1min":0.28100000000000003,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.4569999999999999,"5min":202.54499999999999,"15min":202.54499999999999},"last":2.581}]},{"osd":3,"last update":"Sat Sep 30 09:52:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":3.4159999999999999,"15min":3.7770000000000001},"min":{"1min":0.28399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.2270000000000003,"5min":202.59,"15min":202.59},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.88800000000000001,"5min":3.2349999999999999,"15min":3.6949999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.141,"5min":202.34200000000001,"15min":202.34200000000001},"last":5.8150000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:52:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":5.7919999999999998,"15min":3.133},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.5599999999999996,"5min":247.655,"15min":247.655},"last":1.405},{"interface":"front","average":{"1min":0.71599999999999997,"5min":5.4400000000000004,"15min":3.048},"min":{"1min":0.27200000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.081,"5min":248.012,"15min":248.012},"last":5.7770000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:51:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":5.8070000000000004,"15min":2.9279999999999999},"min":{"1min":0.38500000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.0490000000000004,"5min":243.86500000000001,"15min":243.86500000000001},"last":10.762},{"interface":"front","average":{"1min":1.8939999999999999,"5min":6.093,"15min":3.1629999999999998},"min":{"1min":0.44500000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.1790000000000003,"5min":244.02199999999999,"15min":244.02199999999999},"last":6.4180000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:52:03 2023","interfaces":[{"interface":"back","average":{"1min":3.3250000000000002,"5min":3.8450000000000002,"15min":2.3069999999999999},"min":{"1min":0.34699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":46.732999999999997,"5min":202.70599999999999,"15min":202.70599999999999},"last":5.7300000000000004},{"interface":"front","average":{"1min":4.4370000000000003,"5min":4.274,"15min":2.6389999999999998},"min":{"1min":0.41999999999999998,"5min":0.38800000000000001,"15min":0.34100000000000003},"max":{"1min":46.68,"5min":202.916,"15min":202.916},"last":6.3259999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:52:03 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":6.9059999999999997,"15min":3.423},"min":{"1min":0.39600000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.3440000000000003,"5min":244.911,"15min":244.911},"last":9.1950000000000003},{"interface":"front","average":{"1min":1.9039999999999999,"5min":7.2549999999999999,"15min":3.7850000000000001},"min":{"1min":0.47899999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.2249999999999996,"5min":244.66,"15min":244.66},"last":6.452}]}]},{"osd":3,"up_from":109,"seq":468151435639,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1996988,"kb_used_data":1008048,"kb_used_omap":2,"kb_used_meta":988925,"kb_avail":91748164,"statfs":{"total":95995035648,"available":93950119936,"internally_reserved":0,"allocated":1032241152,"data_stored":1028683225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":1012659559},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":8.3339999999999996,"5min":4.6420000000000003,"15min":4.867},"min":{"1min":0.22500000000000001,"5min":0.191,"15min":0.183},"max":{"1min":103.03,"5min":152.429,"15min":205.73099999999999},"last":0.23200000000000001},{"interface":"front","average":{"1min":7.8680000000000003,"5min":4.8970000000000002,"15min":4.9100000000000001},"min":{"1min":0.27400000000000002,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":103.39700000000001,"5min":150.464,"15min":205.495},"last":2.5289999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:52:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":2.149,"15min":3.3660000000000001},"min":{"1min":0.22800000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.2989999999999999,"5min":15.784000000000001,"15min":170.512},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.863,"15min":3.2999999999999998},"min":{"1min":0.308,"5min":0.23400000000000001,"15min":0.20699999999999999},"max":{"1min":3.3559999999999999,"5min":12.470000000000001,"15min":170.535},"last":3.806}]},{"osd":2,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":3,"15min":5.2729999999999997},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.0720000000000001,"5min":147.87,"15min":206.69},"last":3.847},{"interface":"front","average":{"1min":0.86399999999999999,"5min":3.4100000000000001,"15min":5.2709999999999999},"min":{"1min":0.253,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":3.407,"5min":148.04900000000001,"15min":209.22800000000001},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":3.5329999999999999,"15min":2.3690000000000002},"min":{"1min":0.32400000000000001,"5min":0.29199999999999998,"15min":0.26800000000000002},"max":{"1min":3.4369999999999998,"5min":172.298,"15min":172.298},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.27,"5min":3.738,"15min":2.516},"min":{"1min":0.35299999999999998,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":3.8300000000000001,"5min":172.45099999999999,"15min":172.45099999999999},"last":0.436}]},{"osd":5,"last update":"Sat Sep 30 09:52:36 2023","interfaces":[{"interface":"back","average":{"1min":6.0119999999999996,"5min":2.5670000000000002,"15min":2.0110000000000001},"min":{"1min":0.39600000000000002,"5min":0.33700000000000002,"15min":0.29199999999999998},"max":{"1min":61.426000000000002,"5min":61.426000000000002,"15min":61.426000000000002},"last":6.4550000000000001},{"interface":"front","average":{"1min":6.0979999999999999,"5min":3.1520000000000001,"15min":1.8799999999999999},"min":{"1min":0.46700000000000003,"5min":0.36199999999999999,"15min":0.311},"max":{"1min":61.469000000000001,"5min":61.469000000000001,"15min":61.469000000000001},"last":6.4370000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":6.5800000000000001,"5min":3.0329999999999999,"15min":1.998},"min":{"1min":0.36799999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":75.781000000000006,"5min":75.781000000000006,"15min":75.781000000000006},"last":0.52900000000000003},{"interface":"front","average":{"1min":6.2279999999999998,"5min":3.3540000000000001,"15min":2.1619999999999999},"min":{"1min":0.43099999999999999,"5min":0.35999999999999999,"15min":0.33400000000000002},"max":{"1min":75.754000000000005,"5min":75.754000000000005,"15min":75.754000000000005},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":7.9029999999999996,"5min":3.2629999999999999,"15min":2.7170000000000001},"min":{"1min":0.42099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":90.855999999999995,"5min":90.855999999999995,"15min":213.548},"last":5.4630000000000001},{"interface":"front","average":{"1min":7.0679999999999996,"5min":3.2570000000000001,"15min":2.6240000000000001},"min":{"1min":0.42599999999999999,"5min":0.40899999999999997,"15min":0.36199999999999999},"max":{"1min":91.108000000000004,"5min":91.108000000000004,"15min":213.583},"last":5.4119999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019402,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2279732,"kb_used_data":1696684,"kb_used_omap":0,"kb_used_meta":582912,"kb_avail":91465420,"statfs":{"total":95995035648,"available":93660590080,"internally_reserved":0,"allocated":1737404416,"data_stored":1735395483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":596901888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,8],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:13 2023","interfaces":[{"interface":"back","average":{"1min":3.8610000000000002,"5min":4.4370000000000003,"15min":3.5630000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.21199999999999999},"max":{"1min":60.945,"5min":107.43000000000001,"15min":107.43000000000001},"last":0.87},{"interface":"front","average":{"1min":4.0369999999999999,"5min":4.7949999999999999,"15min":3.7240000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":60.695999999999998,"5min":107.373,"15min":107.373},"last":1.5960000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.9350000000000001,"15min":2.5600000000000001},"min":{"1min":0.30499999999999999,"5min":0.27500000000000002,"15min":0.25},"max":{"1min":12.385,"5min":108.551,"15min":108.551},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.3120000000000001,"5min":3.0619999999999998,"15min":2.6499999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.42,"5min":108.884,"15min":108.884},"last":0.996}]},{"osd":2,"last update":"Sat Sep 30 09:51:57 2023","interfaces":[{"interface":"back","average":{"1min":10.058,"5min":3.661,"15min":4.0499999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":139.53899999999999,"5min":139.53899999999999,"15min":142.88999999999999},"last":6.3799999999999999},{"interface":"front","average":{"1min":9.6950000000000003,"5min":3.6760000000000002,"15min":4.0780000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":139.56399999999999,"5min":139.56399999999999,"15min":142.90700000000001},"last":0.94799999999999995}]},{"osd":3,"last update":"Sat Sep 30 09:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":3.6459999999999999,"15min":2.407},"min":{"1min":0.376,"5min":0.376,"15min":0.32700000000000001},"max":{"1min":4.2290000000000001,"5min":105.142,"15min":105.142},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.0409999999999999,"5min":3.4529999999999998,"15min":2.343},"min":{"1min":0.377,"5min":0.377,"15min":0.29299999999999998},"max":{"1min":4.2190000000000003,"5min":105.157,"15min":105.157},"last":1.6519999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:52:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":2.1339999999999999,"15min":6.6520000000000001},"min":{"1min":0.26400000000000001,"5min":0.249,"15min":0.249},"max":{"1min":8.4329999999999998,"5min":59.530999999999999,"15min":140.11099999999999},"last":1.6299999999999999},{"interface":"front","average":{"1min":2.6789999999999998,"5min":2.1000000000000001,"15min":6.6280000000000001},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.9369999999999994,"5min":60.012,"15min":140.065},"last":6.46}]},{"osd":6,"last update":"Sat Sep 30 09:52:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":1.585,"15min":1.7030000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":13.978,"5min":13.978,"15min":182.10300000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.671,"15min":1.7529999999999999},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":8.5120000000000005,"5min":11.523999999999999,"15min":181.76300000000001},"last":1.843}]},{"osd":7,"last update":"Sat Sep 30 09:51:57 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.5940000000000001,"15min":2.254},"min":{"1min":0.32700000000000001,"5min":0.31900000000000001,"15min":0.27600000000000002},"max":{"1min":7.6559999999999997,"5min":12.676,"15min":161.72999999999999},"last":6.431},{"interface":"front","average":{"1min":1.042,"5min":1.339,"15min":2.1269999999999998},"min":{"1min":0.42599999999999999,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":4.0469999999999997,"5min":8.8529999999999998,"15min":162.60900000000001},"last":0.97299999999999998}]}]},{"osd":5,"up_from":489,"seq":2100239007844,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1245228,"kb_used_data":1008996,"kb_used_omap":6,"kb_used_meta":236089,"kb_avail":92499924,"statfs":{"total":95995035648,"available":94719922176,"internally_reserved":0,"allocated":1033211904,"data_stored":1031207665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":241755231},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,2,2,1,0,1,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":9.0609999999999999,"5min":7.1820000000000004,"15min":17.100000000000001},"min":{"1min":0.24399999999999999,"5min":0.224,"15min":0.22},"max":{"1min":137.846,"5min":137.846,"15min":281.10599999999999},"last":1.552},{"interface":"front","average":{"1min":9.3719999999999999,"5min":6.226,"15min":16.948},"min":{"1min":0.27800000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":137.035,"5min":137.035,"15min":282.07400000000001},"last":3.4940000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":2.8109999999999999,"15min":9.5939999999999994},"min":{"1min":0.30199999999999999,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":11.234999999999999,"5min":129.958,"15min":273.48200000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.006,"5min":2.5449999999999999,"15min":9.0169999999999995},"min":{"1min":0.28899999999999998,"5min":0.255,"15min":0.255},"max":{"1min":8.0820000000000007,"5min":130.05099999999999,"15min":273.50200000000001},"last":1.655}]},{"osd":2,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":4.6269999999999998,"15min":3.2970000000000002},"min":{"1min":0.28899999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.625999999999999,"5min":194.14400000000001,"15min":194.14400000000001},"last":2.5139999999999998},{"interface":"front","average":{"1min":2.1299999999999999,"5min":3.3250000000000002,"15min":3.1699999999999999},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.337999999999999,"5min":198.083,"15min":198.083},"last":3.3839999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":4.8150000000000004,"5min":4.9260000000000002,"15min":10.217000000000001},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.30299999999999999},"max":{"1min":41.872,"5min":130.02199999999999,"15min":274.01799999999997},"last":3.4169999999999998},{"interface":"front","average":{"1min":4.0049999999999999,"5min":4.7030000000000003,"15min":3.194},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":41.829000000000001,"5min":130.07400000000001,"15min":130.07400000000001},"last":2.6190000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":1.738,"5min":4.6139999999999999,"15min":2.1890000000000001},"min":{"1min":0.29599999999999999,"5min":0.26700000000000002,"15min":0.26000000000000001},"max":{"1min":13.601000000000001,"5min":132.94399999999999,"15min":132.94399999999999},"last":2.4350000000000001},{"interface":"front","average":{"1min":2.2770000000000001,"5min":4.7789999999999999,"15min":2.3919999999999999},"min":{"1min":0.34899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":14.223000000000001,"5min":136.083,"15min":136.083},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":4.319,"5min":3.5800000000000001,"15min":2.2469999999999999},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":66.123000000000005,"5min":130.09800000000001,"15min":130.09800000000001},"last":1.583},{"interface":"front","average":{"1min":4.9109999999999996,"5min":3.677,"15min":2.157},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.316},"max":{"1min":66.102999999999994,"5min":130.03399999999999,"15min":130.03399999999999},"last":2.472}]},{"osd":7,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":1.9099999999999999,"15min":1.534},"min":{"1min":0.33800000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":32.578000000000003,"5min":32.578000000000003,"15min":32.578000000000003},"last":1.605},{"interface":"front","average":{"1min":3.2410000000000001,"5min":2.0819999999999999,"15min":1.3560000000000001},"min":{"1min":0.39700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":32.420999999999999,"5min":32.420999999999999,"15min":32.420999999999999},"last":3.5409999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1810173952,"data_stored":1810169880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":625410048,"data_stored":625410048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1590755328,"data_stored":1590755328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1020592128,"data_stored":1020592128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1729761280,"data_stored":1729757208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1008205824,"data_stored":1008205824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1103826944,"data_stored":1103822872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":820903936,"data_stored":820903936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:52:42.861 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 16 2023-09-30T09:52:42.951 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:42.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:43.062 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:43.135 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:43.213 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:43.255 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:43.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:43.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:43.290 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[216815]: pgmap v1621: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 7311/120159 objects degraded (6.084%); 6269/120159 objects misplaced (5.217%); 13 MiB/s, 220 objects/s recovering 2023-09-30T09:52:43.290 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.290 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[224043]: pgmap v1621: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 7311/120159 objects degraded (6.084%); 6269/120159 objects misplaced (5.217%); 13 MiB/s, 220 objects/s recovering 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:52:43.291 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.387 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:43.450 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:43.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:43 smithi167 ceph-mon[192880]: pgmap v1621: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 7311/120159 objects degraded (6.084%); 6269/120159 objects misplaced (5.217%); 13 MiB/s, 220 objects/s recovering 2023-09-30T09:52:43.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:52:43.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:52:43.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:52:43.598 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:43.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:43.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:43.705 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:43.779 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:43.990 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:43.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:43.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:44.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:52:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:52:43] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-09-30T09:52:44.100 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:44.126 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:52:44.146 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:44.193 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:44.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:44.297 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:44.325 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:44.426 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:44.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:44.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1574105475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:44.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:44 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7311/120159 objects degraded (6.084%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:44.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1574105475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:44.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:44 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7311/120159 objects degraded (6.084%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:44.513 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1574105475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:52:44.513 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:44 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7311/120159 objects degraded (6.084%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:44.536 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:44.576 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:44.921 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:44.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:44.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:45.028 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:45.135 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:45.136 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[216815]: pgmap v1622: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 7144/117798 objects degraded (6.065%); 6269/117798 objects misplaced (5.322%); 12 MiB/s, 196 objects/s recovering 2023-09-30T09:52:45.137 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:45.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:45 smithi167 ceph-mon[192880]: pgmap v1622: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 7144/117798 objects degraded (6.065%); 6269/117798 objects misplaced (5.322%); 12 MiB/s, 196 objects/s recovering 2023-09-30T09:52:45.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:45 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:45.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:45 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1574105475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:52:45.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:45 smithi167 ceph-mon[192880]: failed to encode map e610 with expected crc 2023-09-30T09:52:45.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:45 smithi167 ceph-mon[192880]: osdmap e610: 8 total, 8 up, 8 in 2023-09-30T09:52:45.304 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:45.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:45.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:45.328 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:52:45.328 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:52:45.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:52:45.328 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:52:45.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.329 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2300 9.b 9:dee21a30:::benchmark_data_smithi019_560486_object2299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:52:45.329 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.031346+0000", 2023-09-30T09:52:45.329 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 500.29299076299998, 2023-09-30T09:52:45.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0658693860000001, 2023-09-30T09:52:45.329 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.329 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:52:45.329 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.329 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2300 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031346+0000", 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031346+0000", 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031346+0000", 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031390+0000", 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3908e-05 2023-09-30T09:52:45.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031403+0000", 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2999e-05 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031408+0000", 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3580000000000002e-06 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031464+0000", 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5831999999999997e-05 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031593+0000", 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129018 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2", 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031732+0000", 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013851099999999999 2023-09-30T09:52:45.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.033586+0000", 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001854337 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064449+0000", 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.030863595000000001 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:45.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064457+0000", 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8189999999999992e-06 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052844+0000", 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.988386451 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052857+0000", 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2961e-05 2023-09-30T09:52:45.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.090028+0000", 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.037171329 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.091452+0000", 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014238129999999999 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097130+0000", 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0056783509999999999 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097216+0000", 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5702e-05 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2301 9.b 9:d16e4333:::benchmark_data_smithi019_560486_object2300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.058107+0000", 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 500.26623042699998, 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0393504029999998, 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.338 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2301 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058107+0000", 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058107+0000", 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058105+0000", 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058212+0000", 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000106388 2023-09-30T09:52:45.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058246+0000", 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4666999999999999e-05 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058256+0000", 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6940000000000007e-06 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064479+0000", 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0062230080000000004 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064486+0000", 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6390000000000003e-06 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052884+0000", 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98839818000000002 2023-09-30T09:52:45.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052892+0000", 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8879999999999997e-06 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.091815+0000", 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.038923182 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.091981+0000", 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016666200000000001 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.092165+0000", 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018395999999999999 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094158+0000", 2023-09-30T09:52:45.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001992382 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096529+0000", 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023708689999999998 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097255+0000", 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00072591599999999995 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097301+0000", 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6601000000000003e-05 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097457+0000", 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015600400000000001 2023-09-30T09:52:45.346 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2307 9.b 9:d81a73d3:::benchmark_data_smithi019_560486_object2306:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.058514+0000", 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 500.26582374899999, 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0391732079999998, 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.347 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2307 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058514+0000", 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058514+0000", 2023-09-30T09:52:45.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058513+0000", 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058562+0000", 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8896000000000001e-05 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058569+0000", 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1640000000000004e-06 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.058574+0000", 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4900000000000002e-06 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064506+0000", 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0059326819999999999 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.064512+0000", 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1140000000000002e-06 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052928+0000", 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.98841563700000001 2023-09-30T09:52:45.351 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.052936+0000", 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8849999999999999e-06 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.092555+0000", 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.039618919 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.092702+0000", 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000147109 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5", 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.092859+0000", 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000157472 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094391+0000", 2023-09-30T09:52:45.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015314 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096558+0000", 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002167123 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097502+0000", 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00094386399999999999 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097546+0000", 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4351000000000003e-05 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.097687+0000", 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014054399999999999 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79004 9.0 9:0d8925f6:::benchmark_data_smithi019_560486_object79003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 434.79084555600002, 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.377661042, 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79004 2023-09-30T09:52:45.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533491+0000", 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533548+0000", 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7012999999999998e-05 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533561+0000", 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2352e-05 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533565+0000", 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8060000000000002e-06 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533608+0000", 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2279e-05 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533742+0000", 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013385 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533885+0000", 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014328400000000001 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.538697+0000", 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004812584 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.540380+0000", 2023-09-30T09:52:45.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001682944 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911056+0000", 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.37067566800000001 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911112+0000", 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6187999999999998e-05 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911153+0000", 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0515000000000001e-05 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.363 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79005 9.0 9:01cc776d:::benchmark_data_smithi019_560486_object79004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 434.79075917699998, 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.377676597, 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79005 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.364 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:52:45.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:52:45.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533627+0000", 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9091000000000001e-05 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533634+0000", 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3649999999999998e-06 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533638+0000", 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2409999999999997e-06 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536549+0000", 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029111250000000001 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536664+0000", 2023-09-30T09:52:45.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011423500000000001 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536750+0000", 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5932000000000003e-05 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.538813+0000", 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020632480000000002 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.544406+0000", 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055927779999999996 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911189+0000", 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.366783097 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911227+0000", 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8250999999999997e-05 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911255+0000", 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7742e-05 2023-09-30T09:52:45.371 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79017 9.0 9:0d388969:::benchmark_data_smithi019_560486_object79016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 434.77565863400002, 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.37037067400000001, 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.372 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79017 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548677+0000", 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548770+0000", 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2744999999999994e-05 2023-09-30T09:52:45.374 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548786+0000", 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6557000000000001e-05 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548793+0000", 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4880000000000004e-06 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548838+0000", 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5550000000000003e-05 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548961+0000", 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012252199999999999 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.549114+0000", 2023-09-30T09:52:45.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015329499999999999 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.554511+0000", 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0053968749999999998 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.560437+0000", 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0059257829999999996 2023-09-30T09:52:45.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.918969+0000", 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.35853247300000002 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.919009+0000", 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9582999999999997e-05 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.919049+0000", 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0509999999999997e-05 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:172370 9.1 9:8087334a:::benchmark_data_smithi019_560486_object75625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 302.32522086900002, 2023-09-30T09:52:45.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28865046500000002, 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 172370 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999119+0000", 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.807e-06 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999125+0000", 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.959e-06 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999129+0000", 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4599999999999996e-06 2023-09-30T09:52:45.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999170+0000", 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0299999999999997e-05 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999273+0000", 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010336999999999999 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:52:45.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999396+0000", 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012268500000000001 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.005183+0000", 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057871479999999998 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.007181+0000", 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019979009999999998 2023-09-30T09:52:45.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287675+0000", 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28049398800000003 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287725+0000", 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0720999999999998e-05 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287767+0000", 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1523999999999997e-05 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:172371 9.1 9:8d82cc29:::benchmark_data_smithi019_560486_object75626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 302.32519513900002, 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.29074772500000001, 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.386 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 172371 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999144+0000", 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.424e-06 2023-09-30T09:52:45.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999150+0000", 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8680000000000004e-06 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999154+0000", 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7950000000000001e-06 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003063+0000", 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0039091020000000002 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003142+0000", 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9460000000000002e-05 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003210+0000", 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7366000000000005e-05 2023-09-30T09:52:45.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.005317+0000", 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021074150000000001 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.007204+0000", 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018863300000000001 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.391 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289803+0000", 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28259912500000001 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289853+0000", 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0512e-05 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289890+0000", 2023-09-30T09:52:45.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6699999999999998e-05 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187021 9.1 9:8424d638:::benchmark_data_smithi019_560486_object90276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 281.15822522100001, 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34549460300000001, 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.393 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187021 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166111+0000", 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166115+0000", 2023-09-30T09:52:45.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2400000000000001e-06 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166124+0000", 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4719999999999995e-06 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166128+0000", 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3390000000000003e-06 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166156+0000", 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7353999999999999e-05 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166240+0000", 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4368000000000003e-05 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166360+0000", 2023-09-30T09:52:45.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119819 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.174444+0000", 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0080842840000000006 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.174458+0000", 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.397e-05 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511510+0000", 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33705242400000002 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511561+0000", 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0382e-05 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511607+0000", 2023-09-30T09:52:45.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5847000000000003e-05 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187029 9.1 9:8b8b6a3e:::benchmark_data_smithi019_560486_object90284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 281.15715024899998, 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34455740800000001, 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.400 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.401 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.401 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.401 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187029 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:52:45.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167186+0000", 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167191+0000", 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2349999999999999e-06 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167203+0000", 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1552999999999999e-05 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167209+0000", 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0039999999999996e-06 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170626+0000", 2023-09-30T09:52:45.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034176219999999999 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170739+0000", 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112713 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170847+0000", 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000107892 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.175569+0000", 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047219749999999998 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511470+0000", 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33590098299999999 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511656+0000", 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018579499999999999 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511698+0000", 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2017000000000001e-05 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511744+0000", 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6542999999999998e-05 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.407 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187032 9.0 9:01af9635:::benchmark_data_smithi019_560486_object90287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 281.15184533899998, 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33105306600000001, 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.408 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187032 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:52:45.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172491+0000", 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172496+0000", 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4219999999999999e-06 2023-09-30T09:52:45.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172506+0000", 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8460000000000003e-06 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172511+0000", 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6589999999999998e-06 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172557+0000", 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6353999999999997e-05 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172666+0000", 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109045 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172769+0000", 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010234900000000001 2023-09-30T09:52:45.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176428+0000", 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036596440000000001 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176537+0000", 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108589 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503453+0000", 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32691612199999998 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503501+0000", 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8403999999999999e-05 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503545+0000", 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3650999999999999e-05 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187034 9.1 9:85c6d82b:::benchmark_data_smithi019_560486_object90289:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 281.147670205, 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34325798600000001, 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.415 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187034 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:52:45.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176666+0000", 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176672+0000", 2023-09-30T09:52:45.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0789999999999997e-06 2023-09-30T09:52:45.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176683+0000", 2023-09-30T09:52:45.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1401e-05 2023-09-30T09:52:45.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176689+0000", 2023-09-30T09:52:45.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8710000000000002e-06 2023-09-30T09:52:45.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503342+0000", 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32665281600000001 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503509+0000", 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016727300000000001 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503648+0000", 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013894800000000001 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511477+0000", 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0078290040000000005 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511790+0000", 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031249499999999998 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519837+0000", 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.008047874 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519883+0000", 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5791999999999998e-05 2023-09-30T09:52:45.422 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519925+0000", 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1763e-05 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.423 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187035 9.1 9:8a5829f5:::benchmark_data_smithi019_560486_object90290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 281.14763479300001, 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34335321800000002, 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187035 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.424 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176705+0000", 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5059999999999998e-06 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176710+0000", 2023-09-30T09:52:45.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7910000000000001e-06 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176714+0000", 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7469999999999999e-06 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507398+0000", 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33068423200000002 2023-09-30T09:52:45.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507518+0000", 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119772 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507653+0000", 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000134466 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519800+0000", 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012147956999999999 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519964+0000", 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016390900000000001 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:52:45.429 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519994+0000", 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9394999999999999e-05 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.520026+0000", 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2098999999999999e-05 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.520056+0000", 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9793000000000001e-05 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.430 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop_reply(client.46214.0:187032 9.0 e526/483 ondisk, result = 0)", 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 281.14630656600002, 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32552107499999999, 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178030+0000", 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178036+0000", 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.468e-06 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503359+0000", 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32532316900000002 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503369+0000", 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0635999999999999e-05 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503435+0000", 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6155000000000006e-05 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503444+0000", 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5150000000000006e-06 2023-09-30T09:52:45.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503552+0000", 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010775100000000001 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGPush(10.a 574/560 [PushOp(10:50caf5ec:::benchmark_data_smithi019_617837_object16072:head, version: 540'1030, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:50caf5ec:::benchmark_data_smithi019_617837_object16072:head@540'1030, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:55.580297+0000", 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 109.744040445, 2023-09-30T09:52:45.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.173555193, 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580297+0000", 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580297+0000", 2023-09-30T09:52:45.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580297+0000", 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967296 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580324+0000", 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7742999999999998e-05 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580331+0000", 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4250000000000003e-06 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580335+0000", 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5290000000000002e-06 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.582974+0000", 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026384960000000002 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.582981+0000", 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8190000000000003e-06 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.753852+0000", 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.170871355 2023-09-30T09:52:45.439 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGPush(10.b 574/560 [PushOp(10:d9182ae5:::benchmark_data_smithi019_617837_object16071:head, version: 540'1015, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:d9182ae5:::benchmark_data_smithi019_617837_object16071:head@540'1015, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:55.580343+0000", 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 109.743994406, 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.17126106999999999, 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580343+0000", 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580343+0000", 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580343+0000", 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967296 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580371+0000", 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8549999999999999e-05 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580382+0000", 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1122e-05 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580387+0000", 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6779999999999999e-06 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.582336+0000", 2023-09-30T09:52:45.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019489679999999999 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.582341+0000", 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8600000000000001e-06 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751604+0000", 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16926305 2023-09-30T09:52:45.444 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGPush(10.f 574/570 [PushOp(10:f620aa59:::benchmark_data_smithi019_617837_object16955:head, version: 540'1067, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:f620aa59:::benchmark_data_smithi019_617837_object16955:head@540'1067, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:55.584876+0000", 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 109.739461296, 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16918993399999999, 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.584876+0000", 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.584876+0000", 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.584875+0000", 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.584927+0000", 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1319000000000001e-05 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751568+0000", 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16664183299999999 2023-09-30T09:52:45.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751608+0000", 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9156000000000003e-05 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751782+0000", 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000173981 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751810+0000", 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8518e-05 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.754066+0000", 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022557950000000001 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:141187 10.f e607/583 10:f5fa8c0b:::benchmark_data_smithi019_617837_object47787:head v 607'8750, mlcod=607'8748)", 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:28.809320+0000", 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 16.515016812999999, 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.15343026900000001, 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809320+0000", 2023-09-30T09:52:45.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809320+0000", 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809319+0000", 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.451 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809329+0000", 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4239999999999999e-06 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809343+0000", 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3906999999999999e-05 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809350+0000", 2023-09-30T09:52:45.452 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9e-06 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809432+0000", 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2191e-05 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809451+0000", 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9449e-05 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.453 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.962711+0000", 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.15325945099999999 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.962751+0000", 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9928999999999997e-05 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.454 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:141190 10.7 e607/596 10:e09c7f56:::benchmark_data_smithi019_617837_object47790:head v 607'8911, mlcod=586'7848)", 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:28.813334+0000", 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 16.511003059, 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.15209569100000001, 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813334+0000", 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.455 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813334+0000", 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813333+0000", 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.456 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813342+0000", 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9979999999999992e-06 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813353+0000", 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1445e-05 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813360+0000", 2023-09-30T09:52:45.457 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2650000000000002e-06 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.818744+0000", 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005384598 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.818767+0000", 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2548000000000002e-05 2023-09-30T09:52:45.458 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.965390+0000", 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.14662345600000001 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.965430+0000", 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9539000000000002e-05 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.459 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGPush(10.b 607/560 [PushOp(10:d971706d:::benchmark_data_smithi019_617837_object47796:head, version: 547'2907, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:d971706d:::benchmark_data_smithi019_617837_object47796:head@547'2907, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:28.816110+0000", 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 16.508227281, 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.145297395, 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:52:45.460 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.816110+0000", 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.816110+0000", 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.816109+0000", 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:52:45.461 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.816164+0000", 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5049000000000001e-05 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.816186+0000", 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2350000000000001e-05 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.462 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.816198+0000", 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1569000000000001e-05 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.818542+0000", 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002343688 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:52:45.463 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.818551+0000", 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2790000000000004e-06 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.961407+0000", 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.14285649 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.464 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:52:45.465 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:52:45.465 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:52:45.465 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:45.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1574105475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:52:45.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[216815]: failed to encode map e610 with expected crc 2023-09-30T09:52:45.534 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[216815]: osdmap e610: 8 total, 8 up, 8 in 2023-09-30T09:52:45.534 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[224043]: pgmap v1622: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 7144/117798 objects degraded (6.065%); 6269/117798 objects misplaced (5.322%); 12 MiB/s, 196 objects/s recovering 2023-09-30T09:52:45.535 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:52:45.535 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1574105475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:52:45.535 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[224043]: failed to encode map e610 with expected crc 2023-09-30T09:52:45.535 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:45 smithi019 ceph-mon[224043]: osdmap e610: 8 total, 8 up, 8 in 2023-09-30T09:52:45.543 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:45.650 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:45.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:45.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:45.685 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:45.759 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:45.840 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:45.967 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:45.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:45.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:46.082 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:46.153 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:46.166 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:52:46.214 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:52:46.256 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:52:46.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:46 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3008623382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:46.291 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:46.294 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:52:46.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:46.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:46.343 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:52:46.393 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:52:46.401 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:46.434 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:52:46.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:46 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3008623382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:46 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3008623382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:46.485 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:52:46.488 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:52:46.488 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:52:46.488 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:52:46.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:52:46.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:52:46.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:52:46.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:52:46.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:52:46.492 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:47.473 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:47.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:47 smithi019 ceph-mon[216815]: pgmap v1624: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6951/110601 objects degraded (6.285%); 4927/110601 objects misplaced (4.455%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:52:47.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:47 smithi019 ceph-mon[224043]: pgmap v1624: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6951/110601 objects degraded (6.285%); 4927/110601 objects misplaced (4.455%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:52:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:47.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:47.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:47 smithi167 ceph-mon[192880]: pgmap v1624: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6951/110601 objects degraded (6.285%); 4927/110601 objects misplaced (4.455%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:52:47.581 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:47.619 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:52:47.660 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:52:47.671 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:47.833 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:47.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:47.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:47.950 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:48.006 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:48.191 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:48 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:52:48.191 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:48.235 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:48.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:48.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:48.283 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:48 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:52:48.283 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:48.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:48 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:52:48.283 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:52:48.346 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:48.400 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:48.454 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:48.573 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:48.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:48.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:48.687 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:48.705 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:48.772 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:48.867 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:48.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:48.972 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:49.024 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:49.117 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:49.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:49.222 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:49.256 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:49.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:52:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[216815]: pgmap v1625: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 6769/108288 objects degraded (6.251%); 4927/108288 objects misplaced (4.550%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:52:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6769/108288 objects degraded (6.251%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[216815]: osdmap e611: 8 total, 8 up, 8 in 2023-09-30T09:52:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[216815]: failed to encode map e611 with expected crc 2023-09-30T09:52:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1171799469' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:52:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:52:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[224043]: pgmap v1625: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 6769/108288 objects degraded (6.251%); 4927/108288 objects misplaced (4.550%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:52:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6769/108288 objects degraded (6.251%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[224043]: osdmap e611: 8 total, 8 up, 8 in 2023-09-30T09:52:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[224043]: failed to encode map e611 with expected crc 2023-09-30T09:52:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:49 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1171799469' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:52:49.416 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:49.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:49 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:52:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:49 smithi167 ceph-mon[192880]: pgmap v1625: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 6769/108288 objects degraded (6.251%); 4927/108288 objects misplaced (4.550%); 16 MiB/s, 262 objects/s recovering 2023-09-30T09:52:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:49 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6769/108288 objects degraded (6.251%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:49 smithi167 ceph-mon[192880]: osdmap e611: 8 total, 8 up, 8 in 2023-09-30T09:52:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:49 smithi167 ceph-mon[192880]: failed to encode map e611 with expected crc 2023-09-30T09:52:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:49 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1171799469' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:52:49.523 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:52:49.553 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:49.601 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:52:49.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:49.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:52:49.704 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:49.732 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:49.778 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:49.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:49.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:49.881 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:52:49.911 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 10 pgp_num to 16 2023-09-30T09:52:49.913 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:50.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:52:50.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[216815]: failed to encode map e611 with expected crc 2023-09-30T09:52:50.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1171799469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:52:50.196 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[216815]: failed to encode map e612 with expected crc 2023-09-30T09:52:50.284 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:52:50.344 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:52:50.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:50.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:52:50.452 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:50.509 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[216815]: osdmap e612: 8 total, 8 up, 8 in 2023-09-30T09:52:50.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:52:50.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[224043]: failed to encode map e611 with expected crc 2023-09-30T09:52:50.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1171799469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:52:50.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[224043]: failed to encode map e612 with expected crc 2023-09-30T09:52:50.510 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:50 smithi019 ceph-mon[224043]: osdmap e612: 8 total, 8 up, 8 in 2023-09-30T09:52:50.512 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:50.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:50 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:52:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:50 smithi167 ceph-mon[192880]: failed to encode map e611 with expected crc 2023-09-30T09:52:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1171799469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:52:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:50 smithi167 ceph-mon[192880]: failed to encode map e612 with expected crc 2023-09-30T09:52:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:50 smithi167 ceph-mon[192880]: osdmap e612: 8 total, 8 up, 8 in 2023-09-30T09:52:50.951 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:50.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:50.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:51.056 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:51.130 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:51.267 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:52:51.267 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[216815]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.267 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[216815]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.267 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[216815]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.268 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[216815]: pgmap v1628: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 6528/100896 objects degraded (6.470%); 3828/100896 objects misplaced (3.794%); 24 MiB/s, 377 objects/s recovering 2023-09-30T09:52:51.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:52:51.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[224043]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[224043]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[224043]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:51 smithi019 ceph-mon[224043]: pgmap v1628: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 6528/100896 objects degraded (6.470%); 3828/100896 objects misplaced (3.794%); 24 MiB/s, 377 objects/s recovering 2023-09-30T09:52:51.309 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:51.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:51.421 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:51.460 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:51.467 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:51 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:52:51.467 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:51 smithi167 ceph-mon[192880]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.467 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:51 smithi167 ceph-mon[192880]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.467 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:51 smithi167 ceph-mon[192880]: failed to encode map e612 with expected crc 2023-09-30T09:52:51.467 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:51 smithi167 ceph-mon[192880]: pgmap v1628: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 6528/100896 objects degraded (6.470%); 3828/100896 objects misplaced (3.794%); 24 MiB/s, 377 objects/s recovering 2023-09-30T09:52:51.496 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:51.613 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:51.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:51.729 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:51.811 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:51.884 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:51.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:51.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:51.993 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:52.039 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:52.242 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:52:52.253 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:52.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:52.288 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:52:52.288 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:52:52.360 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:52.421 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:52.487 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2756357543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:52.487 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:52.560 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:52.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:52.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:52.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2756357543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:52.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:52.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2756357543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:52.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:52:52.670 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:52.731 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:53.293 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:53.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:53.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:53.400 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:53.429 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:52:53.429 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: pgmap v1629: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6528/100896 objects degraded (6.470%); 3828/100896 objects misplaced (3.794%); 18 MiB/s, 286 objects/s recovering 2023-09-30T09:52:53.429 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:52:53.429 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:53.429 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: osdmap e613: 8 total, 8 up, 8 in 2023-09-30T09:52:53.429 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.429 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.429 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.430 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.430 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.430 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.430 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[224043]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.430 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:52:53.430 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: pgmap v1629: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6528/100896 objects degraded (6.470%); 3828/100896 objects misplaced (3.794%); 18 MiB/s, 286 objects/s recovering 2023-09-30T09:52:53.430 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:52:53.430 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:53.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: osdmap e613: 8 total, 8 up, 8 in 2023-09-30T09:52:53.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:53 smithi019 ceph-mon[216815]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.441 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: pgmap v1629: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6528/100896 objects degraded (6.470%); 3828/100896 objects misplaced (3.794%); 18 MiB/s, 286 objects/s recovering 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: osdmap e613: 8 total, 8 up, 8 in 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:53 smithi167 ceph-mon[192880]: failed to encode map e613 with expected crc 2023-09-30T09:52:53.777 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:53.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:53.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:53.833 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:52:53.887 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:53.948 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:54.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:52:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:52:53] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-30T09:52:54.138 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:54.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:54.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:54.214 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:54 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6528/100896 objects degraded (6.470%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:54.251 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:54.316 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:54.421 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:54.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:54.428 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:54 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6528/100896 objects degraded (6.470%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:54.428 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:54 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6528/100896 objects degraded (6.470%), 5 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:54.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:54.441 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:52:54.484 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:52:54.516 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:52:54.529 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:54.549 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:52:54.560 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:52:54.584 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:54.589 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:52:54.624 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:52:54.656 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:52:54.680 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:54.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:54.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:54.695 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:52:54.698 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:52:54.698 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:52:54.698 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:52:54.698 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:52:54.698 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:52:54.698 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:52:54.698 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:52:54.698 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:52:54.793 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:54.868 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:55.013 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:55.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:55.119 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:55.162 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:55.261 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:55.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:55.365 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:55.412 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:55 smithi019 ceph-mon[216815]: pgmap v1631: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 6469/98643 objects degraded (6.558%); 3828/98643 objects misplaced (3.881%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:52:55.412 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1438072971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:55.412 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:55 smithi019 ceph-mon[224043]: pgmap v1631: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 6469/98643 objects degraded (6.558%); 3828/98643 objects misplaced (3.881%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:52:55.412 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1438072971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:55.418 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:55.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:55 smithi167 ceph-mon[192880]: pgmap v1631: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 6469/98643 objects degraded (6.558%); 3828/98643 objects misplaced (3.881%); 11 MiB/s, 179 objects/s recovering 2023-09-30T09:52:55.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1438072971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:52:55.661 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:55.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:55.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:55.764 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:55.819 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:55.909 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:52:55.917 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:55.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:55.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:56.032 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:52:56.087 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:56.149 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:52:56.150 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:52:56.150 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:52:56.150 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-09-30T09:52:56.150 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-30T09:52:56.181 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:52:56.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:56.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:52:56.294 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:52:56.359 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:56.426 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:52:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:56.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:52:56.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:56 smithi167 ceph-mon[192880]: 1.1c deep-scrub starts 2023-09-30T09:52:56.532 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:56.570 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:56.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:56 smithi019 ceph-mon[224043]: 1.1c deep-scrub starts 2023-09-30T09:52:56.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:56 smithi019 ceph-mon[216815]: 1.1c deep-scrub starts 2023-09-30T09:52:56.601 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:56.693 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:56.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:56.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:56.814 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:56.882 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:57.103 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:57.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:57.160 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:57.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:57.297 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:57.434 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:57.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:57.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:57 smithi167 ceph-mon[192880]: pgmap v1632: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 6271/91452 objects degraded (6.857%); 3298/91452 objects misplaced (3.606%); 14 MiB/s, 217 objects/s recovering 2023-09-30T09:52:57.545 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:57.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:57 smithi019 ceph-mon[216815]: pgmap v1632: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 6271/91452 objects degraded (6.857%); 3298/91452 objects misplaced (3.606%); 14 MiB/s, 217 objects/s recovering 2023-09-30T09:52:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:57 smithi019 ceph-mon[224043]: pgmap v1632: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 6271/91452 objects degraded (6.857%); 3298/91452 objects misplaced (3.606%); 14 MiB/s, 217 objects/s recovering 2023-09-30T09:52:57.598 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:57.747 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:52:57.750 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:57.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:57.853 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:57.894 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:52:57.894 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:52:57.895 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:52:57.895 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:52:57.895 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:52:57.933 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:58.169 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:58.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:58.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:58.277 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:58.363 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:58.511 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:58.520 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:58 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:52:58.522 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:58 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:52:58.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:58 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:52:58.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:58.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:58.634 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:52:58.703 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:58.787 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:52:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:58.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:52:58.891 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:52:58.962 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:59.013 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:52:59.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:59.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:52:59.117 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:52:59.174 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:59.455 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:52:59.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:59.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:52:59.469 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:59 smithi167 ceph-mon[192880]: pgmap v1633: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 6271/88686 objects degraded (7.071%); 3298/88686 objects misplaced (3.719%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:52:59.469 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:52:59 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6271/88686 objects degraded (7.071%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:59.507 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:52:59.568 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:52:59.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:59 smithi019 ceph-mon[224043]: pgmap v1633: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 6271/88686 objects degraded (7.071%); 3298/88686 objects misplaced (3.719%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:52:59.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:52:59 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6271/88686 objects degraded (7.071%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:59.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:59 smithi019 ceph-mon[216815]: pgmap v1633: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 6271/88686 objects degraded (7.071%); 3298/88686 objects misplaced (3.719%); 4.1 MiB/s, 65 objects/s recovering 2023-09-30T09:52:59.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:52:59 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6271/88686 objects degraded (7.071%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:52:59.668 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:52:59.822 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:52:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:52:59.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:52:59.903 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:52:59.949 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:00.030 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:00.155 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:00.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:00.259 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:00.300 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:53:00.311 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:53:00.333 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:00.428 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2567502351' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:00.428 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2567502351' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:00.440 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:00.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:00.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:00.548 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:00.596 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2567502351' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:00.599 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:00.714 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:00.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:00.821 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:00.885 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:01.096 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:01.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:01.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:01.199 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:01.257 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:01.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: pgmap v1634: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 6050/81018 objects degraded (7.467%); 2737/81018 objects misplaced (3.378%); 6.5 MiB/s, 103 objects/s recovering 2023-09-30T09:53:01.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:01.341 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-30T09:53:01.367 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:01.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:01.483 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: pgmap v1634: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 6050/81018 objects degraded (7.467%); 2737/81018 objects misplaced (3.378%); 6.5 MiB/s, 103 objects/s recovering 2023-09-30T09:53:01.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:01.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2567502351' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:53:01.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: osdmap e614: 8 total, 8 up, 8 in 2023-09-30T09:53:01.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2995489199' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:01.524 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:01 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:01.526 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:01.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: pgmap v1634: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 6050/81018 objects degraded (7.467%); 2737/81018 objects misplaced (3.378%); 6.5 MiB/s, 103 objects/s recovering 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2567502351' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: osdmap e614: 8 total, 8 up, 8 in 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2995489199' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:01.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:01.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2567502351' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:53:01.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: osdmap e614: 8 total, 8 up, 8 in 2023-09-30T09:53:01.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: failed to encode map e614 with expected crc 2023-09-30T09:53:01.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2995489199' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:01.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:01 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:01.634 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:01.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:01.743 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:53:01.782 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:02.712 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:53:02.715 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:53:02.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[224043]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[224043]: osdmap e615: 8 total, 8 up, 8 in 2023-09-30T09:53:02.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[224043]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.716 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[224043]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.717 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:53:02.717 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[216815]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.717 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[216815]: osdmap e615: 8 total, 8 up, 8 in 2023-09-30T09:53:02.717 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[216815]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.717 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:02 smithi019 ceph-mon[216815]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:02.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:53:02.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:02 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:53:02.744 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:02 smithi167 ceph-mon[192880]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.745 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:02 smithi167 ceph-mon[192880]: osdmap e615: 8 total, 8 up, 8 in 2023-09-30T09:53:02.745 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:02 smithi167 ceph-mon[192880]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.745 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:02 smithi167 ceph-mon[192880]: failed to encode map e615 with expected crc 2023-09-30T09:53:02.819 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:02.935 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:03.083 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:03.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:03.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:03.192 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:03.250 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:03.351 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:03 smithi167 ceph-mon[192880]: pgmap v1637: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 6050/81018 objects degraded (7.467%); 2737/81018 objects misplaced (3.378%); 7.6 MiB/s, 121 objects/s recovering 2023-09-30T09:53:03.351 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:03.370 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:03.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:03.479 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:03.528 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:03.541 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2297 9.d 9:b0cd7046:::benchmark_data_smithi019_560486_object2296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 518.50766078900006, 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0633178289999998, 2023-09-30T09:53:03.542 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2297 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.543 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.545 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.545 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.545 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:53:03.545 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.545 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.545 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.545 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.545 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031481+0000", 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031516+0000", 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.464e-05 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031523+0000", 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8779999999999998e-06 2023-09-30T09:53:03.546 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.031528+0000", 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6970000000000001e-06 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032166+0000", 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00063765200000000005 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.547 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032280+0000", 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000113811 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.032419+0000", 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013967100000000001 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.033675+0000", 2023-09-30T09:53:03.548 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001255363 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053674+0000", 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.019999847000000001 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:25.053684+0000", 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9990000000000003e-06 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.549 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.059238+0000", 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.00555335 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.059246+0000", 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4940000000000003e-06 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.086761+0000", 2023-09-30T09:53:03.550 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.027514383 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.088028+0000", 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012668429999999999 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094723+0000", 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0066955230000000001 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.551 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.094799+0000", 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6225999999999998e-05 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2293 9.a 9:552d93c3:::benchmark_data_smithi019_560486_object2292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 517.49407445099996, 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.051156384, 2023-09-30T09:53:03.552 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2293 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.553 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045068+0000", 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045066+0000", 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.554 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045217+0000", 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150221 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045234+0000", 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7676000000000002e-05 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045243+0000", 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0429999999999996e-06 2023-09-30T09:53:03.555 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045361+0000", 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011739900000000001 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045370+0000", 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0049999999999993e-06 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.556 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058484+0000", 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013114657 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058494+0000", 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3559999999999999e-06 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.085277+0000", 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.026783555 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.557 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.087303+0000", 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020253559999999999 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096095+0000", 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0087920210000000006 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.558 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096224+0000", 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129453 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:2299 9.a 9:5ff10925:::benchmark_data_smithi019_560486_object2298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 517.49388530500005, 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0514241090000001, 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.559 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "reached pg", 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 2299 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.560 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045257+0000", 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045256+0000", 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045294+0000", 2023-09-30T09:53:03.561 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7457999999999999e-05 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045304+0000", 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0271999999999999e-05 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045310+0000", 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7760000000000003e-06 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.562 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045439+0000", 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128909 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting_for_map not empty", 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.045445+0000", 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0229999999999998e-06 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058514+0000", 2023-09-30T09:53:03.563 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013069717999999999 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:26.058520+0000", 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2920000000000003e-06 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.087342+0000", 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.028822465 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.564 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.088285+0000", 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00094316300000000001 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096601+0000", 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0083155179999999992 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:27.096681+0000", 2023-09-30T09:53:03.565 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0162999999999998e-05 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5657 9.d 9:b0e10006:::benchmark_data_smithi019_560486_object5656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 512.86245063399997, 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.006709437, 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.566 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5657 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.567 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676692+0000", 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676689+0000", 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999976 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676814+0000", 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012537599999999999 2023-09-30T09:53:03.568 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676838+0000", 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3623000000000001e-05 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676847+0000", 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2590000000000006e-06 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.569 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676916+0000", 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8670000000000005e-05 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.676927+0000", 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0849000000000001e-05 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680187+0000", 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0032605560000001 2023-09-30T09:53:03.570 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680317+0000", 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129453 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680486+0000", 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016957800000000001 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.571 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682922+0000", 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0024357670000000001 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683200+0000", 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027740300000000001 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683261+0000", 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1591e-05 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.572 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683346+0000", 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4541000000000003e-05 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683401+0000", 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5316999999999999e-05 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.573 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5664 9.d 9:b51be770:::benchmark_data_smithi019_560486_object5663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 512.861294477, 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.005731259, 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:53:03.575 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5664 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677848+0000", 2023-09-30T09:53:03.576 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677847+0000", 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677904+0000", 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7519999999999998e-05 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.577 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677923+0000", 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8369e-05 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677930+0000", 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4680000000000003e-06 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677977+0000", 2023-09-30T09:53:03.578 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7170999999999999e-05 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.677988+0000", 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0855e-05 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.680921+0000", 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0029331560000001 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.579 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681034+0000", 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011288199999999999 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681184+0000", 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000150094 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.580 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682930+0000", 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017458339999999999 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683222+0000", 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00029221800000000001 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683451+0000", 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00022835100000000001 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.581 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683528+0000", 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6852000000000002e-05 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.683579+0000", 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1418999999999997e-05 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.582 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5673 9.d 9:b96c3c13:::benchmark_data_smithi019_560486_object5672:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 512.85935659899997, 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0071881890000001, 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5673 2023-09-30T09:53:03.583 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679786+0000", 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.584 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679785+0000", 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679870+0000", 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4948000000000001e-05 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.585 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679884+0000", 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4348000000000001e-05 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679891+0000", 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9589999999999998e-06 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679948+0000", 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7120999999999997e-05 2023-09-30T09:53:03.586 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.679955+0000", 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7870000000000004e-06 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681621+0000", 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.001666497 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.587 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681730+0000", 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010867 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.681870+0000", 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013995699999999999 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682935+0000", 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010647829999999999 2023-09-30T09:53:03.588 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.685992+0000", 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030569799999999999 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686739+0000", 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00074698799999999997 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.589 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686937+0000", 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000197858 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686974+0000", 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7280999999999998e-05 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.590 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:5686 9.d 9:b2765571:::benchmark_data_smithi019_560486_object5685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 512.85383812099997, 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.002027996, 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 5686 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.591 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.592 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685304+0000", 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685339+0000", 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5487000000000001e-05 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685347+0000", 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.072e-06 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.593 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685353+0000", 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8860000000000001e-06 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685417+0000", 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4487999999999996e-05 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.594 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:30.685433+0000", 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5132e-05 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682219+0000", 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99678628499999999 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682328+0000", 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109515 2023-09-30T09:53:03.595 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.682483+0000", 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015417100000000001 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.686027+0000", 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0035446449999999999 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.596 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687236+0000", 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012086670000000001 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687247+0000", 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.136e-05 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687295+0000", 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8118999999999998e-05 2023-09-30T09:53:03.597 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.598 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.598 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.598 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:44:31.687332+0000", 2023-09-30T09:53:03.598 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6742999999999997e-05 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129752 10.1 10:8e848273:::benchmark_data_smithi019_617837_object36352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.454555+0000", 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 57.084586825999999, 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90929194899999999, 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.599 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129752 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454555+0000", 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.600 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454555+0000", 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454555+0000", 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454559+0000", 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4240000000000003e-06 2023-09-30T09:53:03.601 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454572+0000", 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2964e-05 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454582+0000", 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0463e-05 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.602 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.455021+0000", 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000438212 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.455034+0000", 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3044000000000001e-05 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349334+0000", 2023-09-30T09:53:03.603 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.89429994800000001 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.351250+0000", 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001916474 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,4", 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.351397+0000", 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014671000000000001 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.604 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.358565+0000", 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0071678640000000004 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363661+0000", 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050959519999999999 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363688+0000", 2023-09-30T09:53:03.605 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7625000000000001e-05 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363746+0000", 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7918999999999999e-05 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363781+0000", 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4502999999999998e-05 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.606 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363847+0000", 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6691000000000002e-05 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129776 10.1 10:81a28789:::benchmark_data_smithi019_617837_object36376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.530436+0000", 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 57.008705847999998, 2023-09-30T09:53:03.607 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83958421100000002, 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129776 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530436+0000", 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.608 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530436+0000", 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530435+0000", 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.609 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530443+0000", 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0320000000000003e-06 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530462+0000", 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8967e-05 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530470+0000", 2023-09-30T09:53:03.610 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7770000000000002e-06 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530534+0000", 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4591000000000006e-05 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530547+0000", 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2830000000000001e-05 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.611 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.353679+0000", 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82313198899999995 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.353769+0000", 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0099999999999995e-05 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,4", 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.353857+0000", 2023-09-30T09:53:03.612 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7779000000000001e-05 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.361026+0000", 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0071691879999999999 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363721+0000", 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026950239999999999 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.613 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363881+0000", 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000160094 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.369883+0000", 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0060018609999999998 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.614 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.369918+0000", 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4863999999999997e-05 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.370021+0000", 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000102556 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.615 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129780 10.1 10:800ff011:::benchmark_data_smithi019_617837_object36380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.533905+0000", 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 57.005236885000002, 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83700672499999995, 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129780 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.616 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533905+0000", 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533905+0000", 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533904+0000", 2023-09-30T09:53:03.617 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533916+0000", 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2052000000000001e-05 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533934+0000", 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7849000000000001e-05 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.618 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533941+0000", 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0700000000000001e-06 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.534007+0000", 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5928999999999998e-05 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.534018+0000", 2023-09-30T09:53:03.619 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0937e-05 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.356020+0000", 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82200270900000005 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.356101+0000", 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0483999999999997e-05 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.620 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,4", 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.356191+0000", 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0212999999999998e-05 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363617+0000", 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0074258939999999997 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363911+0000", 2023-09-30T09:53:03.621 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00029366400000000001 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.369585+0000", 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005674241 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.370811+0000", 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001225627 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.622 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.370849+0000", 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8702999999999999e-05 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.370912+0000", 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2793999999999995e-05 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.623 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140289 10.0 10:01361017:::benchmark_data_smithi019_617837_object46889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:26.932245+0000", 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.606897060999998, 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.181625616, 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140289 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.624 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932245+0000", 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932245+0000", 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.625 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932244+0000", 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932251+0000", 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2239999999999998e-06 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932264+0000", 2023-09-30T09:53:03.626 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2996999999999999e-05 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932276+0000", 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1749999999999999e-05 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.934603+0000", 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023272969999999999 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.627 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.934611+0000", 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2600000000000005e-06 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.073563+0000", 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1389515109999999 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.073729+0000", 2023-09-30T09:53:03.628 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016571399999999999 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.073879+0000", 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015019499999999999 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.082090+0000", 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0082116629999999993 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.629 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113709+0000", 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.031618269999999997 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113772+0000", 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3044000000000002e-05 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113829+0000", 2023-09-30T09:53:03.630 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6829000000000001e-05 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113871+0000", 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2313e-05 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140309 10.0 10:0c5e3186:::benchmark_data_smithi019_617837_object46909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.631 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:26.964625+0000", 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.574517047999997, 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1494339469999999, 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140309 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.632 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964625+0000", 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964625+0000", 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964624+0000", 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.633 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964630+0000", 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3720000000000001e-06 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964642+0000", 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1508999999999999e-05 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.634 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964648+0000", 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9290000000000003e-06 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964895+0000", 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00024736300000000001 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964904+0000", 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7190000000000007e-06 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.635 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.076507+0000", 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1116031259999999 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.076607+0000", 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9716999999999994e-05 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.636 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.076693+0000", 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5878999999999995e-05 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.087652+0000", 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010959652 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113941+0000", 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.026288270999999998 2023-09-30T09:53:03.637 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113976+0000", 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5824999999999997e-05 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114021+0000", 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4641999999999997e-05 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.638 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114059+0000", 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8155999999999999e-05 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140310 10.0 10:0b99e2fd:::benchmark_data_smithi019_617837_object46910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:26.970537+0000", 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.568605054999999, 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.143803983, 2023-09-30T09:53:03.639 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140310 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970537+0000", 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.640 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970537+0000", 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970536+0000", 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.641 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970543+0000", 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5050000000000001e-06 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970562+0000", 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8515000000000001e-05 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970569+0000", 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4039999999999998e-06 2023-09-30T09:53:03.642 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970659+0000", 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0309000000000006e-05 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970667+0000", 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9610000000000005e-06 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.643 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.078368+0000", 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1077000889999999 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.078473+0000", 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105572 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.078544+0000", 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0735000000000006e-05 2023-09-30T09:53:03.644 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.087661+0000", 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091170669999999995 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114213+0000", 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.026552503000000002 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.645 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114250+0000", 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6606999999999997e-05 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114299+0000", 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9483e-05 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114341+0000", 2023-09-30T09:53:03.646 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1709000000000003e-05 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140330 10.0 10:03ecfbaf:::benchmark_data_smithi019_617837_object46930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:26.992195+0000", 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.546946998999999, 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.122355701, 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.647 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140330 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992195+0000", 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.648 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992195+0000", 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992194+0000", 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992201+0000", 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2529999999999998e-06 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.649 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992216+0000", 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5355000000000001e-05 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992223+0000", 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8759999999999999e-06 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.650 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992277+0000", 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4326999999999997e-05 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992286+0000", 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9099999999999994e-06 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.080080+0000", 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.087793515 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.651 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.080175+0000", 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4795000000000003e-05 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.080240+0000", 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5370999999999995e-05 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.652 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093529+0000", 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013288659 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114402+0000", 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.020873769 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114459+0000", 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7014e-05 2023-09-30T09:53:03.653 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114512+0000", 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2544999999999999e-05 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114551+0000", 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8980000000000003e-05 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.654 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140355 10.0 10:03e17385:::benchmark_data_smithi019_617837_object46955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.121630+0000", 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.417512711000001, 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.993256892, 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140355 2023-09-30T09:53:03.655 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121630+0000", 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121630+0000", 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.656 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121629+0000", 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121632+0000", 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8480000000000001e-06 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121694+0000", 2023-09-30T09:53:03.657 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1736000000000007e-05 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121700+0000", 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3439999999999997e-06 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121748+0000", 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7710000000000002e-05 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121755+0000", 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9670000000000004e-06 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.082150+0000", 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.960395675 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.082315+0000", 2023-09-30T09:53:03.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016461500000000001 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.082488+0000", 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000172957 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093537+0000", 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011049392 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114730+0000", 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.021192175000000001 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114794+0000", 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4325000000000005e-05 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114850+0000", 2023-09-30T09:53:03.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6282000000000001e-05 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114886+0000", 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6229999999999997e-05 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140381 10.0 10:07b9ea47:::benchmark_data_smithi019_617837_object46981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.662 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.155244+0000", 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.383898686000002, 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95981043799999999, 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140381 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.663 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155244+0000", 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155244+0000", 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155242+0000", 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.664 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155248+0000", 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2890000000000003e-06 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155263+0000", 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4634000000000001e-05 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155270+0000", 2023-09-30T09:53:03.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7700000000000004e-06 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155315+0000", 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.494e-05 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155322+0000", 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1439999999999997e-06 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.666 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.084861+0000", 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92953928600000002 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.085003+0000", 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014227 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.085141+0000", 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013766399999999999 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.099313+0000", 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014172298999999999 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114944+0000", 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015630511 2023-09-30T09:53:03.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114978+0000", 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3821e-05 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115019+0000", 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1462999999999999e-05 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115054+0000", 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4782e-05 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140433 10.0 10:0f44e937:::benchmark_data_smithi019_617837_object47033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.277027+0000", 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.262115319999999, 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83815227199999998, 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.670 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140433 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277027+0000", 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277027+0000", 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277025+0000", 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277033+0000", 2023-09-30T09:53:03.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.678e-06 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277047+0000", 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3487e-05 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277053+0000", 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8739999999999997e-06 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.673 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277098+0000", 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4750999999999999e-05 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277108+0000", 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4970000000000007e-06 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.087718+0000", 2023-09-30T09:53:03.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81061056200000003 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.087886+0000", 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016749500000000001 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.088030+0000", 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000144036 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.099319+0000", 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011288815000000001 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115090+0000", 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015771866999999998 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115117+0000", 2023-09-30T09:53:03.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6359e-05 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115149+0000", 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2673999999999999e-05 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115179+0000", 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9756000000000001e-05 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.677 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140434 10.0 10:0622d357:::benchmark_data_smithi019_617837_object47034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.279733+0000", 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.259409071999997, 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83556097699999998, 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.678 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140434 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279733+0000", 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279733+0000", 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279731+0000", 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279741+0000", 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0037999999999999e-05 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279766+0000", 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5111000000000001e-05 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279776+0000", 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3179999999999996e-06 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.280109+0000", 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00033358599999999997 2023-09-30T09:53:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.280119+0000", 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9350000000000006e-06 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.090718+0000", 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81059848499999998 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.090857+0000", 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000139507 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.090983+0000", 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000125809 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.101562+0000", 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010579313999999999 2023-09-30T09:53:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115208+0000", 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01364548 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115234+0000", 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5853000000000001e-05 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115265+0000", 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1569000000000002e-05 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115294+0000", 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8830999999999999e-05 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140442 10.0 10:0c0950c7:::benchmark_data_smithi019_617837_object47042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.294201+0000", 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.244940935000002, 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82121004799999997, 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140442 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294201+0000", 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294201+0000", 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294200+0000", 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294209+0000", 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5149999999999995e-06 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294234+0000", 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4958999999999999e-05 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294242+0000", 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.028e-06 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294296+0000", 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3514999999999998e-05 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294305+0000", 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1300000000000007e-06 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093585+0000", 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.79928037600000001 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093731+0000", 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000145909 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093883+0000", 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015131899999999999 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.101573+0000", 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076901340000000004 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115322+0000", 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013749014 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115347+0000", 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5230999999999999e-05 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115384+0000", 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6678999999999999e-05 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115411+0000", 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7801999999999999e-05 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140444 10.0 10:02b09ef4:::benchmark_data_smithi019_617837_object47044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.297861+0000", 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 36.241281372000003, 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81766610399999995, 2023-09-30T09:53:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140444 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297861+0000", 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297861+0000", 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297860+0000", 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297865+0000", 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7089999999999999e-06 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297878+0000", 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3018e-05 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297884+0000", 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2100000000000001e-06 2023-09-30T09:53:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297941+0000", 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7741999999999997e-05 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297948+0000", 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9149999999999999e-06 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.096567+0000", 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.79861880299999999 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.096706+0000", 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013912899999999999 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.096839+0000", 2023-09-30T09:53:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000133022 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.106542+0000", 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097024299999999997 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115440+0000", 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088988499999999998 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115466+0000", 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5695999999999999e-05 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115498+0000", 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1962999999999999e-05 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115527+0000", 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8861999999999999e-05 2023-09-30T09:53:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.701 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.701 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.701 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:03.701 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:03.701 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:53:03.702 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:03 smithi019 ceph-mon[216815]: pgmap v1637: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 6050/81018 objects degraded (7.467%); 2737/81018 objects misplaced (3.378%); 7.6 MiB/s, 121 objects/s recovering 2023-09-30T09:53:03.702 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:03.702 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:03 smithi019 ceph-mon[224043]: pgmap v1637: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 6050/81018 objects degraded (7.467%); 2737/81018 objects misplaced (3.378%); 7.6 MiB/s, 121 objects/s recovering 2023-09-30T09:53:03.702 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:03.703 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:03.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:03.704 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:53:03.704 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:53:03.733 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:53:03.765 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:53:03.804 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:03.830 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:53:03.853 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:03.874 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:53:03.916 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:53:03.961 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:53:03.971 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:03.976 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:53:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:53:03] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-09-30T09:53:03.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:03.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:03.981 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:53:03.981 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:53:03.981 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:53:03.981 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:53:03.982 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:53:03.982 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:53:03.982 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:53:03.982 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:53:04.079 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:04.155 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:04.263 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:04.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:04.369 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:04.394 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3017195416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:04.394 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:04 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6050/81018 objects degraded (7.467%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:04.397 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:04.441 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:04.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:04.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:04.544 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:53:04.574 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3017195416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:04.574 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:04 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6050/81018 objects degraded (7.467%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:04.574 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3017195416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:04.574 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:04 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6050/81018 objects degraded (7.467%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:04.575 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:04.618 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:53:04.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:04.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:53:04.722 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:04.787 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:05.068 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:53:05.095 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:05.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:05.201 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:53:05.222 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:05.341 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:05.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:05 smithi019 ceph-mon[216815]: pgmap v1638: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 6050/78660 objects degraded (7.691%); 2737/78660 objects misplaced (3.480%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:53:05.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:05 smithi019 ceph-mon[224043]: pgmap v1638: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 6050/78660 objects degraded (7.691%); 2737/78660 objects misplaced (3.480%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:53:05.496 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:05.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:05.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:05 smithi167 ceph-mon[192880]: pgmap v1638: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 6050/78660 objects degraded (7.691%); 2737/78660 objects misplaced (3.480%); 3.9 MiB/s, 62 objects/s recovering 2023-09-30T09:53:05.606 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:05.677 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:05.814 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:05.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:05.848 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:05.924 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:05.994 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:05.995 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:06.098 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:06.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:06.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:06.296 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:06.733 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:06.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:06.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:06.841 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:06.932 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:07.055 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:07.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:07.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:07.122 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:53:07.168 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:07.240 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:07.332 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:07.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:07.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:07.440 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:07.508 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:07 smithi167 ceph-mon[192880]: pgmap v1639: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 5834/70668 objects degraded (8.256%); 2148/70668 objects misplaced (3.040%); 7.7 MiB/s, 123 objects/s recovering 2023-09-30T09:53:07.513 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:07.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:07 smithi019 ceph-mon[224043]: pgmap v1639: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 5834/70668 objects degraded (8.256%); 2148/70668 objects misplaced (3.040%); 7.7 MiB/s, 123 objects/s recovering 2023-09-30T09:53:07.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:07 smithi019 ceph-mon[216815]: pgmap v1639: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 5834/70668 objects degraded (8.256%); 2148/70668 objects misplaced (3.040%); 7.7 MiB/s, 123 objects/s recovering 2023-09-30T09:53:07.665 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:07.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:07.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:07.783 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:53:07.848 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:08.091 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:53:08.091 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:53:08.092 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-09-30T09:53:08.092 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:53:08.458 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:53:08.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:08.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:53:08.565 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:08.611 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:08.674 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:08.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:08.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:08.780 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:53:08.879 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:09.077 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:09.227 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:53:09.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:09.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:53:09.354 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:09.400 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:09.446 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:09.478 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:09.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:09.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:09.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:09 smithi019 ceph-mon[224043]: pgmap v1640: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 5834/68004 objects degraded (8.579%); 2148/68004 objects misplaced (3.159%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:53:09.565 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5834/68004 objects degraded (8.579%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:09.565 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:09 smithi019 ceph-mon[216815]: pgmap v1640: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 5834/68004 objects degraded (8.579%); 2148/68004 objects misplaced (3.159%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:53:09.566 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5834/68004 objects degraded (8.579%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:09.588 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:09.607 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:53:09.607 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:53:09.690 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:09.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:09 smithi167 ceph-mon[192880]: pgmap v1640: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 5834/68004 objects degraded (8.579%); 2148/68004 objects misplaced (3.159%); 3.8 MiB/s, 61 objects/s recovering 2023-09-30T09:53:09.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5834/68004 objects degraded (8.579%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:10.404 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:53:10.774 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3979801582' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:10.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3979801582' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:10.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3979801582' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:11.233 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:53:11.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[224043]: pgmap v1641: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5591/59808 objects degraded (9.348%); 1703/59808 objects misplaced (2.847%); 5.6 MiB/s, 103 objects/s recovering 2023-09-30T09:53:11.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:11.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3979801582' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:11.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[224043]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[224043]: osdmap e616: 8 total, 8 up, 8 in 2023-09-30T09:53:11.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[224043]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[224043]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[216815]: pgmap v1641: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5591/59808 objects degraded (9.348%); 1703/59808 objects misplaced (2.847%); 5.6 MiB/s, 103 objects/s recovering 2023-09-30T09:53:11.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:11.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3979801582' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:11.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[216815]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[216815]: osdmap e616: 8 total, 8 up, 8 in 2023-09-30T09:53:11.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[216815]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.592 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:11 smithi019 ceph-mon[216815]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:11 smithi167 ceph-mon[192880]: pgmap v1641: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5591/59808 objects degraded (9.348%); 1703/59808 objects misplaced (2.847%); 5.6 MiB/s, 103 objects/s recovering 2023-09-30T09:53:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:11 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3979801582' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:11 smithi167 ceph-mon[192880]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:11 smithi167 ceph-mon[192880]: osdmap e616: 8 total, 8 up, 8 in 2023-09-30T09:53:11.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:11 smithi167 ceph-mon[192880]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:11 smithi167 ceph-mon[192880]: failed to encode map e616 with expected crc 2023-09-30T09:53:11.975 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:11.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:11.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:12.080 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:53:12.150 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:12.751 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:53:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:12.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:53:12.855 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:12.909 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:13.017 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:13.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:13.127 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:13.186 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:13.277 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:13.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:13.396 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:13.463 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:13 smithi019 ceph-mon[224043]: pgmap v1643: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 5591/59808 objects degraded (9.348%); 1703/59808 objects misplaced (2.847%); 5.4 MiB/s, 100 objects/s recovering 2023-09-30T09:53:13.466 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:13 smithi019 ceph-mon[216815]: pgmap v1643: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 5591/59808 objects degraded (9.348%); 1703/59808 objects misplaced (2.847%); 5.4 MiB/s, 100 objects/s recovering 2023-09-30T09:53:13.479 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:13.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:13 smithi167 ceph-mon[192880]: pgmap v1643: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 5591/59808 objects degraded (9.348%); 1703/59808 objects misplaced (2.847%); 5.4 MiB/s, 100 objects/s recovering 2023-09-30T09:53:13.702 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:13.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:13.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:13.825 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:13.918 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:13.990 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:14.076 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:53:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:53:13] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-09-30T09:53:14.138 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:14.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:14.255 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:14.304 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:14.374 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:53:14.381 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:14.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:14.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:14.418 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:53:14.487 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:14.489 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:53:14.527 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:14.529 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:53:14.548 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:53:14.594 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:53:14.607 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:14.612 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:14 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5591/59808 objects degraded (9.348%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:14.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:14.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:14.634 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:14 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5591/59808 objects degraded (9.348%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:14.635 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:14 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5591/59808 objects degraded (9.348%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:14.635 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:53:14.677 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:53:14.709 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:53:14.714 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:14.746 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:53:14.768 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:14.790 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:53:14.809 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-30T09:53:14.809 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:53:14.810 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:53:14.810 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:53:14.810 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:53:14.810 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:53:14.810 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:53:14.810 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:53:14.854 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:14.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:14.970 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:14.998 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:15 smithi167 ceph-mon[192880]: pgmap v1644: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 5591/57117 objects degraded (9.789%); 1703/57117 objects misplaced (2.982%); 5.4 MiB/s, 100 objects/s recovering 2023-09-30T09:53:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:15 smithi167 ceph-mon[192880]: from='client.36909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:53:15.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2623025261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:15.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:15 smithi019 ceph-mon[216815]: pgmap v1644: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 5591/57117 objects degraded (9.789%); 1703/57117 objects misplaced (2.982%); 5.4 MiB/s, 100 objects/s recovering 2023-09-30T09:53:15.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:15 smithi019 ceph-mon[216815]: from='client.36909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:53:15.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2623025261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:15.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:15 smithi019 ceph-mon[224043]: pgmap v1644: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 5591/57117 objects degraded (9.789%); 1703/57117 objects misplaced (2.982%); 5.4 MiB/s, 100 objects/s recovering 2023-09-30T09:53:15.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:15 smithi019 ceph-mon[224043]: from='client.36909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:53:15.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2623025261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:16.315 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1644,"stamp":"2023-09-30T09:53:14.048671+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1250638512,"num_objects":19039,"num_object_clones":0,"num_object_copies":57117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5591,"num_objects_misplaced":1703,"num_objects_unfound":0,"num_objects_dirty":19039,"num_whiteouts":0,"num_read":5989,"num_read_kb":13222,"num_write":168696,"num_write_kb":5990171,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39930,"num_bytes_recovered":2412119794,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100318,"ondisk_log_size":100318,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":414,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11104080,"kb_used_data":5506316,"kb_used_omap":30,"kb_used_meta":5597409,"kb_avail":738857136,"statfs":{"total":767960285184,"available":756589707264,"internally_reserved":0,"allocated":5638467584,"data_stored":5616010429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":5731747170},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-470614016,"num_objects":-7181,"num_object_clones":0,"num_object_copies":-21543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-459,"num_objects_misplaced":-1034,"num_objects_unfound":0,"num_objects_dirty":-7181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7181,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1009,"num_bytes_recovered":56950784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005721"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":1070,"reported_epoch":614,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:53:00.305769+0000","last_change":"2023-09-30T09:52:16.331889+0000","last_active":"2023-09-30T09:53:00.305769+0000","last_peered":"2023-09-30T09:53:00.305769+0000","last_clean":"2023-09-30T09:52:09.251646+0000","last_became_active":"2023-09-30T09:52:16.331131+0000","last_became_peered":"2023-09-30T09:52:16.331131+0000","last_unstale":"2023-09-30T09:53:00.305769+0000","last_undegraded":"2023-09-30T09:52:16.322829+0000","last_fullsized":"2023-09-30T09:52:16.322421+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:01:47.387781+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":652,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.317641+0000","last_change":"2023-09-30T09:50:41.402505+0000","last_active":"2023-09-30T09:53:00.317641+0000","last_peered":"2023-09-30T09:53:00.317641+0000","last_clean":"2023-09-30T09:53:00.317641+0000","last_became_active":"2023-09-30T09:50:41.402099+0000","last_became_peered":"2023-09-30T09:50:41.402099+0000","last_unstale":"2023-09-30T09:53:00.317641+0000","last_undegraded":"2023-09-30T09:53:00.317641+0000","last_fullsized":"2023-09-30T09:53:00.317641+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:45.502519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":529,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.336282+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:53:00.336282+0000","last_peered":"2023-09-30T09:53:00.336282+0000","last_clean":"2023-09-30T09:53:00.336282+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:53:00.336282+0000","last_undegraded":"2023-09-30T09:53:00.336282+0000","last_fullsized":"2023-09-30T09:53:00.336282+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":1060,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.312281+0000","last_change":"2023-09-30T09:52:09.288155+0000","last_active":"2023-09-30T09:53:00.312281+0000","last_peered":"2023-09-30T09:53:00.312281+0000","last_clean":"2023-09-30T09:53:00.312281+0000","last_became_active":"2023-09-30T09:52:09.287827+0000","last_became_peered":"2023-09-30T09:52:09.287827+0000","last_unstale":"2023-09-30T09:53:00.312281+0000","last_undegraded":"2023-09-30T09:53:00.312281+0000","last_fullsized":"2023-09-30T09:53:00.312281+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:04.226064+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":191,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.306787+0000","last_change":"2023-09-30T09:51:57.147377+0000","last_active":"2023-09-30T09:53:01.306787+0000","last_peered":"2023-09-30T09:53:01.306787+0000","last_clean":"2023-09-30T09:53:01.306787+0000","last_became_active":"2023-09-30T09:51:57.146997+0000","last_became_peered":"2023-09-30T09:51:57.146997+0000","last_unstale":"2023-09-30T09:53:01.306787+0000","last_undegraded":"2023-09-30T09:53:01.306787+0000","last_fullsized":"2023-09-30T09:53:01.306787+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:01.086217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":329,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322790+0000","last_change":"2023-09-30T09:52:06.288110+0000","last_active":"2023-09-30T09:53:00.322790+0000","last_peered":"2023-09-30T09:53:00.322790+0000","last_clean":"2023-09-30T09:53:00.322790+0000","last_became_active":"2023-09-30T09:52:06.287784+0000","last_became_peered":"2023-09-30T09:52:06.287784+0000","last_unstale":"2023-09-30T09:53:00.322790+0000","last_undegraded":"2023-09-30T09:53:00.322790+0000","last_fullsized":"2023-09-30T09:53:00.322790+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:26:56.675676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":625,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320534+0000","last_change":"2023-09-30T09:52:09.270404+0000","last_active":"2023-09-30T09:53:00.320534+0000","last_peered":"2023-09-30T09:53:00.320534+0000","last_clean":"2023-09-30T09:53:00.320534+0000","last_became_active":"2023-09-30T09:52:09.269933+0000","last_became_peered":"2023-09-30T09:52:09.269933+0000","last_unstale":"2023-09-30T09:53:00.320534+0000","last_undegraded":"2023-09-30T09:53:00.320534+0000","last_fullsized":"2023-09-30T09:53:00.320534+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:26.548074+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":36,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.297511+0000","last_change":"2023-09-30T09:51:57.155291+0000","last_active":"2023-09-30T09:53:00.297511+0000","last_peered":"2023-09-30T09:53:00.297511+0000","last_clean":"2023-09-30T09:53:00.297511+0000","last_became_active":"2023-09-30T09:51:57.154983+0000","last_became_peered":"2023-09-30T09:51:57.154983+0000","last_unstale":"2023-09-30T09:53:00.297511+0000","last_undegraded":"2023-09-30T09:53:00.297511+0000","last_fullsized":"2023-09-30T09:53:00.297511+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:44:16.352650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.335067+0000","last_change":"2023-09-30T09:49:31.310014+0000","last_active":"2023-09-30T09:53:00.335067+0000","last_peered":"2023-09-30T09:53:00.335067+0000","last_clean":"2023-09-30T09:53:00.335067+0000","last_became_active":"2023-09-30T09:49:31.309121+0000","last_became_peered":"2023-09-30T09:49:31.309121+0000","last_unstale":"2023-09-30T09:53:00.335067+0000","last_undegraded":"2023-09-30T09:53:00.335067+0000","last_fullsized":"2023-09-30T09:53:00.335067+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:46.598805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":1016,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.321172+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:53:00.321172+0000","last_peered":"2023-09-30T09:53:00.321172+0000","last_clean":"2023-09-30T09:53:00.321172+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:53:00.321172+0000","last_undegraded":"2023-09-30T09:53:00.321172+0000","last_fullsized":"2023-09-30T09:53:00.321172+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":340,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.306338+0000","last_change":"2023-09-30T09:52:06.288453+0000","last_active":"2023-09-30T09:53:00.306338+0000","last_peered":"2023-09-30T09:53:00.306338+0000","last_clean":"2023-09-30T09:53:00.306338+0000","last_became_active":"2023-09-30T09:52:06.287939+0000","last_became_peered":"2023-09-30T09:52:06.287939+0000","last_unstale":"2023-09-30T09:53:00.306338+0000","last_undegraded":"2023-09-30T09:53:00.306338+0000","last_fullsized":"2023-09-30T09:53:00.306338+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:13:58.390726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":215,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.306239+0000","last_change":"2023-09-30T09:49:31.295632+0000","last_active":"2023-09-30T09:53:01.306239+0000","last_peered":"2023-09-30T09:53:01.306239+0000","last_clean":"2023-09-30T09:53:01.306239+0000","last_became_active":"2023-09-30T09:49:31.295269+0000","last_became_peered":"2023-09-30T09:49:31.295269+0000","last_unstale":"2023-09-30T09:53:01.306239+0000","last_undegraded":"2023-09-30T09:53:01.306239+0000","last_fullsized":"2023-09-30T09:53:01.306239+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:32.754585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":617,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.321673+0000","last_change":"2023-09-30T09:51:59.874993+0000","last_active":"2023-09-30T09:53:01.321673+0000","last_peered":"2023-09-30T09:53:01.321673+0000","last_clean":"2023-09-30T09:53:01.321673+0000","last_became_active":"2023-09-30T09:51:59.874470+0000","last_became_peered":"2023-09-30T09:51:59.874470+0000","last_unstale":"2023-09-30T09:53:01.321673+0000","last_undegraded":"2023-09-30T09:53:01.321673+0000","last_fullsized":"2023-09-30T09:53:01.321673+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:44.626194+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":35,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.306742+0000","last_change":"2023-09-30T09:51:57.160642+0000","last_active":"2023-09-30T09:53:00.306742+0000","last_peered":"2023-09-30T09:53:00.306742+0000","last_clean":"2023-09-30T09:53:00.306742+0000","last_became_active":"2023-09-30T09:51:57.160357+0000","last_became_peered":"2023-09-30T09:51:57.160357+0000","last_unstale":"2023-09-30T09:53:00.306742+0000","last_undegraded":"2023-09-30T09:53:00.306742+0000","last_fullsized":"2023-09-30T09:53:00.306742+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:40.417904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":591,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.326497+0000","last_change":"2023-09-30T09:49:31.309241+0000","last_active":"2023-09-30T09:53:00.326497+0000","last_peered":"2023-09-30T09:53:00.326497+0000","last_clean":"2023-09-30T09:53:00.326497+0000","last_became_active":"2023-09-30T09:49:31.308703+0000","last_became_peered":"2023-09-30T09:49:31.308703+0000","last_unstale":"2023-09-30T09:53:00.326497+0000","last_undegraded":"2023-09-30T09:53:00.326497+0000","last_fullsized":"2023-09-30T09:53:00.326497+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:06.877644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":804,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.306119+0000","last_change":"2023-09-30T09:52:09.265362+0000","last_active":"2023-09-30T09:53:00.306119+0000","last_peered":"2023-09-30T09:53:00.306119+0000","last_clean":"2023-09-30T09:53:00.306119+0000","last_became_active":"2023-09-30T09:52:09.265043+0000","last_became_peered":"2023-09-30T09:52:09.265043+0000","last_unstale":"2023-09-30T09:53:00.306119+0000","last_undegraded":"2023-09-30T09:53:00.306119+0000","last_fullsized":"2023-09-30T09:53:00.306119+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:39.738075+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":329,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322478+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:53:00.322478+0000","last_peered":"2023-09-30T09:53:00.322478+0000","last_clean":"2023-09-30T09:53:00.322478+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:53:00.322478+0000","last_undegraded":"2023-09-30T09:53:00.322478+0000","last_fullsized":"2023-09-30T09:53:00.322478+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":57,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.326420+0000","last_change":"2023-09-30T09:50:41.399925+0000","last_active":"2023-09-30T09:53:01.326420+0000","last_peered":"2023-09-30T09:53:01.326420+0000","last_clean":"2023-09-30T09:53:01.326420+0000","last_became_active":"2023-09-30T09:50:41.399642+0000","last_became_peered":"2023-09-30T09:50:41.399642+0000","last_unstale":"2023-09-30T09:53:01.326420+0000","last_undegraded":"2023-09-30T09:53:01.326420+0000","last_fullsized":"2023-09-30T09:53:01.326420+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:45:44.698050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":674,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.323309+0000","last_change":"2023-09-30T09:42:51.699231+0000","last_active":"2023-09-30T09:53:01.323309+0000","last_peered":"2023-09-30T09:53:01.323309+0000","last_clean":"2023-09-30T09:53:01.323309+0000","last_became_active":"2023-09-30T09:42:51.698844+0000","last_became_peered":"2023-09-30T09:42:51.698844+0000","last_unstale":"2023-09-30T09:53:01.323309+0000","last_undegraded":"2023-09-30T09:53:01.323309+0000","last_fullsized":"2023-09-30T09:53:01.323309+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:27.558998+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":388,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320964+0000","last_change":"2023-09-30T09:52:15.347512+0000","last_active":"2023-09-30T09:53:00.320964+0000","last_peered":"2023-09-30T09:53:00.320964+0000","last_clean":"2023-09-30T09:53:00.320964+0000","last_became_active":"2023-09-30T09:52:15.339136+0000","last_became_peered":"2023-09-30T09:52:15.339136+0000","last_unstale":"2023-09-30T09:53:00.320964+0000","last_undegraded":"2023-09-30T09:53:00.320964+0000","last_fullsized":"2023-09-30T09:53:00.320964+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:48.631308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9444,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":656,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.317176+0000","last_change":"2023-09-30T09:49:31.304624+0000","last_active":"2023-09-30T09:53:00.317176+0000","last_peered":"2023-09-30T09:53:00.317176+0000","last_clean":"2023-09-30T09:53:00.317176+0000","last_became_active":"2023-09-30T09:49:31.304106+0000","last_became_peered":"2023-09-30T09:49:31.304106+0000","last_unstale":"2023-09-30T09:53:00.317176+0000","last_undegraded":"2023-09-30T09:53:00.317176+0000","last_fullsized":"2023-09-30T09:53:00.317176+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:42.343641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":895,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.307654+0000","last_change":"2023-09-30T09:51:59.873231+0000","last_active":"2023-09-30T09:53:00.307654+0000","last_peered":"2023-09-30T09:53:00.307654+0000","last_clean":"2023-09-30T09:53:00.307654+0000","last_became_active":"2023-09-30T09:51:59.873003+0000","last_became_peered":"2023-09-30T09:51:59.873003+0000","last_unstale":"2023-09-30T09:53:00.307654+0000","last_undegraded":"2023-09-30T09:53:00.307654+0000","last_fullsized":"2023-09-30T09:53:00.307654+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:04.437753+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":313,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320797+0000","last_change":"2023-09-30T09:52:15.360214+0000","last_active":"2023-09-30T09:53:00.320797+0000","last_peered":"2023-09-30T09:53:00.320797+0000","last_clean":"2023-09-30T09:53:00.320797+0000","last_became_active":"2023-09-30T09:52:15.339270+0000","last_became_peered":"2023-09-30T09:52:15.339270+0000","last_unstale":"2023-09-30T09:53:00.320797+0000","last_undegraded":"2023-09-30T09:53:00.320797+0000","last_fullsized":"2023-09-30T09:53:00.320797+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:37:34.556170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":592,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.321156+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:53:00.321156+0000","last_peered":"2023-09-30T09:53:00.321156+0000","last_clean":"2023-09-30T09:53:00.321156+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:53:00.321156+0000","last_undegraded":"2023-09-30T09:53:00.321156+0000","last_fullsized":"2023-09-30T09:53:00.321156+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":976,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305421+0000","last_change":"2023-09-30T09:49:34.332543+0000","last_active":"2023-09-30T09:53:00.305421+0000","last_peered":"2023-09-30T09:53:00.305421+0000","last_clean":"2023-09-30T09:53:00.305421+0000","last_became_active":"2023-09-30T09:49:34.332215+0000","last_became_peered":"2023-09-30T09:49:34.332215+0000","last_unstale":"2023-09-30T09:53:00.305421+0000","last_undegraded":"2023-09-30T09:53:00.305421+0000","last_fullsized":"2023-09-30T09:53:00.305421+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.827671+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":368,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320625+0000","last_change":"2023-09-30T09:52:06.287263+0000","last_active":"2023-09-30T09:53:00.320625+0000","last_peered":"2023-09-30T09:53:00.320625+0000","last_clean":"2023-09-30T09:53:00.320625+0000","last_became_active":"2023-09-30T09:52:06.286870+0000","last_became_peered":"2023-09-30T09:52:06.286870+0000","last_unstale":"2023-09-30T09:53:00.320625+0000","last_undegraded":"2023-09-30T09:53:00.320625+0000","last_fullsized":"2023-09-30T09:53:00.320625+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:01:20.147214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":35,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305610+0000","last_change":"2023-09-30T09:51:57.157155+0000","last_active":"2023-09-30T09:53:00.305610+0000","last_peered":"2023-09-30T09:53:00.305610+0000","last_clean":"2023-09-30T09:53:00.305610+0000","last_became_active":"2023-09-30T09:51:57.156843+0000","last_became_peered":"2023-09-30T09:51:57.156843+0000","last_unstale":"2023-09-30T09:53:00.305610+0000","last_undegraded":"2023-09-30T09:53:00.305610+0000","last_fullsized":"2023-09-30T09:53:00.305610+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:12:00.333473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":1034,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.307711+0000","last_change":"2023-09-30T09:51:59.874841+0000","last_active":"2023-09-30T09:53:01.307711+0000","last_peered":"2023-09-30T09:53:01.307711+0000","last_clean":"2023-09-30T09:53:01.307711+0000","last_became_active":"2023-09-30T09:51:59.874418+0000","last_became_peered":"2023-09-30T09:51:59.874418+0000","last_unstale":"2023-09-30T09:53:01.307711+0000","last_undegraded":"2023-09-30T09:53:01.307711+0000","last_fullsized":"2023-09-30T09:53:01.307711+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:31:16.097188+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":319,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.326655+0000","last_change":"2023-09-30T09:49:31.309022+0000","last_active":"2023-09-30T09:53:00.326655+0000","last_peered":"2023-09-30T09:53:00.326655+0000","last_clean":"2023-09-30T09:53:00.326655+0000","last_became_active":"2023-09-30T09:49:31.308236+0000","last_became_peered":"2023-09-30T09:49:31.308236+0000","last_unstale":"2023-09-30T09:53:00.326655+0000","last_undegraded":"2023-09-30T09:53:00.326655+0000","last_fullsized":"2023-09-30T09:53:00.326655+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:39.779954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":373,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.326351+0000","last_change":"2023-09-30T09:52:06.288331+0000","last_active":"2023-09-30T09:53:00.326351+0000","last_peered":"2023-09-30T09:53:00.326351+0000","last_clean":"2023-09-30T09:53:00.326351+0000","last_became_active":"2023-09-30T09:52:06.288031+0000","last_became_peered":"2023-09-30T09:52:06.288031+0000","last_unstale":"2023-09-30T09:53:00.326351+0000","last_undegraded":"2023-09-30T09:53:00.326351+0000","last_fullsized":"2023-09-30T09:53:00.326351+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:47:56.726506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":1065,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.307066+0000","last_change":"2023-09-30T09:49:34.332561+0000","last_active":"2023-09-30T09:53:00.307066+0000","last_peered":"2023-09-30T09:53:00.307066+0000","last_clean":"2023-09-30T09:53:00.307066+0000","last_became_active":"2023-09-30T09:49:34.332170+0000","last_became_peered":"2023-09-30T09:49:34.332170+0000","last_unstale":"2023-09-30T09:53:00.307066+0000","last_undegraded":"2023-09-30T09:53:00.307066+0000","last_fullsized":"2023-09-30T09:53:00.307066+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:22.548068+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":40,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305515+0000","last_change":"2023-09-30T09:52:06.314076+0000","last_active":"2023-09-30T09:53:00.305515+0000","last_peered":"2023-09-30T09:53:00.305515+0000","last_clean":"2023-09-30T09:53:00.305515+0000","last_became_active":"2023-09-30T09:52:06.286706+0000","last_became_peered":"2023-09-30T09:52:06.286706+0000","last_unstale":"2023-09-30T09:53:00.305515+0000","last_undegraded":"2023-09-30T09:53:00.305515+0000","last_fullsized":"2023-09-30T09:53:00.305515+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:35.387500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":183,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.310619+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:53:01.310619+0000","last_peered":"2023-09-30T09:53:01.310619+0000","last_clean":"2023-09-30T09:53:01.310619+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:53:01.310619+0000","last_undegraded":"2023-09-30T09:53:01.310619+0000","last_fullsized":"2023-09-30T09:53:01.310619+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"616'10352","reported_seq":10447,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-30T09:53:12.485278+0000","last_change":"2023-09-30T09:48:40.945814+0000","last_active":"2023-09-30T09:53:12.485278+0000","last_peered":"2023-09-30T09:53:12.485278+0000","last_clean":"2023-09-30T09:53:12.485278+0000","last_became_active":"2023-09-30T09:48:40.945431+0000","last_became_peered":"2023-09-30T09:48:40.945431+0000","last_unstale":"2023-09-30T09:53:12.485278+0000","last_undegraded":"2023-09-30T09:53:12.485278+0000","last_fullsized":"2023-09-30T09:53:12.485278+0000","mapping_epoch":536,"log_start":"557'5012","ondisk_log_start":"557'5012","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:59:06.006918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69861376,"num_objects":1066,"num_object_clones":0,"num_object_copies":3198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10352,"num_write_kb":365376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1011,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322368+0000","last_change":"2023-09-30T09:52:09.263017+0000","last_active":"2023-09-30T09:53:00.322368+0000","last_peered":"2023-09-30T09:53:00.322368+0000","last_clean":"2023-09-30T09:53:00.322368+0000","last_became_active":"2023-09-30T09:52:09.261512+0000","last_became_peered":"2023-09-30T09:52:09.261512+0000","last_unstale":"2023-09-30T09:53:00.322368+0000","last_undegraded":"2023-09-30T09:53:00.322368+0000","last_fullsized":"2023-09-30T09:53:00.322368+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:13.359576+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":305,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322801+0000","last_change":"2023-09-30T09:51:57.189281+0000","last_active":"2023-09-30T09:53:00.322801+0000","last_peered":"2023-09-30T09:53:00.322801+0000","last_clean":"2023-09-30T09:53:00.322801+0000","last_became_active":"2023-09-30T09:51:57.165927+0000","last_became_peered":"2023-09-30T09:51:57.165927+0000","last_unstale":"2023-09-30T09:53:00.322801+0000","last_undegraded":"2023-09-30T09:53:00.322801+0000","last_fullsized":"2023-09-30T09:53:00.322801+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:30.207994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":35,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305325+0000","last_change":"2023-09-30T09:51:57.167306+0000","last_active":"2023-09-30T09:53:00.305325+0000","last_peered":"2023-09-30T09:53:00.305325+0000","last_clean":"2023-09-30T09:53:00.305325+0000","last_became_active":"2023-09-30T09:51:57.167040+0000","last_became_peered":"2023-09-30T09:51:57.167040+0000","last_unstale":"2023-09-30T09:53:00.305325+0000","last_undegraded":"2023-09-30T09:53:00.305325+0000","last_fullsized":"2023-09-30T09:53:00.305325+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:27:26.793530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"615'10590","reported_seq":19333,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:09.432431+0000","last_change":"2023-09-30T09:52:34.978315+0000","last_active":"2023-09-30T09:53:09.432431+0000","last_peered":"2023-09-30T09:53:09.432431+0000","last_clean":"2023-09-30T09:53:09.432431+0000","last_became_active":"2023-09-30T09:52:34.912855+0000","last_became_peered":"2023-09-30T09:52:34.912855+0000","last_unstale":"2023-09-30T09:53:09.432431+0000","last_undegraded":"2023-09-30T09:53:09.432431+0000","last_fullsized":"2023-09-30T09:53:09.432431+0000","mapping_epoch":608,"log_start":"556'4747","ondisk_log_start":"556'4747","created":536,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5843,"log_dups_size":0,"ondisk_log_size":5843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:52:19.028693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80478208,"num_objects":1228,"num_object_clones":0,"num_object_copies":3684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10590,"num_write_kb":378176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4301,"num_bytes_recovered":248774656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":1048,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.308465+0000","last_change":"2023-09-30T09:49:34.340125+0000","last_active":"2023-09-30T09:53:01.308465+0000","last_peered":"2023-09-30T09:53:01.308465+0000","last_clean":"2023-09-30T09:53:01.308465+0000","last_became_active":"2023-09-30T09:49:34.339638+0000","last_became_peered":"2023-09-30T09:49:34.339638+0000","last_unstale":"2023-09-30T09:53:01.308465+0000","last_undegraded":"2023-09-30T09:53:01.308465+0000","last_fullsized":"2023-09-30T09:53:01.308465+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:43:43.258960+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":275,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.326159+0000","last_change":"2023-09-30T09:51:57.183013+0000","last_active":"2023-09-30T09:53:00.326159+0000","last_peered":"2023-09-30T09:53:00.326159+0000","last_clean":"2023-09-30T09:53:00.326159+0000","last_became_active":"2023-09-30T09:51:57.161490+0000","last_became_peered":"2023-09-30T09:51:57.161490+0000","last_unstale":"2023-09-30T09:53:00.326159+0000","last_undegraded":"2023-09-30T09:53:00.326159+0000","last_fullsized":"2023-09-30T09:53:00.326159+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:14:47.256298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":675,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.316865+0000","last_change":"2023-09-30T09:50:26.903683+0000","last_active":"2023-09-30T09:53:00.316865+0000","last_peered":"2023-09-30T09:53:00.316865+0000","last_clean":"2023-09-30T09:53:00.316865+0000","last_became_active":"2023-09-30T09:50:26.903262+0000","last_became_peered":"2023-09-30T09:50:26.903262+0000","last_unstale":"2023-09-30T09:53:00.316865+0000","last_undegraded":"2023-09-30T09:53:00.316865+0000","last_fullsized":"2023-09-30T09:53:00.316865+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:40.724403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"615'10652","reported_seq":11617,"reported_epoch":615,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:53:09.108306+0000","last_change":"2023-09-30T09:52:07.362883+0000","last_active":"2023-09-30T09:53:09.108306+0000","last_peered":"2023-09-30T09:53:09.108306+0000","last_clean":"2023-09-30T09:51:55.533937+0000","last_became_active":"2023-09-30T09:51:58.262846+0000","last_became_peered":"2023-09-30T09:51:58.262846+0000","last_unstale":"2023-09-30T09:53:09.108306+0000","last_undegraded":"2023-09-30T09:53:09.108306+0000","last_fullsized":"2023-09-30T09:53:09.108306+0000","mapping_epoch":590,"log_start":"554'4339","ondisk_log_start":"554'4339","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:19:17.909153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83230720,"num_objects":1270,"num_object_clones":0,"num_object_copies":3810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1122,"num_objects_unfound":0,"num_objects_dirty":1270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10652,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":430,"num_bytes_recovered":28180480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":1270}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":697,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305969+0000","last_change":"2023-09-30T09:52:18.342336+0000","last_active":"2023-09-30T09:53:00.305969+0000","last_peered":"2023-09-30T09:53:00.305969+0000","last_clean":"2023-09-30T09:53:00.305969+0000","last_became_active":"2023-09-30T09:52:18.341952+0000","last_became_peered":"2023-09-30T09:52:18.341952+0000","last_unstale":"2023-09-30T09:53:00.305969+0000","last_undegraded":"2023-09-30T09:53:00.305969+0000","last_fullsized":"2023-09-30T09:53:00.305969+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:16.116795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:16.116795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:01:10.275488+0000","scrub_duration":0.045979904000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":384,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.321026+0000","last_change":"2023-09-30T09:51:57.192521+0000","last_active":"2023-09-30T09:53:00.321026+0000","last_peered":"2023-09-30T09:53:00.321026+0000","last_clean":"2023-09-30T09:53:00.321026+0000","last_became_active":"2023-09-30T09:51:57.170876+0000","last_became_peered":"2023-09-30T09:51:57.170876+0000","last_unstale":"2023-09-30T09:53:00.321026+0000","last_undegraded":"2023-09-30T09:53:00.321026+0000","last_fullsized":"2023-09-30T09:53:00.321026+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:44:49.877225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":500,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.306929+0000","last_change":"2023-09-30T09:50:13.889514+0000","last_active":"2023-09-30T09:53:00.306929+0000","last_peered":"2023-09-30T09:53:00.306929+0000","last_clean":"2023-09-30T09:53:00.306929+0000","last_became_active":"2023-09-30T09:50:13.889033+0000","last_became_peered":"2023-09-30T09:50:13.889033+0000","last_unstale":"2023-09-30T09:53:00.306929+0000","last_undegraded":"2023-09-30T09:53:00.306929+0000","last_fullsized":"2023-09-30T09:53:00.306929+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:19.671881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"615'10467","reported_seq":11304,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:09.121079+0000","last_change":"2023-09-30T09:52:18.372502+0000","last_active":"2023-09-30T09:53:09.121079+0000","last_peered":"2023-09-30T09:53:09.121079+0000","last_clean":"2023-09-30T09:53:09.121079+0000","last_became_active":"2023-09-30T09:52:18.341874+0000","last_became_peered":"2023-09-30T09:52:18.341874+0000","last_unstale":"2023-09-30T09:53:09.121079+0000","last_undegraded":"2023-09-30T09:53:09.121079+0000","last_fullsized":"2023-09-30T09:53:09.121079+0000","mapping_epoch":603,"log_start":"547'3115","ondisk_log_start":"547'3115","created":536,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":7352,"log_dups_size":0,"ondisk_log_size":7352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:28:00.684506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76611584,"num_objects":1169,"num_object_clones":0,"num_object_copies":3507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10467,"num_write_kb":372352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":359,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":593,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322716+0000","last_change":"2023-09-30T09:51:59.873050+0000","last_active":"2023-09-30T09:53:00.322716+0000","last_peered":"2023-09-30T09:53:00.322716+0000","last_clean":"2023-09-30T09:53:00.322716+0000","last_became_active":"2023-09-30T09:51:59.872660+0000","last_became_peered":"2023-09-30T09:51:59.872660+0000","last_unstale":"2023-09-30T09:53:00.322716+0000","last_undegraded":"2023-09-30T09:53:00.322716+0000","last_fullsized":"2023-09-30T09:53:00.322716+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:47:10.945792+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":570,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305612+0000","last_change":"2023-09-30T09:49:31.301460+0000","last_active":"2023-09-30T09:53:00.305612+0000","last_peered":"2023-09-30T09:53:00.305612+0000","last_clean":"2023-09-30T09:53:00.305612+0000","last_became_active":"2023-09-30T09:49:31.301114+0000","last_became_peered":"2023-09-30T09:49:31.301114+0000","last_unstale":"2023-09-30T09:53:00.305612+0000","last_undegraded":"2023-09-30T09:53:00.305612+0000","last_fullsized":"2023-09-30T09:53:00.305612+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:42:01.066974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":218,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.323437+0000","last_change":"2023-09-30T09:40:36.592243+0000","last_active":"2023-09-30T09:53:01.323437+0000","last_peered":"2023-09-30T09:53:01.323437+0000","last_clean":"2023-09-30T09:53:01.323437+0000","last_became_active":"2023-09-30T09:40:36.591907+0000","last_became_peered":"2023-09-30T09:40:36.591907+0000","last_unstale":"2023-09-30T09:53:01.323437+0000","last_undegraded":"2023-09-30T09:53:01.323437+0000","last_fullsized":"2023-09-30T09:53:01.323437+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:16:05.039488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"615'10518","reported_seq":10604,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:09.184133+0000","last_change":"2023-09-30T09:48:40.937693+0000","last_active":"2023-09-30T09:53:09.184133+0000","last_peered":"2023-09-30T09:53:09.184133+0000","last_clean":"2023-09-30T09:53:09.184133+0000","last_became_active":"2023-09-30T09:48:40.937364+0000","last_became_peered":"2023-09-30T09:48:40.937364+0000","last_unstale":"2023-09-30T09:53:09.184133+0000","last_undegraded":"2023-09-30T09:53:09.184133+0000","last_fullsized":"2023-09-30T09:53:09.184133+0000","mapping_epoch":536,"log_start":"554'4245","ondisk_log_start":"554'4245","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6273,"log_dups_size":0,"ondisk_log_size":6273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:48:58.223575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82313216,"num_objects":1256,"num_object_clones":0,"num_object_copies":3768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10518,"num_write_kb":376768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1157,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.326577+0000","last_change":"2023-09-30T09:52:09.259618+0000","last_active":"2023-09-30T09:53:00.326577+0000","last_peered":"2023-09-30T09:53:00.326577+0000","last_clean":"2023-09-30T09:53:00.326577+0000","last_became_active":"2023-09-30T09:52:09.259274+0000","last_became_peered":"2023-09-30T09:52:09.259274+0000","last_unstale":"2023-09-30T09:53:00.326577+0000","last_undegraded":"2023-09-30T09:53:00.326577+0000","last_fullsized":"2023-09-30T09:53:00.326577+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:41:35.328752+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.317473+0000","last_change":"2023-09-30T09:51:57.173028+0000","last_active":"2023-09-30T09:53:00.317473+0000","last_peered":"2023-09-30T09:53:00.317473+0000","last_clean":"2023-09-30T09:53:00.317473+0000","last_became_active":"2023-09-30T09:51:57.152483+0000","last_became_peered":"2023-09-30T09:51:57.152483+0000","last_unstale":"2023-09-30T09:53:00.317473+0000","last_undegraded":"2023-09-30T09:53:00.317473+0000","last_fullsized":"2023-09-30T09:53:00.317473+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:11.989575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":632,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.321585+0000","last_change":"2023-09-30T09:49:31.300551+0000","last_active":"2023-09-30T09:53:00.321585+0000","last_peered":"2023-09-30T09:53:00.321585+0000","last_clean":"2023-09-30T09:53:00.321585+0000","last_became_active":"2023-09-30T09:49:31.300231+0000","last_became_peered":"2023-09-30T09:49:31.300231+0000","last_unstale":"2023-09-30T09:53:00.321585+0000","last_undegraded":"2023-09-30T09:53:00.321585+0000","last_fullsized":"2023-09-30T09:53:00.321585+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:45.832564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"615'10578","reported_seq":15076,"reported_epoch":615,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-30T09:53:09.524968+0000","last_change":"2023-09-30T09:52:16.579692+0000","last_active":"2023-09-30T09:53:09.524968+0000","last_peered":"2023-09-30T09:53:09.524968+0000","last_clean":"2023-09-30T09:51:55.471595+0000","last_became_active":"2023-09-30T09:52:16.492103+0000","last_became_peered":"2023-09-30T09:52:16.492103+0000","last_unstale":"2023-09-30T09:53:09.524968+0000","last_undegraded":"2023-09-30T09:52:16.484021+0000","last_fullsized":"2023-09-30T09:53:09.524968+0000","mapping_epoch":601,"log_start":"556'4808","ondisk_log_start":"556'4808","created":536,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:18:38.592310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79691776,"num_objects":1216,"num_object_clones":0,"num_object_copies":3648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10578,"num_write_kb":377408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2180,"num_bytes_recovered":142802944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,4],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1215},{"shards":"3,4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":991,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.297679+0000","last_change":"2023-09-30T09:52:09.249933+0000","last_active":"2023-09-30T09:53:00.297679+0000","last_peered":"2023-09-30T09:53:00.297679+0000","last_clean":"2023-09-30T09:53:00.297679+0000","last_became_active":"2023-09-30T09:52:09.249567+0000","last_became_peered":"2023-09-30T09:52:09.249567+0000","last_unstale":"2023-09-30T09:53:00.297679+0000","last_undegraded":"2023-09-30T09:53:00.297679+0000","last_fullsized":"2023-09-30T09:53:00.297679+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:52:00.664301+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":457,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.317471+0000","last_change":"2023-09-30T09:50:13.887314+0000","last_active":"2023-09-30T09:53:00.317471+0000","last_peered":"2023-09-30T09:53:00.317471+0000","last_clean":"2023-09-30T09:53:00.317471+0000","last_became_active":"2023-09-30T09:50:13.887032+0000","last_became_peered":"2023-09-30T09:50:13.887032+0000","last_unstale":"2023-09-30T09:53:00.317471+0000","last_undegraded":"2023-09-30T09:53:00.317471+0000","last_fullsized":"2023-09-30T09:53:00.317471+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:27:26.626380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":24,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305393+0000","last_change":"2023-09-30T09:52:15.326810+0000","last_active":"2023-09-30T09:53:00.305393+0000","last_peered":"2023-09-30T09:53:00.305393+0000","last_clean":"2023-09-30T09:53:00.305393+0000","last_became_active":"2023-09-30T09:52:15.326507+0000","last_became_peered":"2023-09-30T09:52:15.326507+0000","last_unstale":"2023-09-30T09:53:00.305393+0000","last_undegraded":"2023-09-30T09:53:00.305393+0000","last_fullsized":"2023-09-30T09:53:00.305393+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:51:01.576127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.0","version":"615'10525","reported_seq":22605,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:09.066515+0000","last_change":"2023-09-30T09:52:28.113587+0000","last_active":"2023-09-30T09:53:09.066515+0000","last_peered":"2023-09-30T09:53:09.066515+0000","last_clean":"2023-09-30T09:53:09.066515+0000","last_became_active":"2023-09-30T09:52:28.047541+0000","last_became_peered":"2023-09-30T09:52:28.047541+0000","last_unstale":"2023-09-30T09:53:09.066515+0000","last_undegraded":"2023-09-30T09:53:09.066515+0000","last_fullsized":"2023-09-30T09:53:09.066515+0000","mapping_epoch":606,"log_start":"557'4900","ondisk_log_start":"557'4900","created":536,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:02:13.553694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83951616,"num_objects":1281,"num_object_clones":0,"num_object_copies":3843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10525,"num_write_kb":377792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5978,"num_bytes_recovered":391774208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":982,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.306527+0000","last_change":"2023-09-30T09:52:18.345744+0000","last_active":"2023-09-30T09:53:00.306527+0000","last_peered":"2023-09-30T09:53:00.306527+0000","last_clean":"2023-09-30T09:53:00.306527+0000","last_became_active":"2023-09-30T09:52:18.345422+0000","last_became_peered":"2023-09-30T09:52:18.345422+0000","last_unstale":"2023-09-30T09:53:00.306527+0000","last_undegraded":"2023-09-30T09:53:00.306527+0000","last_fullsized":"2023-09-30T09:53:00.306527+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:49:09.924646+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":616,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.297979+0000","last_change":"2023-09-30T09:52:06.274034+0000","last_active":"2023-09-30T09:53:00.297979+0000","last_peered":"2023-09-30T09:53:00.297979+0000","last_clean":"2023-09-30T09:53:00.297979+0000","last_became_active":"2023-09-30T09:52:06.273748+0000","last_became_peered":"2023-09-30T09:52:06.273748+0000","last_unstale":"2023-09-30T09:53:00.297979+0000","last_undegraded":"2023-09-30T09:53:00.297979+0000","last_fullsized":"2023-09-30T09:53:00.297979+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:22:02.105157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":646,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322589+0000","last_change":"2023-09-30T09:49:31.311818+0000","last_active":"2023-09-30T09:53:00.322589+0000","last_peered":"2023-09-30T09:53:00.322589+0000","last_clean":"2023-09-30T09:53:00.322589+0000","last_became_active":"2023-09-30T09:49:31.311481+0000","last_became_peered":"2023-09-30T09:49:31.311481+0000","last_unstale":"2023-09-30T09:53:00.322589+0000","last_undegraded":"2023-09-30T09:53:00.322589+0000","last_fullsized":"2023-09-30T09:53:00.322589+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:54.941072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"615'10464","reported_seq":14008,"reported_epoch":615,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-30T09:53:09.029850+0000","last_change":"2023-09-30T09:52:16.758569+0000","last_active":"2023-09-30T09:53:09.029850+0000","last_peered":"2023-09-30T09:53:09.029850+0000","last_clean":"2023-09-30T09:52:04.984831+0000","last_became_active":"2023-09-30T09:52:07.326751+0000","last_became_peered":"2023-09-30T09:52:07.326751+0000","last_unstale":"2023-09-30T09:53:09.029850+0000","last_undegraded":"2023-09-30T09:53:09.029850+0000","last_fullsized":"2023-09-30T09:53:09.029850+0000","mapping_epoch":596,"log_start":"557'4876","ondisk_log_start":"557'4876","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5588,"log_dups_size":0,"ondisk_log_size":5588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:12:07.668051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79036416,"num_objects":1206,"num_object_clones":0,"num_object_copies":3618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":581,"num_objects_unfound":0,"num_objects_dirty":1206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10464,"num_write_kb":373440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1720,"num_bytes_recovered":112721920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,4],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1206}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":814,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.303574+0000","last_change":"2023-09-30T09:50:44.462926+0000","last_active":"2023-09-30T09:53:00.303574+0000","last_peered":"2023-09-30T09:53:00.303574+0000","last_clean":"2023-09-30T09:53:00.303574+0000","last_became_active":"2023-09-30T09:50:44.462573+0000","last_became_peered":"2023-09-30T09:50:44.462573+0000","last_unstale":"2023-09-30T09:53:00.303574+0000","last_undegraded":"2023-09-30T09:53:00.303574+0000","last_fullsized":"2023-09-30T09:53:00.303574+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:28.805205+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":183,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.306154+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:53:01.306154+0000","last_peered":"2023-09-30T09:53:01.306154+0000","last_clean":"2023-09-30T09:53:01.306154+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:53:01.306154+0000","last_undegraded":"2023-09-30T09:53:01.306154+0000","last_fullsized":"2023-09-30T09:53:01.306154+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":299,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.321655+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:53:00.321655+0000","last_peered":"2023-09-30T09:53:00.321655+0000","last_clean":"2023-09-30T09:53:00.321655+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:53:00.321655+0000","last_undegraded":"2023-09-30T09:53:00.321655+0000","last_fullsized":"2023-09-30T09:53:00.321655+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"615'10486","reported_seq":10573,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:09.067161+0000","last_change":"2023-09-30T09:48:40.942045+0000","last_active":"2023-09-30T09:53:09.067161+0000","last_peered":"2023-09-30T09:53:09.067161+0000","last_clean":"2023-09-30T09:53:09.067161+0000","last_became_active":"2023-09-30T09:48:40.941786+0000","last_became_peered":"2023-09-30T09:48:40.941786+0000","last_unstale":"2023-09-30T09:53:09.067161+0000","last_undegraded":"2023-09-30T09:53:09.067161+0000","last_fullsized":"2023-09-30T09:53:09.067161+0000","mapping_epoch":536,"log_start":"557'4832","ondisk_log_start":"557'4832","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5654,"log_dups_size":0,"ondisk_log_size":5654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:03:11.354244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85590016,"num_objects":1306,"num_object_clones":0,"num_object_copies":3918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10486,"num_write_kb":377344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":855,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322432+0000","last_change":"2023-09-30T09:50:13.903010+0000","last_active":"2023-09-30T09:53:00.322432+0000","last_peered":"2023-09-30T09:53:00.322432+0000","last_clean":"2023-09-30T09:53:00.322432+0000","last_became_active":"2023-09-30T09:50:13.902355+0000","last_became_peered":"2023-09-30T09:50:13.902355+0000","last_unstale":"2023-09-30T09:53:00.322432+0000","last_undegraded":"2023-09-30T09:53:00.322432+0000","last_fullsized":"2023-09-30T09:53:00.322432+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:35.240830+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'92","reported_seq":849,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.320413+0000","last_change":"2023-09-30T09:40:36.607058+0000","last_active":"2023-09-30T09:53:01.320413+0000","last_peered":"2023-09-30T09:53:01.320413+0000","last_clean":"2023-09-30T09:53:01.320413+0000","last_became_active":"2023-09-30T09:40:36.606523+0000","last_became_peered":"2023-09-30T09:40:36.606523+0000","last_unstale":"2023-09-30T09:53:01.320413+0000","last_undegraded":"2023-09-30T09:53:01.320413+0000","last_fullsized":"2023-09-30T09:53:01.320413+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:56:08.798532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":391,"num_read_kb":366,"num_write":98,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":563,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305691+0000","last_change":"2023-09-30T09:50:41.396047+0000","last_active":"2023-09-30T09:53:00.305691+0000","last_peered":"2023-09-30T09:53:00.305691+0000","last_clean":"2023-09-30T09:53:00.305691+0000","last_became_active":"2023-09-30T09:50:41.395627+0000","last_became_peered":"2023-09-30T09:50:41.395627+0000","last_unstale":"2023-09-30T09:53:00.305691+0000","last_undegraded":"2023-09-30T09:53:00.305691+0000","last_fullsized":"2023-09-30T09:53:00.305691+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:16.050977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":55,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305863+0000","last_change":"2023-09-30T09:52:15.338190+0000","last_active":"2023-09-30T09:53:00.305863+0000","last_peered":"2023-09-30T09:53:00.305863+0000","last_clean":"2023-09-30T09:53:00.305863+0000","last_became_active":"2023-09-30T09:52:15.337907+0000","last_became_peered":"2023-09-30T09:52:15.337907+0000","last_unstale":"2023-09-30T09:53:00.305863+0000","last_undegraded":"2023-09-30T09:53:00.305863+0000","last_fullsized":"2023-09-30T09:53:00.305863+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:20:09.336859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"615'10678","reported_seq":22969,"reported_epoch":615,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-30T09:53:09.514339+0000","last_change":"2023-09-30T09:52:33.171256+0000","last_active":"2023-09-30T09:53:09.514339+0000","last_peered":"2023-09-30T09:53:09.514339+0000","last_clean":"2023-09-30T09:51:56.097639+0000","last_became_active":"2023-09-30T09:51:58.327478+0000","last_became_peered":"2023-09-30T09:51:58.327478+0000","last_unstale":"2023-09-30T09:53:09.514339+0000","last_undegraded":"2023-09-30T09:53:09.514339+0000","last_fullsized":"2023-09-30T09:51:58.140261+0000","mapping_epoch":590,"log_start":"556'4888","ondisk_log_start":"556'4888","created":536,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5790,"log_dups_size":0,"ondisk_log_size":5790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:41:17.448598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79167488,"num_objects":1208,"num_object_clones":0,"num_object_copies":3624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10678,"num_write_kb":380352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6083,"num_bytes_recovered":389480448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"5,6","objects":1208}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":1116,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.307914+0000","last_change":"2023-09-30T09:52:09.283412+0000","last_active":"2023-09-30T09:53:00.307914+0000","last_peered":"2023-09-30T09:53:00.307914+0000","last_clean":"2023-09-30T09:53:00.307914+0000","last_became_active":"2023-09-30T09:52:09.283074+0000","last_became_peered":"2023-09-30T09:52:09.283074+0000","last_unstale":"2023-09-30T09:53:00.307914+0000","last_undegraded":"2023-09-30T09:53:00.307914+0000","last_fullsized":"2023-09-30T09:53:00.307914+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:33:20.204709+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":412,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.306867+0000","last_change":"2023-09-30T09:50:13.889332+0000","last_active":"2023-09-30T09:53:00.306867+0000","last_peered":"2023-09-30T09:53:00.306867+0000","last_clean":"2023-09-30T09:53:00.306867+0000","last_became_active":"2023-09-30T09:50:13.888518+0000","last_became_peered":"2023-09-30T09:50:13.888518+0000","last_unstale":"2023-09-30T09:53:00.306867+0000","last_undegraded":"2023-09-30T09:53:00.306867+0000","last_fullsized":"2023-09-30T09:53:00.306867+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:40:20.465620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":219,"num_read_kb":2265,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":663,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.317258+0000","last_change":"2023-09-30T09:52:15.328572+0000","last_active":"2023-09-30T09:53:00.317258+0000","last_peered":"2023-09-30T09:53:00.317258+0000","last_clean":"2023-09-30T09:53:00.317258+0000","last_became_active":"2023-09-30T09:52:15.328181+0000","last_became_peered":"2023-09-30T09:52:15.328181+0000","last_unstale":"2023-09-30T09:53:00.317258+0000","last_undegraded":"2023-09-30T09:53:00.317258+0000","last_fullsized":"2023-09-30T09:53:00.317258+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:08:51.469327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":283,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305691+0000","last_change":"2023-09-30T09:52:15.332419+0000","last_active":"2023-09-30T09:53:00.305691+0000","last_peered":"2023-09-30T09:53:00.305691+0000","last_clean":"2023-09-30T09:53:00.305691+0000","last_became_active":"2023-09-30T09:52:15.332098+0000","last_became_peered":"2023-09-30T09:52:15.332098+0000","last_unstale":"2023-09-30T09:53:00.305691+0000","last_undegraded":"2023-09-30T09:53:00.305691+0000","last_fullsized":"2023-09-30T09:53:00.305691+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:53:34.176892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"615'10349","reported_seq":12138,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:09.513987+0000","last_change":"2023-09-30T09:52:16.376606+0000","last_active":"2023-09-30T09:53:09.513987+0000","last_peered":"2023-09-30T09:53:09.513987+0000","last_clean":"2023-09-30T09:53:09.513987+0000","last_became_active":"2023-09-30T09:52:16.330391+0000","last_became_peered":"2023-09-30T09:52:16.330391+0000","last_unstale":"2023-09-30T09:53:09.513987+0000","last_undegraded":"2023-09-30T09:53:09.513987+0000","last_fullsized":"2023-09-30T09:53:09.513987+0000","mapping_epoch":601,"log_start":"555'4564","ondisk_log_start":"555'4564","created":536,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5785,"log_dups_size":0,"ondisk_log_size":5785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:02:19.237518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80936960,"num_objects":1235,"num_object_clones":0,"num_object_copies":3705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10349,"num_write_kb":370688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":1074,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320590+0000","last_change":"2023-09-30T09:52:09.253155+0000","last_active":"2023-09-30T09:53:00.320590+0000","last_peered":"2023-09-30T09:53:00.320590+0000","last_clean":"2023-09-30T09:53:00.320590+0000","last_became_active":"2023-09-30T09:52:09.252824+0000","last_became_peered":"2023-09-30T09:52:09.252824+0000","last_unstale":"2023-09-30T09:53:00.320590+0000","last_undegraded":"2023-09-30T09:53:00.320590+0000","last_fullsized":"2023-09-30T09:53:00.320590+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:50.451301+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'109","reported_seq":530,"reported_epoch":615,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:53:01.322769+0000","last_change":"2023-09-30T09:52:16.329223+0000","last_active":"2023-09-30T09:53:01.322769+0000","last_peered":"2023-09-30T09:53:01.322769+0000","last_clean":"2023-09-30T09:23:03.976898+0000","last_became_active":"2023-09-30T09:52:16.328685+0000","last_became_peered":"2023-09-30T09:52:16.328685+0000","last_unstale":"2023-09-30T09:53:01.322769+0000","last_undegraded":"2023-09-30T09:52:16.323100+0000","last_fullsized":"2023-09-30T09:52:16.321702+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:37:30.531480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":1799,"num_write":112,"num_write_kb":1294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.297766+0000","last_change":"2023-09-30T09:50:41.394348+0000","last_active":"2023-09-30T09:53:00.297766+0000","last_peered":"2023-09-30T09:53:00.297766+0000","last_clean":"2023-09-30T09:53:00.297766+0000","last_became_active":"2023-09-30T09:50:41.394017+0000","last_became_peered":"2023-09-30T09:50:41.394017+0000","last_unstale":"2023-09-30T09:53:00.297766+0000","last_undegraded":"2023-09-30T09:53:00.297766+0000","last_fullsized":"2023-09-30T09:53:00.297766+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:37:30.133826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":474,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.306270+0000","last_change":"2023-09-30T09:51:57.162010+0000","last_active":"2023-09-30T09:53:00.306270+0000","last_peered":"2023-09-30T09:53:00.306270+0000","last_clean":"2023-09-30T09:53:00.306270+0000","last_became_active":"2023-09-30T09:51:57.161672+0000","last_became_peered":"2023-09-30T09:51:57.161672+0000","last_unstale":"2023-09-30T09:53:00.306270+0000","last_undegraded":"2023-09-30T09:53:00.306270+0000","last_fullsized":"2023-09-30T09:53:00.306270+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:28.744549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"616'10266","reported_seq":17627,"reported_epoch":616,"state":"active+recovery_wait","last_fresh":"2023-09-30T09:53:12.517454+0000","last_change":"2023-09-30T09:52:49.100747+0000","last_active":"2023-09-30T09:53:12.517454+0000","last_peered":"2023-09-30T09:53:12.517454+0000","last_clean":"2023-09-30T09:49:30.256457+0000","last_became_active":"2023-09-30T09:50:14.978894+0000","last_became_peered":"2023-09-30T09:50:14.978894+0000","last_unstale":"2023-09-30T09:53:12.517454+0000","last_undegraded":"2023-09-30T09:53:12.517454+0000","last_fullsized":"2023-09-30T09:53:12.517454+0000","mapping_epoch":560,"log_start":"557'4904","ondisk_log_start":"557'4904","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5362,"log_dups_size":0,"ondisk_log_size":5362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:58:29.871222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68550656,"num_objects":1046,"num_object_clones":0,"num_object_copies":3138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10266,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3627,"num_bytes_recovered":237699072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":["0","1","4","6"],"object_location_counts":[{"shards":"0,1,6","objects":1046}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'200","reported_seq":1198,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.312430+0000","last_change":"2023-09-30T09:50:42.387920+0000","last_active":"2023-09-30T09:53:00.312430+0000","last_peered":"2023-09-30T09:53:00.312430+0000","last_clean":"2023-09-30T09:53:00.312430+0000","last_became_active":"2023-09-30T09:50:42.387121+0000","last_became_peered":"2023-09-30T09:50:42.387121+0000","last_unstale":"2023-09-30T09:53:00.312430+0000","last_undegraded":"2023-09-30T09:53:00.312430+0000","last_fullsized":"2023-09-30T09:53:00.312430+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:34:05.192090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":575,"num_read_kb":813,"num_write":204,"num_write_kb":285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1037,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.323022+0000","last_change":"2023-09-30T09:51:59.878404+0000","last_active":"2023-09-30T09:53:00.323022+0000","last_peered":"2023-09-30T09:53:00.323022+0000","last_clean":"2023-09-30T09:53:00.323022+0000","last_became_active":"2023-09-30T09:51:59.878042+0000","last_became_peered":"2023-09-30T09:51:59.878042+0000","last_unstale":"2023-09-30T09:53:00.323022+0000","last_undegraded":"2023-09-30T09:53:00.323022+0000","last_fullsized":"2023-09-30T09:53:00.323022+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:53:00.431808+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":35,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305942+0000","last_change":"2023-09-30T09:51:57.166214+0000","last_active":"2023-09-30T09:53:00.305942+0000","last_peered":"2023-09-30T09:53:00.305942+0000","last_clean":"2023-09-30T09:53:00.305942+0000","last_became_active":"2023-09-30T09:51:57.165926+0000","last_became_peered":"2023-09-30T09:51:57.165926+0000","last_unstale":"2023-09-30T09:53:00.305942+0000","last_undegraded":"2023-09-30T09:53:00.305942+0000","last_fullsized":"2023-09-30T09:53:00.305942+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:19:29.596549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":472,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.321099+0000","last_change":"2023-09-30T09:52:15.339945+0000","last_active":"2023-09-30T09:53:00.321099+0000","last_peered":"2023-09-30T09:53:00.321099+0000","last_clean":"2023-09-30T09:53:00.321099+0000","last_became_active":"2023-09-30T09:52:15.339624+0000","last_became_peered":"2023-09-30T09:52:15.339624+0000","last_unstale":"2023-09-30T09:53:00.321099+0000","last_undegraded":"2023-09-30T09:53:00.321099+0000","last_fullsized":"2023-09-30T09:53:00.321099+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:17:03.885650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"616'10414","reported_seq":11252,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-30T09:53:12.512946+0000","last_change":"2023-09-30T09:51:59.899383+0000","last_active":"2023-09-30T09:53:12.512946+0000","last_peered":"2023-09-30T09:53:12.512946+0000","last_clean":"2023-09-30T09:53:12.512946+0000","last_became_active":"2023-09-30T09:51:59.876329+0000","last_became_peered":"2023-09-30T09:51:59.876329+0000","last_unstale":"2023-09-30T09:53:12.512946+0000","last_undegraded":"2023-09-30T09:53:12.512946+0000","last_fullsized":"2023-09-30T09:53:12.512946+0000","mapping_epoch":592,"log_start":"557'5102","ondisk_log_start":"557'5102","created":536,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5312,"log_dups_size":0,"ondisk_log_size":5312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:35:08.375551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68681728,"num_objects":1048,"num_object_clones":0,"num_object_copies":3144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10414,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":360,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":757,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.320395+0000","last_change":"2023-09-30T09:52:09.254399+0000","last_active":"2023-09-30T09:53:01.320395+0000","last_peered":"2023-09-30T09:53:01.320395+0000","last_clean":"2023-09-30T09:53:01.320395+0000","last_became_active":"2023-09-30T09:52:09.253999+0000","last_became_peered":"2023-09-30T09:52:09.253999+0000","last_unstale":"2023-09-30T09:53:01.320395+0000","last_undegraded":"2023-09-30T09:53:01.320395+0000","last_fullsized":"2023-09-30T09:53:01.320395+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:39.230227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.230227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:12.791078+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'110","reported_seq":1023,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322950+0000","last_change":"2023-09-30T09:40:37.605194+0000","last_active":"2023-09-30T09:53:00.322950+0000","last_peered":"2023-09-30T09:53:00.322950+0000","last_clean":"2023-09-30T09:53:00.322950+0000","last_became_active":"2023-09-30T09:40:37.604734+0000","last_became_peered":"2023-09-30T09:40:37.604734+0000","last_unstale":"2023-09-30T09:53:00.322950+0000","last_undegraded":"2023-09-30T09:53:00.322950+0000","last_fullsized":"2023-09-30T09:53:00.322950+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:13:09.916936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":908,"num_read_kb":1346,"num_write":68,"num_write_kb":4396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":240,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:02.533058+0000","last_change":"2023-09-30T09:51:57.155491+0000","last_active":"2023-09-30T09:53:02.533058+0000","last_peered":"2023-09-30T09:53:02.533058+0000","last_clean":"2023-09-30T09:53:02.533058+0000","last_became_active":"2023-09-30T09:51:57.153846+0000","last_became_peered":"2023-09-30T09:51:57.153846+0000","last_unstale":"2023-09-30T09:53:02.533058+0000","last_undegraded":"2023-09-30T09:53:02.533058+0000","last_fullsized":"2023-09-30T09:53:02.533058+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:00:00.681929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":118,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322523+0000","last_change":"2023-09-30T09:52:06.219826+0000","last_active":"2023-09-30T09:53:00.322523+0000","last_peered":"2023-09-30T09:53:00.322523+0000","last_clean":"2023-09-30T09:53:00.322523+0000","last_became_active":"2023-09-30T09:52:06.219645+0000","last_became_peered":"2023-09-30T09:52:06.219645+0000","last_unstale":"2023-09-30T09:53:00.322523+0000","last_undegraded":"2023-09-30T09:53:00.322523+0000","last_fullsized":"2023-09-30T09:53:00.322523+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:08.551862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.d","version":"615'10420","reported_seq":17912,"reported_epoch":615,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-30T09:53:09.196329+0000","last_change":"2023-09-30T09:52:15.367163+0000","last_active":"2023-09-30T09:53:09.196329+0000","last_peered":"2023-09-30T09:53:09.196329+0000","last_clean":"2023-09-30T09:49:30.256680+0000","last_became_active":"2023-09-30T09:52:07.487724+0000","last_became_peered":"2023-09-30T09:52:07.487724+0000","last_unstale":"2023-09-30T09:53:09.196329+0000","last_undegraded":"2023-09-30T09:52:15.364556+0000","last_fullsized":"2023-09-30T09:52:07.219110+0000","mapping_epoch":596,"log_start":"554'4107","ondisk_log_start":"554'4107","created":536,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:07:48.953677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78970904,"num_objects":1206,"num_object_clones":0,"num_object_copies":3618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4364,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1206,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10420,"num_write_kb":371969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3685,"num_bytes_recovered":241500160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":1206},{"shards":"4,6","objects":-921}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":820,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322304+0000","last_change":"2023-09-30T09:52:09.278067+0000","last_active":"2023-09-30T09:53:00.322304+0000","last_peered":"2023-09-30T09:53:00.322304+0000","last_clean":"2023-09-30T09:53:00.322304+0000","last_became_active":"2023-09-30T09:52:09.277739+0000","last_became_peered":"2023-09-30T09:52:09.277739+0000","last_unstale":"2023-09-30T09:53:00.322304+0000","last_undegraded":"2023-09-30T09:53:00.322304+0000","last_fullsized":"2023-09-30T09:53:00.322304+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:42.634023+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'147","reported_seq":1213,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322617+0000","last_change":"2023-09-30T09:52:09.264946+0000","last_active":"2023-09-30T09:53:00.322617+0000","last_peered":"2023-09-30T09:53:00.322617+0000","last_clean":"2023-09-30T09:53:00.322617+0000","last_became_active":"2023-09-30T09:52:09.262504+0000","last_became_peered":"2023-09-30T09:52:09.262504+0000","last_unstale":"2023-09-30T09:53:00.322617+0000","last_undegraded":"2023-09-30T09:53:00.322617+0000","last_fullsized":"2023-09-30T09:53:00.322617+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:10:13.204007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":1876,"num_write":113,"num_write_kb":1100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":70,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305338+0000","last_change":"2023-09-30T09:52:06.230958+0000","last_active":"2023-09-30T09:53:00.305338+0000","last_peered":"2023-09-30T09:53:00.305338+0000","last_clean":"2023-09-30T09:53:00.305338+0000","last_became_active":"2023-09-30T09:52:06.230575+0000","last_became_peered":"2023-09-30T09:52:06.230575+0000","last_unstale":"2023-09-30T09:53:00.305338+0000","last_undegraded":"2023-09-30T09:53:00.305338+0000","last_fullsized":"2023-09-30T09:53:00.305338+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:42:57.084463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":232,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:07.535670+0000","last_change":"2023-09-30T09:44:26.060339+0000","last_active":"2023-09-30T09:53:07.535670+0000","last_peered":"2023-09-30T09:53:07.535670+0000","last_clean":"2023-09-30T09:53:07.535670+0000","last_became_active":"2023-09-30T09:44:26.059952+0000","last_became_peered":"2023-09-30T09:44:26.059952+0000","last_unstale":"2023-09-30T09:53:07.535670+0000","last_undegraded":"2023-09-30T09:53:07.535670+0000","last_fullsized":"2023-09-30T09:53:07.535670+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:37.156772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"615'10533","reported_seq":23267,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:09.162867+0000","last_change":"2023-09-30T09:52:49.938644+0000","last_active":"2023-09-30T09:53:09.162867+0000","last_peered":"2023-09-30T09:53:09.162867+0000","last_clean":"2023-09-30T09:53:09.162867+0000","last_became_active":"2023-09-30T09:52:49.915482+0000","last_became_peered":"2023-09-30T09:52:49.915482+0000","last_unstale":"2023-09-30T09:53:09.162867+0000","last_undegraded":"2023-09-30T09:53:09.162867+0000","last_fullsized":"2023-09-30T09:53:09.162867+0000","mapping_epoch":611,"log_start":"554'4189","ondisk_log_start":"554'4189","created":536,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:02.286775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78839808,"num_objects":1203,"num_object_clones":0,"num_object_copies":3609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10533,"num_write_kb":375552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6290,"num_bytes_recovered":386727936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1229,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305481+0000","last_change":"2023-09-30T09:52:18.344243+0000","last_active":"2023-09-30T09:53:00.305481+0000","last_peered":"2023-09-30T09:53:00.305481+0000","last_clean":"2023-09-30T09:53:00.305481+0000","last_became_active":"2023-09-30T09:52:18.343870+0000","last_became_peered":"2023-09-30T09:52:18.343870+0000","last_unstale":"2023-09-30T09:53:00.305481+0000","last_undegraded":"2023-09-30T09:53:00.305481+0000","last_fullsized":"2023-09-30T09:53:00.305481+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:09:04.148163+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":646,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322589+0000","last_change":"2023-09-30T09:43:45.122311+0000","last_active":"2023-09-30T09:53:00.322589+0000","last_peered":"2023-09-30T09:53:00.322589+0000","last_clean":"2023-09-30T09:53:00.322589+0000","last_became_active":"2023-09-30T09:43:45.121935+0000","last_became_peered":"2023-09-30T09:43:45.121935+0000","last_unstale":"2023-09-30T09:53:00.322589+0000","last_undegraded":"2023-09-30T09:53:00.322589+0000","last_fullsized":"2023-09-30T09:53:00.322589+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:15.924871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'219","reported_seq":1342,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322524+0000","last_change":"2023-09-30T09:50:13.858344+0000","last_active":"2023-09-30T09:53:00.322524+0000","last_peered":"2023-09-30T09:53:00.322524+0000","last_clean":"2023-09-30T09:53:00.322524+0000","last_became_active":"2023-09-30T09:50:13.857963+0000","last_became_peered":"2023-09-30T09:50:13.857963+0000","last_unstale":"2023-09-30T09:53:00.322524+0000","last_undegraded":"2023-09-30T09:53:00.322524+0000","last_fullsized":"2023-09-30T09:53:00.322524+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":219,"log_dups_size":0,"ondisk_log_size":219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:21.850425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1869,"num_read_kb":3003,"num_write":150,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":660,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320668+0000","last_change":"2023-09-30T09:51:57.161323+0000","last_active":"2023-09-30T09:53:00.320668+0000","last_peered":"2023-09-30T09:53:00.320668+0000","last_clean":"2023-09-30T09:53:00.320668+0000","last_became_active":"2023-09-30T09:51:57.160615+0000","last_became_peered":"2023-09-30T09:51:57.160615+0000","last_unstale":"2023-09-30T09:53:00.320668+0000","last_undegraded":"2023-09-30T09:53:00.320668+0000","last_fullsized":"2023-09-30T09:53:00.320668+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:53:15.253650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"616'10493","reported_seq":18492,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-30T09:53:12.378934+0000","last_change":"2023-09-30T09:51:29.884588+0000","last_active":"2023-09-30T09:53:12.378934+0000","last_peered":"2023-09-30T09:53:12.378934+0000","last_clean":"2023-09-30T09:53:12.378934+0000","last_became_active":"2023-09-30T09:51:29.869941+0000","last_became_peered":"2023-09-30T09:51:29.869941+0000","last_unstale":"2023-09-30T09:53:12.378934+0000","last_undegraded":"2023-09-30T09:53:12.378934+0000","last_fullsized":"2023-09-30T09:53:12.378934+0000","mapping_epoch":583,"log_start":"557'4861","ondisk_log_start":"557'4861","created":536,"last_epoch_clean":584,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:48:39.911195+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.911195+0000","objects_scrubbed":0,"log_size":5632,"log_dups_size":0,"ondisk_log_size":5632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:34:20.008267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69926912,"num_objects":1067,"num_object_clones":0,"num_object_copies":3201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10493,"num_write_kb":369920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3740,"num_bytes_recovered":219152384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":1043,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.308374+0000","last_change":"2023-09-30T09:51:59.873401+0000","last_active":"2023-09-30T09:53:01.308374+0000","last_peered":"2023-09-30T09:53:01.308374+0000","last_clean":"2023-09-30T09:53:01.308374+0000","last_became_active":"2023-09-30T09:51:59.873066+0000","last_became_peered":"2023-09-30T09:51:59.873066+0000","last_unstale":"2023-09-30T09:53:01.308374+0000","last_undegraded":"2023-09-30T09:53:01.308374+0000","last_fullsized":"2023-09-30T09:53:01.308374+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:48:11.565720+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'91","reported_seq":920,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322724+0000","last_change":"2023-09-30T09:50:41.393850+0000","last_active":"2023-09-30T09:53:00.322724+0000","last_peered":"2023-09-30T09:53:00.322724+0000","last_clean":"2023-09-30T09:53:00.322724+0000","last_became_active":"2023-09-30T09:50:41.393481+0000","last_became_peered":"2023-09-30T09:50:41.393481+0000","last_unstale":"2023-09-30T09:53:00.322724+0000","last_undegraded":"2023-09-30T09:53:00.322724+0000","last_fullsized":"2023-09-30T09:53:00.322724+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:47:48.909491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":518,"num_read_kb":1739,"num_write":93,"num_write_kb":1207,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":195,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.322666+0000","last_change":"2023-09-30T09:51:57.187258+0000","last_active":"2023-09-30T09:53:00.322666+0000","last_peered":"2023-09-30T09:53:00.322666+0000","last_clean":"2023-09-30T09:53:00.322666+0000","last_became_active":"2023-09-30T09:51:57.158336+0000","last_became_peered":"2023-09-30T09:51:57.158336+0000","last_unstale":"2023-09-30T09:53:00.322666+0000","last_undegraded":"2023-09-30T09:53:00.322666+0000","last_fullsized":"2023-09-30T09:53:00.322666+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:10:59.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":593,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.307817+0000","last_change":"2023-09-30T09:52:06.382180+0000","last_active":"2023-09-30T09:53:00.307817+0000","last_peered":"2023-09-30T09:53:00.307817+0000","last_clean":"2023-09-30T09:53:00.307817+0000","last_became_active":"2023-09-30T09:52:06.381759+0000","last_became_peered":"2023-09-30T09:52:06.381759+0000","last_unstale":"2023-09-30T09:53:00.307817+0000","last_undegraded":"2023-09-30T09:53:00.307817+0000","last_fullsized":"2023-09-30T09:53:00.307817+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:35:44.180146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":974,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320745+0000","last_change":"2023-09-30T09:51:59.878763+0000","last_active":"2023-09-30T09:53:00.320745+0000","last_peered":"2023-09-30T09:53:00.320745+0000","last_clean":"2023-09-30T09:53:00.320745+0000","last_became_active":"2023-09-30T09:51:59.878357+0000","last_became_peered":"2023-09-30T09:51:59.878357+0000","last_unstale":"2023-09-30T09:53:00.320745+0000","last_undegraded":"2023-09-30T09:53:00.320745+0000","last_fullsized":"2023-09-30T09:53:00.320745+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:31:08.208858+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":366,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.317352+0000","last_change":"2023-09-30T09:51:57.164777+0000","last_active":"2023-09-30T09:53:00.317352+0000","last_peered":"2023-09-30T09:53:00.317352+0000","last_clean":"2023-09-30T09:53:00.317352+0000","last_became_active":"2023-09-30T09:51:57.164468+0000","last_became_peered":"2023-09-30T09:51:57.164468+0000","last_unstale":"2023-09-30T09:53:00.317352+0000","last_undegraded":"2023-09-30T09:53:00.317352+0000","last_fullsized":"2023-09-30T09:53:00.317352+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:14.012018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":35,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.321348+0000","last_change":"2023-09-30T09:51:57.164689+0000","last_active":"2023-09-30T09:53:00.321348+0000","last_peered":"2023-09-30T09:53:00.321348+0000","last_clean":"2023-09-30T09:53:00.321348+0000","last_became_active":"2023-09-30T09:51:57.164384+0000","last_became_peered":"2023-09-30T09:51:57.164384+0000","last_unstale":"2023-09-30T09:53:00.321348+0000","last_undegraded":"2023-09-30T09:53:00.321348+0000","last_fullsized":"2023-09-30T09:53:00.321348+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:47:39.321245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":761,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.326262+0000","last_change":"2023-09-30T09:50:13.849267+0000","last_active":"2023-09-30T09:53:00.326262+0000","last_peered":"2023-09-30T09:53:00.326262+0000","last_clean":"2023-09-30T09:53:00.326262+0000","last_became_active":"2023-09-30T09:50:13.848637+0000","last_became_peered":"2023-09-30T09:50:13.848637+0000","last_unstale":"2023-09-30T09:53:00.326262+0000","last_undegraded":"2023-09-30T09:53:00.326262+0000","last_fullsized":"2023-09-30T09:53:00.326262+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.674372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.674372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:06:24.775922+0000","scrub_duration":0.0075955099999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":543,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305527+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:53:00.305527+0000","last_peered":"2023-09-30T09:53:00.305527+0000","last_clean":"2023-09-30T09:53:00.305527+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:53:00.305527+0000","last_undegraded":"2023-09-30T09:53:00.305527+0000","last_fullsized":"2023-09-30T09:53:00.305527+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":29,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320461+0000","last_change":"2023-09-30T09:52:06.292310+0000","last_active":"2023-09-30T09:53:00.320461+0000","last_peered":"2023-09-30T09:53:00.320461+0000","last_clean":"2023-09-30T09:53:00.320461+0000","last_became_active":"2023-09-30T09:52:06.291977+0000","last_became_peered":"2023-09-30T09:52:06.291977+0000","last_unstale":"2023-09-30T09:53:00.320461+0000","last_undegraded":"2023-09-30T09:53:00.320461+0000","last_fullsized":"2023-09-30T09:53:00.320461+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:00.274025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":513,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.305816+0000","last_change":"2023-09-30T09:49:31.313229+0000","last_active":"2023-09-30T09:53:00.305816+0000","last_peered":"2023-09-30T09:53:00.305816+0000","last_clean":"2023-09-30T09:53:00.305816+0000","last_became_active":"2023-09-30T09:49:31.301137+0000","last_became_peered":"2023-09-30T09:49:31.301137+0000","last_unstale":"2023-09-30T09:53:00.305816+0000","last_undegraded":"2023-09-30T09:53:00.305816+0000","last_fullsized":"2023-09-30T09:53:00.305816+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:57.362379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":219,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.319585+0000","last_change":"2023-09-30T09:52:06.388361+0000","last_active":"2023-09-30T09:53:01.319585+0000","last_peered":"2023-09-30T09:53:01.319585+0000","last_clean":"2023-09-30T09:53:01.319585+0000","last_became_active":"2023-09-30T09:52:06.382251+0000","last_became_peered":"2023-09-30T09:52:06.382251+0000","last_unstale":"2023-09-30T09:53:01.319585+0000","last_undegraded":"2023-09-30T09:53:01.319585+0000","last_fullsized":"2023-09-30T09:53:01.319585+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:23:31.835279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":766,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.320747+0000","last_change":"2023-09-30T09:52:56.538817+0000","last_active":"2023-09-30T09:53:00.320747+0000","last_peered":"2023-09-30T09:53:00.320747+0000","last_clean":"2023-09-30T09:53:00.320747+0000","last_became_active":"2023-09-30T09:50:29.883457+0000","last_became_peered":"2023-09-30T09:50:29.883457+0000","last_unstale":"2023-09-30T09:53:00.320747+0000","last_undegraded":"2023-09-30T09:53:00.320747+0000","last_fullsized":"2023-09-30T09:53:00.320747+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:14.071789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:14.071789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":88,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-30T09:53:01.307405+0000","last_change":"2023-09-30T09:52:06.286929+0000","last_active":"2023-09-30T09:53:01.307405+0000","last_peered":"2023-09-30T09:53:01.307405+0000","last_clean":"2023-09-30T09:53:01.307405+0000","last_became_active":"2023-09-30T09:52:06.286569+0000","last_became_peered":"2023-09-30T09:52:06.286569+0000","last_unstale":"2023-09-30T09:53:01.307405+0000","last_undegraded":"2023-09-30T09:53:01.307405+0000","last_fullsized":"2023-09-30T09:53:01.307405+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:29.837107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":50,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.298059+0000","last_change":"2023-09-30T09:50:41.404832+0000","last_active":"2023-09-30T09:53:00.298059+0000","last_peered":"2023-09-30T09:53:00.298059+0000","last_clean":"2023-09-30T09:53:00.298059+0000","last_became_active":"2023-09-30T09:50:41.404445+0000","last_became_peered":"2023-09-30T09:50:41.404445+0000","last_unstale":"2023-09-30T09:53:00.298059+0000","last_undegraded":"2023-09-30T09:53:00.298059+0000","last_fullsized":"2023-09-30T09:53:00.298059+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:09:27.041434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":777,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.307358+0000","last_change":"2023-09-30T09:52:09.241112+0000","last_active":"2023-09-30T09:53:00.307358+0000","last_peered":"2023-09-30T09:53:00.307358+0000","last_clean":"2023-09-30T09:53:00.307358+0000","last_became_active":"2023-09-30T09:52:09.240703+0000","last_became_peered":"2023-09-30T09:52:09.240703+0000","last_unstale":"2023-09-30T09:53:00.307358+0000","last_undegraded":"2023-09-30T09:53:00.307358+0000","last_fullsized":"2023-09-30T09:53:00.307358+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:40:40.400090+0000","scrub_duration":0.004695744,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":314,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.306197+0000","last_change":"2023-09-30T09:52:06.308608+0000","last_active":"2023-09-30T09:53:00.306197+0000","last_peered":"2023-09-30T09:53:00.306197+0000","last_clean":"2023-09-30T09:53:00.306197+0000","last_became_active":"2023-09-30T09:52:06.287432+0000","last_became_peered":"2023-09-30T09:52:06.287432+0000","last_unstale":"2023-09-30T09:53:00.306197+0000","last_undegraded":"2023-09-30T09:53:00.306197+0000","last_fullsized":"2023-09-30T09:53:00.306197+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:30:09.888951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":659,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-30T09:53:00.316650+0000","last_change":"2023-09-30T09:50:26.905427+0000","last_active":"2023-09-30T09:53:00.316650+0000","last_peered":"2023-09-30T09:53:00.316650+0000","last_clean":"2023-09-30T09:53:00.316650+0000","last_became_active":"2023-09-30T09:50:26.905100+0000","last_became_peered":"2023-09-30T09:50:26.905100+0000","last_unstale":"2023-09-30T09:53:00.316650+0000","last_undegraded":"2023-09-30T09:53:00.316650+0000","last_fullsized":"2023-09-30T09:53:00.316650+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:38.036208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1245839384,"num_objects":19011,"num_object_clones":0,"num_object_copies":57033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5580,"num_objects_misplaced":1703,"num_objects_unfound":0,"num_objects_dirty":19011,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167785,"num_write_kb":5977409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39577,"num_bytes_recovered":2398814208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5486227456,"data_stored":5486215240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94296,"ondisk_log_size":94296,"up":48,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":13644,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":599313,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":709030,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234497,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1048776,"kb_used_data":421504,"kb_used_omap":0,"kb_used_meta":627072,"kb_avail":92696376,"statfs":{"total":95995035648,"available":94921089024,"internally_reserved":0,"allocated":431620096,"data_stored":429576376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":642121728},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.5209999999999999,"15min":5.2839999999999998},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.22900000000000001},"max":{"1min":14.641999999999999,"5min":45.895000000000003,"15min":215.416},"last":2.226},{"interface":"front","average":{"1min":0.80800000000000005,"5min":2.661,"15min":5.3259999999999996},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.1799999999999997,"5min":46.451000000000001,"15min":215.041},"last":6.2960000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:52:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.823,"15min":3.77},"min":{"1min":0.29299999999999998,"5min":0.26700000000000002,"15min":0.25},"max":{"1min":6.2809999999999997,"5min":10.749000000000001,"15min":157.578},"last":1.98},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.946,"15min":3.7890000000000001},"min":{"1min":0.32700000000000001,"5min":0.29599999999999999,"15min":0.29199999999999998},"max":{"1min":4.4260000000000002,"5min":12.013999999999999,"15min":157.589},"last":7.2919999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:52:52 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":4.6239999999999997,"15min":4.6310000000000002},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.467000000000001,"5min":187.74600000000001,"15min":187.74600000000001},"last":7.3529999999999998},{"interface":"front","average":{"1min":1.873,"5min":4.6589999999999998,"15min":4.601},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0720000000000001,"5min":187.583,"15min":187.583},"last":6.2199999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:52:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.867,"15min":4.9790000000000001},"min":{"1min":0.34799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.3520000000000003,"5min":13.303000000000001,"15min":248.691},"last":2.0139999999999998},{"interface":"front","average":{"1min":1.0629999999999999,"5min":2.2730000000000001,"15min":5.2169999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.21,"5min":13.257,"15min":249.36500000000001},"last":6.2599999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:52:52 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":3.9649999999999999,"15min":2.6709999999999998},"min":{"1min":0.36199999999999999,"5min":0.27200000000000002,"15min":0.23799999999999999},"max":{"1min":5.125,"5min":238.511,"15min":238.511},"last":2.7250000000000001},{"interface":"front","average":{"1min":2.327,"5min":4.4349999999999996,"15min":2.7429999999999999},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.252},"max":{"1min":9.6709999999999994,"5min":238.387,"15min":238.387},"last":9.5600000000000005}]},{"osd":5,"last update":"Sat Sep 30 09:52:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":4.165,"15min":7.9400000000000004},"min":{"1min":0.39000000000000001,"5min":0.32700000000000001,"15min":0.28399999999999997},"max":{"1min":9.3659999999999997,"5min":187.374,"15min":197.75700000000001},"last":9.6029999999999998},{"interface":"front","average":{"1min":2.0590000000000002,"5min":3.8500000000000001,"15min":7.8220000000000001},"min":{"1min":0.32400000000000001,"5min":0.309,"15min":0.309},"max":{"1min":11.863,"5min":187.416,"15min":197.63300000000001},"last":7.3280000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:52:52 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":1.631,"15min":2.3279999999999998},"min":{"1min":0.38600000000000001,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":9.4730000000000008,"5min":9.4730000000000008,"15min":206.244},"last":2.1930000000000001},{"interface":"front","average":{"1min":1.946,"5min":1.702,"15min":2.2930000000000001},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.29599999999999999},"max":{"1min":7.0339999999999998,"5min":12.058,"15min":202.78800000000001},"last":6.3220000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496135,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1197056,"kb_used_data":549560,"kb_used_omap":0,"kb_used_meta":647488,"kb_avail":92548096,"statfs":{"total":95995035648,"available":94769250304,"internally_reserved":0,"allocated":562749440,"data_stored":560713265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":663027712},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":5.9130000000000003,"15min":6.5599999999999996},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.3609999999999998,"5min":192.511,"15min":192.511},"last":3.1739999999999999},{"interface":"front","average":{"1min":0.87,"5min":6.2089999999999996,"15min":6.6340000000000003},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.246},"max":{"1min":4.3630000000000004,"5min":192.346,"15min":192.346},"last":2.2280000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:53:02 2023","interfaces":[{"interface":"back","average":{"1min":3.323,"5min":3.2010000000000001,"15min":2.218},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.954000000000001,"5min":103.038,"15min":103.038},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.1549999999999998,"5min":3.1869999999999998,"15min":2.3239999999999998},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.7960000000000003,"5min":103.05,"15min":103.05},"last":0.39900000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:53:02 2023","interfaces":[{"interface":"back","average":{"1min":2.9910000000000001,"5min":3.7029999999999998,"15min":4.569},"min":{"1min":0.46999999999999997,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":12.044,"5min":109.88500000000001,"15min":207.02500000000001},"last":0.504},{"interface":"front","average":{"1min":2.8980000000000001,"5min":3.3900000000000001,"15min":4.5119999999999996},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30499999999999999},"max":{"1min":10.923999999999999,"5min":105.81699999999999,"15min":207.114},"last":0.42499999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":2.706,"15min":2.7869999999999999},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":6.3330000000000002,"5min":106.402,"15min":107.11799999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.359,"5min":3.177,"15min":2.9390000000000001},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.31900000000000001},"max":{"1min":4.4290000000000003,"5min":106.34699999999999,"15min":106.982},"last":0.48999999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":2.286,"15min":1.6739999999999999},"min":{"1min":0.28899999999999998,"5min":0.28100000000000003,"15min":0.24399999999999999},"max":{"1min":6.976,"5min":30.568000000000001,"15min":63.798000000000002},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.268,"5min":1.919,"15min":1.532},"min":{"1min":0.32200000000000001,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":7.5330000000000004,"5min":29.27,"15min":63.133000000000003},"last":0.377}]},{"osd":5,"last update":"Sat Sep 30 09:52:48 2023","interfaces":[{"interface":"back","average":{"1min":3.198,"5min":2.1779999999999999,"15min":2.0169999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.27100000000000002},"max":{"1min":11.972,"5min":19.212,"15min":19.212},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.1509999999999998,"5min":1.6619999999999999,"15min":2.1589999999999998},"min":{"1min":0.40799999999999997,"5min":0.33400000000000002,"15min":0.26900000000000002},"max":{"1min":7.4279999999999999,"5min":12.731,"15min":16.443000000000001},"last":0.52800000000000002}]},{"osd":7,"last update":"Sat Sep 30 09:53:02 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":2.2919999999999998,"15min":1.4750000000000001},"min":{"1min":0.41599999999999998,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":12.202,"5min":39.868000000000002,"15min":39.868000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":3.343,"5min":1.901,"15min":1.3069999999999999},"min":{"1min":0.436,"5min":0.38800000000000001,"15min":0.27500000000000002},"max":{"1min":10.843,"5min":12.625,"15min":12.625},"last":0.56100000000000005}]}]},{"osd":1,"up_from":301,"seq":1292785156333,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1018940,"kb_used_data":287476,"kb_used_omap":9,"kb_used_meta":731446,"kb_avail":92726212,"statfs":{"total":95995035648,"available":94951641088,"internally_reserved":0,"allocated":294375424,"data_stored":290858749,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":749000722},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:53:07 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":10.128,"15min":5.7249999999999996},"min":{"1min":0.216,"5min":0.20399999999999999,"15min":0.17100000000000001},"max":{"1min":21.786000000000001,"5min":191.62700000000001,"15min":191.62700000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":3.2429999999999999,"5min":10.32,"15min":5.7240000000000002},"min":{"1min":0.222,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":21.821999999999999,"5min":191.58799999999999,"15min":191.58799999999999},"last":0.222}]},{"osd":2,"last update":"Sat Sep 30 09:53:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":4.1660000000000004,"15min":3.2869999999999999},"min":{"1min":0.24099999999999999,"5min":0.21099999999999999,"15min":0.184},"max":{"1min":16.443000000000001,"5min":189.14500000000001,"15min":189.14500000000001},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.8029999999999999,"5min":4.0979999999999999,"15min":3.2109999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":6.2560000000000002,"5min":189.333,"15min":189.333},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:53:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":2.1739999999999999,"15min":3.7149999999999999},"min":{"1min":0.27900000000000003,"5min":0.23799999999999999,"15min":0.23400000000000001},"max":{"1min":8.0619999999999994,"5min":31.151,"15min":199.41},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.548,"5min":1.7569999999999999,"15min":3.4470000000000001},"min":{"1min":0.245,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.7560000000000002,"5min":31.727,"15min":199.58099999999999},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:53:07 2023","interfaces":[{"interface":"back","average":{"1min":3.77,"5min":1.762,"15min":2.254},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26800000000000002},"max":{"1min":47.332000000000001,"5min":47.332000000000001,"15min":142.77099999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":5.5229999999999997,"5min":2.714,"15min":2.6160000000000001},"min":{"1min":0.38800000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":47.606999999999999,"5min":47.606999999999999,"15min":144.898},"last":0.90200000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:52:44 2023","interfaces":[{"interface":"back","average":{"1min":9.7449999999999992,"5min":3.8620000000000001,"15min":2.6669999999999998},"min":{"1min":0.38500000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":165.88300000000001,"5min":165.88300000000001,"15min":165.88300000000001},"last":0.93700000000000006},{"interface":"front","average":{"1min":9.7829999999999995,"5min":3.2690000000000001,"15min":2.0110000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":165.863,"5min":165.863,"15min":165.863},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:53:07 2023","interfaces":[{"interface":"back","average":{"1min":7.0659999999999998,"5min":2.7919999999999998,"15min":2.4089999999999998},"min":{"1min":0.33800000000000002,"5min":0.33700000000000002,"15min":0.29899999999999999},"max":{"1min":97.281999999999996,"5min":97.281999999999996,"15min":97.281999999999996},"last":1.5029999999999999},{"interface":"front","average":{"1min":7.4649999999999999,"5min":3.0030000000000001,"15min":2.4830000000000001},"min":{"1min":0.47499999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":97.242000000000004,"5min":97.242000000000004,"15min":97.242000000000004},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:53:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.548,"15min":1.587},"min":{"1min":0.38500000000000001,"5min":0.36899999999999999,"15min":0.28299999999999997},"max":{"1min":6.452,"5min":9.8390000000000004,"15min":15.587},"last":1.7529999999999999},{"interface":"front","average":{"1min":2.0419999999999998,"5min":1.556,"15min":1.6060000000000001},"min":{"1min":0.503,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":6.4219999999999997,"5min":9.8109999999999999,"15min":15.226000000000001},"last":0.50600000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926232,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1895996,"kb_used_data":1045936,"kb_used_omap":5,"kb_used_meta":850042,"kb_avail":91849156,"statfs":{"total":95995035648,"available":94053535744,"internally_reserved":0,"allocated":1071038464,"data_stored":1067302126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":870443261},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:53:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":1.421,"15min":1.6140000000000001},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.191},"max":{"1min":5.1539999999999999,"5min":11.641,"15min":57.482999999999997},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.369,"15min":1.984},"min":{"1min":0.25700000000000001,"5min":0.22500000000000001,"15min":0.19800000000000001},"max":{"1min":8.9770000000000003,"5min":23.503,"15min":57.140999999999998},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:52:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":3.7490000000000001,"15min":4.0300000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20799999999999999},"max":{"1min":9.0470000000000006,"5min":173.185,"15min":173.185},"last":1.9059999999999999},{"interface":"front","average":{"1min":1.704,"5min":3.556,"15min":4.0789999999999997},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.249},"max":{"1min":8.9510000000000005,"5min":173.22499999999999,"15min":173.22499999999999},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:52:20 2023","interfaces":[{"interface":"back","average":{"1min":3.8599999999999999,"5min":2.7490000000000001,"15min":3.496},"min":{"1min":0.37,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":53.192999999999998,"5min":53.192999999999998,"15min":125.562},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.6619999999999999,"5min":2.4140000000000001,"15min":2.9670000000000001},"min":{"1min":0.371,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":53.207000000000001,"5min":53.207000000000001,"15min":117.295},"last":0.376}]},{"osd":4,"last update":"Sat Sep 30 09:52:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":2.8279999999999998,"15min":2.4009999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3149999999999995,"5min":67.945999999999998,"15min":72.403999999999996},"last":1.4159999999999999},{"interface":"front","average":{"1min":1.363,"5min":2.835,"15min":2.484},"min":{"1min":0.36199999999999999,"5min":0.35999999999999999,"15min":0.29699999999999999},"max":{"1min":10.215,"5min":67.792000000000002,"15min":72.162999999999997},"last":0.51900000000000002}]},{"osd":5,"last update":"Sat Sep 30 09:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":2.6309999999999998,"15min":1.9319999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.492,"5min":71.376000000000005,"15min":71.376000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.369,"5min":2.5720000000000001,"15min":1.6120000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30499999999999999},"max":{"1min":4.4459999999999997,"5min":71.706000000000003,"15min":71.706000000000003},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":4.0880000000000001,"15min":2.4889999999999999},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.33200000000000002},"max":{"1min":2.1259999999999999,"5min":240.946,"15min":240.946},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":4.5540000000000003,"15min":2.9399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.34999999999999998},"max":{"1min":3.8479999999999999,"5min":240.977,"15min":240.977},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:52:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":3.359,"15min":2.1960000000000002},"min":{"1min":0.43099999999999999,"5min":0.41499999999999998,"15min":0.36099999999999999},"max":{"1min":9.1460000000000008,"5min":146.828,"15min":146.828},"last":2.8250000000000002},{"interface":"front","average":{"1min":0.93000000000000005,"5min":3.843,"15min":2.694},"min":{"1min":0.436,"5min":0.41599999999999998,"15min":0.33100000000000002},"max":{"1min":3.8889999999999998,"5min":146.876,"15min":146.876},"last":0.61099999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599170,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2059580,"kb_used_data":1140724,"kb_used_omap":5,"kb_used_meta":918842,"kb_avail":91685572,"statfs":{"total":95995035648,"available":93886025728,"internally_reserved":0,"allocated":1168101376,"data_stored":1164645087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":940894861},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:53:05 2023","interfaces":[{"interface":"back","average":{"1min":14.224,"5min":5.1580000000000004,"15min":4.8159999999999998},"min":{"1min":0.19800000000000001,"5min":0.188,"15min":0.188},"max":{"1min":217.00999999999999,"5min":217.00999999999999,"15min":217.00999999999999},"last":1.6930000000000001},{"interface":"front","average":{"1min":14.706,"5min":5.5940000000000003,"15min":5.0570000000000004},"min":{"1min":0.29299999999999998,"5min":0.222,"15min":0.217},"max":{"1min":217.09,"5min":217.09,"15min":217.09},"last":2.984}]},{"osd":1,"last update":"Sat Sep 30 09:53:05 2023","interfaces":[{"interface":"back","average":{"1min":6.4130000000000003,"5min":3.3260000000000001,"15min":3.8410000000000002},"min":{"1min":0.23699999999999999,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":80.828999999999994,"5min":99.638999999999996,"15min":202.369},"last":0.38},{"interface":"front","average":{"1min":7.673,"5min":3.613,"15min":4.0030000000000001},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.21099999999999999},"max":{"1min":83.114999999999995,"5min":102.131,"15min":202.54499999999999},"last":1.8029999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:52:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":3.4159999999999999,"15min":3.7770000000000001},"min":{"1min":0.28399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.2270000000000003,"5min":202.59,"15min":202.59},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.88800000000000001,"5min":3.2349999999999999,"15min":3.6949999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.141,"5min":202.34200000000001,"15min":202.34200000000001},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:53:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":1.671,"15min":3.141},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.96,"5min":14.637,"15min":247.655},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.6379999999999999,"5min":1.171,"15min":3.0840000000000001},"min":{"1min":0.39500000000000002,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":5.7770000000000001,"5min":5.7990000000000004,"15min":248.012},"last":0.63200000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:52:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":3.835,"15min":2.9889999999999999},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":10.762,"5min":202.65600000000001,"15min":243.86500000000001},"last":2.9319999999999999},{"interface":"front","average":{"1min":3.0289999999999999,"5min":4.1210000000000004,"15min":3.2770000000000001},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":15.164999999999999,"5min":202.86799999999999,"15min":244.02199999999999},"last":0.68000000000000005}]},{"osd":6,"last update":"Sat Sep 30 09:53:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7989999999999999,"5min":2.1320000000000001,"15min":2.3660000000000001},"min":{"1min":0.41999999999999998,"5min":0.32800000000000001,"15min":0.29399999999999998},"max":{"1min":6.5300000000000002,"5min":46.732999999999997,"15min":202.70599999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.794,"5min":2.536,"15min":2.6659999999999999},"min":{"1min":0.47099999999999997,"5min":0.38800000000000001,"15min":0.34100000000000003},"max":{"1min":6.8470000000000004,"5min":46.68,"15min":202.916},"last":0.754}]},{"osd":7,"last update":"Sat Sep 30 09:53:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7069999999999999,"5min":1.641,"15min":3.5169999999999999},"min":{"1min":0.44500000000000001,"5min":0.38500000000000001,"15min":0.32000000000000001},"max":{"1min":9.1950000000000003,"5min":9.1950000000000003,"15min":244.911},"last":0.69899999999999995},{"interface":"front","average":{"1min":3.7090000000000001,"5min":2.294,"15min":3.879},"min":{"1min":0.496,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":10.782,"5min":10.782,"15min":244.66},"last":0.71899999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435646,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487140,"kb_used_data":498204,"kb_used_omap":2,"kb_used_meta":988925,"kb_avail":92258012,"statfs":{"total":95995035648,"available":94472204288,"internally_reserved":0,"allocated":510160896,"data_stored":506553348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":1012659559},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":8.3339999999999996,"5min":4.6420000000000003,"15min":4.867},"min":{"1min":0.22500000000000001,"5min":0.191,"15min":0.183},"max":{"1min":103.03,"5min":152.429,"15min":205.73099999999999},"last":3.2229999999999999},{"interface":"front","average":{"1min":7.8680000000000003,"5min":4.8970000000000002,"15min":4.9100000000000001},"min":{"1min":0.27400000000000002,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":103.39700000000001,"5min":150.464,"15min":205.495},"last":4.3710000000000004}]},{"osd":1,"last update":"Sat Sep 30 09:53:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.982,"15min":2.968},"min":{"1min":0.26000000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.5739999999999998,"5min":15.784000000000001,"15min":170.512},"last":6.3440000000000003},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.633,"15min":2.9359999999999999},"min":{"1min":0.24399999999999999,"5min":0.23400000000000001,"15min":0.20699999999999999},"max":{"1min":6.6280000000000001,"5min":11.497999999999999,"15min":170.535},"last":8.7309999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":3,"15min":5.2729999999999997},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.0720000000000001,"5min":147.87,"15min":206.69},"last":8.7040000000000006},{"interface":"front","average":{"1min":0.86399999999999999,"5min":3.4100000000000001,"15min":5.2709999999999999},"min":{"1min":0.253,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":3.407,"5min":148.04900000000001,"15min":209.22800000000001},"last":3.1720000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":3.5329999999999999,"15min":2.3690000000000002},"min":{"1min":0.32400000000000001,"5min":0.29199999999999998,"15min":0.26800000000000002},"max":{"1min":3.4369999999999998,"5min":172.298,"15min":172.298},"last":3.0779999999999998},{"interface":"front","average":{"1min":1.27,"5min":3.738,"15min":2.516},"min":{"1min":0.35299999999999998,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":3.8300000000000001,"5min":172.45099999999999,"15min":172.45099999999999},"last":4.2999999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:52:36 2023","interfaces":[{"interface":"back","average":{"1min":6.0119999999999996,"5min":2.5670000000000002,"15min":2.0110000000000001},"min":{"1min":0.39600000000000002,"5min":0.33700000000000002,"15min":0.29199999999999998},"max":{"1min":61.426000000000002,"5min":61.426000000000002,"15min":61.426000000000002},"last":8.6319999999999997},{"interface":"front","average":{"1min":6.0979999999999999,"5min":3.1520000000000001,"15min":1.8799999999999999},"min":{"1min":0.46700000000000003,"5min":0.36199999999999999,"15min":0.311},"max":{"1min":61.469000000000001,"5min":61.469000000000001,"15min":61.469000000000001},"last":4.3419999999999996}]},{"osd":6,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":6.5800000000000001,"5min":3.0329999999999999,"15min":1.998},"min":{"1min":0.36799999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":75.781000000000006,"5min":75.781000000000006,"15min":75.781000000000006},"last":3.198},{"interface":"front","average":{"1min":6.2279999999999998,"5min":3.3540000000000001,"15min":2.1619999999999999},"min":{"1min":0.43099999999999999,"5min":0.35999999999999999,"15min":0.33400000000000002},"max":{"1min":75.754000000000005,"5min":75.754000000000005,"15min":75.754000000000005},"last":4.3959999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:52:23 2023","interfaces":[{"interface":"back","average":{"1min":7.9029999999999996,"5min":3.2629999999999999,"15min":2.7170000000000001},"min":{"1min":0.42099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":90.855999999999995,"5min":90.855999999999995,"15min":213.548},"last":8.6760000000000002},{"interface":"front","average":{"1min":7.0679999999999996,"5min":3.2570000000000001,"15min":2.6240000000000001},"min":{"1min":0.42599999999999999,"5min":0.40899999999999997,"15min":0.36199999999999999},"max":{"1min":91.108000000000004,"5min":91.108000000000004,"15min":213.583},"last":3.1240000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019408,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1675516,"kb_used_data":1076404,"kb_used_omap":0,"kb_used_meta":599104,"kb_avail":92069636,"statfs":{"total":95995035648,"available":94279307264,"internally_reserved":0,"allocated":1102237696,"data_stored":1100205144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":613482496},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:13 2023","interfaces":[{"interface":"back","average":{"1min":3.8610000000000002,"5min":4.4370000000000003,"15min":3.5630000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.21199999999999999},"max":{"1min":60.945,"5min":107.43000000000001,"15min":107.43000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":4.0369999999999999,"5min":4.7949999999999999,"15min":3.7240000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":60.695999999999998,"5min":107.373,"15min":107.373},"last":0.20799999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.9350000000000001,"15min":2.5600000000000001},"min":{"1min":0.30499999999999999,"5min":0.27500000000000002,"15min":0.25},"max":{"1min":12.385,"5min":108.551,"15min":108.551},"last":0.318},{"interface":"front","average":{"1min":1.3120000000000001,"5min":3.0619999999999998,"15min":2.6499999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.42,"5min":108.884,"15min":108.884},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:52:57 2023","interfaces":[{"interface":"back","average":{"1min":3.7000000000000002,"5min":3.9740000000000002,"15min":4.1980000000000004},"min":{"1min":0.373,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":12.676,"5min":139.53899999999999,"15min":142.88999999999999},"last":6.7199999999999998},{"interface":"front","average":{"1min":2.339,"5min":3.5840000000000001,"15min":4.1020000000000003},"min":{"1min":0.39000000000000001,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":14.071,"5min":139.56399999999999,"15min":142.90700000000001},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":3.6459999999999999,"15min":2.407},"min":{"1min":0.376,"5min":0.376,"15min":0.32700000000000001},"max":{"1min":4.2290000000000001,"5min":105.142,"15min":105.142},"last":1.206},{"interface":"front","average":{"1min":1.0409999999999999,"5min":3.4529999999999998,"15min":2.343},"min":{"1min":0.377,"5min":0.377,"15min":0.29299999999999998},"max":{"1min":4.2190000000000003,"5min":105.157,"15min":105.157},"last":1.0760000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:52:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":2.1339999999999999,"15min":6.6520000000000001},"min":{"1min":0.26400000000000001,"5min":0.249,"15min":0.249},"max":{"1min":8.4329999999999998,"5min":59.530999999999999,"15min":140.11099999999999},"last":6.9989999999999997},{"interface":"front","average":{"1min":2.6789999999999998,"5min":2.1000000000000001,"15min":6.6280000000000001},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.9369999999999994,"5min":60.012,"15min":140.065},"last":6.1840000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:52:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":1.585,"15min":1.7030000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":13.978,"5min":13.978,"15min":182.10300000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.671,"15min":1.7529999999999999},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":8.5120000000000005,"5min":11.523999999999999,"15min":181.76300000000001},"last":1.583}]},{"osd":7,"last update":"Sat Sep 30 09:52:57 2023","interfaces":[{"interface":"back","average":{"1min":4.1769999999999996,"5min":2.2080000000000002,"15min":2.4340000000000002},"min":{"1min":0.48699999999999999,"5min":0.31900000000000001,"15min":0.27600000000000002},"max":{"1min":12.715999999999999,"5min":12.715999999999999,"15min":161.72999999999999},"last":6.2149999999999999},{"interface":"front","average":{"1min":2.2559999999999998,"5min":1.466,"15min":2.1960000000000002},"min":{"1min":0.36699999999999999,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":14.018000000000001,"5min":14.018000000000001,"15min":162.60900000000001},"last":0.627}]}]},{"osd":5,"up_from":489,"seq":2100239007850,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":721076,"kb_used_data":486508,"kb_used_omap":6,"kb_used_meta":234489,"kb_avail":93024076,"statfs":{"total":95995035648,"available":95256653824,"internally_reserved":0,"allocated":498184192,"data_stored":496156334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":240116831},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":9.0609999999999999,"5min":7.1820000000000004,"15min":17.100000000000001},"min":{"1min":0.24399999999999999,"5min":0.224,"15min":0.22},"max":{"1min":137.846,"5min":137.846,"15min":281.10599999999999},"last":3.3180000000000001},{"interface":"front","average":{"1min":9.3719999999999999,"5min":6.226,"15min":16.948},"min":{"1min":0.27800000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":137.035,"5min":137.035,"15min":282.07400000000001},"last":3.581}]},{"osd":1,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":2.8109999999999999,"15min":9.5939999999999994},"min":{"1min":0.30199999999999999,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":11.234999999999999,"5min":129.958,"15min":273.48200000000003},"last":2.173},{"interface":"front","average":{"1min":1.006,"5min":2.5449999999999999,"15min":9.0169999999999995},"min":{"1min":0.28899999999999998,"5min":0.255,"15min":0.255},"max":{"1min":8.0820000000000007,"5min":130.05099999999999,"15min":273.50200000000001},"last":3.3420000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":4.6269999999999998,"15min":3.2970000000000002},"min":{"1min":0.28899999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.625999999999999,"5min":194.14400000000001,"15min":194.14400000000001},"last":5.3109999999999999},{"interface":"front","average":{"1min":2.1299999999999999,"5min":3.3250000000000002,"15min":3.1699999999999999},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.337999999999999,"5min":198.083,"15min":198.083},"last":2.242}]},{"osd":3,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":4.8150000000000004,"5min":4.9260000000000002,"15min":10.217000000000001},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.30299999999999999},"max":{"1min":41.872,"5min":130.02199999999999,"15min":274.01799999999997},"last":6.3109999999999999},{"interface":"front","average":{"1min":4.0049999999999999,"5min":4.7030000000000003,"15min":3.194},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":41.829000000000001,"5min":130.07400000000001,"15min":130.07400000000001},"last":4.3810000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":1.738,"5min":4.6139999999999999,"15min":2.1890000000000001},"min":{"1min":0.29599999999999999,"5min":0.26700000000000002,"15min":0.26000000000000001},"max":{"1min":13.601000000000001,"5min":132.94399999999999,"15min":132.94399999999999},"last":3.302},{"interface":"front","average":{"1min":2.2770000000000001,"5min":4.7789999999999999,"15min":2.3919999999999999},"min":{"1min":0.34899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":14.223000000000001,"5min":136.083,"15min":136.083},"last":1.579}]},{"osd":6,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":4.319,"5min":3.5800000000000001,"15min":2.2469999999999999},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":66.123000000000005,"5min":130.09800000000001,"15min":130.09800000000001},"last":3.2869999999999999},{"interface":"front","average":{"1min":4.9109999999999996,"5min":3.677,"15min":2.157},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.316},"max":{"1min":66.102999999999994,"5min":130.03399999999999,"15min":130.03399999999999},"last":3.3540000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":1.9099999999999999,"15min":1.534},"min":{"1min":0.33800000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":32.578000000000003,"5min":32.578000000000003,"15min":32.578000000000003},"last":3.2559999999999998},{"interface":"front","average":{"1min":3.2410000000000001,"5min":2.0819999999999999,"15min":1.3560000000000001},"min":{"1min":0.39700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":32.420999999999999,"5min":32.420999999999999,"15min":32.420999999999999},"last":3.335}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1040388096,"data_stored":1040384024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":278724608,"data_stored":278724608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1142423552,"data_stored":1142423552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":499318784,"data_stored":499318784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1092227072,"data_stored":1092223000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":476971008,"data_stored":476971008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":547295232,"data_stored":547291160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":408879104,"data_stored":408879104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:53:16.323 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 16 2023-09-30T09:53:17.079 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:17.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:17.191 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:17.292 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:17.312 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:17.374 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:17.517 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:17.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:17 smithi019 ceph-mon[224043]: pgmap v1645: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1012 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 5364/48438 objects degraded (11.074%); 1374/48438 objects misplaced (2.837%); 3.8 MiB/s, 107 objects/s recovering 2023-09-30T09:53:17.521 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:17 smithi019 ceph-mon[216815]: pgmap v1645: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1012 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 5364/48438 objects degraded (11.074%); 1374/48438 objects misplaced (2.837%); 3.8 MiB/s, 107 objects/s recovering 2023-09-30T09:53:17.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:17.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:17.637 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:17.723 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:17.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:17 smithi167 ceph-mon[192880]: pgmap v1645: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1012 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 5364/48438 objects degraded (11.074%); 1374/48438 objects misplaced (2.837%); 3.8 MiB/s, 107 objects/s recovering 2023-09-30T09:53:17.837 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:17.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:17.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:17.851 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:53:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:53:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:53:17.852 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:53:17.852 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:53:17.945 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:18.004 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:18.102 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:18.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:18.215 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:18.319 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:18.447 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:18.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:18.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:18.564 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:53:18.626 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:18.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:18.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:18.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:19.092 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:53:19.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:19 smithi019 ceph-mon[216815]: pgmap v1646: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 958 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 5364/45828 objects degraded (11.705%); 1374/45828 objects misplaced (2.998%); 3.8 MiB/s, 107 objects/s recovering 2023-09-30T09:53:19.445 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:19 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5364/45828 objects degraded (11.705%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:19 smithi019 ceph-mon[224043]: pgmap v1646: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 958 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 5364/45828 objects degraded (11.705%); 1374/45828 objects misplaced (2.998%); 3.8 MiB/s, 107 objects/s recovering 2023-09-30T09:53:19.446 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:19 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5364/45828 objects degraded (11.705%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:19.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:19 smithi167 ceph-mon[192880]: pgmap v1646: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 958 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 5364/45828 objects degraded (11.705%); 1374/45828 objects misplaced (2.998%); 3.8 MiB/s, 107 objects/s recovering 2023-09-30T09:53:19.773 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:19 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5364/45828 objects degraded (11.705%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:19.907 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:53:20.206 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:53:20.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:20.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:53:20.317 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:20.353 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:20.520 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:20.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:20.628 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:20.710 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:20.928 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:20.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:21.037 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:21.122 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:21.449 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:21 smithi019 ceph-mon[224043]: pgmap v1647: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 794 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5151/37974 objects degraded (13.565%); 1074/37974 objects misplaced (2.828%); 2.7 MiB/s, 118 objects/s recovering 2023-09-30T09:53:21.450 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:21 smithi019 ceph-mon[216815]: pgmap v1647: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 794 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5151/37974 objects degraded (13.565%); 1074/37974 objects misplaced (2.828%); 2.7 MiB/s, 118 objects/s recovering 2023-09-30T09:53:21.468 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:21.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:21.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:21.496 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:21.578 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:21.669 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:21.739 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:21.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:21 smithi167 ceph-mon[192880]: pgmap v1647: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 794 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5151/37974 objects degraded (13.565%); 1074/37974 objects misplaced (2.828%); 2.7 MiB/s, 118 objects/s recovering 2023-09-30T09:53:21.809 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:21.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:21.924 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:21.981 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:22.056 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:22.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:22.164 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:22.245 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:22.502 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:53:22.515 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:22.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:22.527 INFO:teuthology.orchestra.run.smithi019.stderr:set pool 10 pgp_num to 16 2023-09-30T09:53:22.627 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:22.707 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:22.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2008400243' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:53:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:22 smithi167 ceph-mon[192880]: pgmap v1648: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 794 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5151/37974 objects degraded (13.565%); 1074/37974 objects misplaced (2.828%); 2.4 MiB/s, 101 objects/s recovering 2023-09-30T09:53:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1110304614' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:53:22.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:22 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:53:22.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2008400243' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:53:22.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:22 smithi019 ceph-mon[216815]: pgmap v1648: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 794 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5151/37974 objects degraded (13.565%); 1074/37974 objects misplaced (2.828%); 2.4 MiB/s, 101 objects/s recovering 2023-09-30T09:53:22.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1110304614' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:53:22.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:22 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:53:22.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2008400243' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:53:22.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:22 smithi019 ceph-mon[224043]: pgmap v1648: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 794 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5151/37974 objects degraded (13.565%); 1074/37974 objects misplaced (2.828%); 2.4 MiB/s, 101 objects/s recovering 2023-09-30T09:53:22.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1110304614' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:53:22.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:22 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-30T09:53:22.843 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:22.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:22.960 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:23.015 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:23.548 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:23.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:23.655 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:23.710 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:23.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:23.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2008400243' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:53:23.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:53:23.777 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: osdmap e617: 8 total, 8 up, 8 in 2023-09-30T09:53:23.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.778 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:23 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:53:23.804 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:23.804 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2008400243' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:53:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:53:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: osdmap e617: 8 total, 8 up, 8 in 2023-09-30T09:53:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.805 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:53:23.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:23.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2008400243' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:53:23.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-30T09:53:23.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: osdmap e617: 8 total, 8 up, 8 in 2023-09-30T09:53:23.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.808 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: failed to encode map e617 with expected crc 2023-09-30T09:53:23.809 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:23 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:53:23.821 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:23.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:23.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:23.933 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:53:23.984 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:24.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:53:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:53:23] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-09-30T09:53:24.488 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5151/37974 objects degraded (13.565%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:24.488 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:24 smithi019 ceph-mon[216815]: pgmap v1650: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 736 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 5151/35199 objects degraded (14.634%); 1074/35199 objects misplaced (3.051%); 2.7 MiB/s, 118 objects/s recovering 2023-09-30T09:53:24.488 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:24 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:53:24.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5151/37974 objects degraded (13.565%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:24 smithi167 ceph-mon[192880]: pgmap v1650: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 736 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 5151/35199 objects degraded (14.634%); 1074/35199 objects misplaced (3.051%); 2.7 MiB/s, 118 objects/s recovering 2023-09-30T09:53:24.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:24 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:53:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5151/37974 objects degraded (13.565%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:24 smithi019 ceph-mon[224043]: pgmap v1650: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 736 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 5151/35199 objects degraded (14.634%); 1074/35199 objects misplaced (3.051%); 2.7 MiB/s, 118 objects/s recovering 2023-09-30T09:53:24.836 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:24 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:53:25.292 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:53:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:25.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:53:25.404 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:25.472 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:25.575 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:25.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:25.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:25.681 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:25.739 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:25.841 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:25.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:25.952 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:26.017 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:26.105 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:26.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:26.164 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0: {} 2023-09-30T09:53:26.207 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:53:26.212 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:26.252 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:26.252 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:53:26.279 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1695131548' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:26.289 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:53:26.324 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:53:26.333 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:26.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:26.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:26.370 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:53:26.400 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:53:26.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 2023-09-30T09:53:26.472 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:53:26.476 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:53:26.477 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:53:26.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-30T09:53:26.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:53:26.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:53:26.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:53:26.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:53:26.489 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:53:26.490 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:53:26.490 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:53:26.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1695131548' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:26.516 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1695131548' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:26.538 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:27.111 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-30T09:53:27.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:27.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.0' 2023-09-30T09:53:27.216 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:27.269 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:27.363 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:27.365 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:27 smithi019 ceph-mon[216815]: pgmap v1651: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 564 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 4895/26919 objects degraded (18.184%); 744/26919 objects misplaced (2.764%); 2.9 MiB/s, 125 objects/s recovering 2023-09-30T09:53:27.365 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:27 smithi019 ceph-mon[224043]: pgmap v1651: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 564 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 4895/26919 objects degraded (18.184%); 744/26919 objects misplaced (2.764%); 2.9 MiB/s, 125 objects/s recovering 2023-09-30T09:53:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:27.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:27.467 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:28.070 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:53:28.070 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:27 smithi167 ceph-mon[192880]: pgmap v1651: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 564 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 4895/26919 objects degraded (18.184%); 744/26919 objects misplaced (2.764%); 2.9 MiB/s, 125 objects/s recovering 2023-09-30T09:53:28.127 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:28.210 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:28.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:28.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:28.317 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:53:28.330 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:28.411 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:28.501 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:28.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:28.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:28.615 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:28.680 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:28.716 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:28.779 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:28.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:28.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:28.891 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:28.961 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:29.042 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:53:29.065 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:29.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:29.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:29.174 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:29.234 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:29.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:29 smithi019 ceph-mon[224043]: pgmap v1652: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 503 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4895/24027 objects degraded (20.373%); 744/24027 objects misplaced (3.097%); 2.9 MiB/s, 125 objects/s recovering 2023-09-30T09:53:29.267 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:29 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4895/24027 objects degraded (20.373%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:29.267 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:29 smithi019 ceph-mon[216815]: pgmap v1652: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 503 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4895/24027 objects degraded (20.373%); 744/24027 objects misplaced (3.097%); 2.9 MiB/s, 125 objects/s recovering 2023-09-30T09:53:29.267 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:29 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4895/24027 objects degraded (20.373%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:29 smithi167 ceph-mon[192880]: pgmap v1652: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 503 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4895/24027 objects degraded (20.373%); 744/24027 objects misplaced (3.097%); 2.9 MiB/s, 125 objects/s recovering 2023-09-30T09:53:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:29 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4895/24027 objects degraded (20.373%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:30.206 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:30.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:30.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:30.269 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:53:30.312 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:30.366 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:30.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:30 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:53:30.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:30 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:53:30.590 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:30 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:53:30.643 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:53:30.644 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:53:30.644 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:53:30.644 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-30T09:53:30.644 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.0 2023-09-30T09:53:31.059 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:31.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:31.085 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:53:30 smithi019 systemd[1]: Stopping Ceph osd.0 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:53:31.162 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:31.167 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:31.222 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:31.275 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:31 smithi167 ceph-mon[192880]: pgmap v1653: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 321 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4649/15282 objects degraded (30.421%); 425/15282 objects misplaced (2.781%); 3.1 MiB/s, 130 objects/s recovering 2023-09-30T09:53:31.421 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:31.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:31.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:31.528 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:31.583 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:31 smithi019 ceph-mon[224043]: pgmap v1653: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 321 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4649/15282 objects degraded (30.421%); 425/15282 objects misplaced (2.781%); 3.1 MiB/s, 130 objects/s recovering 2023-09-30T09:53:31.583 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:53:31 smithi019 conmon[237636]: 2023-09-30T09:53:31.322+0000 7fbda3233700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:53:31.584 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:53:31 smithi019 conmon[237636]: 2023-09-30T09:53:31.322+0000 7fbda3233700 -1 osd.0 617 *** Got signal Terminated *** 2023-09-30T09:53:31.584 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:53:31 smithi019 conmon[237636]: 2023-09-30T09:53:31.322+0000 7fbda3233700 -1 osd.0 617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:53:31.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:31 smithi019 ceph-mon[216815]: pgmap v1653: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 321 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4649/15282 objects degraded (30.421%); 425/15282 objects misplaced (2.781%); 3.1 MiB/s, 130 objects/s recovering 2023-09-30T09:53:31.601 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:31.692 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:31.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:31.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:31.796 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:31.876 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:31.980 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:31.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:32.092 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:32.169 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:32.210 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:53:32.225 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:32.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:32 smithi019 ceph-mon[216815]: osd.0 marked itself down and dead 2023-09-30T09:53:32.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4284721367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:32.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:32 smithi019 ceph-mon[224043]: osd.0 marked itself down and dead 2023-09-30T09:53:32.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4284721367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:32.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:32 smithi167 ceph-mon[192880]: osd.0 marked itself down and dead 2023-09-30T09:53:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4284721367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:32.328 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:32.356 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:32.401 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:32.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:32.505 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:32.532 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:32.576 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:32.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:32.680 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:32.706 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:32.751 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:32.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:32.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:32.854 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:32.881 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:32.925 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:32.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:33.028 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:33.054 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:33.101 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:33.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:33.205 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:33.244 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:33.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[216815]: pgmap v1654: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 321 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4649/15282 objects degraded (30.421%); 425/15282 objects misplaced (2.781%); 3.1 MiB/s, 130 objects/s recovering 2023-09-30T09:53:33.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[216815]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:53:33.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:53:33.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4284721367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:33.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[216815]: osdmap e618: 8 total, 7 up, 8 in 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[216815]: failed to encode map e618 with expected crc 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[224043]: pgmap v1654: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 321 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4649/15282 objects degraded (30.421%); 425/15282 objects misplaced (2.781%); 3.1 MiB/s, 130 objects/s recovering 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[224043]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4284721367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[224043]: osdmap e618: 8 total, 7 up, 8 in 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[224043]: failed to encode map e618 with expected crc 2023-09-30T09:53:33.461 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:33.462 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:33.512 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:33.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:33.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:33.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:33 smithi167 ceph-mon[192880]: pgmap v1654: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 321 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4649/15282 objects degraded (30.421%); 425/15282 objects misplaced (2.781%); 3.1 MiB/s, 130 objects/s recovering 2023-09-30T09:53:33.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:33 smithi167 ceph-mon[192880]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-30T09:53:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:33 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:53:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:33 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4284721367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:33 smithi167 ceph-mon[192880]: osdmap e618: 8 total, 7 up, 8 in 2023-09-30T09:53:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:33 smithi167 ceph-mon[192880]: failed to encode map e618 with expected crc 2023-09-30T09:53:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:33.616 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:33.648 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:33.708 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:33.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:33.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:33.733 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:53:33 smithi019 bash[679781]: 0112d2fb79a7e5df14657ee7f1b14d117480bcd776678714ee681127d0c04d9f 2023-09-30T09:53:33.813 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:33.849 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:33.896 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:33.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:34.000 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:34.044 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:34.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:53:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:53:33] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-09-30T09:53:34.119 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:34.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:34.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:34.224 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:34.252 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:34.298 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:34.301 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:53:34.301 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.301 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.301 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.301 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.301 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: osdmap e619: 8 total, 7 up, 8 in 2023-09-30T09:53:34.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: 10.4 continuing backfill to osd.1 from (558'5439,617'11752] 10:21c76c17:::benchmark_data_smithi019_617837_object60406:head to 617'11752 2023-09-30T09:53:34.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: 10.1 continuing backfill to osd.1 from (562'5876,617'11520] 10:88902c24:::benchmark_data_smithi019_617837_object47797:head to 617'11520 2023-09-30T09:53:34.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4649/15282 objects degraded (30.421%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:34.302 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:34 smithi167 ceph-mon[192880]: osdmap e620: 8 total, 7 up, 8 in 2023-09-30T09:53:34.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:34.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:34.405 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:34.431 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:34.475 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:34.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:34.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:34.578 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:34.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:53:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: osdmap e619: 8 total, 7 up, 8 in 2023-09-30T09:53:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: 10.4 continuing backfill to osd.1 from (558'5439,617'11752] 10:21c76c17:::benchmark_data_smithi019_617837_object60406:head to 617'11752 2023-09-30T09:53:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: 10.1 continuing backfill to osd.1 from (562'5876,617'11520] 10:88902c24:::benchmark_data_smithi019_617837_object47797:head to 617'11520 2023-09-30T09:53:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4649/15282 objects degraded (30.421%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:34.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[224043]: osdmap e620: 8 total, 7 up, 8 in 2023-09-30T09:53:34.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:53:34.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: osdmap e619: 8 total, 7 up, 8 in 2023-09-30T09:53:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: failed to encode map e619 with expected crc 2023-09-30T09:53:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: 10.4 continuing backfill to osd.1 from (558'5439,617'11752] 10:21c76c17:::benchmark_data_smithi019_617837_object60406:head to 617'11752 2023-09-30T09:53:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: 10.1 continuing backfill to osd.1 from (562'5876,617'11520] 10:88902c24:::benchmark_data_smithi019_617837_object47797:head to 617'11520 2023-09-30T09:53:34.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4649/15282 objects degraded (30.421%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-30T09:53:34.589 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:34 smithi019 ceph-mon[216815]: osdmap e620: 8 total, 7 up, 8 in 2023-09-30T09:53:34.604 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:34.649 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:34.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:34.754 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:34.782 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:34.824 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:34.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:34.928 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:34.956 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:34.958 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:53:34.981 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:53:35.007 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:53:35.008 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:35.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:35.038 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:53:35.071 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:53:35.112 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:35.113 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:53:35.149 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:53:35.149 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-30T09:53:35.150 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:53:35.150 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:53:35.150 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:53:35.150 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:53:35.150 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:53:35.150 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:53:35.150 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:53:35.179 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:35.216 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.216 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.216 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.216 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.216 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:53:35.297 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:35.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:35.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:35.412 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:35.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:53:35.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: pgmap v1658: 119 pgs: 1 remapped+peering, 4 peering, 1 stale+active+recovery_wait, 6 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 308 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 4639/14640 objects degraded (31.687%); 425/14640 objects misplaced (2.903%); 2.6 MiB/s, 115 objects/s recovering 2023-09-30T09:53:35.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: osdmap e621: 8 total, 7 up, 8 in 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1226935653' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: failed to encode map e620 with expected crc 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: pgmap v1658: 119 pgs: 1 remapped+peering, 4 peering, 1 stale+active+recovery_wait, 6 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 308 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 4639/14640 objects degraded (31.687%); 425/14640 objects misplaced (2.903%); 2.6 MiB/s, 115 objects/s recovering 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:53:35.433 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: osdmap e621: 8 total, 7 up, 8 in 2023-09-30T09:53:35.434 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1226935653' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:35.472 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:35.494 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:53:35.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: pgmap v1658: 119 pgs: 1 remapped+peering, 4 peering, 1 stale+active+recovery_wait, 6 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 308 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 4639/14640 objects degraded (31.687%); 425/14640 objects misplaced (2.903%); 2.6 MiB/s, 115 objects/s recovering 2023-09-30T09:53:35.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-30T09:53:35.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: osdmap e621: 8 total, 7 up, 8 in 2023-09-30T09:53:35.525 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1226935653' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:53:36.246 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:53:36.356 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:36.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:36.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:36.459 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:36.511 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:36 smithi167 ceph-mon[192880]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:36 smithi167 ceph-mon[192880]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:36 smithi167 ceph-mon[192880]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.546 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:36.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:36 smithi019 ceph-mon[224043]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:36 smithi019 ceph-mon[224043]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:36 smithi019 ceph-mon[224043]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:36 smithi019 ceph-mon[216815]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:36 smithi019 ceph-mon[216815]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:36 smithi019 ceph-mon[216815]: failed to encode map e621 with expected crc 2023-09-30T09:53:36.787 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:36.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:36.897 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:36.972 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:37.089 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:37.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:37.128 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:53:37.128 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:53:37.128 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.128 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186375 10.a 10:5e12c2a0:::benchmark_data_smithi019_617837_object92975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:53:37.128 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.106477+0000", 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.020330866, 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.020362871000000001, 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186375 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.129 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106477+0000", 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106477+0000", 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.130 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106475+0000", 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106482+0000", 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5860000000000001e-06 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.131 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106495+0000", 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3251000000000001e-05 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106502+0000", 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1060000000000002e-06 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.132 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106612+0000", 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010917999999999999 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106751+0000", 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000139782 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:53:37.133 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.107285+0000", 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00053378799999999999 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116715+0000", 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0094294110000000004 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:37.134 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.123777+0000", 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0070626109999999999 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186376 10.3 10:c2551dc8:::benchmark_data_smithi019_617837_object92976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.106548+0000", 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.020260133, 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.020386400999999998, 2023-09-30T09:53:37.135 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186376 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106548+0000", 2023-09-30T09:53:37.136 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.137 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.137 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106548+0000", 2023-09-30T09:53:37.137 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.137 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.137 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106547+0000", 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106551+0000", 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2799999999999997e-06 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.138 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106560+0000", 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0620000000000007e-06 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.106566+0000", 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4500000000000003e-06 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.139 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.107902+0000", 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013361880000000001 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for degraded object", 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.108631+0000", 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00072842800000000004 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.121269+0000", 2023-09-30T09:53:37.140 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012638149 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.121344+0000", 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5591000000000002e-05 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.121473+0000", 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000129185 2023-09-30T09:53:37.141 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186377 10.a 10:51a5a2e0:::benchmark_data_smithi019_617837_object92977:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.107804+0000", 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.019003932000000001, 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.019213974000000002, 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.142 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186377 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.107804+0000", 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.143 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.107804+0000", 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.107803+0000", 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.144 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.107809+0000", 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0839999999999997e-06 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.107822+0000", 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2642e-05 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.145 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.107828+0000", 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9229999999999996e-06 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.111507+0000", 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036786409999999999 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.146 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.147 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.111596+0000", 2023-09-30T09:53:37.147 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.8703999999999996e-05 2023-09-30T09:53:37.147 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.147 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.147 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:53:37.147 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.111665+0000", 2023-09-30T09:53:37.147 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9089000000000002e-05 2023-09-30T09:53:37.147 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.123128+0000", 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011463235 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.123782+0000", 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00065424700000000005 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.148 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:186379 10.6 e621/618 10:623adc22:::benchmark_data_smithi019_617837_object92979:head v 621'11393, mlcod=621'11391)", 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.112857+0000", 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.013950519, 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014242652, 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.149 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112857+0000", 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112857+0000", 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.150 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112857+0000", 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112866+0000", 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1200000000000008e-06 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.151 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112881+0000", 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5268999999999999e-05 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112887+0000", 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3049999999999998e-06 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.152 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.113001+0000", 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011320299999999999 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.113024+0000", 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3711999999999999e-05 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.153 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:186381 10.e e621/611 10:77497476:::benchmark_data_smithi019_617837_object92981:head v 621'11712, mlcod=621'11710)", 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.112902+0000", 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.013905496999999999, 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014256247, 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112902+0000", 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.154 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112902+0000", 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112902+0000", 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.155 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112906+0000", 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1790000000000001e-06 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112912+0000", 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6080000000000003e-06 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.156 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.112915+0000", 2023-09-30T09:53:37.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3900000000000002e-06 2023-09-30T09:53:37.157 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.157 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.157 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.115893+0000", 2023-09-30T09:53:37.157 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029781640000000002 2023-09-30T09:53:37.157 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.157 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.115912+0000", 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8949000000000001e-05 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.158 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186385 10.a 10:517e001a:::benchmark_data_smithi019_617837_object92985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.116554+0000", 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.010254058, 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010663677, 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186385 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.159 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.160 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.160 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116554+0000", 2023-09-30T09:53:37.160 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.160 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.160 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.160 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.160 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116554+0000", 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116552+0000", 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116558+0000", 2023-09-30T09:53:37.161 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2360000000000004e-06 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116567+0000", 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8290000000000003e-06 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116573+0000", 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5269999999999998e-06 2023-09-30T09:53:37.162 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116742+0000", 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016937899999999999 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116851+0000", 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108917 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.163 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116961+0000", 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109591 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.123766+0000", 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0068056109999999996 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.164 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.123787+0000", 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0106999999999999e-05 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186386 10.a 10:559f23f7:::benchmark_data_smithi019_617837_object92986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.116586+0000", 2023-09-30T09:53:37.165 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.010222088000000001, 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010714906999999999, 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "waiting for sub ops", 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186386 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.166 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116586+0000", 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116586+0000", 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.167 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116585+0000", 2023-09-30T09:53:37.168 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:37.168 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.168 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.168 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.168 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116588+0000", 2023-09-30T09:53:37.168 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4660000000000002e-06 2023-09-30T09:53:37.168 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116594+0000", 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2389999999999999e-06 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.116598+0000", 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9389999999999999e-06 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.169 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.119961+0000", 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033628019999999998 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120068+0000", 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010733199999999999 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.170 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120177+0000", 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108906 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.124799+0000", 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046223210000000004 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.171 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:186383 10.6 e621/618 10:6766d453:::benchmark_data_smithi019_617837_object92983:head v 621'11394, mlcod=621'11391)", 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.120183+0000", 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0066245510000000002, 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0072056730000000001, 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.172 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120183+0000", 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120183+0000", 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120183+0000", 2023-09-30T09:53:37.173 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120189+0000", 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7629999999999997e-06 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120195+0000", 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9839999999999998e-06 2023-09-30T09:53:37.174 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.175 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.175 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.175 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120199+0000", 2023-09-30T09:53:37.175 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8659999999999999e-06 2023-09-30T09:53:37.175 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.175 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.176 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.176 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.124936+0000", 2023-09-30T09:53:37.176 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047374629999999999 2023-09-30T09:53:37.176 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.176 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.176 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.124954+0000", 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8247e-05 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:186388 10.e e621/611 10:7d319505:::benchmark_data_smithi019_617837_object92988:head v 621'11713, mlcod=621'11710)", 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.120341+0000", 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 0.0064671210000000002, 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007105976, 2023-09-30T09:53:37.177 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:53:37.178 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:53:37.178 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:53:37.178 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.178 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:53:37.179 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120341+0000", 2023-09-30T09:53:37.179 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.179 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.179 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.179 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:53:37.179 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120341+0000", 2023-09-30T09:53:37.179 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:53:37.179 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120340+0000", 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120346+0000", 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1739999999999997e-06 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.180 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120358+0000", 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1297e-05 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.120364+0000", 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9290000000000003e-06 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.125427+0000", 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050629409999999996 2023-09-30T09:53:37.181 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.125445+0000", 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8941e-05 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: ], 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 9 2023-09-30T09:53:37.182 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:53:37.198 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:37.240 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:37.348 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:37 smithi167 ceph-mon[192880]: pgmap v1660: 119 pgs: 2 remapped+peering, 4 peering, 1 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 140 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3013/6567 objects degraded (45.881%); 5.1 MiB/s, 212 objects/s recovering 2023-09-30T09:53:37.415 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:37.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:37.525 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:37.580 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:37.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:37 smithi019 ceph-mon[216815]: pgmap v1660: 119 pgs: 2 remapped+peering, 4 peering, 1 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 140 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3013/6567 objects degraded (45.881%); 5.1 MiB/s, 212 objects/s recovering 2023-09-30T09:53:37.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:37 smithi019 ceph-mon[224043]: pgmap v1660: 119 pgs: 2 remapped+peering, 4 peering, 1 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 140 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3013/6567 objects degraded (45.881%); 5.1 MiB/s, 212 objects/s recovering 2023-09-30T09:53:37.623 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:37.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:37.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:37.726 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:37.753 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:37.796 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:37.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:37.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:37.884 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Removed 93397 objects 2023-09-30T09:53:37.885 INFO:tasks.radosbench.radosbench.0.smithi019.stdout:Clean up completed and total clean up time :196.117 2023-09-30T09:53:37.899 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:37.931 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:38.247 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:53:38.382 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:38.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:38.419 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-30T09:53:38.420 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-30T09:53:38.486 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:38.516 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:38.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:38 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-30T09:53:38.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:38 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-30T09:53:38.771 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:38 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-30T09:53:38.914 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-30T09:53:38.914 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:38.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:38.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:39.019 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:39.024 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:53:39.024 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-30T09:53:39.055 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:39.105 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:39.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:39.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:39.211 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:39.239 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:39.286 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:39.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:39.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:39.302 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:39.354 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:39.390 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:39.421 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:39.481 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:39.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:39.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:39.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:39 smithi167 ceph-mon[192880]: pgmap v1661: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 129 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 3014/6075 objects degraded (49.613%); 3.5 MiB/s, 146 objects/s recovering 2023-09-30T09:53:39.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:39 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 3014/6075 objects degraded (49.613%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:39.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:39 smithi019 ceph-mon[224043]: pgmap v1661: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 129 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 3014/6075 objects degraded (49.613%); 3.5 MiB/s, 146 objects/s recovering 2023-09-30T09:53:39.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:39 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 3014/6075 objects degraded (49.613%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:39.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:39 smithi019 ceph-mon[216815]: pgmap v1661: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 129 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 3014/6075 objects degraded (49.613%); 3.5 MiB/s, 146 objects/s recovering 2023-09-30T09:53:39.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:39 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 3014/6075 objects degraded (49.613%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:39.586 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:39.625 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:39.682 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:39.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:39.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:39.785 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:39.816 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:39.937 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:53:40.164 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:40.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:40.268 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:40.299 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:40.306 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:53:40.390 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:53:40.557 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2172919065' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:40.557 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:40 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:40.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2172919065' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:40.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:40 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2172919065' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:40.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:40 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:53:40.598 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:40.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:40.703 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:40.731 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:40.776 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:40.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:40.880 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:40.909 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:41.006 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:41.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:41.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:41.111 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:41.138 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:41.185 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:41.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:41.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:41.288 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:41.315 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:41.358 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:41.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:41.462 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:41.486 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:41 smithi167 ceph-mon[192880]: pgmap v1662: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 94 active+clean; 5.0 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1961/102 objects degraded (1922.549%); 4.8 MiB/s, 192 objects/s recovering 2023-09-30T09:53:41.486 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:41 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:41.486 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:41 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:53:41.486 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:41 smithi167 ceph-mon[192880]: failed to encode map e622 with expected crc 2023-09-30T09:53:41.486 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:41 smithi167 ceph-mon[192880]: osdmap e622: 8 total, 7 up, 8 in 2023-09-30T09:53:41.486 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:41 smithi167 ceph-mon[192880]: failed to encode map e622 with expected crc 2023-09-30T09:53:41.488 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:41.535 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:41.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:41.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:41.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[224043]: pgmap v1662: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 94 active+clean; 5.0 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1961/102 objects degraded (1922.549%); 4.8 MiB/s, 192 objects/s recovering 2023-09-30T09:53:41.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[224043]: failed to encode map e622 with expected crc 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[224043]: osdmap e622: 8 total, 7 up, 8 in 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[224043]: failed to encode map e622 with expected crc 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[216815]: pgmap v1662: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 94 active+clean; 5.0 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1961/102 objects degraded (1922.549%); 4.8 MiB/s, 192 objects/s recovering 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[216815]: failed to encode map e622 with expected crc 2023-09-30T09:53:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[216815]: osdmap e622: 8 total, 7 up, 8 in 2023-09-30T09:53:41.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:41 smithi019 ceph-mon[216815]: failed to encode map e622 with expected crc 2023-09-30T09:53:41.639 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:41.669 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:42.056 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:42.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:42.161 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:42.190 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:42.345 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:42.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:42.431 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:53:42.450 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:42.454 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:53:42.474 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:53:42.479 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:42.493 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:53:42.515 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:53:42.531 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:42.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:42.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:42.535 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:53:42.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:53:42.556 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-30T09:53:42.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:53:42.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:53:42.556 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:53:42.557 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:53:42.557 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:53:42.557 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:53:42.557 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:53:42.635 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:42.664 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:42.807 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:42.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:42.912 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:42.943 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:43.141 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:43.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:43.248 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:43.281 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:43.304 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:53:43.305 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:53:43.310 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'unique_pool_5' removed 2023-09-30T09:53:43.367 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:43.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:43.496 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:43 smithi167 ceph-mon[192880]: pgmap v1664: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 94 active+clean; 5.0 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 1961/102 objects degraded (1922.549%); 4.1 MiB/s, 165 objects/s recovering 2023-09-30T09:53:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1822129930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1788080821' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:53:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:43 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:53:43.539 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:43 smithi019 ceph-mon[216815]: pgmap v1664: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 94 active+clean; 5.0 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 1961/102 objects degraded (1922.549%); 4.1 MiB/s, 165 objects/s recovering 2023-09-30T09:53:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1822129930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1788080821' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:53:43.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:43 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:53:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:43 smithi019 ceph-mon[224043]: pgmap v1664: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 94 active+clean; 5.0 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 1961/102 objects degraded (1922.549%); 4.1 MiB/s, 165 objects/s recovering 2023-09-30T09:53:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1822129930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1788080821' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-30T09:53:43.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:43 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:53:43.685 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:43.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:43.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:43.700 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:53:43.701 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:53:43.792 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:43.821 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:43.865 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:43.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:43.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:43.969 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:43.998 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:44.025 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [0] live_osds: [2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:53:44.025 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:53:44.026 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:53:44.026 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-09-30T09:53:44.026 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-30T09:53:44.042 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:44.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:44.054 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:53:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:53:43] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-09-30T09:53:44.084 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:53:44.146 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:44.174 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:44.217 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:44.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:44.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:44.245 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-30T09:53:44.245 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-30T09:53:44.245 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-30T09:53:44.246 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-30T09:53:44.246 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-30T09:53:44.246 DEBUG:teuthology.orchestra.run.smithi019:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:53:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T09:53:44.269 INFO:teuthology.orchestra.run.smithi019.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-30T09:53:44.269 DEBUG:teuthology.orchestra.run.smithi019:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:53:44.320 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:44.342 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-30T09:53:44.343 DEBUG:teuthology.orchestra.run.smithi019:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-30T09:53:44.404 INFO:tasks.workunit:timeout=3h 2023-09-30T09:53:44.405 INFO:tasks.workunit:cleanup=True 2023-09-30T09:53:44.405 DEBUG:teuthology.orchestra.run.smithi019:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-30T09:53:44.417 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:44.456 INFO:tasks.workunit.client.0.smithi019.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-30T09:53:44.461 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:44.527 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:44.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:44.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1788080821' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:53:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[216815]: failed to encode map e623 with expected crc 2023-09-30T09:53:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[216815]: osdmap e623: 8 total, 7 up, 8 in 2023-09-30T09:53:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[216815]: failed to encode map e623 with expected crc 2023-09-30T09:53:44.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 1961/102 objects degraded (1922.549%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1788080821' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:53:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[224043]: failed to encode map e623 with expected crc 2023-09-30T09:53:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[224043]: osdmap e623: 8 total, 7 up, 8 in 2023-09-30T09:53:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[224043]: failed to encode map e623 with expected crc 2023-09-30T09:53:44.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:44 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 1961/102 objects degraded (1922.549%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:44.627 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:44.637 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:44.663 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:44 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1788080821' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-30T09:53:44.663 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:44 smithi167 ceph-mon[192880]: failed to encode map e623 with expected crc 2023-09-30T09:53:44.663 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:44 smithi167 ceph-mon[192880]: osdmap e623: 8 total, 7 up, 8 in 2023-09-30T09:53:44.663 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:44 smithi167 ceph-mon[192880]: failed to encode map e623 with expected crc 2023-09-30T09:53:44.664 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:44 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 1961/102 objects degraded (1922.549%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-30T09:53:44.665 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:44.811 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:44.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:44.883 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:53:44.915 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:44.946 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:44.991 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:44.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:44.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:45.096 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:45.125 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:45.217 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:45.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:45.321 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:53:45.322 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:45.354 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:45.494 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:45 smithi019 ceph-mon[216815]: pgmap v1666: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:45.494 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:45 smithi019 ceph-mon[224043]: pgmap v1666: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:45.500 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:45.501 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:45 smithi167 ceph-mon[192880]: pgmap v1666: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:45.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:45.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:45.603 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:45.630 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:45.701 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:53:45.729 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:45.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:45.833 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:45.861 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:46.668 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:46.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:46.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:46.777 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:46.778 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:46.805 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:47.002 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:47.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:47.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:47.105 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:47.138 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:47.362 INFO:teuthology.orchestra.run.smithi167.stdout:ERROR: (22) Invalid argument 2023-09-30T09:53:47.362 INFO:teuthology.orchestra.run.smithi167.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:53:47.394 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:47 smithi167 ceph-mon[192880]: pgmap v1667: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:47.451 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:47.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:47.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:47.533 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:47 smithi019 ceph-mon[224043]: pgmap v1667: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:47.533 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:47 smithi019 ceph-mon[216815]: pgmap v1667: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:47.556 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:47.586 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:47.984 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:47.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:47.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:48.089 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:48.120 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:48.327 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-30T09:53:48.330 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:53:48.344 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:48.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:48.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:48.353 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-30T09:53:48.421 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3715707446' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:48.422 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1138847355' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:48.422 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:48.422 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:48.422 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:48.423 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3715707446' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:48.423 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1138847355' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:48.423 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:48.423 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:48.423 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:48.452 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:48.480 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:48.584 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:48.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:48.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:48.587 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3715707446' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:53:48.587 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1138847355' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:48.587 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:53:48.587 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:48.588 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:53:48.688 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:48.717 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:49.567 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:49.569 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[216815]: pgmap v1668: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:49.569 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:49.569 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3715707446' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:53:49.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:49.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[216815]: failed to encode map e624 with expected crc 2023-09-30T09:53:49.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[216815]: osdmap e624: 8 total, 7 up, 8 in 2023-09-30T09:53:49.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:53:49.570 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[224043]: pgmap v1668: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:49.570 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:49.570 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3715707446' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:53:49.570 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:49.571 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[224043]: failed to encode map e624 with expected crc 2023-09-30T09:53:49.571 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[224043]: osdmap e624: 8 total, 7 up, 8 in 2023-09-30T09:53:49.571 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:49 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:53:49.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:49.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:49.672 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:49.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:49 smithi167 ceph-mon[192880]: pgmap v1668: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:49.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:49 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:53:49.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:49 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3715707446' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:53:49.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:53:49.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:49 smithi167 ceph-mon[192880]: failed to encode map e624 with expected crc 2023-09-30T09:53:49.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:49 smithi167 ceph-mon[192880]: osdmap e624: 8 total, 7 up, 8 in 2023-09-30T09:53:49.700 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:49 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:53:49.702 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:49.752 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:49.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:49.859 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:49.885 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:49.936 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:49.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:49.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:50.044 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:50.048 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:50.071 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:50.097 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:53:50.114 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:50.116 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:53:50.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:50.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:50.137 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:53:50.156 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:53:50.175 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:53:50.195 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:53:50.217 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:53:50.217 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-30T09:53:50.217 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:53:50.218 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:53:50.218 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:53:50.218 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:53:50.218 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:53:50.218 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:53:50.218 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:53:50.220 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:53:50.246 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:50.288 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:53:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:50.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:53:50.394 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:50.420 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:50.462 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:50.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:50.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:50.567 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:50.598 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:50.695 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:50.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:50.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:50 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:50 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2720239838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:50.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:50 smithi167 ceph-mon[192880]: pgmap v1670: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:50.798 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:50.827 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:50.834 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:50.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:50 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:50.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:50 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2720239838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:50.835 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:50 smithi019 ceph-mon[224043]: pgmap v1670: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:50.835 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:50.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:50 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:50.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:50 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2720239838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:53:50.837 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:50 smithi019 ceph-mon[216815]: pgmap v1670: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:50.873 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:50.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:50.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:50.978 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:51.006 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:51.048 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:51.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:51.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:51.154 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:51.181 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:51.223 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:51.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:51.332 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:51.358 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:51.399 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:51.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:51.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:51.505 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:51.507 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:53:51.507 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:53:51.540 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:51.641 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:51.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:51.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:51.745 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:51.767 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:53:51.773 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:51.916 INFO:tasks.workunit.client.0.smithi019.stderr:Updating files: 92% (10463/11322) Updating files: 93% (10530/11322) Updating files: 94% (10643/11322) Updating files: 95% (10756/11322) Updating files: 96% (10870/11322) Updating files: 97% (10983/11322) Updating files: 98% (11096/11322) Updating files: 99% (11209/11322) Updating files: 100% (11322/11322) Updating files: 100% (11322/11322), done. 2023-09-30T09:53:51.933 DEBUG:teuthology.orchestra.run.smithi019:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-30T09:53:52.092 INFO:tasks.workunit.client.0.smithi019.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-30T09:53:52.094 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-30T09:53:52.094 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-30T09:53:52.279 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:52.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:52.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:52.383 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:52.412 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:52.440 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-30T09:53:52.786 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-30T09:53:52.832 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-30T09:53:52.834 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-30T09:53:52.834 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-30T09:53:52.901 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-30T09:53:52.933 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:53:52.933 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-30T09:53:53.057 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-30T09:53:53.058 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-30T09:53:53.058 DEBUG:teuthology.orchestra.run.smithi019:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-30T09:53:53.508 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:53:53.511 INFO:tasks.workunit.client.0.smithi019.stdout:Running main() from gmock_main.cc 2023-09-30T09:53:53.511 INFO:tasks.workunit.client.0.smithi019.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-30T09:53:53.511 INFO:tasks.workunit.client.0.smithi019.stdout:[----------] Global test environment set-up. 2023-09-30T09:53:53.511 INFO:tasks.workunit.client.0.smithi019.stdout:[----------] 70 tests from TestClsRbd 2023-09-30T09:53:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:53 smithi167 ceph-mon[192880]: pgmap v1671: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:53.537 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:53 smithi019 ceph-mon[216815]: pgmap v1671: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:53.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:53 smithi019 ceph-mon[224043]: pgmap v1671: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:54.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:53:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:53:53] "GET /metrics HTTP/1.1" 200 34129 "" "Prometheus/2.43.0" 2023-09-30T09:53:54.715 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [0] live_osds: [2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:53:54.715 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:53:54.715 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-30T09:53:54.715 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.387148 2023-09-30T09:53:54.716 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0.38714772109616347 2023-09-30T09:53:54.752 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:54.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:54.856 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:54.884 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:55.034 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:55.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:55.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:55.138 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:55.154 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-30T09:53:55.168 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_all_features (15 ms) 2023-09-30T09:53:55.169 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.copyup 2023-09-30T09:53:55.178 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:55.187 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:55.225 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:55.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:55.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:55.330 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:55.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[224043]: pgmap v1672: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:55.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[224043]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[224043]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[224043]: osdmap e625: 8 total, 7 up, 8 in 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1675423789' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi019-685904-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[224043]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[216815]: pgmap v1672: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[216815]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[216815]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[216815]: osdmap e625: 8 total, 7 up, 8 in 2023-09-30T09:53:55.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1675423789' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi019-685904-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:53:55.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:55 smithi019 ceph-mon[216815]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.359 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:55.367 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.copyup (199 ms) 2023-09-30T09:53:55.367 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-30T09:53:55.387 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.sparse_copyup (20 ms) 2023-09-30T09:53:55.388 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-30T09:53:55.406 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:53:55.410 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:55 smithi167 ceph-mon[192880]: pgmap v1672: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:55.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:55 smithi167 ceph-mon[192880]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:55 smithi167 ceph-mon[192880]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:55 smithi167 ceph-mon[192880]: osdmap e625: 8 total, 7 up, 8 in 2023-09-30T09:53:55.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1675423789' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi019-685904-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-30T09:53:55.411 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:55 smithi167 ceph-mon[192880]: failed to encode map e625 with expected crc 2023-09-30T09:53:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:55.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:53:55.456 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_and_set_id (68 ms) 2023-09-30T09:53:55.456 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-30T09:53:55.457 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:55.467 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.add_remove_child (12 ms) 2023-09-30T09:53:55.468 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-30T09:53:55.497 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.directory_methods (30 ms) 2023-09-30T09:53:55.498 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.create 2023-09-30T09:53:55.513 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:53:55.541 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.create (44 ms) 2023-09-30T09:53:55.542 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_features 2023-09-30T09:53:55.543 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-09-30T09:53:55.543 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-30T09:53:55.545 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-09-30T09:53:55.545 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-30T09:53:55.545 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:55.546 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-09-30T09:53:55.547 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-30T09:53:55.549 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-09-30T09:53:55.549 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-30T09:53:55.550 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-30T09:53:55.550 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-30T09:53:55.556 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_data_pool (5 ms) 2023-09-30T09:53:55.556 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.get_size 2023-09-30T09:53:55.561 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.get_size (6 ms) 2023-09-30T09:53:55.562 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.set_size 2023-09-30T09:53:55.567 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-09-30T09:53:55.567 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-30T09:53:55.589 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.protection_status (22 ms) 2023-09-30T09:53:55.589 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-30T09:53:55.597 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2023-09-30T09:53:55.597 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-30T09:53:55.632 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:53:55.651 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.parents_v1 (54 ms) 2023-09-30T09:53:55.651 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-30T09:53:55.684 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.parents_v2 (33 ms) 2023-09-30T09:53:55.684 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-30T09:53:55.708 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.snapshots (24 ms) 2023-09-30T09:53:55.708 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-30T09:53:55.714 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.snapid_race (6 ms) 2023-09-30T09:53:55.714 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-30T09:53:55.724 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2023-09-30T09:53:55.724 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-30T09:53:55.728 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.object_map_save (5 ms) 2023-09-30T09:53:55.729 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-30T09:53:55.737 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2023-09-30T09:53:55.737 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-30T09:53:55.743 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:53:55.745 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.object_map_update (8 ms) 2023-09-30T09:53:55.745 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-30T09:53:55.745 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-30T09:53:55.745 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-30T09:53:55.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:55.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:53:55.755 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.object_map_snap_add (11 ms) 2023-09-30T09:53:55.756 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-30T09:53:55.762 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2023-09-30T09:53:55.763 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.flags 2023-09-30T09:53:55.768 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-09-30T09:53:55.768 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.metadata 2023-09-30T09:53:55.794 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.metadata (26 ms) 2023-09-30T09:53:55.794 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.set_features 2023-09-30T09:53:55.802 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2023-09-30T09:53:55.802 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.mirror 2023-09-30T09:53:55.849 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:55.876 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:55.887 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.mirror (85 ms) 2023-09-30T09:53:55.887 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-30T09:53:55.906 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.mirror_image (19 ms) 2023-09-30T09:53:55.906 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-30T09:53:55.925 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:55.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:55.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:56.030 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:56.064 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:56.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1675423789' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi019-685904-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-30T09:53:56.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: osdmap e626: 8 total, 7 up, 8 in 2023-09-30T09:53:56.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.157 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.158 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.158 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:56.158 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:56 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:56.358 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:53:56.358 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:53:56.358 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 0 2023-09-30T09:53:56.359 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:53:56.403 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:56.409 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1675423789' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi019-685904-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-30T09:53:56.409 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.409 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: osdmap e626: 8 total, 7 up, 8 in 2023-09-30T09:53:56.409 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.410 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.410 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.410 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.410 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:56.410 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:56.410 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1675423789' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi019-685904-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-30T09:53:56.410 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.410 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: osdmap e626: 8 total, 7 up, 8 in 2023-09-30T09:53:56.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: failed to encode map e626 with expected crc 2023-09-30T09:53:56.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:56.411 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:56 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:56.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:56.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:56.541 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:56.590 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:56.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:56.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:56.695 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:56.727 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:57.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:57 smithi167 ceph-mon[192880]: pgmap v1675: 135 pgs: 16 creating+peering, 16 unknown, 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:57 smithi167 ceph-mon[192880]: failed to encode map e627 with expected crc 2023-09-30T09:53:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:57 smithi167 ceph-mon[192880]: failed to encode map e627 with expected crc 2023-09-30T09:53:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:57 smithi167 ceph-mon[192880]: osdmap e627: 8 total, 7 up, 8 in 2023-09-30T09:53:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:53:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:53:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:57.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[216815]: pgmap v1675: 135 pgs: 16 creating+peering, 16 unknown, 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:57.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[216815]: failed to encode map e627 with expected crc 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[216815]: failed to encode map e627 with expected crc 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[216815]: osdmap e627: 8 total, 7 up, 8 in 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[224043]: pgmap v1675: 135 pgs: 16 creating+peering, 16 unknown, 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[224043]: failed to encode map e627 with expected crc 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[224043]: failed to encode map e627 with expected crc 2023-09-30T09:53:57.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[224043]: osdmap e627: 8 total, 7 up, 8 in 2023-09-30T09:53:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:53:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:53:57.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:53:58.232 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:58.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:58.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:58.339 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:53:58.371 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:58.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:58 smithi167 ceph-mon[192880]: failed to encode map e627 with expected crc 2023-09-30T09:53:58.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:58 smithi019 ceph-mon[216815]: failed to encode map e627 with expected crc 2023-09-30T09:53:58.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:58 smithi019 ceph-mon[224043]: failed to encode map e627 with expected crc 2023-09-30T09:53:58.744 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.mirror_image_status (2838 ms) 2023-09-30T09:53:58.744 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-30T09:53:58.834 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.mirror_image_map (90 ms) 2023-09-30T09:53:58.834 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-30T09:53:58.842 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.mirror_instances (9 ms) 2023-09-30T09:53:58.843 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-30T09:53:58.855 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2023-09-30T09:53:58.855 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-30T09:53:58.858 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-09-30T09:53:58.858 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-30T09:53:58.861 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2023-09-30T09:53:58.861 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-30T09:53:58.865 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.dir_add_already_existing (4 ms) 2023-09-30T09:53:58.865 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-30T09:53:58.872 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-09-30T09:53:58.872 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-30T09:53:58.875 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-09-30T09:53:58.875 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-30T09:53:58.879 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-09-30T09:53:58.879 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-30T09:53:58.881 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-09-30T09:53:58.881 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-30T09:53:58.884 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_image_remove (4 ms) 2023-09-30T09:53:58.885 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-30T09:53:58.887 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-09-30T09:53:58.888 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-30T09:53:58.892 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-09-30T09:53:58.892 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-30T09:53:58.895 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-09-30T09:53:58.895 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-30T09:53:58.898 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-09-30T09:53:58.898 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-30T09:53:58.901 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-30T09:53:58.901 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-30T09:53:58.902 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-30T09:53:58.903 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-30T09:53:58.904 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-30T09:53:58.905 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-30T09:53:58.908 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-30T09:53:58.908 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-30T09:53:58.910 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-09-30T09:53:58.910 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-30T09:53:58.912 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-09-30T09:53:58.913 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-30T09:53:58.916 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-30T09:53:58.916 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-30T09:53:58.932 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (17 ms) 2023-09-30T09:53:58.932 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-30T09:53:59.175 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:53:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:59.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:53:59.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:59 smithi019 ceph-mon[216815]: pgmap v1677: 135 pgs: 1 active+clean+remapped, 16 creating+peering, 13 unknown, 11 active+undersized, 4 active+undersized+degraded, 90 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 31 KiB/s rd, 8.2 KiB/s wr, 47 op/s; 4/93 objects degraded (4.301%) 2023-09-30T09:53:59.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:53:59 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4/93 objects degraded (4.301%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:53:59.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:59 smithi019 ceph-mon[224043]: pgmap v1677: 135 pgs: 1 active+clean+remapped, 16 creating+peering, 13 unknown, 11 active+undersized, 4 active+undersized+degraded, 90 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 31 KiB/s rd, 8.2 KiB/s wr, 47 op/s; 4/93 objects degraded (4.301%) 2023-09-30T09:53:59.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:53:59 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4/93 objects degraded (4.301%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:53:59.281 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:53:59.307 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:59.350 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:53:59.352 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (419 ms) 2023-09-30T09:53:59.352 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-30T09:53:59.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:59.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:53:59.356 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-30T09:53:59.356 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-30T09:53:59.358 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-30T09:53:59.358 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-30T09:53:59.366 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2023-09-30T09:53:59.366 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.op_features 2023-09-30T09:53:59.372 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2023-09-30T09:53:59.372 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-30T09:53:59.395 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.clone_parent (23 ms) 2023-09-30T09:53:59.395 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-30T09:53:59.403 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2023-09-30T09:53:59.403 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-30T09:53:59.417 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.clone_child (14 ms) 2023-09-30T09:53:59.417 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-30T09:53:59.424 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-09-30T09:53:59.424 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.migration 2023-09-30T09:53:59.435 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.migration (11 ms) 2023-09-30T09:53:59.435 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-30T09:53:59.447 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.migration_v1 (13 ms) 2023-09-30T09:53:59.448 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-30T09:53:59.456 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:53:59.479 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:59 smithi167 ceph-mon[192880]: pgmap v1677: 135 pgs: 1 active+clean+remapped, 16 creating+peering, 13 unknown, 11 active+undersized, 4 active+undersized+degraded, 90 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 31 KiB/s rd, 8.2 KiB/s wr, 47 op/s; 4/93 objects degraded (4.301%) 2023-09-30T09:53:59.480 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:53:59 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4/93 objects degraded (4.301%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:53:59.481 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:59.524 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:53:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:59.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:53:59.629 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:53:59.666 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:59.712 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-30T09:53:59.825 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:53:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:53:59.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:53:59.929 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:53:59.957 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:53:59.999 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:00.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:00.104 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:00.134 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:00.172 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:54:00.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/444721177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:00.287 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:00.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/444721177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:00.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/444721177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:00.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:00.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:00.384 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:54:00.384 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:54:00.384 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:54:00.390 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:54:00.392 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:00.413 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:54:00.423 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:00.437 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:54:00.460 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:54:00.460 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-30T09:54:00.460 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:54:00.460 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:54:00.460 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:54:00.461 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:54:00.461 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:54:00.461 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:54:00.461 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:54:00.576 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:00.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:00.681 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:00.710 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:00.755 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:00.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:00.861 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:00.888 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:00.932 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:00.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:01.037 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:01.063 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:01.105 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:01.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:01.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:01.176 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1728 ms) 2023-09-30T09:54:01.176 INFO:tasks.workunit.client.0.smithi019.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-30T09:54:01.180 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.3 primary-affinity to 0.387148 (8253722) 2023-09-30T09:54:01.209 INFO:tasks.workunit.client.0.smithi019.stdout:[ OK ] TestClsRbd.sparsify (34 ms) 2023-09-30T09:54:01.210 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:01.236 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:01.278 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:01.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:01.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:01.383 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:01.417 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:01.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: pgmap v1678: 135 pgs: 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 113 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.33k op/s; 15/234 objects degraded (6.410%) 2023-09-30T09:54:01.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:01.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/444721177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:01.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: failed to encode map e628 with expected crc 2023-09-30T09:54:01.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: osdmap e628: 8 total, 7 up, 8 in 2023-09-30T09:54:01.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: failed to encode map e628 with expected crc 2023-09-30T09:54:01.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2821201726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4028657069' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]: dispatch 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]: dispatch 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: pgmap v1678: 135 pgs: 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 113 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.33k op/s; 15/234 objects degraded (6.410%) 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/444721177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: failed to encode map e628 with expected crc 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: osdmap e628: 8 total, 7 up, 8 in 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: failed to encode map e628 with expected crc 2023-09-30T09:54:01.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2821201726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:01.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4028657069' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]: dispatch 2023-09-30T09:54:01.488 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:01 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]: dispatch 2023-09-30T09:54:01.516 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:01.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:01.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:01.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: pgmap v1678: 135 pgs: 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 113 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.33k op/s; 15/234 objects degraded (6.410%) 2023-09-30T09:54:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/444721177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: failed to encode map e628 with expected crc 2023-09-30T09:54:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: osdmap e628: 8 total, 7 up, 8 in 2023-09-30T09:54:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: failed to encode map e628 with expected crc 2023-09-30T09:54:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2821201726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4028657069' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]: dispatch 2023-09-30T09:54:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:01 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]: dispatch 2023-09-30T09:54:01.619 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:01.646 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:02.192 INFO:tasks.workunit.client.0.smithi019.stdout:[----------] 70 tests from TestClsRbd (6056 ms total) 2023-09-30T09:54:02.192 INFO:tasks.workunit.client.0.smithi019.stdout: 2023-09-30T09:54:02.192 INFO:tasks.workunit.client.0.smithi019.stdout:[----------] Global test environment tear-down 2023-09-30T09:54:02.192 INFO:tasks.workunit.client.0.smithi019.stdout:[==========] 70 tests from 1 test suite ran. (8681 ms total) 2023-09-30T09:54:02.192 INFO:tasks.workunit.client.0.smithi019.stdout:[ PASSED ] 70 tests. 2023-09-30T09:54:02.194 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-30T09:54:02.194 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-30T09:54:02.224 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-30T09:54:02.224 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-30T09:54:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]': finished 2023-09-30T09:54:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[224043]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[224043]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[224043]: osdmap e629: 8 total, 7 up, 8 in 2023-09-30T09:54:02.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[224043]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[224043]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]': finished 2023-09-30T09:54:02.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[216815]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[216815]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[216815]: osdmap e629: 8 total, 7 up, 8 in 2023-09-30T09:54:02.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[216815]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:02 smithi019 ceph-mon[216815]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.302 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:02.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:02.325 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:54:02.325 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:54:02.407 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:02.435 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:02.507 DEBUG:teuthology.parallel:result is None 2023-09-30T09:54:02.508 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:54:02.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:02 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.38714772109616347}]': finished 2023-09-30T09:54:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:02 smithi167 ceph-mon[192880]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:02 smithi167 ceph-mon[192880]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:02 smithi167 ceph-mon[192880]: osdmap e629: 8 total, 7 up, 8 in 2023-09-30T09:54:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:02 smithi167 ceph-mon[192880]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:02 smithi167 ceph-mon[192880]: failed to encode map e629 with expected crc 2023-09-30T09:54:02.530 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:02.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:02.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:02.559 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:54:02.559 DEBUG:teuthology.orchestra.run.smithi019:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:54:02.636 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:02.644 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:54:02.644 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-30T09:54:02.645 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-30T09:54:02.645 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-30T09:54:02.645 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-30T09:54:02.646 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-30T09:54:02.646 DEBUG:teuthology.orchestra.run.smithi019:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:54:02.676 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T09:54:02.697 INFO:teuthology.orchestra.run.smithi019.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-30T09:54:02.697 DEBUG:teuthology.orchestra.run.smithi019:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:54:02.706 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:02.726 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-30T09:54:02.726 DEBUG:teuthology.orchestra.run.smithi019:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-30T09:54:02.783 INFO:tasks.workunit:timeout=3h 2023-09-30T09:54:02.783 INFO:tasks.workunit:cleanup=True 2023-09-30T09:54:02.783 DEBUG:teuthology.orchestra.run.smithi019:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-30T09:54:02.815 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:02.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:02.833 INFO:tasks.workunit.client.0.smithi019.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-30T09:54:02.919 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:02.950 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:02.996 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:02.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:03.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:03.100 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:03.131 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:03.378 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:03.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: pgmap v1681: 135 pgs: 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 113 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.33k op/s; 15/234 objects degraded (6.410%) 2023-09-30T09:54:03.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: osdmap e630: 8 total, 7 up, 8 in 2023-09-30T09:54:03.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.380 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: 1.1e scrub starts 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: 1.1e scrub ok 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: pgmap v1681: 135 pgs: 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 113 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.33k op/s; 15/234 objects degraded (6.410%) 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: osdmap e630: 8 total, 7 up, 8 in 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.381 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: 1.1e scrub starts 2023-09-30T09:54:03.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: 1.1e scrub ok 2023-09-30T09:54:03.382 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:03.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:03.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:03.484 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:03.512 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: pgmap v1681: 135 pgs: 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 113 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.33k op/s; 15/234 objects degraded (6.410%) 2023-09-30T09:54:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: osdmap e630: 8 total, 7 up, 8 in 2023-09-30T09:54:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: failed to encode map e630 with expected crc 2023-09-30T09:54:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: 1.1e scrub starts 2023-09-30T09:54:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: 1.1e scrub ok 2023-09-30T09:54:03.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:03.558 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:03.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:03.662 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:03.695 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:03.734 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:54:03.744 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:03.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:03.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:03.851 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:03.882 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:03.937 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:03.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:04.022 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:54:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:54:03] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2023-09-30T09:54:04.039 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:04.066 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:04.107 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:04.111 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:04.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:04.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:04.214 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:04.246 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:04.326 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:54:04.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:04 smithi019 ceph-mon[224043]: 1.1c scrub starts 2023-09-30T09:54:04.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:04 smithi019 ceph-mon[224043]: 1.1c scrub ok 2023-09-30T09:54:04.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:04 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 15/234 objects degraded (6.410%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:04 smithi019 ceph-mon[216815]: 1.1c scrub starts 2023-09-30T09:54:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:04 smithi019 ceph-mon[216815]: 1.1c scrub ok 2023-09-30T09:54:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:04 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 15/234 objects degraded (6.410%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:04.407 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:04.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:04.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:04.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:04 smithi167 ceph-mon[192880]: 1.1c scrub starts 2023-09-30T09:54:04.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:04 smithi167 ceph-mon[192880]: 1.1c scrub ok 2023-09-30T09:54:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:04 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 15/234 objects degraded (6.410%), 10 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:04.540 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:04.584 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:04.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:04.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:04.690 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:04.716 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:04.758 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:04.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:04.863 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:04.889 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:04.931 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:04.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:04.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:05.036 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:05.067 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:05.346 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:54:05.346 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:54:05.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:05 smithi167 ceph-mon[192880]: pgmap v1683: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:05.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:05 smithi019 ceph-mon[216815]: pgmap v1683: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:05.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:05 smithi019 ceph-mon[224043]: pgmap v1683: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:05.667 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:05.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:05.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:05.770 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:05.802 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:05.849 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:05.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:05.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:05.956 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:05.984 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:06.028 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:06.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:06.133 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:06.160 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:06.166 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-30T09:54:06.178 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:06.202 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:06.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:06.307 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:06.334 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:06.375 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:06.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:06.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:06.480 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:06.506 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:06.548 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:06.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:06.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:06.655 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:06.685 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:06.749 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:06.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:06.853 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:06.886 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:06.940 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:06.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:06.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:07.044 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:07.073 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:07.120 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:07.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:07.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:07.190 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:54:07.214 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:07 smithi019 ceph-mon[216815]: pgmap v1684: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:07.214 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:07 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3528714220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:54:07.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:07 smithi019 ceph-mon[224043]: pgmap v1684: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:07.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:07 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3528714220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:54:07.224 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:07.254 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:07.302 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:07.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:07.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:07.407 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:07.434 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:07.478 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:07.482 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:07 smithi167 ceph-mon[192880]: pgmap v1684: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:07.482 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:07 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3528714220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:54:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:07.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:07.584 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [0] live_osds: [2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:54:07.584 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:54:07.584 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-30T09:54:07.585 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:54:07.585 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:07.612 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:07.653 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:07.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:07.758 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:07.784 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:07.827 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:07.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:07.934 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:07.960 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:08.002 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:08.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:08.109 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:08.137 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:08.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:54:08.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3528714220' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:54:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[216815]: failed to encode map e631 with expected crc 2023-09-30T09:54:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[216815]: osdmap e631: 8 total, 7 up, 8 in 2023-09-30T09:54:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[216815]: failed to encode map e631 with expected crc 2023-09-30T09:54:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:54:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3528714220' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:54:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[224043]: failed to encode map e631 with expected crc 2023-09-30T09:54:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[224043]: osdmap e631: 8 total, 7 up, 8 in 2023-09-30T09:54:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:08 smithi019 ceph-mon[224043]: failed to encode map e631 with expected crc 2023-09-30T09:54:08.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:08 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:54:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:08 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3528714220' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:54:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:08 smithi167 ceph-mon[192880]: failed to encode map e631 with expected crc 2023-09-30T09:54:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:08 smithi167 ceph-mon[192880]: osdmap e631: 8 total, 7 up, 8 in 2023-09-30T09:54:08.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:08 smithi167 ceph-mon[192880]: failed to encode map e631 with expected crc 2023-09-30T09:54:08.623 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:08.737 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:08.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:08.855 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:08.885 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:09.115 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:09.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:09.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:09.220 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:09.252 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:09.294 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:09.301 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:09.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:09.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:09.406 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:09.435 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:09.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:09 smithi019 ceph-mon[216815]: pgmap v1686: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:09.461 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:09.462 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:09 smithi019 ceph-mon[224043]: pgmap v1686: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:09.462 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:09.479 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:09.484 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:09 smithi167 ceph-mon[192880]: pgmap v1686: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:09.485 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:09.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:09.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:09.586 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:09.613 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:09.655 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:09.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:09.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:09.763 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:09.789 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:09.831 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:09.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:09.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:09.883 DEBUG:teuthology.orchestra.run.smithi019:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-30T09:54:09.901 INFO:tasks.workunit.client.0.smithi019.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-30T09:54:09.903 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-30T09:54:09.904 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-30T09:54:09.937 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:09.970 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:10.083 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:54:10.084 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:54:10.084 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:54:10.084 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:54:10.084 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:54:10.092 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:54:10.112 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:54:10.112 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-30T09:54:10.112 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:54:10.112 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:54:10.113 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:54:10.113 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:54:10.113 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:54:10.113 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:54:10.113 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:54:10.115 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-30T09:54:10.241 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-30T09:54:10.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:10 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2097511470' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:10.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:10 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2097511470' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:10.353 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-30T09:54:10.356 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-30T09:54:10.356 INFO:tasks.workunit.client.0.smithi019.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-30T09:54:10.473 INFO:tasks.workunit.client.0.smithi019.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-30T09:54:10.476 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2023-09-30T09:54:10.476 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-30T09:54:10.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:10 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2097511470' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:10.535 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-30T09:54:10.536 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-30T09:54:10.536 DEBUG:teuthology.orchestra.run.smithi019:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-30T09:54:10.596 INFO:tasks.workunit.client.0.smithi019.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-30T09:54:10.596 INFO:tasks.workunit.client.0.smithi019.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-30T09:54:10.596 INFO:tasks.workunit.client.0.smithi019.stderr:+ TMPDIR=/tmp/rbd_import_export_690681 2023-09-30T09:54:10.596 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm -rf /tmp/rbd_import_export_690681 2023-09-30T09:54:10.597 INFO:tasks.workunit.client.0.smithi019.stderr:+ mkdir /tmp/rbd_import_export_690681 2023-09-30T09:54:10.598 INFO:tasks.workunit.client.0.smithi019.stderr:+ trap 'rm -rf /tmp/rbd_import_export_690681' INT TERM EXIT 2023-09-30T09:54:10.598 INFO:tasks.workunit.client.0.smithi019.stderr:+ mkdir foo.690681 2023-09-30T09:54:10.598 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import foo.690681 foo.dir 2023-09-30T09:54:10.728 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-30T09:54:10.819 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: cannot import a directory 2023-09-30T09:54:10.819 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 0% complete...failed. 2023-09-30T09:54:10.819 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: import failed: (21) Is a directory 2023-09-30T09:54:10.821 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:54:10.821 INFO:tasks.workunit.client.0.smithi019.stderr:+ rmdir foo.690681 2023-09-30T09:54:10.822 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_690681/img bs=1k count=1 seek=10 2023-09-30T09:54:10.823 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:54:10.823 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:54:10.823 INFO:tasks.workunit.client.0.smithi019.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.0318e-05 s, 20.4 MB/s 2023-09-30T09:54:10.823 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_690681/img bs=1k count=10 seek=100 2023-09-30T09:54:10.824 INFO:tasks.workunit.client.0.smithi019.stderr:10+0 records in 2023-09-30T09:54:10.824 INFO:tasks.workunit.client.0.smithi019.stderr:10+0 records out 2023-09-30T09:54:10.824 INFO:tasks.workunit.client.0.smithi019.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.8353e-05 s, 175 MB/s 2023-09-30T09:54:10.824 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_690681/img bs=1k count=100 seek=1000 2023-09-30T09:54:10.825 INFO:tasks.workunit.client.0.smithi019.stderr:70+1 records in 2023-09-30T09:54:10.825 INFO:tasks.workunit.client.0.smithi019.stderr:70+1 records out 2023-09-30T09:54:10.825 INFO:tasks.workunit.client.0.smithi019.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000205242 s, 351 MB/s 2023-09-30T09:54:10.825 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_690681/img bs=1k seek=10000 2023-09-30T09:54:10.825 INFO:tasks.workunit.client.0.smithi019.stderr:139+1 records in 2023-09-30T09:54:10.826 INFO:tasks.workunit.client.0.smithi019.stderr:139+1 records out 2023-09-30T09:54:10.826 INFO:tasks.workunit.client.0.smithi019.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.00035146 s, 408 MB/s 2023-09-30T09:54:10.826 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_690681/img bs=1k seek=100000 2023-09-30T09:54:10.852 INFO:tasks.workunit.client.0.smithi019.stderr:70+1 records in 2023-09-30T09:54:10.853 INFO:tasks.workunit.client.0.smithi019.stderr:70+1 records out 2023-09-30T09:54:10.853 INFO:tasks.workunit.client.0.smithi019.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0261029 s, 2.8 MB/s 2023-09-30T09:54:10.853 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_690681/img bs=1k seek=1000000 2023-09-30T09:54:10.853 INFO:tasks.workunit.client.0.smithi019.stderr:189+1 records in 2023-09-30T09:54:10.854 INFO:tasks.workunit.client.0.smithi019.stderr:189+1 records out 2023-09-30T09:54:10.854 INFO:tasks.workunit.client.0.smithi019.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000487512 s, 397 MB/s 2023-09-30T09:54:10.854 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm testimg 2023-09-30T09:54:10.895 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 0% complete...failed. 2023-09-30T09:54:10.895 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: delete error: (2) No such file or directory 2023-09-30T09:54:10.900 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:54:10.900 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format /tmp/rbd_import_export_690681/img testimg 2023-09-30T09:54:11.471 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-30T09:54:11.472 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:11.474 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg /tmp/rbd_import_export_690681/img2 2023-09-30T09:54:11.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:11 smithi167 ceph-mon[192880]: pgmap v1687: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:11.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:11 smithi019 ceph-mon[216815]: pgmap v1687: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:11.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:11 smithi019 ceph-mon[224043]: pgmap v1687: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:11.727 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:11.730 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg - 2023-09-30T09:54:12.484 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:12.487 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm testimg 2023-09-30T09:54:12.596 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-30T09:54:12.599 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp /tmp/rbd_import_export_690681/img /tmp/rbd_import_export_690681/img2 2023-09-30T09:54:12.973 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:13.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:13.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:13 smithi019 ceph-mon[216815]: pgmap v1688: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:13.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:13 smithi019 ceph-mon[224043]: pgmap v1688: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:13.384 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp /tmp/rbd_import_export_690681/img /tmp/rbd_import_export_690681/img3 2023-09-30T09:54:13.437 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:13.469 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:13 smithi167 ceph-mon[192880]: pgmap v1688: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-30T09:54:13.471 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:13.523 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:13.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:13.631 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:13.657 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:13.700 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:13.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:13.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:13.805 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:13.831 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:13.873 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:13.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:13.899 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img2 /tmp/rbd_import_export_690681/img3 2023-09-30T09:54:13.979 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:14.005 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:14.049 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:14.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:14.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:14.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:54:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:54:13] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2023-09-30T09:54:14.154 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:14.180 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:14.225 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:14.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:14.331 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:14.361 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:15.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:14 smithi167 ceph-mon[192880]: pgmap v1689: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s rd, 0 op/s; 4/87 objects degraded (4.598%); 1/87 objects misplaced (1.149%) 2023-09-30T09:54:15.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:14 smithi019 ceph-mon[216815]: pgmap v1689: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s rd, 0 op/s; 4/87 objects degraded (4.598%); 1/87 objects misplaced (1.149%) 2023-09-30T09:54:15.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:14 smithi019 ceph-mon[224043]: pgmap v1689: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s rd, 0 op/s; 4/87 objects degraded (4.598%); 1/87 objects misplaced (1.149%) 2023-09-30T09:54:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:15 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4/87 objects degraded (4.598%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:15 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4/87 objects degraded (4.598%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:15 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4/87 objects degraded (4.598%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:16 smithi167 ceph-mon[192880]: pgmap v1690: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s; 4/87 objects degraded (4.598%); 1/87 objects misplaced (1.149%) 2023-09-30T09:54:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:16 smithi019 ceph-mon[216815]: pgmap v1690: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s; 4/87 objects degraded (4.598%); 1/87 objects misplaced (1.149%) 2023-09-30T09:54:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:16 smithi019 ceph-mon[224043]: pgmap v1690: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s; 4/87 objects degraded (4.598%); 1/87 objects misplaced (1.149%) 2023-09-30T09:54:18.860 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format - testimg 2023-09-30T09:54:19.269 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:19.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:19.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:19.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:19 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:54:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:19 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:19.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:19 smithi019 ceph-mon[216815]: pgmap v1691: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 109 KiB/s rd, 47 KiB/s wr, 28 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:19 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:54:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:19 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:19 smithi019 ceph-mon[224043]: pgmap v1691: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 109 KiB/s rd, 47 KiB/s wr, 28 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:19.374 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:19.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:19 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:54:19.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:19 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:19.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:19 smithi167 ceph-mon[192880]: pgmap v1691: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 109 KiB/s rd, 47 KiB/s wr, 28 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:19.412 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:19.463 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:19.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:19.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:54:19.493 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:54:19.571 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:19.599 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:19.632 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:19.632 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:19.634 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg /tmp/rbd_import_export_690681/img2 2023-09-30T09:54:19.642 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:19.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:19.749 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:19.786 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:19.906 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:19.911 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg - 2023-09-30T09:54:20.192 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:20.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:20.223 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:20.298 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:20.325 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46214.0:2308 9.6 9:6c010bf7:::benchmark_data_smithi019_560486_object2307:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:44:25.073425+0000", 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 595.28300710999997, 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0051149229999998, 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46214", 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 2308 2023-09-30T09:54:20.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073425+0000", 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073425+0000", 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073427+0000", 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.53e-06 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073444+0000", 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7336000000000001e-05 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073457+0000", 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2767e-05 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073463+0000", 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7910000000000001e-06 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073513+0000", 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9795e-05 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:25.073528+0000", 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5732000000000001e-05 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.048589+0000", 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97506089299999998 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:26.048605+0000", 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6209e-05 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.067517+0000", 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0189112579999999 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.067667+0000", 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014982999999999999 2023-09-30T09:54:20.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,5", 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.067825+0000", 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015861299999999999 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.069034+0000", 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001208771 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.069105+0000", 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0778000000000006e-05 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.078279+0000", 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091738259999999995 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.078342+0000", 2023-09-30T09:54:20.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3880000000000005e-05 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:44:27.078540+0000", 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019791399999999999 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(10.9 e586,578 10:99390b1e:::benchmark_data_smithi019_617837_object2246:head,565'5928)", 2023-09-30T09:54:20.368 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:37.119010+0000", 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 163.237422487, 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0682279239999999, 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:37.119010+0000", 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:37.119010+0000", 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:37.119011+0000", 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2600000000000001e-07 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:37.119017+0000", 2023-09-30T09:54:20.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.562e-06 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:37.119032+0000", 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4267000000000001e-05 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:37.119039+0000", 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2269999999999996e-06 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:40.158912+0000", 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0398735530000001 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:40.187238+0000", 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028325488999999999 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129550 10.9 10:95b81bb4:::benchmark_data_smithi019_617837_object36150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:04.439852+0000", 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 135.916580713, 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.010737803, 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129550 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.373 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.439852+0000", 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.439852+0000", 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.439853+0000", 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3069999999999999e-06 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.439858+0000", 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9130000000000002e-06 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.439870+0000", 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1681999999999999e-05 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.439877+0000", 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1899999999999998e-06 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.178976+0000", 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.73909930000000001 2023-09-30T09:54:20.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.204681+0000", 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.025704893999999999 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.226490+0000", 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.021808455000000001 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.377 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.275957+0000", 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.049467292000000003 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.304653+0000", 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028696023000000001 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.304671+0000", 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8542999999999998e-05 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.319524+0000", 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.014852981 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.319869+0000", 2023-09-30T09:54:20.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00034499200000000002 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.450401+0000", 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.130531489 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.450590+0000", 2023-09-30T09:54:20.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018874199999999999 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129582 10.9 10:995c3693:::benchmark_data_smithi019_617837_object36182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:04.545505+0000", 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 135.81092799800001, 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.940017906, 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:54:20.382 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129582 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.545505+0000", 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.545505+0000", 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.545506+0000", 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7659999999999999e-06 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.545512+0000", 2023-09-30T09:54:20.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8499999999999999e-06 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.545527+0000", 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.453e-05 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.545535+0000", 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4290000000000001e-06 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.246620+0000", 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.70108532899999998 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.256810+0000", 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010189948000000001 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,7", 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.266483+0000", 2023-09-30T09:54:20.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.009672656 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.276824+0000", 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010341037000000001 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305432+0000", 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.028608043999999999 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305446+0000", 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4121000000000001e-05 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.450638+0000", 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1451919770000001 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for missing object", 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.450997+0000", 2023-09-30T09:54:20.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035882900000000002 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485326+0000", 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.034328833000000003 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485522+0000", 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019655700000000001 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.389 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129591 10.9 10:9303dcc2:::benchmark_data_smithi019_617837_object36191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:04.581889+0000", 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 135.77454378600001, 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9043803619999999, 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.390 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129591 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.581889+0000", 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.581889+0000", 2023-09-30T09:54:20.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.581890+0000", 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4950000000000001e-06 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.581897+0000", 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7440000000000001e-06 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.581909+0000", 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1911e-05 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.581922+0000", 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3132999999999999e-05 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305510+0000", 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.72358831599999995 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305523+0000", 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2581e-05 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.451393+0000", 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1458698649999999 2023-09-30T09:54:20.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.451685+0000", 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00029218700000000001 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.451952+0000", 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00026671399999999998 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.484118+0000", 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.032166696000000002 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485621+0000", 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015024400000000001 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485753+0000", 2023-09-30T09:54:20.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000132433 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485925+0000", 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017167899999999999 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.486269+0000", 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000344168 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.397 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129593 10.9 10:934de09a:::benchmark_data_smithi019_617837_object36193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:04.586537+0000", 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 135.76989568900001, 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.900487228, 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.398 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129593 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.586537+0000", 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.586537+0000", 2023-09-30T09:54:20.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.586538+0000", 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2190000000000001e-06 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.586543+0000", 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6619999999999996e-06 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.586557+0000", 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3987e-05 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.586563+0000", 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2639999999999997e-06 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305552+0000", 2023-09-30T09:54:20.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.71898905700000004 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305563+0000", 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1049e-05 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.455081+0000", 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.149517559 2023-09-30T09:54:20.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.455326+0000", 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024527300000000001 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.455604+0000", 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027789999999999998 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485249+0000", 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.029645241999999999 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.486372+0000", 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011231469999999999 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.486497+0000", 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000124478 2023-09-30T09:54:20.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.486652+0000", 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015482500000000001 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.487024+0000", 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00037256600000000001 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.405 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129600 10.9 10:9cc115b5:::benchmark_data_smithi019_617837_object36200:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:04.724334+0000", 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 135.632098173, 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.763285668, 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.406 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129600 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.724334+0000", 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.407 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.724334+0000", 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.724336+0000", 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8300000000000001e-06 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.724342+0000", 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6790000000000002e-06 2023-09-30T09:54:20.408 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.724357+0000", 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4834999999999999e-05 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.724365+0000", 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6219999999999998e-06 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.409 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305594+0000", 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.58122832000000002 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305599+0000", 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1569999999999999e-06 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.410 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.458250+0000", 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1526513940000001 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.458498+0000", 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00024743900000000001 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.458761+0000", 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00026359400000000002 2023-09-30T09:54:20.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485270+0000", 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.026509027000000001 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.487129+0000", 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018589399999999999 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.487245+0000", 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115479 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.487354+0000", 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000109582 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.487620+0000", 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00026577000000000003 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129617 10.9 10:95eacad5:::benchmark_data_smithi019_617837_object36217:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:04.809121+0000", 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 135.547311799, 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6790295200000001, 2023-09-30T09:54:20.414 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129617 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.809121+0000", 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.809121+0000", 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.809122+0000", 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.623e-06 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.809129+0000", 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6050000000000003e-06 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.809146+0000", 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6979000000000001e-05 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.809155+0000", 2023-09-30T09:54:20.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3070000000000008e-06 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305640+0000", 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.49648456600000002 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305651+0000", 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.096e-05 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.479614+0000", 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.17396364 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.479946+0000", 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033121300000000002 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:54:20.419 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.480366+0000", 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00041998600000000001 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485664+0000", 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005298105 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.487704+0000", 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0020405000000000002 2023-09-30T09:54:20.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.487792+0000", 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8200000000000003e-05 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.487896+0000", 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000103164 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.488150+0000", 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000254672 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129629 10.9 10:960a506f:::benchmark_data_smithi019_617837_object36229:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:04.836801+0000", 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 135.51963105499999, 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.661294311, 2023-09-30T09:54:20.422 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129629 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.836801+0000", 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.423 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.836801+0000", 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.836803+0000", 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.823e-06 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.836809+0000", 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.122e-06 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.836827+0000", 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7325000000000001e-05 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:04.836837+0000", 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7899999999999994e-06 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305685+0000", 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.468848294 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:20.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.305690+0000", 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1050000000000003e-06 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.488203+0000", 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.182512988 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.488475+0000", 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027167499999999998 2023-09-30T09:54:20.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,7", 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.488774+0000", 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00029900899999999998 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.497518+0000", 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0087445519999999992 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.497557+0000", 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9195e-05 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.497639+0000", 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1766000000000003e-05 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.429 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.497791+0000", 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000152146 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.498096+0000", 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00030452099999999998 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:138215 10.5 10:ac5dc606:::benchmark_data_smithi019_617837_object44815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:23.125097+0000", 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 117.231336042, 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.18707847899999999, 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 138215 2023-09-30T09:54:20.431 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125097+0000", 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125097+0000", 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.432 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125098+0000", 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.646e-06 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125104+0000", 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5890000000000002e-06 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.433 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125116+0000", 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2334000000000001e-05 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125123+0000", 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8689999999999998e-06 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125174+0000", 2023-09-30T09:54:20.434 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1489000000000001e-05 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125274+0000", 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9882000000000004e-05 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.125338+0000", 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4030999999999997e-05 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.127811+0000", 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0024724080000000002 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.127911+0000", 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010034099999999999 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.436 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.312076+0000", 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.18416511699999999 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.312135+0000", 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8566000000000001e-05 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.312175+0000", 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0207000000000003e-05 2023-09-30T09:54:20.437 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:140227 10.5 10:abb122a2:::benchmark_data_smithi019_617837_object46827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e605)", 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:26.748526+0000", 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 113.607906052, 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.181308833, 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.438 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 140227 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748526+0000", 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.439 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748526+0000", 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748528+0000", 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1909999999999999e-06 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.440 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748533+0000", 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2100000000000001e-06 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748548+0000", 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4608e-05 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748554+0000", 2023-09-30T09:54:20.441 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5350000000000001e-06 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748618+0000", 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3769999999999994e-05 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748726+0000", 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000108022 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.442 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.748826+0000", 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000100566 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.751471+0000", 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026441160000000002 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.443 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.751670+0000", 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019916000000000001 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.929712+0000", 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.17804197599999999 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.929781+0000", 2023-09-30T09:54:20.444 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9109000000000005e-05 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:26.929835+0000", 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4570000000000001e-05 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:150189 10.9 e609/601 10:9c288ecb:::benchmark_data_smithi019_617837_object56789:head v 609'9320, mlcod=609'9318)", 2023-09-30T09:54:20.445 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:44.127876+0000", 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 96.228556979999993, 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20166214399999999, 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.127876+0000", 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.446 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.127876+0000", 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.127876+0000", 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1100000000000004e-07 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.127887+0000", 2023-09-30T09:54:20.447 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.006e-05 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.127896+0000", 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4700000000000008e-06 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.127907+0000", 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0808000000000001e-05 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.448 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.127972+0000", 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5123999999999995e-05 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.127996+0000", 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4148e-05 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.329459+0000", 2023-09-30T09:54:20.449 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.201463067 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.329538+0000", 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8555999999999998e-05 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:150191 10.9 e609/601 10:987fe4da:::benchmark_data_smithi019_617837_object56791:head v 609'9321, mlcod=609'9318)", 2023-09-30T09:54:20.450 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:44.131162+0000", 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 96.225270813999998, 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.20349303099999999, 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.131162+0000", 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.451 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.131162+0000", 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.131162+0000", 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6800000000000001e-07 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.131169+0000", 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4300000000000003e-06 2023-09-30T09:54:20.452 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.329447+0000", 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.198278125 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.329467+0000", 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0271e-05 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.453 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.329573+0000", 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000106467 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.329600+0000", 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7056e-05 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.454 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.334600+0000", 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0049995359999999997 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:44.334655+0000", 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4778000000000003e-05 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.455 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:159173 10.5 10:a4390122:::benchmark_data_smithi019_617837_object65773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e613)", 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:57.596561+0000", 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 82.759871224999998, 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.22297714799999999, 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 159173 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.456 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596561+0000", 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596561+0000", 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.457 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596563+0000", 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.248e-06 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596568+0000", 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9899999999999997e-06 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.458 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596578+0000", 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0861e-05 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596584+0000", 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8619999999999999e-06 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596637+0000", 2023-09-30T09:54:20.459 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2327000000000002e-05 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596725+0000", 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8302999999999998e-05 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.596796+0000", 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0778999999999994e-05 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.460 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.599690+0000", 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002894148 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.599865+0000", 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017510799999999999 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.461 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.819433+0000", 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.21956763700000001 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.819492+0000", 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9750000000000002e-05 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:57.819538+0000", 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6134999999999998e-05 2023-09-30T09:54:20.462 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:161349 10.5 10:ada3d3f7:::benchmark_data_smithi019_617837_object67949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e614)", 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:00.958698+0000", 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 79.397734145000001, 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.173431627, 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.463 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 161349 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.958698+0000", 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.464 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.958698+0000", 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.958700+0000", 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9989999999999998e-06 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.958706+0000", 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5300000000000004e-06 2023-09-30T09:54:20.465 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.958719+0000", 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2629000000000001e-05 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.958728+0000", 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9830000000000002e-06 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.466 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.958801+0000", 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3628999999999996e-05 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.958952+0000", 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000150354 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 0,3", 2023-09-30T09:54:20.467 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:00.959074+0000", 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012234400000000001 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:01.128024+0000", 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.16894968399999999 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:01.128070+0000", 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6186000000000001e-05 2023-09-30T09:54:20.468 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:01.132032+0000", 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0039627559999999996 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:01.132087+0000", 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4908999999999999e-05 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.469 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:01.132130+0000", 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2623999999999997e-05 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:185181 10.5 10:a8140a98:::benchmark_data_smithi019_617837_object91781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:35.559177+0000", 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 44.797255847999999, 2023-09-30T09:54:20.470 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.189868225, 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 185181 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559177+0000", 2023-09-30T09:54:20.471 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559177+0000", 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559178+0000", 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4130000000000001e-06 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.472 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559190+0000", 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1934e-05 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559207+0000", 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7289e-05 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.473 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559216+0000", 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9190000000000005e-06 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559274+0000", 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7293000000000002e-05 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559362+0000", 2023-09-30T09:54:20.474 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8283999999999997e-05 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3", 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.559434+0000", 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2235000000000002e-05 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.562942+0000", 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0035077189999999999 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.475 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.748864+0000", 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.18592207899999999 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.748933+0000", 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9109000000000005e-05 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.476 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.749045+0000", 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000111951 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:186751 10.5 10:a524a895:::benchmark_data_smithi019_617837_object93351:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.539651+0000", 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 42.816781505999998, 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31566933800000002, 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.477 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 186751 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.539651+0000", 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.478 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.539651+0000", 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.539652+0000", 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0189999999999999e-06 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.479 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.539656+0000", 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3760000000000001e-06 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.539667+0000", 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.01e-05 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.539672+0000", 2023-09-30T09:54:20.480 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4099999999999999e-06 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.539726+0000", 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4494999999999999e-05 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.539883+0000", 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015667 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.481 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3", 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.540029+0000", 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000145921 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.543999+0000", 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0039699510000000002 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.482 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.855146+0000", 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31114718800000002 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.855226+0000", 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.9453e-05 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.855320+0000", 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4754999999999997e-05 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.483 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:186756 10.5 10:ac3838a2:::benchmark_data_smithi019_617837_object93356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.552601+0000", 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 42.803831698000003, 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30654986899999997, 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.484 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 186756 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552601+0000", 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552601+0000", 2023-09-30T09:54:20.485 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552602+0000", 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8999999999999995e-07 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552605+0000", 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7979999999999999e-06 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.486 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552619+0000", 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3107e-05 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552624+0000", 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.9359999999999996e-06 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.487 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552674+0000", 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9181000000000002e-05 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552765+0000", 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1077999999999998e-05 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3", 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.552835+0000", 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9888000000000006e-05 2023-09-30T09:54:20.488 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555858+0000", 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003023338 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.858991+0000", 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30313316299999998 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.489 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.859054+0000", 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2736000000000004e-05 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.859151+0000", 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6854000000000005e-05 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.490 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:186758 10.5 10:a07d56da:::benchmark_data_smithi019_617837_object93358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.555836+0000", 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 42.800596183000003, 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.303537108, 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.491 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 186758 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555836+0000", 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555836+0000", 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.492 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555837+0000", 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0470000000000001e-06 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555843+0000", 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2859999999999999e-06 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.493 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555855+0000", 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2147000000000001e-05 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555863+0000", 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1950000000000003e-06 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.494 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555925+0000", 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2065999999999998e-05 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.556036+0000", 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000110937 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3", 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.556117+0000", 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1161000000000004e-05 2023-09-30T09:54:20.495 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.561236+0000", 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0051191040000000002 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.859232+0000", 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.29799552800000001 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.496 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.859290+0000", 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8239999999999998e-05 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.859373+0000", 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3397000000000001e-05 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.497 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:186759 10.5 10:ac728139:::benchmark_data_smithi019_617837_object93359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.555968+0000", 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 42.800464757999997, 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31363837999999999, 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 186759 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.498 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555968+0000", 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555968+0000", 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.499 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555969+0000", 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4499999999999996e-07 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555973+0000", 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2749999999999997e-06 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:20.500 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555983+0000", 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7880000000000001e-06 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.555992+0000", 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6860000000000001e-06 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.558794+0000", 2023-09-30T09:54:20.501 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002802115 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.558890+0000", 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5307000000000001e-05 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 3", 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.558962+0000", 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2739999999999998e-05 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.502 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.561391+0000", 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002428281 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.869470+0000", 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30807973700000002 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.503 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.869516+0000", 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5649000000000003e-05 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.869606+0000", 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0056999999999995e-05 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:20.504 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:20.505 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:54:20.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:20.506 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:20.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:20.607 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:20.644 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:20.733 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:20.736 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm testimg 2023-09-30T09:54:20.832 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-30T09:54:20.836 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp /tmp/rbd_import_export_690681/img /tmp/rbd_import_export_690681/img2 2023-09-30T09:54:21.495 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-30T09:54:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:21 smithi167 ceph-mon[192880]: pgmap v1692: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 100 KiB/s rd, 42 KiB/s wr, 26 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:21.522 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:21 smithi019 ceph-mon[216815]: pgmap v1692: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 100 KiB/s rd, 42 KiB/s wr, 26 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:21.522 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:21 smithi019 ceph-mon[224043]: pgmap v1692: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 100 KiB/s rd, 42 KiB/s wr, 26 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:21.526 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp /tmp/rbd_import_export_690681/img /tmp/rbd_import_export_690681/img3 2023-09-30T09:54:21.802 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:21.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:21.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:21.907 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:21.940 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:21.990 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:21.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:21.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:22.062 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img /tmp/rbd_import_export_690681/img2 /tmp/rbd_import_export_690681/img3 2023-09-30T09:54:22.096 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:22.124 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:22.167 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:22.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:22.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:22.274 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:22.302 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:23.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:23 smithi167 ceph-mon[192880]: pgmap v1693: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 100 KiB/s rd, 42 KiB/s wr, 26 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:23.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:23 smithi019 ceph-mon[216815]: pgmap v1693: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 100 KiB/s rd, 42 KiB/s wr, 26 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:23.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:23 smithi019 ceph-mon[224043]: pgmap v1693: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 100 KiB/s rd, 42 KiB/s wr, 26 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:24.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:54:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:54:23] "GET /metrics HTTP/1.1" 200 34119 "" "Prometheus/2.43.0" 2023-09-30T09:54:24.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:24 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:24.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:24 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:24.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:24 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:25.477 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:25 smithi167 ceph-mon[192880]: pgmap v1694: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 127 KiB/s rd, 45 KiB/s wr, 63 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:25.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:25 smithi019 ceph-mon[216815]: pgmap v1694: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 127 KiB/s rd, 45 KiB/s wr, 63 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:25.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:25 smithi019 ceph-mon[224043]: pgmap v1694: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 127 KiB/s rd, 45 KiB/s wr, 63 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:27.481 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd help export 2023-09-30T09:54:27.481 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -q export-format 2023-09-30T09:54:27.501 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_690681/img bs=1k count=10 seek=100 2023-09-30T09:54:27.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:27 smithi167 ceph-mon[192880]: pgmap v1695: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 210 KiB/s rd, 86 KiB/s wr, 66 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:27.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:27 smithi019 ceph-mon[216815]: pgmap v1695: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 210 KiB/s rd, 86 KiB/s wr, 66 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:27.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:27 smithi019 ceph-mon[224043]: pgmap v1695: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 210 KiB/s rd, 86 KiB/s wr, 66 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:27.585 INFO:tasks.workunit.client.0.smithi019.stderr:10+0 records in 2023-09-30T09:54:27.585 INFO:tasks.workunit.client.0.smithi019.stderr:10+0 records out 2023-09-30T09:54:27.585 INFO:tasks.workunit.client.0.smithi019.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.9079e-05 s, 129 MB/s 2023-09-30T09:54:27.585 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format /tmp/rbd_import_export_690681/img testimg 2023-09-30T09:54:27.647 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:27.647 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:27.649 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap create testimg@snap 2023-09-30T09:54:28.092 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:28.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:28.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:28.174 INFO:tasks.workunit.client.0.smithi019.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-30T09:54:28.182 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd image-meta set testimg key1 value1 2023-09-30T09:54:28.196 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:28.229 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:28.235 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-30T09:54:28.237 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd image-meta list testimg 2023-09-30T09:54:28.278 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:28.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:28.284 INFO:tasks.workunit.client.0.smithi019.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-30T09:54:28.285 INFO:tasks.workunit.client.0.smithi019.stderr: 2023-09-30T09:54:28.285 INFO:tasks.workunit.client.0.smithi019.stderr:Key Value 2023-09-30T09:54:28.285 INFO:tasks.workunit.client.0.smithi019.stderr:key1 value1' 2023-09-30T09:54:28.285 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_690681/img_v2 2023-09-30T09:54:28.329 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:28.332 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_690681/img_v2 testimg_import 2023-09-30T09:54:28.384 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:28.411 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:28.454 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:28.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:28.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:28.561 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:28.592 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:29.182 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:54:29.568 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:29 smithi167 ceph-mon[192880]: pgmap v1696: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 111 KiB/s rd, 44 KiB/s wr, 41 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:29.568 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:29 smithi167 ceph-mon[192880]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.568 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:29 smithi167 ceph-mon[192880]: osdmap e632: 8 total, 7 up, 8 in 2023-09-30T09:54:29.568 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:29 smithi167 ceph-mon[192880]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.568 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:29 smithi167 ceph-mon[192880]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.568 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout:rbd image 'testimg_import': 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout: size 110 KiB in 1 objects 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout: order 22 (4 MiB objects) 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout: snapshot_count: 1 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout: id: b7ec5ad5e03e 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout: block_name_prefix: rbd_data.b7ec5ad5e03e 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout: format: 2 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-30T09:54:29.569 INFO:tasks.workunit.client.0.smithi019.stdout: op_features: 2023-09-30T09:54:29.570 INFO:tasks.workunit.client.0.smithi019.stdout: flags: 2023-09-30T09:54:29.570 INFO:tasks.workunit.client.0.smithi019.stdout: create_timestamp: Sat Sep 30 09:54:28 2023 2023-09-30T09:54:29.570 INFO:tasks.workunit.client.0.smithi019.stdout: access_timestamp: Sat Sep 30 09:54:28 2023 2023-09-30T09:54:29.570 INFO:tasks.workunit.client.0.smithi019.stdout: modify_timestamp: Sat Sep 30 09:54:28 2023 2023-09-30T09:54:29.570 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import@snap 2023-09-30T09:54:29.570 INFO:tasks.workunit.client.0.smithi019.stdout:rbd image 'testimg_import': 2023-09-30T09:54:29.570 INFO:tasks.workunit.client.0.smithi019.stdout: size 110 KiB in 1 objects 2023-09-30T09:54:29.571 INFO:tasks.workunit.client.0.smithi019.stdout: order 22 (4 MiB objects) 2023-09-30T09:54:29.571 INFO:tasks.workunit.client.0.smithi019.stdout: snapshot_count: 1 2023-09-30T09:54:29.571 INFO:tasks.workunit.client.0.smithi019.stdout: id: b7ec5ad5e03e 2023-09-30T09:54:29.571 INFO:tasks.workunit.client.0.smithi019.stdout: block_name_prefix: rbd_data.b7ec5ad5e03e 2023-09-30T09:54:29.571 INFO:tasks.workunit.client.0.smithi019.stdout: format: 2 2023-09-30T09:54:29.571 INFO:tasks.workunit.client.0.smithi019.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-30T09:54:29.571 INFO:tasks.workunit.client.0.smithi019.stdout: op_features: 2023-09-30T09:54:29.573 INFO:tasks.workunit.client.0.smithi019.stdout: flags: 2023-09-30T09:54:29.573 INFO:tasks.workunit.client.0.smithi019.stdout: create_timestamp: Sat Sep 30 09:54:28 2023 2023-09-30T09:54:29.573 INFO:tasks.workunit.client.0.smithi019.stdout: access_timestamp: Sat Sep 30 09:54:28 2023 2023-09-30T09:54:29.573 INFO:tasks.workunit.client.0.smithi019.stdout: modify_timestamp: Sat Sep 30 09:54:28 2023 2023-09-30T09:54:29.574 INFO:tasks.workunit.client.0.smithi019.stdout: protected: False 2023-09-30T09:54:29.574 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd image-meta list testimg_import 2023-09-30T09:54:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:29.575 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:29.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr: 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr:Key Value 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr:key1 value1' 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr: 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr:Key Value 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr: 2023-09-30T09:54:29.577 INFO:tasks.workunit.client.0.smithi019.stderr:Key Value 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:key1 value1' ']' 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg_import /tmp/rbd_import_export_690681/img_import 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete...done. 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_690681/img /tmp/rbd_import_export_690681/img_import 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_690681/img /tmp/rbd_import_export_690681/img_import 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/img --format %b 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=24 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/img_import --format %b 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=224 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=200 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=200 2023-09-30T09:54:29.578 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-30T09:54:29.579 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_690681/img_snap 2023-09-30T09:54:29.579 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete...done. 2023-09-30T09:54:29.579 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_690681/img_snap_import 2023-09-30T09:54:29.579 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:29.580 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete...done. 2023-09-30T09:54:29.580 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_690681/img_snap /tmp/rbd_import_export_690681/img_snap_import 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_690681/img_snap /tmp/rbd_import_export_690681/img_snap_import 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/img_snap --format %b 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=224 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/img_snap_import --format %b 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=224 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img_v2 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img_import 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img_snap 2023-09-30T09:54:29.581 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img_snap_import 2023-09-30T09:54:29.582 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap rm testimg_import@snap 2023-09-30T09:54:29.582 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:29.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[224043]: pgmap v1696: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 111 KiB/s rd, 44 KiB/s wr, 41 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:29.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[224043]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[224043]: osdmap e632: 8 total, 7 up, 8 in 2023-09-30T09:54:29.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[224043]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[224043]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[216815]: pgmap v1696: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 111 KiB/s rd, 44 KiB/s wr, 41 op/s; 4/84 objects degraded (4.762%) 2023-09-30T09:54:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[216815]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[216815]: osdmap e632: 8 total, 7 up, 8 in 2023-09-30T09:54:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[216815]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:29 smithi019 ceph-mon[216815]: failed to encode map e632 with expected crc 2023-09-30T09:54:29.663 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:54:29.725 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:54:29.725 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:29.795 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:30.046 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:30.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:30.155 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:30.195 INFO:tasks.workunit.client.0.smithi019.stderr: Removing snap: 100% complete...done. 2023-09-30T09:54:30.195 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:54:30.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:30 smithi019 ceph-mon[216815]: osdmap e633: 8 total, 7 up, 8 in 2023-09-30T09:54:30.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:30 smithi019 ceph-mon[216815]: failed to encode map e633 with expected crc 2023-09-30T09:54:30.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:30 smithi019 ceph-mon[216815]: failed to encode map e633 with expected crc 2023-09-30T09:54:30.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:30 smithi019 ceph-mon[216815]: failed to encode map e633 with expected crc 2023-09-30T09:54:30.223 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:30 smithi019 ceph-mon[224043]: osdmap e633: 8 total, 7 up, 8 in 2023-09-30T09:54:30.223 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:30 smithi019 ceph-mon[224043]: failed to encode map e633 with expected crc 2023-09-30T09:54:30.223 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:30 smithi019 ceph-mon[224043]: failed to encode map e633 with expected crc 2023-09-30T09:54:30.223 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:30 smithi019 ceph-mon[224043]: failed to encode map e633 with expected crc 2023-09-30T09:54:30.227 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:30.296 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:30.299 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap rm testimg@snap 2023-09-30T09:54:30.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:30 smithi167 ceph-mon[192880]: osdmap e633: 8 total, 7 up, 8 in 2023-09-30T09:54:30.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:30 smithi167 ceph-mon[192880]: failed to encode map e633 with expected crc 2023-09-30T09:54:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:30 smithi167 ceph-mon[192880]: failed to encode map e633 with expected crc 2023-09-30T09:54:30.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:30 smithi167 ceph-mon[192880]: failed to encode map e633 with expected crc 2023-09-30T09:54:31.000 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1698,"stamp":"2023-09-30T09:54:29.156670+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790288,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":6926,"num_read_kb":15957,"num_write":1060,"num_write_kb":13779,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":373,"num_bytes_recovered":13514604,"num_keys_recovered":50,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6193,"ondisk_log_size":6193,"up":294,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6995752,"kb_used_data":1309728,"kb_used_omap":30,"kb_used_meta":5685921,"kb_avail":742965464,"statfs":{"total":767960285184,"available":760796635136,"internally_reserved":0,"allocated":1341161472,"data_stored":1318312107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31390,"internal_metadata":5822383458},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":105,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":680,"num_read_kb":1549,"num_write":102,"num_write_kb":545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.072635"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":1100,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183554+0000","last_change":"2023-09-30T09:53:33.962998+0000","last_active":"2023-09-30T09:54:02.183554+0000","last_peered":"2023-09-30T09:54:02.183554+0000","last_clean":"2023-09-30T09:54:02.183554+0000","last_became_active":"2023-09-30T09:53:33.962583+0000","last_became_peered":"2023-09-30T09:53:33.962583+0000","last_unstale":"2023-09-30T09:54:02.183554+0000","last_undegraded":"2023-09-30T09:54:02.183554+0000","last_fullsized":"2023-09-30T09:54:02.183554+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:02:25.528383+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":668,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.204332+0000","last_change":"2023-09-30T09:50:41.402505+0000","last_active":"2023-09-30T09:54:02.204332+0000","last_peered":"2023-09-30T09:54:02.204332+0000","last_clean":"2023-09-30T09:54:02.204332+0000","last_became_active":"2023-09-30T09:50:41.402099+0000","last_became_peered":"2023-09-30T09:50:41.402099+0000","last_unstale":"2023-09-30T09:54:02.204332+0000","last_undegraded":"2023-09-30T09:54:02.204332+0000","last_fullsized":"2023-09-30T09:54:02.204332+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:45.502519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":545,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.205677+0000","last_change":"2023-09-30T09:43:45.122832+0000","last_active":"2023-09-30T09:54:02.205677+0000","last_peered":"2023-09-30T09:54:02.205677+0000","last_clean":"2023-09-30T09:54:02.205677+0000","last_became_active":"2023-09-30T09:43:45.122323+0000","last_became_peered":"2023-09-30T09:43:45.122323+0000","last_unstale":"2023-09-30T09:54:02.205677+0000","last_undegraded":"2023-09-30T09:54:02.205677+0000","last_fullsized":"2023-09-30T09:54:02.205677+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:13.670809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":1076,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183963+0000","last_change":"2023-09-30T09:52:09.288155+0000","last_active":"2023-09-30T09:54:02.183963+0000","last_peered":"2023-09-30T09:54:02.183963+0000","last_clean":"2023-09-30T09:54:02.183963+0000","last_became_active":"2023-09-30T09:52:09.287827+0000","last_became_peered":"2023-09-30T09:52:09.287827+0000","last_unstale":"2023-09-30T09:54:02.183963+0000","last_undegraded":"2023-09-30T09:54:02.183963+0000","last_fullsized":"2023-09-30T09:54:02.183963+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:35:04.226064+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":206,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.187625+0000","last_change":"2023-09-30T09:51:57.147377+0000","last_active":"2023-09-30T09:54:02.187625+0000","last_peered":"2023-09-30T09:54:02.187625+0000","last_clean":"2023-09-30T09:54:02.187625+0000","last_became_active":"2023-09-30T09:51:57.146997+0000","last_became_peered":"2023-09-30T09:51:57.146997+0000","last_unstale":"2023-09-30T09:54:02.187625+0000","last_undegraded":"2023-09-30T09:54:02.187625+0000","last_fullsized":"2023-09-30T09:54:02.187625+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:57:01.086217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":346,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384882+0000","last_change":"2023-09-30T09:52:06.288110+0000","last_active":"2023-09-30T09:54:07.384882+0000","last_peered":"2023-09-30T09:54:07.384882+0000","last_clean":"2023-09-30T09:54:07.384882+0000","last_became_active":"2023-09-30T09:52:06.287784+0000","last_became_peered":"2023-09-30T09:52:06.287784+0000","last_unstale":"2023-09-30T09:54:07.384882+0000","last_undegraded":"2023-09-30T09:54:07.384882+0000","last_fullsized":"2023-09-30T09:54:07.384882+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:26:56.675676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":642,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384239+0000","last_change":"2023-09-30T09:52:09.270404+0000","last_active":"2023-09-30T09:54:07.384239+0000","last_peered":"2023-09-30T09:54:07.384239+0000","last_clean":"2023-09-30T09:54:07.384239+0000","last_became_active":"2023-09-30T09:52:09.269933+0000","last_became_peered":"2023-09-30T09:52:09.269933+0000","last_unstale":"2023-09-30T09:54:07.384239+0000","last_undegraded":"2023-09-30T09:54:07.384239+0000","last_fullsized":"2023-09-30T09:54:07.384239+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.857444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.857444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:36:26.548074+0000","scrub_duration":0.0014606759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":53,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.186801+0000","last_change":"2023-09-30T09:51:57.155291+0000","last_active":"2023-09-30T09:54:07.186801+0000","last_peered":"2023-09-30T09:54:07.186801+0000","last_clean":"2023-09-30T09:54:07.186801+0000","last_became_active":"2023-09-30T09:51:57.154983+0000","last_became_peered":"2023-09-30T09:51:57.154983+0000","last_unstale":"2023-09-30T09:54:07.186801+0000","last_undegraded":"2023-09-30T09:54:07.186801+0000","last_fullsized":"2023-09-30T09:54:07.186801+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:44:16.352650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.194923+0000","last_change":"2023-09-30T09:49:31.310014+0000","last_active":"2023-09-30T09:54:02.194923+0000","last_peered":"2023-09-30T09:54:02.194923+0000","last_clean":"2023-09-30T09:54:02.194923+0000","last_became_active":"2023-09-30T09:49:31.309121+0000","last_became_peered":"2023-09-30T09:49:31.309121+0000","last_unstale":"2023-09-30T09:54:02.194923+0000","last_undegraded":"2023-09-30T09:54:02.194923+0000","last_fullsized":"2023-09-30T09:54:02.194923+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:46.598805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":1033,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384732+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:54:07.384732+0000","last_peered":"2023-09-30T09:54:07.384732+0000","last_clean":"2023-09-30T09:54:07.384732+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:54:07.384732+0000","last_undegraded":"2023-09-30T09:54:07.384732+0000","last_fullsized":"2023-09-30T09:54:07.384732+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":356,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.184700+0000","last_change":"2023-09-30T09:52:06.288453+0000","last_active":"2023-09-30T09:54:02.184700+0000","last_peered":"2023-09-30T09:54:02.184700+0000","last_clean":"2023-09-30T09:54:02.184700+0000","last_became_active":"2023-09-30T09:52:06.287939+0000","last_became_peered":"2023-09-30T09:52:06.287939+0000","last_unstale":"2023-09-30T09:54:02.184700+0000","last_undegraded":"2023-09-30T09:54:02.184700+0000","last_fullsized":"2023-09-30T09:54:02.184700+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:13:58.390726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":230,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183538+0000","last_change":"2023-09-30T09:49:31.295632+0000","last_active":"2023-09-30T09:54:02.183538+0000","last_peered":"2023-09-30T09:54:02.183538+0000","last_clean":"2023-09-30T09:54:02.183538+0000","last_became_active":"2023-09-30T09:49:31.295269+0000","last_became_peered":"2023-09-30T09:49:31.295269+0000","last_unstale":"2023-09-30T09:54:02.183538+0000","last_undegraded":"2023-09-30T09:54:02.183538+0000","last_fullsized":"2023-09-30T09:54:02.183538+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:32.754585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":633,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.181838+0000","last_change":"2023-09-30T09:51:59.874993+0000","last_active":"2023-09-30T09:54:02.181838+0000","last_peered":"2023-09-30T09:54:02.181838+0000","last_clean":"2023-09-30T09:54:02.181838+0000","last_became_active":"2023-09-30T09:51:59.874470+0000","last_became_peered":"2023-09-30T09:51:59.874470+0000","last_unstale":"2023-09-30T09:54:02.181838+0000","last_undegraded":"2023-09-30T09:54:02.181838+0000","last_fullsized":"2023-09-30T09:54:02.181838+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:51:44.626194+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":51,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.184488+0000","last_change":"2023-09-30T09:51:57.160642+0000","last_active":"2023-09-30T09:54:02.184488+0000","last_peered":"2023-09-30T09:54:02.184488+0000","last_clean":"2023-09-30T09:54:02.184488+0000","last_became_active":"2023-09-30T09:51:57.160357+0000","last_became_peered":"2023-09-30T09:51:57.160357+0000","last_unstale":"2023-09-30T09:54:02.184488+0000","last_undegraded":"2023-09-30T09:54:02.184488+0000","last_fullsized":"2023-09-30T09:54:02.184488+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:40.417904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":608,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384206+0000","last_change":"2023-09-30T09:49:31.309241+0000","last_active":"2023-09-30T09:54:07.384206+0000","last_peered":"2023-09-30T09:54:07.384206+0000","last_clean":"2023-09-30T09:54:07.384206+0000","last_became_active":"2023-09-30T09:49:31.308703+0000","last_became_peered":"2023-09-30T09:49:31.308703+0000","last_unstale":"2023-09-30T09:54:07.384206+0000","last_undegraded":"2023-09-30T09:54:07.384206+0000","last_fullsized":"2023-09-30T09:54:07.384206+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:06.877644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":820,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183794+0000","last_change":"2023-09-30T09:52:09.265362+0000","last_active":"2023-09-30T09:54:02.183794+0000","last_peered":"2023-09-30T09:54:02.183794+0000","last_clean":"2023-09-30T09:54:02.183794+0000","last_became_active":"2023-09-30T09:52:09.265043+0000","last_became_peered":"2023-09-30T09:52:09.265043+0000","last_unstale":"2023-09-30T09:54:02.183794+0000","last_undegraded":"2023-09-30T09:54:02.183794+0000","last_fullsized":"2023-09-30T09:54:02.183794+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:50:39.738075+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":346,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384968+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:54:07.384968+0000","last_peered":"2023-09-30T09:54:07.384968+0000","last_clean":"2023-09-30T09:54:07.384968+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:54:07.384968+0000","last_undegraded":"2023-09-30T09:54:07.384968+0000","last_fullsized":"2023-09-30T09:54:07.384968+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183302+0000","last_change":"2023-09-30T09:50:41.399925+0000","last_active":"2023-09-30T09:54:02.183302+0000","last_peered":"2023-09-30T09:54:02.183302+0000","last_clean":"2023-09-30T09:54:02.183302+0000","last_became_active":"2023-09-30T09:50:41.399642+0000","last_became_peered":"2023-09-30T09:50:41.399642+0000","last_unstale":"2023-09-30T09:54:02.183302+0000","last_undegraded":"2023-09-30T09:54:02.183302+0000","last_fullsized":"2023-09-30T09:54:02.183302+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:45:44.698050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":477,"reported_epoch":630,"state":"active+undersized","last_fresh":"2023-09-30T09:54:02.183076+0000","last_change":"2023-09-30T09:53:33.219615+0000","last_active":"2023-09-30T09:54:02.183076+0000","last_peered":"2023-09-30T09:54:02.183076+0000","last_clean":"2023-09-30T09:32:04.392210+0000","last_became_active":"2023-09-30T09:53:33.219615+0000","last_became_peered":"2023-09-30T09:53:33.219615+0000","last_unstale":"2023-09-30T09:54:02.183076+0000","last_undegraded":"2023-09-30T09:54:02.183076+0000","last_fullsized":"2023-09-30T09:53:33.216753+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:12:14.708037+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":405,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.383971+0000","last_change":"2023-09-30T09:52:15.347512+0000","last_active":"2023-09-30T09:54:07.383971+0000","last_peered":"2023-09-30T09:54:07.383971+0000","last_clean":"2023-09-30T09:54:07.383971+0000","last_became_active":"2023-09-30T09:52:15.339136+0000","last_became_peered":"2023-09-30T09:52:15.339136+0000","last_unstale":"2023-09-30T09:54:07.383971+0000","last_undegraded":"2023-09-30T09:54:07.383971+0000","last_fullsized":"2023-09-30T09:54:07.383971+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:27:48.631308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9444,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":672,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.204736+0000","last_change":"2023-09-30T09:49:31.304624+0000","last_active":"2023-09-30T09:54:02.204736+0000","last_peered":"2023-09-30T09:54:02.204736+0000","last_clean":"2023-09-30T09:54:02.204736+0000","last_became_active":"2023-09-30T09:49:31.304106+0000","last_became_peered":"2023-09-30T09:49:31.304106+0000","last_unstale":"2023-09-30T09:54:02.204736+0000","last_undegraded":"2023-09-30T09:54:02.204736+0000","last_fullsized":"2023-09-30T09:54:02.204736+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:42.343641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":911,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182395+0000","last_change":"2023-09-30T09:51:59.873231+0000","last_active":"2023-09-30T09:54:02.182395+0000","last_peered":"2023-09-30T09:54:02.182395+0000","last_clean":"2023-09-30T09:54:02.182395+0000","last_became_active":"2023-09-30T09:51:59.873003+0000","last_became_peered":"2023-09-30T09:51:59.873003+0000","last_unstale":"2023-09-30T09:54:02.182395+0000","last_undegraded":"2023-09-30T09:54:02.182395+0000","last_fullsized":"2023-09-30T09:54:02.182395+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:04.437753+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":330,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.383927+0000","last_change":"2023-09-30T09:52:15.360214+0000","last_active":"2023-09-30T09:54:07.383927+0000","last_peered":"2023-09-30T09:54:07.383927+0000","last_clean":"2023-09-30T09:54:07.383927+0000","last_became_active":"2023-09-30T09:52:15.339270+0000","last_became_peered":"2023-09-30T09:52:15.339270+0000","last_unstale":"2023-09-30T09:54:07.383927+0000","last_undegraded":"2023-09-30T09:54:07.383927+0000","last_fullsized":"2023-09-30T09:54:07.383927+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:37:34.556170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":609,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384612+0000","last_change":"2023-09-30T09:43:45.122512+0000","last_active":"2023-09-30T09:54:07.384612+0000","last_peered":"2023-09-30T09:54:07.384612+0000","last_clean":"2023-09-30T09:54:07.384612+0000","last_became_active":"2023-09-30T09:43:45.122185+0000","last_became_peered":"2023-09-30T09:43:45.122185+0000","last_unstale":"2023-09-30T09:54:07.384612+0000","last_undegraded":"2023-09-30T09:54:07.384612+0000","last_fullsized":"2023-09-30T09:54:07.384612+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:35.881186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":992,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182497+0000","last_change":"2023-09-30T09:49:34.332543+0000","last_active":"2023-09-30T09:54:02.182497+0000","last_peered":"2023-09-30T09:54:02.182497+0000","last_clean":"2023-09-30T09:54:02.182497+0000","last_became_active":"2023-09-30T09:49:34.332215+0000","last_became_peered":"2023-09-30T09:49:34.332215+0000","last_unstale":"2023-09-30T09:54:02.182497+0000","last_undegraded":"2023-09-30T09:54:02.182497+0000","last_fullsized":"2023-09-30T09:54:02.182497+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.827671+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":385,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384336+0000","last_change":"2023-09-30T09:52:06.287263+0000","last_active":"2023-09-30T09:54:07.384336+0000","last_peered":"2023-09-30T09:54:07.384336+0000","last_clean":"2023-09-30T09:54:07.384336+0000","last_became_active":"2023-09-30T09:52:06.286870+0000","last_became_peered":"2023-09-30T09:52:06.286870+0000","last_unstale":"2023-09-30T09:54:07.384336+0000","last_undegraded":"2023-09-30T09:54:07.384336+0000","last_fullsized":"2023-09-30T09:54:07.384336+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:01:20.147214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":51,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183215+0000","last_change":"2023-09-30T09:51:57.157155+0000","last_active":"2023-09-30T09:54:02.183215+0000","last_peered":"2023-09-30T09:54:02.183215+0000","last_clean":"2023-09-30T09:54:02.183215+0000","last_became_active":"2023-09-30T09:51:57.156843+0000","last_became_peered":"2023-09-30T09:51:57.156843+0000","last_unstale":"2023-09-30T09:54:02.183215+0000","last_undegraded":"2023-09-30T09:54:02.183215+0000","last_fullsized":"2023-09-30T09:54:02.183215+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:12:00.333473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":1054,"reported_epoch":630,"state":"active+undersized","last_fresh":"2023-09-30T09:54:02.184234+0000","last_change":"2023-09-30T09:53:33.220329+0000","last_active":"2023-09-30T09:54:02.184234+0000","last_peered":"2023-09-30T09:54:02.184234+0000","last_clean":"2023-09-30T09:53:22.476149+0000","last_became_active":"2023-09-30T09:53:33.220329+0000","last_became_peered":"2023-09-30T09:53:33.220329+0000","last_unstale":"2023-09-30T09:54:02.184234+0000","last_undegraded":"2023-09-30T09:54:02.184234+0000","last_fullsized":"2023-09-30T09:53:33.218054+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:08:48.485687+0000","scrub_duration":0.0031022789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":336,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384300+0000","last_change":"2023-09-30T09:49:31.309022+0000","last_active":"2023-09-30T09:54:07.384300+0000","last_peered":"2023-09-30T09:54:07.384300+0000","last_clean":"2023-09-30T09:54:07.384300+0000","last_became_active":"2023-09-30T09:49:31.308236+0000","last_became_peered":"2023-09-30T09:49:31.308236+0000","last_unstale":"2023-09-30T09:54:07.384300+0000","last_undegraded":"2023-09-30T09:54:07.384300+0000","last_fullsized":"2023-09-30T09:54:07.384300+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:39.779954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":358,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.429757+0000","last_change":"2023-09-30T09:54:02.200326+0000","last_active":"2023-09-30T09:54:02.429757+0000","last_peered":"2023-09-30T09:54:02.429757+0000","last_clean":"2023-09-30T09:54:02.429757+0000","last_became_active":"2023-09-30T09:54:02.199993+0000","last_became_peered":"2023-09-30T09:54:02.199993+0000","last_unstale":"2023-09-30T09:54:02.429757+0000","last_undegraded":"2023-09-30T09:54:02.429757+0000","last_fullsized":"2023-09-30T09:54:02.429757+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:52:25.196897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":1081,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.184727+0000","last_change":"2023-09-30T09:49:34.332561+0000","last_active":"2023-09-30T09:54:02.184727+0000","last_peered":"2023-09-30T09:54:02.184727+0000","last_clean":"2023-09-30T09:54:02.184727+0000","last_became_active":"2023-09-30T09:49:34.332170+0000","last_became_peered":"2023-09-30T09:49:34.332170+0000","last_unstale":"2023-09-30T09:54:02.184727+0000","last_undegraded":"2023-09-30T09:54:02.184727+0000","last_fullsized":"2023-09-30T09:54:02.184727+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:22.548068+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":56,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183113+0000","last_change":"2023-09-30T09:52:06.314076+0000","last_active":"2023-09-30T09:54:02.183113+0000","last_peered":"2023-09-30T09:54:02.183113+0000","last_clean":"2023-09-30T09:54:02.183113+0000","last_became_active":"2023-09-30T09:52:06.286706+0000","last_became_peered":"2023-09-30T09:52:06.286706+0000","last_unstale":"2023-09-30T09:54:02.183113+0000","last_undegraded":"2023-09-30T09:54:02.183113+0000","last_fullsized":"2023-09-30T09:54:02.183113+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:35.387500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":198,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.186680+0000","last_change":"2023-09-30T09:42:48.644216+0000","last_active":"2023-09-30T09:54:02.186680+0000","last_peered":"2023-09-30T09:54:02.186680+0000","last_clean":"2023-09-30T09:54:02.186680+0000","last_became_active":"2023-09-30T09:42:48.639733+0000","last_became_peered":"2023-09-30T09:42:48.639733+0000","last_unstale":"2023-09-30T09:54:02.186680+0000","last_undegraded":"2023-09-30T09:54:02.186680+0000","last_fullsized":"2023-09-30T09:54:02.186680+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:50:55.344260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1028,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384828+0000","last_change":"2023-09-30T09:52:09.263017+0000","last_active":"2023-09-30T09:54:07.384828+0000","last_peered":"2023-09-30T09:54:07.384828+0000","last_clean":"2023-09-30T09:54:07.384828+0000","last_became_active":"2023-09-30T09:52:09.261512+0000","last_became_peered":"2023-09-30T09:52:09.261512+0000","last_unstale":"2023-09-30T09:54:07.384828+0000","last_undegraded":"2023-09-30T09:54:07.384828+0000","last_fullsized":"2023-09-30T09:54:07.384828+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:13.359576+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":322,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.385129+0000","last_change":"2023-09-30T09:51:57.189281+0000","last_active":"2023-09-30T09:54:07.385129+0000","last_peered":"2023-09-30T09:54:07.385129+0000","last_clean":"2023-09-30T09:54:07.385129+0000","last_became_active":"2023-09-30T09:51:57.165927+0000","last_became_peered":"2023-09-30T09:51:57.165927+0000","last_unstale":"2023-09-30T09:54:07.385129+0000","last_undegraded":"2023-09-30T09:54:07.385129+0000","last_fullsized":"2023-09-30T09:54:07.385129+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:39:30.207994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182350+0000","last_change":"2023-09-30T09:51:57.167306+0000","last_active":"2023-09-30T09:54:02.182350+0000","last_peered":"2023-09-30T09:54:02.182350+0000","last_clean":"2023-09-30T09:54:02.182350+0000","last_became_active":"2023-09-30T09:51:57.167040+0000","last_became_peered":"2023-09-30T09:51:57.167040+0000","last_unstale":"2023-09-30T09:54:02.182350+0000","last_undegraded":"2023-09-30T09:54:02.182350+0000","last_fullsized":"2023-09-30T09:54:02.182350+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:27:26.793530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":1069,"reported_epoch":631,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:54:07.503066+0000","last_change":"2023-09-30T09:53:33.225468+0000","last_active":"2023-09-30T09:54:07.503066+0000","last_peered":"2023-09-30T09:54:07.503066+0000","last_clean":"2023-09-30T09:53:22.480768+0000","last_became_active":"2023-09-30T09:53:33.225468+0000","last_became_peered":"2023-09-30T09:53:33.225468+0000","last_unstale":"2023-09-30T09:54:07.503066+0000","last_undegraded":"2023-09-30T09:53:33.219487+0000","last_fullsized":"2023-09-30T09:53:33.219252+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:55:11.466094+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":292,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384993+0000","last_change":"2023-09-30T09:51:57.183013+0000","last_active":"2023-09-30T09:54:07.384993+0000","last_peered":"2023-09-30T09:54:07.384993+0000","last_clean":"2023-09-30T09:54:07.384993+0000","last_became_active":"2023-09-30T09:51:57.161490+0000","last_became_peered":"2023-09-30T09:51:57.161490+0000","last_unstale":"2023-09-30T09:54:07.384993+0000","last_undegraded":"2023-09-30T09:54:07.384993+0000","last_fullsized":"2023-09-30T09:54:07.384993+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:14:47.256298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":691,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.204251+0000","last_change":"2023-09-30T09:50:26.903683+0000","last_active":"2023-09-30T09:54:02.204251+0000","last_peered":"2023-09-30T09:54:02.204251+0000","last_clean":"2023-09-30T09:54:02.204251+0000","last_became_active":"2023-09-30T09:50:26.903262+0000","last_became_peered":"2023-09-30T09:50:26.903262+0000","last_unstale":"2023-09-30T09:54:02.204251+0000","last_undegraded":"2023-09-30T09:54:02.204251+0000","last_fullsized":"2023-09-30T09:54:02.204251+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:40.724403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":713,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183686+0000","last_change":"2023-09-30T09:52:18.342336+0000","last_active":"2023-09-30T09:54:02.183686+0000","last_peered":"2023-09-30T09:54:02.183686+0000","last_clean":"2023-09-30T09:54:02.183686+0000","last_became_active":"2023-09-30T09:52:18.341952+0000","last_became_peered":"2023-09-30T09:52:18.341952+0000","last_unstale":"2023-09-30T09:54:02.183686+0000","last_undegraded":"2023-09-30T09:54:02.183686+0000","last_fullsized":"2023-09-30T09:54:02.183686+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:16.116795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:16.116795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:01:10.275488+0000","scrub_duration":0.045979904000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":401,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384611+0000","last_change":"2023-09-30T09:51:57.192521+0000","last_active":"2023-09-30T09:54:07.384611+0000","last_peered":"2023-09-30T09:54:07.384611+0000","last_clean":"2023-09-30T09:54:07.384611+0000","last_became_active":"2023-09-30T09:51:57.170876+0000","last_became_peered":"2023-09-30T09:51:57.170876+0000","last_unstale":"2023-09-30T09:54:07.384611+0000","last_undegraded":"2023-09-30T09:54:07.384611+0000","last_fullsized":"2023-09-30T09:54:07.384611+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:44:49.877225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":516,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.184346+0000","last_change":"2023-09-30T09:50:13.889514+0000","last_active":"2023-09-30T09:54:02.184346+0000","last_peered":"2023-09-30T09:54:02.184346+0000","last_clean":"2023-09-30T09:54:02.184346+0000","last_became_active":"2023-09-30T09:50:13.889033+0000","last_became_peered":"2023-09-30T09:50:13.889033+0000","last_unstale":"2023-09-30T09:54:02.184346+0000","last_undegraded":"2023-09-30T09:54:02.184346+0000","last_fullsized":"2023-09-30T09:54:02.184346+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:40:19.671881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":610,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384881+0000","last_change":"2023-09-30T09:51:59.873050+0000","last_active":"2023-09-30T09:54:07.384881+0000","last_peered":"2023-09-30T09:54:07.384881+0000","last_clean":"2023-09-30T09:54:07.384881+0000","last_became_active":"2023-09-30T09:51:59.872660+0000","last_became_peered":"2023-09-30T09:51:59.872660+0000","last_unstale":"2023-09-30T09:54:07.384881+0000","last_undegraded":"2023-09-30T09:54:07.384881+0000","last_fullsized":"2023-09-30T09:54:07.384881+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:47:10.945792+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":586,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182964+0000","last_change":"2023-09-30T09:49:31.301460+0000","last_active":"2023-09-30T09:54:02.182964+0000","last_peered":"2023-09-30T09:54:02.182964+0000","last_clean":"2023-09-30T09:54:02.182964+0000","last_became_active":"2023-09-30T09:49:31.301114+0000","last_became_peered":"2023-09-30T09:49:31.301114+0000","last_unstale":"2023-09-30T09:54:02.182964+0000","last_undegraded":"2023-09-30T09:54:02.182964+0000","last_fullsized":"2023-09-30T09:54:02.182964+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:42:01.066974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":234,"reported_epoch":630,"state":"active+undersized","last_fresh":"2023-09-30T09:54:02.204586+0000","last_change":"2023-09-30T09:53:33.232468+0000","last_active":"2023-09-30T09:54:02.204586+0000","last_peered":"2023-09-30T09:54:02.204586+0000","last_clean":"2023-09-30T09:32:28.396366+0000","last_became_active":"2023-09-30T09:53:33.232468+0000","last_became_peered":"2023-09-30T09:53:33.232468+0000","last_unstale":"2023-09-30T09:54:02.204586+0000","last_undegraded":"2023-09-30T09:54:02.204586+0000","last_fullsized":"2023-09-30T09:53:33.220447+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:12:17.944153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1174,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384220+0000","last_change":"2023-09-30T09:52:09.259618+0000","last_active":"2023-09-30T09:54:07.384220+0000","last_peered":"2023-09-30T09:54:07.384220+0000","last_clean":"2023-09-30T09:54:07.384220+0000","last_became_active":"2023-09-30T09:52:09.259274+0000","last_became_peered":"2023-09-30T09:52:09.259274+0000","last_unstale":"2023-09-30T09:54:07.384220+0000","last_undegraded":"2023-09-30T09:54:07.384220+0000","last_fullsized":"2023-09-30T09:54:07.384220+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:41:35.328752+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":126,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.205588+0000","last_change":"2023-09-30T09:51:57.173028+0000","last_active":"2023-09-30T09:54:02.205588+0000","last_peered":"2023-09-30T09:54:02.205588+0000","last_clean":"2023-09-30T09:54:02.205588+0000","last_became_active":"2023-09-30T09:51:57.152483+0000","last_became_peered":"2023-09-30T09:51:57.152483+0000","last_unstale":"2023-09-30T09:54:02.205588+0000","last_undegraded":"2023-09-30T09:54:02.205588+0000","last_fullsized":"2023-09-30T09:54:02.205588+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:14:11.989575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":648,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.204554+0000","last_change":"2023-09-30T09:49:31.300551+0000","last_active":"2023-09-30T09:54:02.204554+0000","last_peered":"2023-09-30T09:54:02.204554+0000","last_clean":"2023-09-30T09:54:02.204554+0000","last_became_active":"2023-09-30T09:49:31.300231+0000","last_became_peered":"2023-09-30T09:49:31.300231+0000","last_unstale":"2023-09-30T09:54:02.204554+0000","last_undegraded":"2023-09-30T09:54:02.204554+0000","last_fullsized":"2023-09-30T09:54:02.204554+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:45.832564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":1007,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.180072+0000","last_change":"2023-09-30T09:52:09.249933+0000","last_active":"2023-09-30T09:54:02.180072+0000","last_peered":"2023-09-30T09:54:02.180072+0000","last_clean":"2023-09-30T09:54:02.180072+0000","last_became_active":"2023-09-30T09:52:09.249567+0000","last_became_peered":"2023-09-30T09:52:09.249567+0000","last_unstale":"2023-09-30T09:54:02.180072+0000","last_undegraded":"2023-09-30T09:54:02.180072+0000","last_fullsized":"2023-09-30T09:54:02.180072+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:52:00.664301+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":473,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.204307+0000","last_change":"2023-09-30T09:50:13.887314+0000","last_active":"2023-09-30T09:54:02.204307+0000","last_peered":"2023-09-30T09:54:02.204307+0000","last_clean":"2023-09-30T09:54:02.204307+0000","last_became_active":"2023-09-30T09:50:13.887032+0000","last_became_peered":"2023-09-30T09:50:13.887032+0000","last_unstale":"2023-09-30T09:54:02.204307+0000","last_undegraded":"2023-09-30T09:54:02.204307+0000","last_fullsized":"2023-09-30T09:54:02.204307+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:27:26.626380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":40,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182915+0000","last_change":"2023-09-30T09:52:15.326810+0000","last_active":"2023-09-30T09:54:02.182915+0000","last_peered":"2023-09-30T09:54:02.182915+0000","last_clean":"2023-09-30T09:54:02.182915+0000","last_became_active":"2023-09-30T09:52:15.326507+0000","last_became_peered":"2023-09-30T09:52:15.326507+0000","last_unstale":"2023-09-30T09:54:02.182915+0000","last_undegraded":"2023-09-30T09:54:02.182915+0000","last_fullsized":"2023-09-30T09:54:02.182915+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:51:01.576127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":998,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183595+0000","last_change":"2023-09-30T09:52:18.345744+0000","last_active":"2023-09-30T09:54:02.183595+0000","last_peered":"2023-09-30T09:54:02.183595+0000","last_clean":"2023-09-30T09:54:02.183595+0000","last_became_active":"2023-09-30T09:52:18.345422+0000","last_became_peered":"2023-09-30T09:52:18.345422+0000","last_unstale":"2023-09-30T09:54:02.183595+0000","last_undegraded":"2023-09-30T09:54:02.183595+0000","last_fullsized":"2023-09-30T09:54:02.183595+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:49:09.924646+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":633,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.187265+0000","last_change":"2023-09-30T09:52:06.274034+0000","last_active":"2023-09-30T09:54:07.187265+0000","last_peered":"2023-09-30T09:54:07.187265+0000","last_clean":"2023-09-30T09:54:07.187265+0000","last_became_active":"2023-09-30T09:52:06.273748+0000","last_became_peered":"2023-09-30T09:52:06.273748+0000","last_unstale":"2023-09-30T09:54:07.187265+0000","last_undegraded":"2023-09-30T09:54:07.187265+0000","last_fullsized":"2023-09-30T09:54:07.187265+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:22:02.105157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":662,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.205897+0000","last_change":"2023-09-30T09:49:31.311818+0000","last_active":"2023-09-30T09:54:02.205897+0000","last_peered":"2023-09-30T09:54:02.205897+0000","last_clean":"2023-09-30T09:54:02.205897+0000","last_became_active":"2023-09-30T09:49:31.311481+0000","last_became_peered":"2023-09-30T09:49:31.311481+0000","last_unstale":"2023-09-30T09:54:02.205897+0000","last_undegraded":"2023-09-30T09:54:02.205897+0000","last_fullsized":"2023-09-30T09:54:02.205897+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:54.941072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":830,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183217+0000","last_change":"2023-09-30T09:50:44.462926+0000","last_active":"2023-09-30T09:54:02.183217+0000","last_peered":"2023-09-30T09:54:02.183217+0000","last_clean":"2023-09-30T09:54:02.183217+0000","last_became_active":"2023-09-30T09:50:44.462573+0000","last_became_peered":"2023-09-30T09:50:44.462573+0000","last_unstale":"2023-09-30T09:54:02.183217+0000","last_undegraded":"2023-09-30T09:54:02.183217+0000","last_fullsized":"2023-09-30T09:54:02.183217+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:52.780499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:52.780499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:18:28.805205+0000","scrub_duration":0.0035425679999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":199,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183055+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:54:02.183055+0000","last_peered":"2023-09-30T09:54:02.183055+0000","last_clean":"2023-09-30T09:54:02.183055+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:54:02.183055+0000","last_undegraded":"2023-09-30T09:54:02.183055+0000","last_fullsized":"2023-09-30T09:54:02.183055+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":315,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.205810+0000","last_change":"2023-09-30T09:43:45.125522+0000","last_active":"2023-09-30T09:54:02.205810+0000","last_peered":"2023-09-30T09:54:02.205810+0000","last_clean":"2023-09-30T09:54:02.205810+0000","last_became_active":"2023-09-30T09:43:45.125177+0000","last_became_peered":"2023-09-30T09:43:45.125177+0000","last_unstale":"2023-09-30T09:54:02.205810+0000","last_undegraded":"2023-09-30T09:54:02.205810+0000","last_fullsized":"2023-09-30T09:54:02.205810+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:16:57.813498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":872,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384891+0000","last_change":"2023-09-30T09:50:13.903010+0000","last_active":"2023-09-30T09:54:07.384891+0000","last_peered":"2023-09-30T09:54:07.384891+0000","last_clean":"2023-09-30T09:54:07.384891+0000","last_became_active":"2023-09-30T09:50:13.902355+0000","last_became_peered":"2023-09-30T09:50:13.902355+0000","last_unstale":"2023-09-30T09:54:07.384891+0000","last_undegraded":"2023-09-30T09:54:07.384891+0000","last_fullsized":"2023-09-30T09:54:07.384891+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:02:35.240830+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"632'116","reported_seq":635,"reported_epoch":632,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:54:28.327019+0000","last_change":"2023-09-30T09:53:33.229223+0000","last_active":"2023-09-30T09:54:28.327019+0000","last_peered":"2023-09-30T09:54:28.327019+0000","last_clean":"2023-09-30T09:38:16.639268+0000","last_became_active":"2023-09-30T09:53:33.229223+0000","last_became_peered":"2023-09-30T09:53:33.229223+0000","last_unstale":"2023-09-30T09:54:28.327019+0000","last_undegraded":"2023-09-30T09:53:33.224682+0000","last_fullsized":"2023-09-30T09:53:33.224570+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:19:25.962789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":70,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":438,"num_read_kb":957,"num_write":123,"num_write_kb":352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":3}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":579,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183597+0000","last_change":"2023-09-30T09:50:41.396047+0000","last_active":"2023-09-30T09:54:02.183597+0000","last_peered":"2023-09-30T09:54:02.183597+0000","last_clean":"2023-09-30T09:54:02.183597+0000","last_became_active":"2023-09-30T09:50:41.395627+0000","last_became_peered":"2023-09-30T09:50:41.395627+0000","last_unstale":"2023-09-30T09:54:02.183597+0000","last_undegraded":"2023-09-30T09:54:02.183597+0000","last_fullsized":"2023-09-30T09:54:02.183597+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:16.050977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182661+0000","last_change":"2023-09-30T09:52:15.338190+0000","last_active":"2023-09-30T09:54:02.182661+0000","last_peered":"2023-09-30T09:54:02.182661+0000","last_clean":"2023-09-30T09:54:02.182661+0000","last_became_active":"2023-09-30T09:52:15.337907+0000","last_became_peered":"2023-09-30T09:52:15.337907+0000","last_unstale":"2023-09-30T09:54:02.182661+0000","last_undegraded":"2023-09-30T09:54:02.182661+0000","last_fullsized":"2023-09-30T09:54:02.182661+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:20:09.336859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":1132,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182660+0000","last_change":"2023-09-30T09:52:09.283412+0000","last_active":"2023-09-30T09:54:02.182660+0000","last_peered":"2023-09-30T09:54:02.182660+0000","last_clean":"2023-09-30T09:54:02.182660+0000","last_became_active":"2023-09-30T09:52:09.283074+0000","last_became_peered":"2023-09-30T09:52:09.283074+0000","last_unstale":"2023-09-30T09:54:02.182660+0000","last_undegraded":"2023-09-30T09:54:02.182660+0000","last_fullsized":"2023-09-30T09:54:02.182660+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:33:20.204709+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'8","reported_seq":454,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:20.789954+0000","last_change":"2023-09-30T09:50:13.889332+0000","last_active":"2023-09-30T09:54:20.789954+0000","last_peered":"2023-09-30T09:54:20.789954+0000","last_clean":"2023-09-30T09:54:20.789954+0000","last_became_active":"2023-09-30T09:50:13.888518+0000","last_became_peered":"2023-09-30T09:50:13.888518+0000","last_unstale":"2023-09-30T09:54:20.789954+0000","last_undegraded":"2023-09-30T09:54:20.789954+0000","last_fullsized":"2023-09-30T09:54:20.789954+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:40:20.465620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061647000000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":245,"num_read_kb":2291,"num_write":8,"num_write_kb":2052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":679,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.204838+0000","last_change":"2023-09-30T09:52:15.328572+0000","last_active":"2023-09-30T09:54:02.204838+0000","last_peered":"2023-09-30T09:54:02.204838+0000","last_clean":"2023-09-30T09:54:02.204838+0000","last_became_active":"2023-09-30T09:52:15.328181+0000","last_became_peered":"2023-09-30T09:52:15.328181+0000","last_unstale":"2023-09-30T09:54:02.204838+0000","last_undegraded":"2023-09-30T09:54:02.204838+0000","last_fullsized":"2023-09-30T09:54:02.204838+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:08:51.469327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":299,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183056+0000","last_change":"2023-09-30T09:52:15.332419+0000","last_active":"2023-09-30T09:54:02.183056+0000","last_peered":"2023-09-30T09:54:02.183056+0000","last_clean":"2023-09-30T09:54:02.183056+0000","last_became_active":"2023-09-30T09:52:15.332098+0000","last_became_peered":"2023-09-30T09:52:15.332098+0000","last_unstale":"2023-09-30T09:54:02.183056+0000","last_undegraded":"2023-09-30T09:54:02.183056+0000","last_fullsized":"2023-09-30T09:54:02.183056+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:53:34.176892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":1098,"reported_epoch":631,"state":"active+undersized","last_fresh":"2023-09-30T09:54:07.384326+0000","last_change":"2023-09-30T09:53:33.231789+0000","last_active":"2023-09-30T09:54:07.384326+0000","last_peered":"2023-09-30T09:54:07.384326+0000","last_clean":"2023-09-30T09:53:22.493359+0000","last_became_active":"2023-09-30T09:53:33.231789+0000","last_became_peered":"2023-09-30T09:53:33.231789+0000","last_unstale":"2023-09-30T09:54:07.384326+0000","last_undegraded":"2023-09-30T09:54:07.384326+0000","last_fullsized":"2023-09-30T09:53:33.225900+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:49:22.697339+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"631'167","reported_seq":921,"reported_epoch":631,"state":"active+clean+remapped","last_fresh":"2023-09-30T09:54:20.828593+0000","last_change":"2023-09-30T09:53:33.978406+0000","last_active":"2023-09-30T09:54:20.828593+0000","last_peered":"2023-09-30T09:54:20.828593+0000","last_clean":"2023-09-30T09:54:20.828593+0000","last_became_active":"2023-09-30T09:53:33.967298+0000","last_became_peered":"2023-09-30T09:53:33.967298+0000","last_unstale":"2023-09-30T09:54:20.828593+0000","last_undegraded":"2023-09-30T09:54:20.828593+0000","last_fullsized":"2023-09-30T09:54:20.828593+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:31:43.604921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082354300000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":914,"num_read_kb":2130,"num_write":166,"num_write_kb":1328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,2],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":73,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.180164+0000","last_change":"2023-09-30T09:50:41.394348+0000","last_active":"2023-09-30T09:54:02.180164+0000","last_peered":"2023-09-30T09:54:02.180164+0000","last_clean":"2023-09-30T09:54:02.180164+0000","last_became_active":"2023-09-30T09:50:41.394017+0000","last_became_peered":"2023-09-30T09:50:41.394017+0000","last_unstale":"2023-09-30T09:54:02.180164+0000","last_undegraded":"2023-09-30T09:54:02.180164+0000","last_fullsized":"2023-09-30T09:54:02.180164+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:37:30.133826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":490,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.184576+0000","last_change":"2023-09-30T09:51:57.162010+0000","last_active":"2023-09-30T09:54:02.184576+0000","last_peered":"2023-09-30T09:54:02.184576+0000","last_clean":"2023-09-30T09:54:02.184576+0000","last_became_active":"2023-09-30T09:51:57.161672+0000","last_became_peered":"2023-09-30T09:51:57.161672+0000","last_unstale":"2023-09-30T09:54:02.184576+0000","last_undegraded":"2023-09-30T09:54:02.184576+0000","last_fullsized":"2023-09-30T09:54:02.184576+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:13:28.744549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"631'211","reported_seq":1236,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:20.832242+0000","last_change":"2023-09-30T09:50:42.387920+0000","last_active":"2023-09-30T09:54:20.832242+0000","last_peered":"2023-09-30T09:54:20.832242+0000","last_clean":"2023-09-30T09:54:20.832242+0000","last_became_active":"2023-09-30T09:50:42.387121+0000","last_became_peered":"2023-09-30T09:50:42.387121+0000","last_unstale":"2023-09-30T09:54:20.832242+0000","last_undegraded":"2023-09-30T09:54:20.832242+0000","last_fullsized":"2023-09-30T09:54:20.832242+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:34:05.192090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038762699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":590,"num_read_kb":1106,"num_write":214,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1054,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384317+0000","last_change":"2023-09-30T09:51:59.878404+0000","last_active":"2023-09-30T09:54:07.384317+0000","last_peered":"2023-09-30T09:54:07.384317+0000","last_clean":"2023-09-30T09:54:07.384317+0000","last_became_active":"2023-09-30T09:51:59.878042+0000","last_became_peered":"2023-09-30T09:51:59.878042+0000","last_unstale":"2023-09-30T09:54:07.384317+0000","last_undegraded":"2023-09-30T09:54:07.384317+0000","last_fullsized":"2023-09-30T09:54:07.384317+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:53:00.431808+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":51,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183121+0000","last_change":"2023-09-30T09:51:57.166214+0000","last_active":"2023-09-30T09:54:02.183121+0000","last_peered":"2023-09-30T09:54:02.183121+0000","last_clean":"2023-09-30T09:54:02.183121+0000","last_became_active":"2023-09-30T09:51:57.165926+0000","last_became_peered":"2023-09-30T09:51:57.165926+0000","last_unstale":"2023-09-30T09:54:02.183121+0000","last_undegraded":"2023-09-30T09:54:02.183121+0000","last_fullsized":"2023-09-30T09:54:02.183121+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:19:29.596549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":496,"reported_epoch":631,"state":"active+undersized","last_fresh":"2023-09-30T09:54:07.384735+0000","last_change":"2023-09-30T09:53:33.233762+0000","last_active":"2023-09-30T09:54:07.384735+0000","last_peered":"2023-09-30T09:54:07.384735+0000","last_clean":"2023-09-30T09:53:22.494019+0000","last_became_active":"2023-09-30T09:53:33.233762+0000","last_became_peered":"2023-09-30T09:53:33.233762+0000","last_unstale":"2023-09-30T09:54:07.384735+0000","last_undegraded":"2023-09-30T09:54:07.384735+0000","last_fullsized":"2023-09-30T09:53:33.225638+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:23:46.252190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":773,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.189025+0000","last_change":"2023-09-30T09:52:09.254399+0000","last_active":"2023-09-30T09:54:07.189025+0000","last_peered":"2023-09-30T09:54:07.189025+0000","last_clean":"2023-09-30T09:54:07.189025+0000","last_became_active":"2023-09-30T09:52:09.253999+0000","last_became_peered":"2023-09-30T09:52:09.253999+0000","last_unstale":"2023-09-30T09:54:07.189025+0000","last_undegraded":"2023-09-30T09:54:07.189025+0000","last_fullsized":"2023-09-30T09:54:07.189025+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:39.230227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.230227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:06:12.791078+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"632'153","reported_seq":1193,"reported_epoch":632,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:54:28.397627+0000","last_change":"2023-09-30T09:53:33.234273+0000","last_active":"2023-09-30T09:54:28.397627+0000","last_peered":"2023-09-30T09:54:28.397627+0000","last_clean":"2023-09-30T09:53:22.495638+0000","last_became_active":"2023-09-30T09:53:33.234273+0000","last_became_peered":"2023-09-30T09:53:33.234273+0000","last_unstale":"2023-09-30T09:54:28.397627+0000","last_undegraded":"2023-09-30T09:53:33.226701+0000","last_fullsized":"2023-09-30T09:53:33.226602+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":153,"log_dups_size":0,"ondisk_log_size":153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:36:19.434789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042610199999999998,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1185,"num_read_kb":1727,"num_write":106,"num_write_kb":4485,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":225,"reported_epoch":630,"state":"active+undersized","last_fresh":"2023-09-30T09:54:02.186929+0000","last_change":"2023-09-30T09:53:33.222521+0000","last_active":"2023-09-30T09:54:02.186929+0000","last_peered":"2023-09-30T09:54:02.186929+0000","last_clean":"2023-09-30T09:51:52.132453+0000","last_became_active":"2023-09-30T09:53:33.222521+0000","last_became_peered":"2023-09-30T09:53:33.222521+0000","last_unstale":"2023-09-30T09:54:02.186929+0000","last_undegraded":"2023-09-30T09:54:02.186929+0000","last_fullsized":"2023-09-30T09:53:33.219169+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:24:45.559011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":135,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384668+0000","last_change":"2023-09-30T09:52:06.219826+0000","last_active":"2023-09-30T09:54:07.384668+0000","last_peered":"2023-09-30T09:54:07.384668+0000","last_clean":"2023-09-30T09:54:07.384668+0000","last_became_active":"2023-09-30T09:52:06.219645+0000","last_became_peered":"2023-09-30T09:52:06.219645+0000","last_unstale":"2023-09-30T09:54:07.384668+0000","last_undegraded":"2023-09-30T09:54:07.384668+0000","last_fullsized":"2023-09-30T09:54:07.384668+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:18:08.551862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":836,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.205725+0000","last_change":"2023-09-30T09:52:09.278067+0000","last_active":"2023-09-30T09:54:02.205725+0000","last_peered":"2023-09-30T09:54:02.205725+0000","last_clean":"2023-09-30T09:54:02.205725+0000","last_became_active":"2023-09-30T09:52:09.277739+0000","last_became_peered":"2023-09-30T09:52:09.277739+0000","last_unstale":"2023-09-30T09:54:02.205725+0000","last_undegraded":"2023-09-30T09:54:02.205725+0000","last_fullsized":"2023-09-30T09:54:02.205725+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:18:42.634023+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"632'153","reported_seq":1239,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-30T09:54:28.394060+0000","last_change":"2023-09-30T09:52:09.264946+0000","last_active":"2023-09-30T09:54:28.394060+0000","last_peered":"2023-09-30T09:54:28.394060+0000","last_clean":"2023-09-30T09:54:28.394060+0000","last_became_active":"2023-09-30T09:52:09.262504+0000","last_became_peered":"2023-09-30T09:52:09.262504+0000","last_unstale":"2023-09-30T09:54:28.394060+0000","last_undegraded":"2023-09-30T09:54:28.394060+0000","last_fullsized":"2023-09-30T09:54:28.394060+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":153,"log_dups_size":0,"ondisk_log_size":153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:10:13.204007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033952899999999998,"stat_sum":{"num_bytes":35,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1002,"num_read_kb":2028,"num_write":120,"num_write_kb":1176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":86,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182713+0000","last_change":"2023-09-30T09:52:06.230958+0000","last_active":"2023-09-30T09:54:02.182713+0000","last_peered":"2023-09-30T09:54:02.182713+0000","last_clean":"2023-09-30T09:54:02.182713+0000","last_became_active":"2023-09-30T09:52:06.230575+0000","last_became_peered":"2023-09-30T09:52:06.230575+0000","last_unstale":"2023-09-30T09:54:02.182713+0000","last_undegraded":"2023-09-30T09:54:02.182713+0000","last_fullsized":"2023-09-30T09:54:02.182713+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:42:57.084463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":303,"reported_epoch":630,"state":"active+undersized","last_fresh":"2023-09-30T09:54:02.184228+0000","last_change":"2023-09-30T09:53:33.278694+0000","last_active":"2023-09-30T09:54:02.184228+0000","last_peered":"2023-09-30T09:54:02.184228+0000","last_clean":"2023-09-30T09:41:40.702643+0000","last_became_active":"2023-09-30T09:53:33.278694+0000","last_became_peered":"2023-09-30T09:53:33.278694+0000","last_unstale":"2023-09-30T09:54:02.184228+0000","last_undegraded":"2023-09-30T09:54:02.184228+0000","last_fullsized":"2023-09-30T09:53:33.220990+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:07:21.166526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1245,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182959+0000","last_change":"2023-09-30T09:52:18.344243+0000","last_active":"2023-09-30T09:54:02.182959+0000","last_peered":"2023-09-30T09:54:02.182959+0000","last_clean":"2023-09-30T09:54:02.182959+0000","last_became_active":"2023-09-30T09:52:18.343870+0000","last_became_peered":"2023-09-30T09:52:18.343870+0000","last_unstale":"2023-09-30T09:54:02.182959+0000","last_undegraded":"2023-09-30T09:54:02.182959+0000","last_fullsized":"2023-09-30T09:54:02.182959+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:09:04.148163+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":670,"reported_epoch":631,"state":"active+undersized+degraded","last_fresh":"2023-09-30T09:54:07.385008+0000","last_change":"2023-09-30T09:53:33.229507+0000","last_active":"2023-09-30T09:54:07.385008+0000","last_peered":"2023-09-30T09:54:07.385008+0000","last_clean":"2023-09-30T09:53:22.495209+0000","last_became_active":"2023-09-30T09:53:33.229507+0000","last_became_peered":"2023-09-30T09:53:33.229507+0000","last_unstale":"2023-09-30T09:54:07.385008+0000","last_undegraded":"2023-09-30T09:53:33.224686+0000","last_fullsized":"2023-09-30T09:53:33.224578+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:59:05.445248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"631'227","reported_seq":1370,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:20.821109+0000","last_change":"2023-09-30T09:50:13.858344+0000","last_active":"2023-09-30T09:54:20.821109+0000","last_peered":"2023-09-30T09:54:20.821109+0000","last_clean":"2023-09-30T09:54:20.821109+0000","last_became_active":"2023-09-30T09:50:13.857963+0000","last_became_peered":"2023-09-30T09:50:13.857963+0000","last_unstale":"2023-09-30T09:54:20.821109+0000","last_undegraded":"2023-09-30T09:54:20.821109+0000","last_fullsized":"2023-09-30T09:54:20.821109+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":227,"log_dups_size":0,"ondisk_log_size":227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:46:21.850425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081945,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1873,"num_read_kb":3663,"num_write":155,"num_write_kb":2627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":677,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384191+0000","last_change":"2023-09-30T09:51:57.161323+0000","last_active":"2023-09-30T09:54:07.384191+0000","last_peered":"2023-09-30T09:54:07.384191+0000","last_clean":"2023-09-30T09:54:07.384191+0000","last_became_active":"2023-09-30T09:51:57.160615+0000","last_became_peered":"2023-09-30T09:51:57.160615+0000","last_unstale":"2023-09-30T09:54:07.384191+0000","last_undegraded":"2023-09-30T09:54:07.384191+0000","last_fullsized":"2023-09-30T09:54:07.384191+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:53:15.253650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":1059,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.183510+0000","last_change":"2023-09-30T09:51:59.873401+0000","last_active":"2023-09-30T09:54:02.183510+0000","last_peered":"2023-09-30T09:54:02.183510+0000","last_clean":"2023-09-30T09:54:02.183510+0000","last_became_active":"2023-09-30T09:51:59.873066+0000","last_became_peered":"2023-09-30T09:51:59.873066+0000","last_unstale":"2023-09-30T09:54:02.183510+0000","last_undegraded":"2023-09-30T09:54:02.183510+0000","last_fullsized":"2023-09-30T09:54:02.183510+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:48:11.565720+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"632'112","reported_seq":1028,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-30T09:54:28.325556+0000","last_change":"2023-09-30T09:50:41.393850+0000","last_active":"2023-09-30T09:54:28.325556+0000","last_peered":"2023-09-30T09:54:28.325556+0000","last_clean":"2023-09-30T09:54:28.325556+0000","last_became_active":"2023-09-30T09:50:41.393481+0000","last_became_peered":"2023-09-30T09:50:41.393481+0000","last_unstale":"2023-09-30T09:54:28.325556+0000","last_undegraded":"2023-09-30T09:54:28.325556+0000","last_fullsized":"2023-09-30T09:54:28.325556+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:47:48.909491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038117799999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":679,"num_read_kb":2055,"num_write":107,"num_write_kb":1299,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":211,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.206090+0000","last_change":"2023-09-30T09:51:57.187258+0000","last_active":"2023-09-30T09:54:02.206090+0000","last_peered":"2023-09-30T09:54:02.206090+0000","last_clean":"2023-09-30T09:54:02.206090+0000","last_became_active":"2023-09-30T09:51:57.158336+0000","last_became_peered":"2023-09-30T09:51:57.158336+0000","last_unstale":"2023-09-30T09:54:02.206090+0000","last_undegraded":"2023-09-30T09:54:02.206090+0000","last_fullsized":"2023-09-30T09:54:02.206090+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:10:59.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":616,"reported_epoch":630,"state":"active+undersized","last_fresh":"2023-09-30T09:54:02.182706+0000","last_change":"2023-09-30T09:53:33.225281+0000","last_active":"2023-09-30T09:54:02.182706+0000","last_peered":"2023-09-30T09:54:02.182706+0000","last_clean":"2023-09-30T09:53:22.478832+0000","last_became_active":"2023-09-30T09:53:33.225281+0000","last_became_peered":"2023-09-30T09:53:33.225281+0000","last_unstale":"2023-09-30T09:54:02.182706+0000","last_undegraded":"2023-09-30T09:54:02.182706+0000","last_fullsized":"2023-09-30T09:53:33.219446+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:51:48.936561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":991,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384088+0000","last_change":"2023-09-30T09:51:59.878763+0000","last_active":"2023-09-30T09:54:07.384088+0000","last_peered":"2023-09-30T09:54:07.384088+0000","last_clean":"2023-09-30T09:54:07.384088+0000","last_became_active":"2023-09-30T09:51:59.878357+0000","last_became_peered":"2023-09-30T09:51:59.878357+0000","last_unstale":"2023-09-30T09:54:07.384088+0000","last_undegraded":"2023-09-30T09:54:07.384088+0000","last_fullsized":"2023-09-30T09:54:07.384088+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:31:08.208858+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":382,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.205031+0000","last_change":"2023-09-30T09:51:57.164777+0000","last_active":"2023-09-30T09:54:02.205031+0000","last_peered":"2023-09-30T09:54:02.205031+0000","last_clean":"2023-09-30T09:54:02.205031+0000","last_became_active":"2023-09-30T09:51:57.164468+0000","last_became_peered":"2023-09-30T09:51:57.164468+0000","last_unstale":"2023-09-30T09:54:02.205031+0000","last_undegraded":"2023-09-30T09:54:02.205031+0000","last_fullsized":"2023-09-30T09:54:02.205031+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:26:14.012018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":51,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.204504+0000","last_change":"2023-09-30T09:51:57.164689+0000","last_active":"2023-09-30T09:54:02.204504+0000","last_peered":"2023-09-30T09:54:02.204504+0000","last_clean":"2023-09-30T09:54:02.204504+0000","last_became_active":"2023-09-30T09:51:57.164384+0000","last_became_peered":"2023-09-30T09:51:57.164384+0000","last_unstale":"2023-09-30T09:54:02.204504+0000","last_undegraded":"2023-09-30T09:54:02.204504+0000","last_fullsized":"2023-09-30T09:54:02.204504+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:47:39.321245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":622,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.680525+0000","last_change":"2023-09-30T09:54:02.680525+0000","last_active":"2023-09-30T09:54:02.680525+0000","last_peered":"2023-09-30T09:54:02.680525+0000","last_clean":"2023-09-30T09:54:02.680525+0000","last_became_active":"2023-09-30T09:54:02.200078+0000","last_became_peered":"2023-09-30T09:54:02.200078+0000","last_unstale":"2023-09-30T09:54:02.680525+0000","last_undegraded":"2023-09-30T09:54:02.680525+0000","last_fullsized":"2023-09-30T09:54:02.680525+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:54:02.680294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:54:02.680294+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:00:33.759005+0000","scrub_duration":0.0025974169999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":559,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.182853+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:54:02.182853+0000","last_peered":"2023-09-30T09:54:02.182853+0000","last_clean":"2023-09-30T09:54:02.182853+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:54:02.182853+0000","last_undegraded":"2023-09-30T09:54:02.182853+0000","last_fullsized":"2023-09-30T09:54:02.182853+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":46,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384345+0000","last_change":"2023-09-30T09:52:06.292310+0000","last_active":"2023-09-30T09:54:07.384345+0000","last_peered":"2023-09-30T09:54:07.384345+0000","last_clean":"2023-09-30T09:54:07.384345+0000","last_became_active":"2023-09-30T09:52:06.291977+0000","last_became_peered":"2023-09-30T09:52:06.291977+0000","last_unstale":"2023-09-30T09:54:07.384345+0000","last_undegraded":"2023-09-30T09:54:07.384345+0000","last_fullsized":"2023-09-30T09:54:07.384345+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:39:00.274025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":529,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.184528+0000","last_change":"2023-09-30T09:49:31.313229+0000","last_active":"2023-09-30T09:54:02.184528+0000","last_peered":"2023-09-30T09:54:02.184528+0000","last_clean":"2023-09-30T09:54:02.184528+0000","last_became_active":"2023-09-30T09:49:31.301137+0000","last_became_peered":"2023-09-30T09:49:31.301137+0000","last_unstale":"2023-09-30T09:54:02.184528+0000","last_undegraded":"2023-09-30T09:54:02.184528+0000","last_fullsized":"2023-09-30T09:54:02.184528+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:57.362379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":211,"reported_epoch":630,"state":"active+undersized","last_fresh":"2023-09-30T09:54:02.184385+0000","last_change":"2023-09-30T09:53:33.227572+0000","last_active":"2023-09-30T09:54:02.184385+0000","last_peered":"2023-09-30T09:54:02.184385+0000","last_clean":"2023-09-30T09:52:03.179751+0000","last_became_active":"2023-09-30T09:53:33.227572+0000","last_became_peered":"2023-09-30T09:53:33.227572+0000","last_unstale":"2023-09-30T09:54:02.184385+0000","last_undegraded":"2023-09-30T09:54:02.184385+0000","last_fullsized":"2023-09-30T09:53:33.218373+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:03:53.943606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":808,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-30T09:54:07.384386+0000","last_change":"2023-09-30T09:54:03.339280+0000","last_active":"2023-09-30T09:54:07.384386+0000","last_peered":"2023-09-30T09:54:07.384386+0000","last_clean":"2023-09-30T09:54:07.384386+0000","last_became_active":"2023-09-30T09:50:29.883457+0000","last_became_peered":"2023-09-30T09:50:29.883457+0000","last_unstale":"2023-09-30T09:54:07.384386+0000","last_undegraded":"2023-09-30T09:54:07.384386+0000","last_fullsized":"2023-09-30T09:54:07.384386+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:54:03.339198+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:54:03.339198+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:05.265754+0000","scrub_duration":0.0015674910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":103,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.186225+0000","last_change":"2023-09-30T09:52:06.286929+0000","last_active":"2023-09-30T09:54:02.186225+0000","last_peered":"2023-09-30T09:54:02.186225+0000","last_clean":"2023-09-30T09:54:02.186225+0000","last_became_active":"2023-09-30T09:52:06.286569+0000","last_became_peered":"2023-09-30T09:52:06.286569+0000","last_unstale":"2023-09-30T09:54:02.186225+0000","last_undegraded":"2023-09-30T09:54:02.186225+0000","last_fullsized":"2023-09-30T09:54:02.186225+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:35:29.837107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":72,"reported_epoch":630,"state":"active+undersized","last_fresh":"2023-09-30T09:54:02.179274+0000","last_change":"2023-09-30T09:53:33.225001+0000","last_active":"2023-09-30T09:54:02.179274+0000","last_peered":"2023-09-30T09:54:02.179274+0000","last_clean":"2023-09-30T09:53:22.483469+0000","last_became_active":"2023-09-30T09:53:33.225001+0000","last_became_peered":"2023-09-30T09:53:33.225001+0000","last_unstale":"2023-09-30T09:54:02.179274+0000","last_undegraded":"2023-09-30T09:54:02.179274+0000","last_fullsized":"2023-09-30T09:53:33.221367+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:05:51.921002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":793,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.179557+0000","last_change":"2023-09-30T09:52:09.241112+0000","last_active":"2023-09-30T09:54:02.179557+0000","last_peered":"2023-09-30T09:54:02.179557+0000","last_clean":"2023-09-30T09:54:02.179557+0000","last_became_active":"2023-09-30T09:52:09.240703+0000","last_became_peered":"2023-09-30T09:52:09.240703+0000","last_unstale":"2023-09-30T09:54:02.179557+0000","last_undegraded":"2023-09-30T09:54:02.179557+0000","last_fullsized":"2023-09-30T09:54:02.179557+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:40:40.400090+0000","scrub_duration":0.004695744,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":330,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.184307+0000","last_change":"2023-09-30T09:52:06.308608+0000","last_active":"2023-09-30T09:54:02.184307+0000","last_peered":"2023-09-30T09:54:02.184307+0000","last_clean":"2023-09-30T09:54:02.184307+0000","last_became_active":"2023-09-30T09:52:06.287432+0000","last_became_peered":"2023-09-30T09:52:06.287432+0000","last_unstale":"2023-09-30T09:54:02.184307+0000","last_undegraded":"2023-09-30T09:54:02.184307+0000","last_fullsized":"2023-09-30T09:54:02.184307+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:30:09.888951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":675,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-30T09:54:02.204252+0000","last_change":"2023-09-30T09:50:26.905427+0000","last_active":"2023-09-30T09:54:02.204252+0000","last_peered":"2023-09-30T09:54:02.204252+0000","last_clean":"2023-09-30T09:54:02.204252+0000","last_became_active":"2023-09-30T09:50:26.905100+0000","last_became_peered":"2023-09-30T09:50:26.905100+0000","last_unstale":"2023-09-30T09:54:02.204252+0000","last_undegraded":"2023-09-30T09:54:02.204252+0000","last_fullsized":"2023-09-30T09:54:02.204252+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:38.036208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":90,"acting":90,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":13644,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28232,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":94,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":89,"acting":89,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194428,"num_objects":9,"num_object_clones":0,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":6926,"num_read_kb":15957,"num_write":999,"num_write_kb":13750,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":252,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12619776,"data_stored":12593314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":1147,"ondisk_log_size":1147,"up":21,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234513,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":637184,"kb_used_data":11320,"kb_used_omap":0,"kb_used_meta":625856,"kb_avail":93107968,"statfs":{"total":95995035648,"available":95342559232,"internally_reserved":0,"allocated":11591680,"data_stored":9498632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":640876544},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:53:20 2023","interfaces":[{"interface":"back","average":{"1min":7.1180000000000003,"5min":3.3370000000000002,"15min":5.6710000000000003},"min":{"1min":0.33200000000000002,"5min":0.25600000000000001,"15min":0.22900000000000001},"max":{"1min":100.176,"5min":100.176,"15min":215.416},"last":0.78600000000000003},{"interface":"front","average":{"1min":7.0099999999999998,"5min":3.4729999999999999,"15min":5.7119999999999997},"min":{"1min":0.308,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":99.890000000000001,"5min":99.890000000000001,"15min":215.041},"last":1.0820000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.4259999999999999,"15min":3.4900000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.2440000000000002,"5min":8.9320000000000004,"15min":157.578},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.619,"15min":3.5009999999999999},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.875,"5min":9.6709999999999994,"15min":157.589},"last":0.69499999999999995}]},{"osd":2,"last update":"Sat Sep 30 09:53:53 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":4.5629999999999997,"15min":4.3360000000000003},"min":{"1min":0.33000000000000002,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":7.3529999999999998,"5min":187.74600000000001,"15min":187.74600000000001},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.514,"5min":4.5789999999999997,"15min":4.3310000000000004},"min":{"1min":0.31,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.2199999999999998,"5min":187.583,"15min":187.583},"last":0.68300000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:53:32 2023","interfaces":[{"interface":"back","average":{"1min":7.4930000000000003,"5min":2.8900000000000001,"15min":5.399},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":102.139,"5min":102.139,"15min":248.691},"last":0.57699999999999996},{"interface":"front","average":{"1min":6.6909999999999998,"5min":2.9169999999999998,"15min":5.5659999999999998},"min":{"1min":0.378,"5min":0.314,"15min":0.314},"max":{"1min":99.713999999999999,"5min":99.713999999999999,"15min":249.36500000000001},"last":0.63900000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:53:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":4.0780000000000003,"15min":2.7650000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":7.1660000000000004,"5min":238.511,"15min":238.511},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.762,"5min":4.7089999999999996,"15min":2.8980000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":9.5600000000000005,"5min":238.387,"15min":238.387},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:53:37 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":4.3239999999999998,"15min":7.3150000000000004},"min":{"1min":0.36899999999999999,"5min":0.33100000000000002,"15min":0.28399999999999997},"max":{"1min":16.314,"5min":187.374,"15min":197.75700000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.262,"5min":3.9750000000000001,"15min":7.1840000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.6970000000000001,"5min":187.416,"15min":197.63300000000001},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:53:53 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.714,"15min":1.7050000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":6.0010000000000003,"5min":9.4730000000000008,"15min":129.14500000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.724,"5min":1.8620000000000001,"15min":1.6990000000000001},"min":{"1min":0.35699999999999998,"5min":0.32200000000000001,"15min":0.29599999999999999},"max":{"1min":11.960000000000001,"5min":12.058,"15min":128.494},"last":0.54600000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496150,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":664548,"kb_used_data":7260,"kb_used_omap":0,"kb_used_meta":657280,"kb_avail":93080604,"statfs":{"total":95995035648,"available":95314538496,"internally_reserved":0,"allocated":7434240,"data_stored":5361296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":673054720},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:53:26 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":5.6040000000000001,"15min":6.6399999999999997},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.016,"5min":192.511,"15min":192.511},"last":4.0780000000000003},{"interface":"front","average":{"1min":2.1360000000000001,"5min":5.6820000000000004,"15min":6.7009999999999996},"min":{"1min":0.39200000000000002,"5min":0.248,"15min":0.248},"max":{"1min":5.9630000000000001,"5min":192.346,"15min":192.346},"last":3.9369999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":3.2290000000000001,"15min":1.9530000000000001},"min":{"1min":0.25800000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.4500000000000002,"5min":103.038,"15min":103.038},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.3640000000000001,"5min":3.2970000000000002,"15min":2.097},"min":{"1min":0.33800000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.7549999999999999,"5min":103.05,"15min":103.05},"last":0.61799999999999999}]},{"osd":2,"last update":"Sat Sep 30 09:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":3.7080000000000002,"15min":4.5650000000000004},"min":{"1min":0.32200000000000001,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":3.0870000000000002,"5min":109.88500000000001,"15min":207.02500000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.446,"5min":3.516,"15min":4.5140000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.3209999999999997,"5min":105.81699999999999,"15min":207.114},"last":0.53100000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:53:26 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":3.1480000000000001,"15min":2.8959999999999999},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":10.894,"5min":106.402,"15min":107.11799999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":3.5169999999999999,"5min":3.4409999999999998,"15min":3.0590000000000002},"min":{"1min":0.44,"5min":0.33000000000000002,"15min":0.31900000000000001},"max":{"1min":10.625999999999999,"5min":106.34699999999999,"15min":106.982},"last":0.67800000000000005}]},{"osd":4,"last update":"Sat Sep 30 09:53:26 2023","interfaces":[{"interface":"back","average":{"1min":2.7160000000000002,"5min":2.2869999999999999,"15min":1.7889999999999999},"min":{"1min":0.36099999999999999,"5min":0.28699999999999998,"15min":0.24399999999999999},"max":{"1min":7.2320000000000002,"5min":30.568000000000001,"15min":63.798000000000002},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.0230000000000001,"15min":1.6279999999999999},"min":{"1min":0.311,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":10.724,"5min":29.27,"15min":63.133000000000003},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:53:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":2.278,"15min":1.988},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"max":{"1min":6.9429999999999996,"5min":19.212,"15min":19.212},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.4289999999999998,"5min":1.9850000000000001,"15min":2.1459999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.26900000000000002},"max":{"1min":18.135999999999999,"5min":18.135999999999999,"15min":18.135999999999999},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Sep 30 09:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":2.4369999999999998,"15min":1.5229999999999999},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.26500000000000001},"max":{"1min":11.191000000000001,"5min":39.868000000000002,"15min":39.868000000000002},"last":0.58099999999999996},{"interface":"front","average":{"1min":1,"5min":1.982,"15min":1.3240000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.27500000000000002},"max":{"1min":3.3700000000000001,"5min":12.625,"15min":12.625},"last":0.66600000000000004}]}]},{"osd":1,"up_from":301,"seq":1292785156348,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":762512,"kb_used_data":12996,"kb_used_omap":9,"kb_used_meta":749494,"kb_avail":92982640,"statfs":{"total":95995035648,"available":95214223360,"internally_reserved":0,"allocated":13307904,"data_stored":9714700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":767481874},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:53:07 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":10.128,"15min":5.7249999999999996},"min":{"1min":0.216,"5min":0.20399999999999999,"15min":0.17100000000000001},"max":{"1min":21.786000000000001,"5min":191.62700000000001,"15min":191.62700000000001},"last":0.626},{"interface":"front","average":{"1min":3.2429999999999999,"5min":10.32,"15min":5.7240000000000002},"min":{"1min":0.222,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":21.821999999999999,"5min":191.58799999999999,"15min":191.58799999999999},"last":0.37}]},{"osd":2,"last update":"Sat Sep 30 09:54:09 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":4.1500000000000004,"15min":3.2170000000000001},"min":{"1min":0.22800000000000001,"5min":0.21099999999999999,"15min":0.184},"max":{"1min":4.4180000000000001,"5min":189.14500000000001,"15min":189.14500000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.4650000000000001,"5min":4.1189999999999998,"15min":3.202},"min":{"1min":0.25800000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":11.973000000000001,"5min":189.333,"15min":189.333},"last":0.52700000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:54:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":2.0190000000000001,"15min":3.625},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.23400000000000001},"max":{"1min":7.0439999999999996,"5min":31.151,"15min":199.41},"last":0.498},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.728,"15min":3.3809999999999998},"min":{"1min":0.23599999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.766,"5min":31.727,"15min":199.58099999999999},"last":0.65400000000000003}]},{"osd":4,"last update":"Sat Sep 30 09:54:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.7470000000000001,"15min":2.2429999999999999},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.26800000000000002},"max":{"1min":10.602,"5min":47.332000000000001,"15min":142.77099999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.536,"15min":2.5470000000000002},"min":{"1min":0.36799999999999999,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":8.4540000000000006,"5min":47.606999999999999,"15min":144.898},"last":0.69899999999999995}]},{"osd":5,"last update":"Sat Sep 30 09:53:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0179999999999998,"5min":3.5489999999999999,"15min":2.673},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.0949999999999998,"5min":165.88300000000001,"15min":165.88300000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.952,"5min":3.2709999999999999,"15min":2.0590000000000002},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.313000000000001,"5min":165.863,"15min":165.863},"last":0.77400000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:54:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":2.7989999999999999,"15min":2.3959999999999999},"min":{"1min":0.37,"5min":0.33700000000000002,"15min":0.29899999999999999},"max":{"1min":8.2590000000000003,"5min":97.281999999999996,"15min":97.281999999999996},"last":0.83299999999999996},{"interface":"front","average":{"1min":2.0579999999999998,"5min":3.1400000000000001,"15min":2.492},"min":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":10.641,"5min":97.242000000000004,"15min":97.242000000000004},"last":0.872}]},{"osd":7,"last update":"Sat Sep 30 09:54:09 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.579,"15min":1.524},"min":{"1min":0.41599999999999998,"5min":0.36899999999999999,"15min":0.28299999999999997},"max":{"1min":4.9989999999999997,"5min":9.8390000000000004,"15min":9.8390000000000004},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.242,"5min":1.609,"15min":1.5589999999999999},"min":{"1min":0.35699999999999998,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":6.9299999999999997,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":0.81399999999999995}]}]},{"osd":0,"up_from":94,"seq":403726926235,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1538372,"kb_used_data":684156,"kb_used_omap":5,"kb_used_meta":854202,"kb_avail":92206780,"statfs":{"total":95995035648,"available":94419742720,"internally_reserved":0,"allocated":700575744,"data_stored":696835464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":874703101},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:53:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":1.421,"15min":1.6140000000000001},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.191},"max":{"1min":5.1539999999999999,"5min":11.641,"15min":57.482999999999997},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.369,"15min":1.984},"min":{"1min":0.25700000000000001,"5min":0.22500000000000001,"15min":0.19800000000000001},"max":{"1min":8.9770000000000003,"5min":23.503,"15min":57.140999999999998},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:52:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":3.7490000000000001,"15min":4.0300000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20799999999999999},"max":{"1min":9.0470000000000006,"5min":173.185,"15min":173.185},"last":0.316},{"interface":"front","average":{"1min":1.704,"5min":3.556,"15min":4.0789999999999997},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.249},"max":{"1min":8.9510000000000005,"5min":173.22499999999999,"15min":173.22499999999999},"last":0.49199999999999999}]},{"osd":3,"last update":"Sat Sep 30 09:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0739999999999998,"5min":2.5779999999999998,"15min":3.4180000000000001},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":7.2910000000000004,"5min":53.192999999999998,"15min":125.562},"last":1.129},{"interface":"front","average":{"1min":2.3090000000000002,"5min":2.5680000000000001,"15min":2.9580000000000002},"min":{"1min":0.33900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.5490000000000004,"5min":53.207000000000001,"15min":117.295},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.5310000000000001,"5min":2.4990000000000001,"15min":2.4399999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.0039999999999996,"5min":67.945999999999998,"15min":72.403999999999996},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.907,"5min":2.6899999999999999,"15min":2.4489999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.29699999999999999},"max":{"1min":7.0410000000000004,"5min":67.792000000000002,"15min":72.162999999999997},"last":1.167}]},{"osd":5,"last update":"Sat Sep 30 09:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":2.6309999999999998,"15min":1.9319999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.492,"5min":71.376000000000005,"15min":71.376000000000005},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.369,"5min":2.5720000000000001,"15min":1.6120000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30499999999999999},"max":{"1min":4.4459999999999997,"5min":71.706000000000003,"15min":71.706000000000003},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Sep 30 09:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":1.9990000000000001,"15min":2.5539999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.33200000000000002},"max":{"1min":6.7359999999999998,"5min":11.512,"15min":240.946},"last":0.65600000000000003},{"interface":"front","average":{"1min":2.8210000000000002,"5min":1.889,"15min":2.9809999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.34999999999999998},"max":{"1min":9.0280000000000005,"5min":10.750999999999999,"15min":240.977},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Sep 30 09:53:20 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":2.1040000000000001,"15min":2.29},"min":{"1min":0.48499999999999999,"5min":0.41499999999999998,"15min":0.36099999999999999},"max":{"1min":6.8170000000000002,"5min":11.571,"15min":146.828},"last":0.628},{"interface":"front","average":{"1min":3.585,"5min":2.0049999999999999,"15min":2.758},"min":{"1min":0.43099999999999999,"5min":0.41599999999999998,"15min":0.33100000000000002},"max":{"1min":10.452999999999999,"5min":11.029,"15min":146.876},"last":0.52200000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599185,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1216216,"kb_used_data":291788,"kb_used_omap":5,"kb_used_meta":924410,"kb_avail":92528936,"statfs":{"total":95995035648,"available":94749630464,"internally_reserved":0,"allocated":298790912,"data_stored":295250514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":946596493},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:53:05 2023","interfaces":[{"interface":"back","average":{"1min":14.224,"5min":5.1580000000000004,"15min":4.8159999999999998},"min":{"1min":0.19800000000000001,"5min":0.188,"15min":0.188},"max":{"1min":217.00999999999999,"5min":217.00999999999999,"15min":217.00999999999999},"last":0.221},{"interface":"front","average":{"1min":14.706,"5min":5.5940000000000003,"15min":5.0570000000000004},"min":{"1min":0.29299999999999998,"5min":0.222,"15min":0.217},"max":{"1min":217.09,"5min":217.09,"15min":217.09},"last":1.8280000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":2.4209999999999998,"15min":3.8519999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20399999999999999},"max":{"1min":9.2439999999999998,"5min":80.828999999999994,"15min":202.369},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":2.6150000000000002,"15min":3.9780000000000002},"min":{"1min":0.27500000000000002,"5min":0.255,"15min":0.21099999999999999},"max":{"1min":1.873,"5min":83.114999999999995,"15min":202.54499999999999},"last":0.254}]},{"osd":3,"last update":"Sat Sep 30 09:53:24 2023","interfaces":[{"interface":"back","average":{"1min":15.901,"5min":4.1660000000000004,"15min":4.5039999999999996},"min":{"1min":0.29699999999999999,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":231.00700000000001,"5min":231.00700000000001,"15min":231.00700000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":15.836,"5min":3.9540000000000002,"15min":4.431},"min":{"1min":0.26600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":230.91499999999999,"5min":230.91499999999999,"15min":230.91499999999999},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.714,"15min":3.145},"min":{"1min":0.38300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.468,"5min":14.637,"15min":247.655},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.2190000000000001,"15min":3.1019999999999999},"min":{"1min":0.34699999999999998,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":4.4249999999999998,"5min":5.7990000000000004,"15min":248.012},"last":0.44500000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:53:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8280000000000001,"5min":1.835,"15min":3.0390000000000001},"min":{"1min":0.38700000000000001,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":5.923,"5min":10.762,"15min":243.86500000000001},"last":0.377},{"interface":"front","average":{"1min":2.1320000000000001,"5min":2.133,"15min":3.331},"min":{"1min":0.36599999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.0109999999999992,"5min":15.164999999999999,"15min":244.02199999999999},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":2.077,"15min":2.3730000000000002},"min":{"1min":0.41699999999999998,"5min":0.32800000000000001,"15min":0.29399999999999998},"max":{"1min":2.9510000000000001,"5min":46.732999999999997,"15min":202.70599999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.228,"5min":2.617,"15min":2.6840000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":4.7450000000000001,"5min":46.68,"15min":202.916},"last":0.504}]},{"osd":7,"last update":"Sat Sep 30 09:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.6659999999999999,"15min":3.536},"min":{"1min":0.39500000000000002,"5min":0.38500000000000001,"15min":0.32000000000000001},"max":{"1min":4.7569999999999997,"5min":9.1950000000000003,"15min":244.911},"last":0.495},{"interface":"front","average":{"1min":1.091,"5min":2.2709999999999999,"15min":3.8879999999999999},"min":{"1min":0.40999999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.6859999999999999,"5min":10.782,"15min":244.66},"last":0.42999999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435661,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1025580,"kb_used_data":18596,"kb_used_omap":2,"kb_used_meta":1006973,"kb_avail":92719572,"statfs":{"total":95995035648,"available":94944841728,"internally_reserved":0,"allocated":19042304,"data_stored":15359205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":1031140711},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:53:23 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":4.4779999999999998,"15min":4.8869999999999996},"min":{"1min":0.23100000000000001,"5min":0.22,"15min":0.183},"max":{"1min":7.2080000000000002,"5min":152.429,"15min":205.73099999999999},"last":3.7970000000000002},{"interface":"front","average":{"1min":2.2160000000000002,"5min":4.4279999999999999,"15min":4.9299999999999997},"min":{"1min":0.24199999999999999,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":6.4640000000000004,"5min":150.464,"15min":205.495},"last":2.625}]},{"osd":1,"last update":"Sat Sep 30 09:54:07 2023","interfaces":[{"interface":"back","average":{"1min":4.306,"5min":2.3260000000000001,"15min":3.2149999999999999},"min":{"1min":0.254,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":63.744999999999997,"5min":63.744999999999997,"15min":170.512},"last":0.439},{"interface":"front","average":{"1min":4.3470000000000004,"5min":2.2549999999999999,"15min":3.1379999999999999},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.20699999999999999},"max":{"1min":63.762,"5min":63.762,"15min":170.535},"last":0.54300000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":2.7730000000000001,"15min":4.391},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":4.3719999999999999,"5min":147.87,"15min":206.69},"last":0.373},{"interface":"front","average":{"1min":0.72099999999999997,"5min":2.968,"15min":4.4850000000000003},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":4.8789999999999996,"5min":148.04900000000001,"15min":209.22800000000001},"last":0.33200000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":3.4159999999999999,"15min":2.3690000000000002},"min":{"1min":0.32400000000000001,"5min":0.28499999999999998,"15min":0.26800000000000002},"max":{"1min":4.7409999999999997,"5min":172.298,"15min":172.298},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.304,"5min":3.8140000000000001,"15min":2.6560000000000001},"min":{"1min":0.38,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":6.681,"5min":172.45099999999999,"15min":172.45099999999999},"last":0.505}]},{"osd":5,"last update":"Sat Sep 30 09:53:40 2023","interfaces":[{"interface":"back","average":{"1min":3.5369999999999999,"5min":2.9329999999999998,"15min":2.145},"min":{"1min":0.41599999999999998,"5min":0.33700000000000002,"15min":0.29199999999999998},"max":{"1min":33.273000000000003,"5min":61.426000000000002,"15min":61.426000000000002},"last":0.46200000000000002},{"interface":"front","average":{"1min":4.5460000000000003,"5min":3.4399999999999999,"15min":2.1200000000000001},"min":{"1min":0.52100000000000002,"5min":0.36199999999999999,"15min":0.311},"max":{"1min":33.256,"5min":61.469000000000001,"15min":61.469000000000001},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:54:25 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":3.0649999999999999,"15min":2.044},"min":{"1min":0.36799999999999999,"5min":0.312,"15min":0.312},"max":{"1min":4.7889999999999997,"5min":75.781000000000006,"15min":75.781000000000006},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.0089999999999999,"5min":3.2669999999999999,"15min":2.282},"min":{"1min":0.48299999999999998,"5min":0.35999999999999999,"15min":0.33400000000000002},"max":{"1min":4.1230000000000002,"5min":75.754000000000005,"15min":75.754000000000005},"last":0.67200000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:54:25 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":3.2959999999999998,"15min":2.718},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.32900000000000001},"max":{"1min":4.3310000000000004,"5min":90.855999999999995,"15min":213.548},"last":0.621},{"interface":"front","average":{"1min":1.3129999999999999,"5min":3.2210000000000001,"15min":2.6899999999999999},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.8499999999999996,"5min":91.108000000000004,"15min":213.583},"last":0.69199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019423,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":894160,"kb_used_data":276424,"kb_used_omap":0,"kb_used_meta":617728,"kb_avail":92850992,"statfs":{"total":95995035648,"available":95079415808,"internally_reserved":0,"allocated":283058176,"data_stored":281000483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":632553472},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5209999999999999,"5min":4.0499999999999998,"15min":3.5089999999999999},"min":{"1min":0.246,"5min":0.222,"15min":0.21199999999999999},"max":{"1min":14.045999999999999,"5min":107.43000000000001,"15min":107.43000000000001},"last":3.6200000000000001},{"interface":"front","average":{"1min":3.0779999999999998,"5min":4.3040000000000003,"15min":3.7549999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":12.773,"5min":107.373,"15min":107.373},"last":3.4319999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:54:19 2023","interfaces":[{"interface":"back","average":{"1min":3.1259999999999999,"5min":2.0840000000000001,"15min":2.335},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":32.536000000000001,"5min":32.536000000000001,"15min":108.551},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.9550000000000001,"5min":2.129,"15min":2.496},"min":{"1min":0.29499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":32.637,"5min":32.637,"15min":108.884},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:53:58 2023","interfaces":[{"interface":"back","average":{"1min":9.5879999999999992,"5min":5.5999999999999996,"15min":4.7699999999999996},"min":{"1min":0.317,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":105.20699999999999,"5min":139.53899999999999,"15min":142.88999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":8.7729999999999997,"5min":5.0800000000000001,"15min":4.6189999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":105.246,"5min":139.56399999999999,"15min":142.90700000000001},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:54:19 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":2.8660000000000001,"15min":2.464},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32700000000000001},"max":{"1min":29.675999999999998,"5min":53.082999999999998,"15min":105.142},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.52,"5min":2.6120000000000001,"15min":2.359},"min":{"1min":0.315,"5min":0.309,"15min":0.29299999999999998},"max":{"1min":29.664999999999999,"5min":53.097000000000001,"15min":105.157},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 30 09:53:35 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":2.355,"15min":6.0860000000000003},"min":{"1min":0.27500000000000002,"5min":0.249,"15min":0.249},"max":{"1min":6.9989999999999997,"5min":59.530999999999999,"15min":140.11099999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.5990000000000002,"5min":2.448,"15min":6.0830000000000002},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.583,"5min":60.012,"15min":140.065},"last":0.30599999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:53:35 2023","interfaces":[{"interface":"back","average":{"1min":5.4059999999999997,"5min":2.3599999999999999,"15min":1.393},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":71.864999999999995,"5min":71.864999999999995,"15min":71.864999999999995},"last":0.436},{"interface":"front","average":{"1min":6.5129999999999999,"5min":2.7679999999999998,"15min":1.5289999999999999},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":72.018000000000001,"5min":72.018000000000001,"15min":72.018000000000001},"last":0.34699999999999998}]},{"osd":7,"last update":"Sat Sep 30 09:53:58 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":2.5310000000000001,"15min":2.5600000000000001},"min":{"1min":0.32100000000000001,"5min":0.31900000000000001,"15min":0.27600000000000002},"max":{"1min":16.858000000000001,"5min":16.858000000000001,"15min":161.72999999999999},"last":0.374},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.6319999999999999,"15min":2.2679999999999998},"min":{"1min":0.34999999999999998,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":9.8109999999999999,"5min":14.018000000000001,"15min":162.60900000000001},"last":0.32400000000000001}]}]},{"osd":5,"up_from":489,"seq":2100239007865,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":257180,"kb_used_data":7188,"kb_used_omap":6,"kb_used_meta":249977,"kb_avail":93487972,"statfs":{"total":95995035648,"available":95731683328,"internally_reserved":0,"allocated":7360512,"data_stored":5291813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":255976543},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:52:34 2023","interfaces":[{"interface":"back","average":{"1min":9.0609999999999999,"5min":7.1820000000000004,"15min":17.100000000000001},"min":{"1min":0.24399999999999999,"5min":0.224,"15min":0.22},"max":{"1min":137.846,"5min":137.846,"15min":281.10599999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":9.3719999999999999,"5min":6.226,"15min":16.948},"min":{"1min":0.27800000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":137.035,"5min":137.035,"15min":282.07400000000001},"last":0.312}]},{"osd":1,"last update":"Sat Sep 30 09:53:36 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":2.8660000000000001,"15min":8.7200000000000006},"min":{"1min":0.29299999999999998,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":7.9690000000000003,"5min":129.958,"15min":273.48200000000003},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.609,"5min":2.7629999999999999,"15min":8.2509999999999994},"min":{"1min":0.33200000000000002,"5min":0.28799999999999998,"15min":0.255},"max":{"1min":7.3109999999999999,"5min":130.05099999999999,"15min":273.50200000000001},"last":0.69799999999999995}]},{"osd":2,"last update":"Sat Sep 30 09:53:36 2023","interfaces":[{"interface":"back","average":{"1min":2.972,"5min":3.0659999999999998,"15min":3.4319999999999999},"min":{"1min":0.32300000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.2590000000000003,"5min":129.98400000000001,"15min":194.14400000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":3,"5min":1.8620000000000001,"15min":3.2669999999999999},"min":{"1min":0.42599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.065,"5min":11.337999999999999,"15min":198.083},"last":0.51800000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:53:36 2023","interfaces":[{"interface":"back","average":{"1min":3.2349999999999999,"5min":5.0890000000000004,"15min":9.4489999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.30299999999999999},"max":{"1min":7.9169999999999998,"5min":130.02199999999999,"15min":274.01799999999997},"last":0.56499999999999995},{"interface":"front","average":{"1min":3.0539999999999998,"5min":4.7270000000000003,"15min":3.282},"min":{"1min":0.38600000000000001,"5min":0.309,"15min":0.29499999999999998},"max":{"1min":11.775,"5min":130.07400000000001,"15min":130.07400000000001},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:53:36 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":4.6829999999999998,"15min":2.2509999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":9.2929999999999993,"5min":132.94399999999999,"15min":132.94399999999999},"last":0.63},{"interface":"front","average":{"1min":1.278,"5min":4.7519999999999998,"15min":2.3999999999999999},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.4059999999999997,"5min":136.083,"15min":136.083},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 30 09:53:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":3.5299999999999998,"15min":2.254},"min":{"1min":0.33800000000000002,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":7.8330000000000002,"5min":130.09800000000001,"15min":130.09800000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":2.085,"5min":3.7200000000000002,"15min":2.2029999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.3599999999999994,"5min":130.03399999999999,"15min":130.03399999999999},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:53:36 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":1.835,"15min":1.53},"min":{"1min":0.376,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.2559999999999998,"5min":32.578000000000003,"15min":32.578000000000003},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.1110000000000002,"5min":2.0600000000000001,"15min":1.4350000000000001},"min":{"1min":0.29799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.3970000000000002,"5min":32.420999999999999,"15min":32.420999999999999},"last":0.71299999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4218880,"data_stored":4204649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5891,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:54:31.154 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 81 2023-09-30T09:54:31.170 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[216815]: from='client.37134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:54:31.211 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[216815]: pgmap v1699: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 200 KiB/s rd, 71 KiB/s wr, 102 op/s; 7/111 objects degraded (6.306%) 2023-09-30T09:54:31.211 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 7/111 objects degraded (6.306%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:31.211 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[216815]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[216815]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.212 INFO:tasks.workunit.client.0.smithi019.stderr: Removing snap: 100% complete...done. 2023-09-30T09:54:31.212 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm testimg 2023-09-30T09:54:31.304 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:31.308 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --order 20 /tmp/rbd_import_export_690681/img testimg 2023-09-30T09:54:31.374 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:31.374 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:54:31.377 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_690681/img_v2 2023-09-30T09:54:31.420 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:31.424 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_690681/img_v2 testimg_import 2023-09-30T09:54:31.494 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:54:31.498 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import 2023-09-30T09:54:31.498 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep order 2023-09-30T09:54:31.498 INFO:tasks.workunit.client.0.smithi019.stderr:+ awk '{print $2}' 2023-09-30T09:54:31.498 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 20 2023-09-30T09:54:31.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:31 smithi167 ceph-mon[192880]: from='client.37134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:54:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:31 smithi167 ceph-mon[192880]: pgmap v1699: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 200 KiB/s rd, 71 KiB/s wr, 102 op/s; 7/111 objects degraded (6.306%) 2023-09-30T09:54:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:31 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 7/111 objects degraded (6.306%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:31 smithi167 ceph-mon[192880]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:31 smithi167 ceph-mon[192880]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:31 smithi167 ceph-mon[192880]: osdmap e634: 8 total, 7 up, 8 in 2023-09-30T09:54:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:31 smithi167 ceph-mon[192880]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.544 INFO:tasks.workunit.client.0.smithi019.stdout:20 2023-09-30T09:54:31.544 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img_v2 2023-09-30T09:54:31.544 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:54:31.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[216815]: osdmap e634: 8 total, 7 up, 8 in 2023-09-30T09:54:31.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[216815]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[224043]: from='client.37134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:54:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[224043]: pgmap v1699: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 200 KiB/s rd, 71 KiB/s wr, 102 op/s; 7/111 objects degraded (6.306%) 2023-09-30T09:54:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 7/111 objects degraded (6.306%), 4 pgs degraded (PG_DEGRADED) 2023-09-30T09:54:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[224043]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[224043]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[224043]: osdmap e634: 8 total, 7 up, 8 in 2023-09-30T09:54:31.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:31 smithi019 ceph-mon[224043]: failed to encode map e634 with expected crc 2023-09-30T09:54:31.649 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:31.650 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg 2023-09-30T09:54:31.738 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:31.741 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_690681/img testimg 2023-09-30T09:54:31.790 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:31.793 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd info testimg 2023-09-30T09:54:31.793 INFO:tasks.workunit.client.0.smithi019.stderr:++ grep features 2023-09-30T09:54:31.830 INFO:tasks.workunit.client.0.smithi019.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-30T09:54:31.831 INFO:tasks.workunit.client.0.smithi019.stderr: op_features: ' 2023-09-30T09:54:31.831 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_690681/img_v2 2023-09-30T09:54:31.869 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:31.872 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_690681/img_v2 testimg_import 2023-09-30T09:54:31.928 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:54:31.932 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd info testimg_import 2023-09-30T09:54:31.932 INFO:tasks.workunit.client.0.smithi019.stderr:++ grep features 2023-09-30T09:54:31.971 INFO:tasks.workunit.client.0.smithi019.stderr:+ FEATURES_AFTER=' features: layering 2023-09-30T09:54:31.971 INFO:tasks.workunit.client.0.smithi019.stderr: op_features: ' 2023-09-30T09:54:31.971 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' ' features: layering 2023-09-30T09:54:31.972 INFO:tasks.workunit.client.0.smithi019.stderr: op_features: ' '!=' ' features: layering 2023-09-30T09:54:31.972 INFO:tasks.workunit.client.0.smithi019.stderr: op_features: ' ']' 2023-09-30T09:54:31.972 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img_v2 2023-09-30T09:54:31.972 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:54:31.987 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:31.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:32.042 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:32.045 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg 2023-09-30T09:54:32.091 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:32.118 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:32.119 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:32.122 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_690681/img testimg 2023-09-30T09:54:32.161 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:32.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:32.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:32.188 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is set 2023-09-30T09:54:32.195 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:32.198 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_690681/img_v2 2023-09-30T09:54:32.216 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[224043]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.216 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[224043]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.216 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[224043]: osdmap e635: 8 total, 7 up, 8 in 2023-09-30T09:54:32.216 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[224043]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.216 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[224043]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.216 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2413566490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:54:32.217 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:54:32.217 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[216815]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.217 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[216815]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.217 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[216815]: osdmap e635: 8 total, 7 up, 8 in 2023-09-30T09:54:32.217 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[216815]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.217 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[216815]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.217 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2413566490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:54:32.218 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:32 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:54:32.239 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:32.242 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_690681/img_v2 testimg_import 2023-09-30T09:54:32.268 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:32.295 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:32.315 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:54:32.319 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import 2023-09-30T09:54:32.319 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 'stripe unit' 2023-09-30T09:54:32.319 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-30T09:54:32.340 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:32.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:32.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:32.357 INFO:tasks.workunit.client.0.smithi019.stdout: stripe unit: 4 KiB 2023-09-30T09:54:32.358 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import 2023-09-30T09:54:32.358 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 'stripe count' 2023-09-30T09:54:32.358 INFO:tasks.workunit.client.0.smithi019.stderr:+ awk '{print $3}' 2023-09-30T09:54:32.358 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 1000 2023-09-30T09:54:32.400 INFO:tasks.workunit.client.0.smithi019.stdout:1000 2023-09-30T09:54:32.400 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/img_v2 2023-09-30T09:54:32.400 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:54:32.446 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:32.482 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:32.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:32 smithi167 ceph-mon[192880]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:32 smithi167 ceph-mon[192880]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:32 smithi167 ceph-mon[192880]: osdmap e635: 8 total, 7 up, 8 in 2023-09-30T09:54:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:32 smithi167 ceph-mon[192880]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:32 smithi167 ceph-mon[192880]: failed to encode map e635 with expected crc 2023-09-30T09:54:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:32 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2413566490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:54:32.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:32 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-30T09:54:32.522 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-30T09:54:32.525 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg 2023-09-30T09:54:32.617 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-30T09:54:32.619 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_690681/img testimg 2023-09-30T09:54:32.680 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:32.682 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap create testimg@snap1 2023-09-30T09:54:33.201 INFO:tasks.workunit.client.0.smithi019.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-30T09:54:33.208 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap create testimg@snap2 2023-09-30T09:54:33.266 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:33.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[224043]: pgmap v1702: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 68 KiB/s rd, 10 KiB/s wr, 82 op/s; 7/111 objects degraded (6.306%) 2023-09-30T09:54:33.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[224043]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:33.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[224043]: failed to encode map e636 with expected crc 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[224043]: failed to encode map e636 with expected crc 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[224043]: osdmap e636: 8 total, 7 up, 8 in 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[216815]: pgmap v1702: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 68 KiB/s rd, 10 KiB/s wr, 82 op/s; 7/111 objects degraded (6.306%) 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[216815]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:54:33.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[216815]: failed to encode map e636 with expected crc 2023-09-30T09:54:33.270 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[216815]: failed to encode map e636 with expected crc 2023-09-30T09:54:33.270 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[216815]: osdmap e636: 8 total, 7 up, 8 in 2023-09-30T09:54:33.270 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:54:33.270 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:33.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:33.371 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:33.398 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:33.441 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:33.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:33 smithi167 ceph-mon[192880]: pgmap v1702: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 68 KiB/s rd, 10 KiB/s wr, 82 op/s; 7/111 objects degraded (6.306%) 2023-09-30T09:54:33.447 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:33 smithi167 ceph-mon[192880]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:33.448 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:33 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-30T09:54:33.448 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:33 smithi167 ceph-mon[192880]: failed to encode map e636 with expected crc 2023-09-30T09:54:33.448 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:33 smithi167 ceph-mon[192880]: failed to encode map e636 with expected crc 2023-09-30T09:54:33.448 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:33 smithi167 ceph-mon[192880]: osdmap e636: 8 total, 7 up, 8 in 2023-09-30T09:54:33.448 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:54:33.448 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:33.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:33.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:33.550 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:33.575 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:33.618 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:33.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:33.716 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:54:33.723 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:33.740 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:54:33.749 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:33.769 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:54:33.788 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:54:33.791 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:33.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:33.809 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:54:33.828 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:54:33.848 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:54:33.848 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-30T09:54:33.848 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-30T09:54:33.848 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-30T09:54:33.849 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-30T09:54:33.849 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-30T09:54:33.849 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-30T09:54:33.849 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-30T09:54:33.849 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-30T09:54:33.902 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:33.928 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:33.969 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:33.970 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:33.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:34.006 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:54:34.006 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:54:34.077 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:34.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:54:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:54:33] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2023-09-30T09:54:34.114 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:34.205 INFO:tasks.workunit.client.0.smithi019.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-30T09:54:34.211 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap protect testimg@snap2 2023-09-30T09:54:34.254 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_690681/snap_protect 2023-09-30T09:54:34.296 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:34.299 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_690681/snap_protect testimg_import 2023-09-30T09:54:34.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:34 smithi167 ceph-mon[192880]: failed to encode map e637 with expected crc 2023-09-30T09:54:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:34 smithi167 ceph-mon[192880]: osdmap e637: 8 total, 7 up, 8 in 2023-09-30T09:54:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:34 smithi167 ceph-mon[192880]: failed to encode map e637 with expected crc 2023-09-30T09:54:34.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/568210317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:34.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:34 smithi019 ceph-mon[216815]: failed to encode map e637 with expected crc 2023-09-30T09:54:34.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:34 smithi019 ceph-mon[216815]: osdmap e637: 8 total, 7 up, 8 in 2023-09-30T09:54:34.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:34 smithi019 ceph-mon[216815]: failed to encode map e637 with expected crc 2023-09-30T09:54:34.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/568210317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:34 smithi019 ceph-mon[224043]: failed to encode map e637 with expected crc 2023-09-30T09:54:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:34 smithi019 ceph-mon[224043]: osdmap e637: 8 total, 7 up, 8 in 2023-09-30T09:54:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:34 smithi019 ceph-mon[224043]: failed to encode map e637 with expected crc 2023-09-30T09:54:34.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/568210317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:35.234 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:35.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:35.338 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:35.363 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-30T09:54:35.364 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:35.408 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:35.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:35.446 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:35 smithi019 ceph-mon[216815]: pgmap v1705: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s rd, 6.0 KiB/s wr, 86 op/s; 7/120 objects degraded (5.833%); 3/120 objects misplaced (2.500%) 2023-09-30T09:54:35.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:35 smithi019 ceph-mon[216815]: failed to encode map e638 with expected crc 2023-09-30T09:54:35.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:35 smithi019 ceph-mon[216815]: osdmap e638: 8 total, 7 up, 8 in 2023-09-30T09:54:35.447 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:35 smithi019 ceph-mon[216815]: failed to encode map e638 with expected crc 2023-09-30T09:54:35.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:35 smithi019 ceph-mon[224043]: pgmap v1705: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s rd, 6.0 KiB/s wr, 86 op/s; 7/120 objects degraded (5.833%); 3/120 objects misplaced (2.500%) 2023-09-30T09:54:35.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:35 smithi019 ceph-mon[224043]: failed to encode map e638 with expected crc 2023-09-30T09:54:35.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:35 smithi019 ceph-mon[224043]: osdmap e638: 8 total, 7 up, 8 in 2023-09-30T09:54:35.447 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:35 smithi019 ceph-mon[224043]: failed to encode map e638 with expected crc 2023-09-30T09:54:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:54:35.467 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:54:35.515 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:35.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:35 smithi167 ceph-mon[192880]: pgmap v1705: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s rd, 6.0 KiB/s wr, 86 op/s; 7/120 objects degraded (5.833%); 3/120 objects misplaced (2.500%) 2023-09-30T09:54:35.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:35 smithi167 ceph-mon[192880]: failed to encode map e638 with expected crc 2023-09-30T09:54:35.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:35 smithi167 ceph-mon[192880]: osdmap e638: 8 total, 7 up, 8 in 2023-09-30T09:54:35.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:35 smithi167 ceph-mon[192880]: failed to encode map e638 with expected crc 2023-09-30T09:54:35.533 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-30T09:54:35.542 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:35.583 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:35.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:35.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:35.691 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:35.717 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:35.758 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:35.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:35.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:35.865 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:35.891 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:35.933 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:35.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:35.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:36.038 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:36.064 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:36.106 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:36.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:36.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:36.211 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:36.231 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-30T09:54:36.235 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import@snap1 2023-09-30T09:54:36.235 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 'protected: False' 2023-09-30T09:54:36.242 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:36.278 INFO:tasks.workunit.client.0.smithi019.stdout: protected: False 2023-09-30T09:54:36.279 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd info testimg_import@snap2 2023-09-30T09:54:36.279 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep 'protected: True' 2023-09-30T09:54:36.322 INFO:tasks.workunit.client.0.smithi019.stdout: protected: True 2023-09-30T09:54:36.322 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/snap_protect 2023-09-30T09:54:36.322 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap unprotect testimg@snap2 2023-09-30T09:54:36.370 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-30T09:54:36.393 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:36.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:36.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:36.423 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap purge testimg 2023-09-30T09:54:36.502 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:36.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:36 smithi167 ceph-mon[192880]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:36 smithi167 ceph-mon[192880]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:36 smithi167 ceph-mon[192880]: osdmap e639: 8 total, 7 up, 8 in 2023-09-30T09:54:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:36 smithi167 ceph-mon[192880]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.528 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:36.543 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:36.571 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:36.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:36.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:36.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:36 smithi019 ceph-mon[224043]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:36 smithi019 ceph-mon[224043]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:36 smithi019 ceph-mon[224043]: osdmap e639: 8 total, 7 up, 8 in 2023-09-30T09:54:36.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:36 smithi019 ceph-mon[224043]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:36 smithi019 ceph-mon[216815]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:36 smithi019 ceph-mon[216815]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:36 smithi019 ceph-mon[216815]: osdmap e639: 8 total, 7 up, 8 in 2023-09-30T09:54:36.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:36 smithi019 ceph-mon[216815]: failed to encode map e639 with expected crc 2023-09-30T09:54:36.677 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:36.703 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:36.745 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:36.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:36.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:36.810 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:36.853 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:36.878 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:36.920 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:36.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:36.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:37.025 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:37.059 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:37.112 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:37.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:37.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:37.125 INFO:teuthology.orchestra.run.smithi019.stderr:Error EINVAL: specified pgp_num 81 > pg_num 31 2023-09-30T09:54:37.217 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:37.245 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:37.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[216815]: pgmap v1708: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 429 KiB/s rd, 47 KiB/s wr, 570 op/s; 4/105 objects degraded (3.810%); 3/105 objects misplaced (2.857%) 2023-09-30T09:54:37.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4/105 objects degraded (3.810%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:37.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[216815]: failed to encode map e640 with expected crc 2023-09-30T09:54:37.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[216815]: failed to encode map e640 with expected crc 2023-09-30T09:54:37.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[216815]: osdmap e640: 8 total, 7 up, 8 in 2023-09-30T09:54:37.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2375891698' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:54:37.280 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[224043]: pgmap v1708: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 429 KiB/s rd, 47 KiB/s wr, 570 op/s; 4/105 objects degraded (3.810%); 3/105 objects misplaced (2.857%) 2023-09-30T09:54:37.280 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4/105 objects degraded (3.810%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:37.280 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[224043]: failed to encode map e640 with expected crc 2023-09-30T09:54:37.280 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[224043]: failed to encode map e640 with expected crc 2023-09-30T09:54:37.280 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[224043]: osdmap e640: 8 total, 7 up, 8 in 2023-09-30T09:54:37.280 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:37 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2375891698' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:54:37.285 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:37.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:37.392 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:37.418 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:37.460 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:37.463 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:37 smithi167 ceph-mon[192880]: pgmap v1708: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 429 KiB/s rd, 47 KiB/s wr, 570 op/s; 4/105 objects degraded (3.810%); 3/105 objects misplaced (2.857%) 2023-09-30T09:54:37.463 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:37 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4/105 objects degraded (3.810%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:37.464 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:37 smithi167 ceph-mon[192880]: failed to encode map e640 with expected crc 2023-09-30T09:54:37.464 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:37 smithi167 ceph-mon[192880]: failed to encode map e640 with expected crc 2023-09-30T09:54:37.464 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:37 smithi167 ceph-mon[192880]: osdmap e640: 8 total, 7 up, 8 in 2023-09-30T09:54:37.464 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:37 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2375891698' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-30T09:54:37.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:37.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:37.467 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-30T09:54:37.565 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:37.595 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:38.191 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:54:38.227 INFO:tasks.workunit.client.0.smithi019.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-30T09:54:38.234 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd snap purge testimg_import 2023-09-30T09:54:38.378 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:38.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:38.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:38.482 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:38.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:38 smithi019 ceph-mon[216815]: osdmap e641: 8 total, 7 up, 8 in 2023-09-30T09:54:38.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:38 smithi019 ceph-mon[216815]: failed to encode map e641 with expected crc 2023-09-30T09:54:38.509 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:38 smithi019 ceph-mon[216815]: failed to encode map e641 with expected crc 2023-09-30T09:54:38.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:38 smithi019 ceph-mon[224043]: osdmap e641: 8 total, 7 up, 8 in 2023-09-30T09:54:38.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:38 smithi019 ceph-mon[224043]: failed to encode map e641 with expected crc 2023-09-30T09:54:38.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:38 smithi019 ceph-mon[224043]: failed to encode map e641 with expected crc 2023-09-30T09:54:38.511 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:38.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:38 smithi167 ceph-mon[192880]: osdmap e641: 8 total, 7 up, 8 in 2023-09-30T09:54:38.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:38 smithi167 ceph-mon[192880]: failed to encode map e641 with expected crc 2023-09-30T09:54:38.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:38 smithi167 ceph-mon[192880]: failed to encode map e641 with expected crc 2023-09-30T09:54:38.978 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:38.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:38.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:39.082 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:39.114 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:39.163 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:39.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:39.270 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:39.298 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:39.342 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:39.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:39.447 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:39.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:39 smithi019 ceph-mon[216815]: pgmap v1711: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 388 KiB/s rd, 42 KiB/s wr, 517 op/s; 4/102 objects degraded (3.922%); 2/102 objects misplaced (1.961%) 2023-09-30T09:54:39.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:39 smithi019 ceph-mon[216815]: failed to encode map e642 with expected crc 2023-09-30T09:54:39.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:39 smithi019 ceph-mon[216815]: osdmap e642: 8 total, 7 up, 8 in 2023-09-30T09:54:39.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:39 smithi019 ceph-mon[216815]: failed to encode map e642 with expected crc 2023-09-30T09:54:39.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:39 smithi019 ceph-mon[224043]: pgmap v1711: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 388 KiB/s rd, 42 KiB/s wr, 517 op/s; 4/102 objects degraded (3.922%); 2/102 objects misplaced (1.961%) 2023-09-30T09:54:39.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:39 smithi019 ceph-mon[224043]: failed to encode map e642 with expected crc 2023-09-30T09:54:39.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:39 smithi019 ceph-mon[224043]: osdmap e642: 8 total, 7 up, 8 in 2023-09-30T09:54:39.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:39 smithi019 ceph-mon[224043]: failed to encode map e642 with expected crc 2023-09-30T09:54:39.481 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:39.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:39 smithi167 ceph-mon[192880]: pgmap v1711: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 388 KiB/s rd, 42 KiB/s wr, 517 op/s; 4/102 objects degraded (3.922%); 2/102 objects misplaced (1.961%) 2023-09-30T09:54:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:39 smithi167 ceph-mon[192880]: failed to encode map e642 with expected crc 2023-09-30T09:54:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:39 smithi167 ceph-mon[192880]: osdmap e642: 8 total, 7 up, 8 in 2023-09-30T09:54:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:39 smithi167 ceph-mon[192880]: failed to encode map e642 with expected crc 2023-09-30T09:54:39.556 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:39.646 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:39.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:39.751 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:39.784 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:39.887 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:39.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:39.992 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 2023-09-30T09:54:40.023 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:40.124 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-30T09:54:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:40.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.2' 2023-09-30T09:54:40.232 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:40.234 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is set 2023-09-30T09:54:40.241 INFO:tasks.workunit.client.0.smithi019.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-30T09:54:40.250 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg 2023-09-30T09:54:40.265 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:40.317 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:40.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[224043]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[224043]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[224043]: osdmap e643: 8 total, 7 up, 8 in 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[224043]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/132263682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[216815]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[216815]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[216815]: osdmap e643: 8 total, 7 up, 8 in 2023-09-30T09:54:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[216815]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:40 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/132263682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:54:40.353 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:40.356 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd remove testimg_import 2023-09-30T09:54:40.422 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:40.447 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:40.451 INFO:tasks.workunit.client.0.smithi019.stderr:+ tiered=0 2023-09-30T09:54:40.451 INFO:tasks.workunit.client.0.smithi019.stderr:+ ceph osd dump 2023-09-30T09:54:40.451 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep '^pool' 2023-09-30T09:54:40.451 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep ''\''rbd'\''' 2023-09-30T09:54:40.451 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep tier 2023-09-30T09:54:40.455 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:40.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:40 smithi167 ceph-mon[192880]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:40 smithi167 ceph-mon[192880]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:40 smithi167 ceph-mon[192880]: osdmap e643: 8 total, 7 up, 8 in 2023-09-30T09:54:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:40 smithi167 ceph-mon[192880]: failed to encode map e643 with expected crc 2023-09-30T09:54:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:40 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/132263682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:54:41.062 INFO:teuthology.orchestra.run.smithi019.stdout:ERROR: (22) Invalid argument 2023-09-30T09:54:41.062 INFO:teuthology.orchestra.run.smithi019.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-30T09:54:41.228 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[216815]: pgmap v1714: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 145 KiB/s rd, 12 KiB/s wr, 196 op/s; 4/111 objects degraded (3.604%); 2/111 objects misplaced (1.802%) 2023-09-30T09:54:41.228 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[216815]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:41.228 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/132263682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:54:41.228 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[216815]: failed to encode map e644 with expected crc 2023-09-30T09:54:41.228 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[216815]: osdmap e644: 8 total, 7 up, 8 in 2023-09-30T09:54:41.228 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[216815]: failed to encode map e644 with expected crc 2023-09-30T09:54:41.229 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1557588056' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:54:41.363 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_690681/sparse1 2023-09-30T09:54:41.369 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:54:41.369 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:54:41.369 INFO:tasks.workunit.client.0.smithi019.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00507648 s, 207 MB/s 2023-09-30T09:54:41.369 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_690681/sparse2 2023-09-30T09:54:41.375 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:54:41.375 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:54:41.375 INFO:tasks.workunit.client.0.smithi019.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00504408 s, 208 MB/s 2023-09-30T09:54:41.375 INFO:tasks.workunit.client.0.smithi019.stderr:+ truncate /tmp/rbd_import_export_690681/sparse2 -s 2M 2023-09-30T09:54:41.412 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse1 2023-09-30T09:54:41.452 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 0% complete...failed. 2023-09-30T09:54:41.452 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: delete error: (2) No such file or directory 2023-09-30T09:54:41.455 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:54:41.455 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_690681/sparse1 2023-09-30T09:54:41.519 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-30T09:54:41.520 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:41.520 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:54:41.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:41 smithi167 ceph-mon[192880]: pgmap v1714: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 145 KiB/s rd, 12 KiB/s wr, 196 op/s; 4/111 objects degraded (3.604%); 2/111 objects misplaced (1.802%) 2023-09-30T09:54:41.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:41 smithi167 ceph-mon[192880]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/132263682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:54:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:41 smithi167 ceph-mon[192880]: failed to encode map e644 with expected crc 2023-09-30T09:54:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:41 smithi167 ceph-mon[192880]: osdmap e644: 8 total, 7 up, 8 in 2023-09-30T09:54:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:41 smithi167 ceph-mon[192880]: failed to encode map e644 with expected crc 2023-09-30T09:54:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:41 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1557588056' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:54:41.523 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd ls -l 2023-09-30T09:54:41.523 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep sparse1 2023-09-30T09:54:41.523 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-30T09:54:41.562 INFO:tasks.workunit.client.0.smithi019.stdout:sparse1 2 MiB 2 2023-09-30T09:54:41.562 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse1 2023-09-30T09:54:41.562 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse1 2023-09-30T09:54:41.563 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse1 2023-09-30T09:54:41.563 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:54:41.563 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:54:41.577 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[224043]: pgmap v1714: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 145 KiB/s rd, 12 KiB/s wr, 196 op/s; 4/111 objects degraded (3.604%); 2/111 objects misplaced (1.802%) 2023-09-30T09:54:41.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[224043]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:41.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/132263682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-30T09:54:41.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[224043]: failed to encode map e644 with expected crc 2023-09-30T09:54:41.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[224043]: osdmap e644: 8 total, 7 up, 8 in 2023-09-30T09:54:41.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[224043]: failed to encode map e644 with expected crc 2023-09-30T09:54:41.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:41 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1557588056' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:54:41.602 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.b92156835347 2023-09-30T09:54:41.603 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.b92156835347 2023-09-30T09:54:41.603 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse1 2023-09-30T09:54:41.603 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.b92156835347\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:54:41.604 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse1 2023-09-30T09:54:41.604 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:54:41.604 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse1 2023-09-30T09:54:41.604 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:54:41.604 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:54:41.642 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:54:41.642 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:54:41.642 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:54:41.642 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:54:41.642 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:54:41.733 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=1 2023-09-30T09:54:41.733 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 1 2023-09-30T09:54:41.733 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-30T09:54:41.733 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse1 /tmp/rbd_import_export_690681/sparse1.out 2023-09-30T09:54:41.771 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:41.774 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_690681/sparse1 /tmp/rbd_import_export_690681/sparse1.out 2023-09-30T09:54:41.774 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_690681/sparse1 /tmp/rbd_import_export_690681/sparse1.out 2023-09-30T09:54:41.776 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse1 --format %b 2023-09-30T09:54:41.777 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=2048 2023-09-30T09:54:41.778 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse1.out --format %b 2023-09-30T09:54:41.778 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=2048 2023-09-30T09:54:41.779 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:41.779 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:41.779 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:54:41.779 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/sparse1.out 2023-09-30T09:54:41.779 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse1 2023-09-30T09:54:41.873 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-30T09:54:41.875 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse2 2023-09-30T09:54:41.917 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 0% complete...failed. 2023-09-30T09:54:41.917 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: delete error: (2) No such file or directory 2023-09-30T09:54:41.919 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:54:41.919 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_690681/sparse2 2023-09-30T09:54:42.136 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-30T09:54:42.136 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:42.137 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:54:42.139 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd ls -l 2023-09-30T09:54:42.139 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep sparse2 2023-09-30T09:54:42.140 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-30T09:54:42.160 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:42.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:42.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:42.178 INFO:tasks.workunit.client.0.smithi019.stdout:sparse2 2 MiB 2 2023-09-30T09:54:42.180 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse2 2023-09-30T09:54:42.180 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse2 2023-09-30T09:54:42.180 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse2 2023-09-30T09:54:42.180 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:54:42.180 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:54:42.218 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.b954db152996 2023-09-30T09:54:42.218 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.b954db152996 2023-09-30T09:54:42.218 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse2 2023-09-30T09:54:42.218 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.b954db152996\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:54:42.218 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse2 2023-09-30T09:54:42.219 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:54:42.224 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse2 2023-09-30T09:54:42.224 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:54:42.224 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:54:42.251 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-30T09:54:42.265 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:42.269 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:54:42.269 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:54:42.269 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:54:42.269 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:54:42.269 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:54:42.291 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:42.305 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=0 2023-09-30T09:54:42.305 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 0 2023-09-30T09:54:42.305 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-30T09:54:42.305 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse2 /tmp/rbd_import_export_690681/sparse2.out 2023-09-30T09:54:42.333 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:42.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:42.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:42.343 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:42.348 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_690681/sparse2 /tmp/rbd_import_export_690681/sparse2.out 2023-09-30T09:54:42.348 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_690681/sparse2 /tmp/rbd_import_export_690681/sparse2.out 2023-09-30T09:54:42.350 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse2 --format %b 2023-09-30T09:54:42.351 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=2048 2023-09-30T09:54:42.351 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse2.out --format %b 2023-09-30T09:54:42.353 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=2048 2023-09-30T09:54:42.353 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:42.353 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:42.353 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:54:42.353 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/sparse2.out 2023-09-30T09:54:42.353 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse2 2023-09-30T09:54:42.434 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:42 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1557588056' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:54:42.435 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:42 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1557588056' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:54:42.439 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:42.456 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-30T09:54:42.460 INFO:tasks.workunit.client.0.smithi019.stderr:+ truncate /tmp/rbd_import_export_690681/sparse1 -s 10M 2023-09-30T09:54:42.461 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-30T09:54:42.471 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:42.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:42 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1557588056' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-30T09:54:42.559 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:42.560 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:42.560 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:54:42.563 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd ls -l 2023-09-30T09:54:42.563 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep sparse1 2023-09-30T09:54:42.563 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-30T09:54:42.598 INFO:tasks.workunit.client.0.smithi019.stdout:sparse1 10 MiB 2 2023-09-30T09:54:42.599 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse1 2023-09-30T09:54:42.599 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse1 2023-09-30T09:54:42.600 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse1 2023-09-30T09:54:42.600 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:54:42.600 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:54:42.638 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.b9819d7d9009 2023-09-30T09:54:42.639 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.b9819d7d9009 2023-09-30T09:54:42.640 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse1 2023-09-30T09:54:42.640 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse1 2023-09-30T09:54:42.640 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.b9819d7d9009\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:54:42.640 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:54:42.640 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse1 2023-09-30T09:54:42.640 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:54:42.640 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:54:42.684 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:54:42.685 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:54:42.685 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:54:42.685 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:54:42.685 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:54:42.726 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=1 2023-09-30T09:54:42.726 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 1 2023-09-30T09:54:42.727 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-30T09:54:42.727 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse1 /tmp/rbd_import_export_690681/sparse1.out 2023-09-30T09:54:42.765 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:42.769 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_690681/sparse1 /tmp/rbd_import_export_690681/sparse1.out 2023-09-30T09:54:42.769 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_690681/sparse1 /tmp/rbd_import_export_690681/sparse1.out 2023-09-30T09:54:42.777 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse1 --format %b 2023-09-30T09:54:42.778 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=2048 2023-09-30T09:54:42.779 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse1.out --format %b 2023-09-30T09:54:42.780 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=2048 2023-09-30T09:54:42.780 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:42.780 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:42.780 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:54:42.780 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/sparse1.out 2023-09-30T09:54:42.781 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse1 2023-09-30T09:54:42.827 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:42.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:42.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:42.879 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-30T09:54:42.882 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_690681/sparse2 oflag=append conv=notrunc 2023-09-30T09:54:42.894 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:54:42.894 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:54:42.894 INFO:tasks.workunit.client.0.smithi019.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0106536 s, 197 MB/s 2023-09-30T09:54:42.895 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-30T09:54:42.898 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1: {} 2023-09-30T09:54:42.921 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2: {} 2023-09-30T09:54:42.932 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:42.948 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3: {} 2023-09-30T09:54:42.967 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:42.974 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4: {} 2023-09-30T09:54:42.992 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:42.993 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:42.993 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:42.993 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:54:42.997 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd ls -l 2023-09-30T09:54:42.998 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep sparse2 2023-09-30T09:54:42.998 INFO:tasks.workunit.client.0.smithi019.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-30T09:54:43.005 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5: {} 2023-09-30T09:54:43.026 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:43.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:43.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:43.033 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6: {} 2023-09-30T09:54:43.044 INFO:tasks.workunit.client.0.smithi019.stdout:sparse2 4 MiB 2 2023-09-30T09:54:43.045 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse2 2023-09-30T09:54:43.045 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse2 2023-09-30T09:54:43.046 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse2 2023-09-30T09:54:43.046 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:54:43.046 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:54:43.061 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7: {} 2023-09-30T09:54:43.061 INFO:teuthology.orchestra.run.smithi019.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-30T09:54:43.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-30T09:54:43.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-30T09:54:43.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-30T09:54:43.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-30T09:54:43.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-30T09:54:43.061 INFO:teuthology.orchestra.run.smithi019.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-30T09:54:43.062 INFO:teuthology.orchestra.run.smithi019.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-30T09:54:43.090 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.92b547bbad77 2023-09-30T09:54:43.090 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.92b547bbad77 2023-09-30T09:54:43.091 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse2 2023-09-30T09:54:43.091 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.92b547bbad77\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:54:43.091 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse2 2023-09-30T09:54:43.091 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:54:43.091 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse2 2023-09-30T09:54:43.092 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:54:43.092 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:54:43.128 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:54:43.128 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:54:43.129 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:54:43.129 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:54:43.129 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:54:43.132 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 2023-09-30T09:54:43.166 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:43.168 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects='0 2023-09-30T09:54:43.168 INFO:tasks.workunit.client.0.smithi019.stderr:2 2023-09-30T09:54:43.168 INFO:tasks.workunit.client.0.smithi019.stderr:3' 2023-09-30T09:54:43.168 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 0 2 3 2023-09-30T09:54:43.168 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-30T09:54:43.169 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse2 /tmp/rbd_import_export_690681/sparse2.out 2023-09-30T09:54:43.193 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 0, 3, 6, 2] out_osds: [] dead_osds: [0] live_osds: [2, 3, 4, 6, 7, 1, 5] 2023-09-30T09:54:43.193 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-30T09:54:43.193 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-30T09:54:43.193 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-30T09:54:43.193 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-30T09:54:43.194 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-30T09:54:43.212 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-30T09:54:43.215 INFO:tasks.workunit.client.0.smithi019.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_690681/sparse2 /tmp/rbd_import_export_690681/sparse2.out 2023-09-30T09:54:43.216 INFO:tasks.workunit.client.0.smithi019.stderr:+ cmp -l /tmp/rbd_import_export_690681/sparse2 /tmp/rbd_import_export_690681/sparse2.out 2023-09-30T09:54:43.219 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse2 --format %b 2023-09-30T09:54:43.220 INFO:tasks.workunit.client.0.smithi019.stderr:+ origsize=6144 2023-09-30T09:54:43.220 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse2.out --format %b 2023-09-30T09:54:43.221 INFO:tasks.workunit.client.0.smithi019.stderr:+ exportsize=6144 2023-09-30T09:54:43.221 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:43.221 INFO:tasks.workunit.client.0.smithi019.stderr:+ difference=0 2023-09-30T09:54:43.221 INFO:tasks.workunit.client.0.smithi019.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-30T09:54:43.222 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/sparse2.out 2023-09-30T09:54:43.222 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse2 2023-09-30T09:54:43.316 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-30T09:54:43.316 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-30T09:54:43.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:43.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.1' 2023-09-30T09:54:43.320 INFO:tasks.workunit.client.0.smithi019.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-30T09:54:43.320 INFO:tasks.workunit.client.0.smithi019.stdout:partially-sparse file imports to partially-sparse image 2023-09-30T09:54:43.320 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_690681/sparse1 sparse 2023-09-30T09:54:43.386 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-30T09:54:43.388 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:43.388 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --order is deprecated, use --object-size 2023-09-30T09:54:43.391 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse 2023-09-30T09:54:43.391 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse 2023-09-30T09:54:43.391 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse 2023-09-30T09:54:43.391 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:54:43.391 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:54:43.420 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:43.428 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.92e233479586 2023-09-30T09:54:43.429 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.92e233479586 2023-09-30T09:54:43.429 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse 2023-09-30T09:54:43.429 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse 2023-09-30T09:54:43.429 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.92e233479586\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:54:43.429 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:54:43.429 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse 2023-09-30T09:54:43.430 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:54:43.430 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:54:43.447 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:43.467 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:54:43.467 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:54:43.468 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:54:43.468 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:54:43.468 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:54:43.501 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=1 2023-09-30T09:54:43.501 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 1 2023-09-30T09:54:43.501 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-30T09:54:43.501 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse 2023-09-30T09:54:43.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:43 smithi167 ceph-mon[192880]: pgmap v1716: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 119 KiB/s rd, 9.6 KiB/s wr, 161 op/s; 4/111 objects degraded (3.604%); 2/111 objects misplaced (1.802%) 2023-09-30T09:54:43.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/875293161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:43.546 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:43.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:43.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:43.564 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:54:43.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:43 smithi019 ceph-mon[224043]: pgmap v1716: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 119 KiB/s rd, 9.6 KiB/s wr, 161 op/s; 4/111 objects degraded (3.604%); 2/111 objects misplaced (1.802%) 2023-09-30T09:54:43.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/875293161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:43.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:43 smithi019 ceph-mon[216815]: pgmap v1716: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 119 KiB/s rd, 9.6 KiB/s wr, 161 op/s; 4/111 objects degraded (3.604%); 2/111 objects misplaced (1.802%) 2023-09-30T09:54:43.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/875293161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-30T09:54:43.598 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-30T09:54:43.601 INFO:tasks.workunit.client.0.smithi019.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-30T09:54:43.601 INFO:tasks.workunit.client.0.smithi019.stdout:zeros import through stdin to sparse image 2023-09-30T09:54:43.601 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-30T09:54:43.602 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd import --new-format - sparse 2023-09-30T09:54:43.650 INFO:tasks.workunit.client.0.smithi019.stderr:4+0 records in 2023-09-30T09:54:43.650 INFO:tasks.workunit.client.0.smithi019.stderr:4+0 records out 2023-09-30T09:54:43.650 INFO:tasks.workunit.client.0.smithi019.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0485792 s, 86.3 MB/s 2023-09-30T09:54:43.653 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:43.662 INFO:tasks.workunit.client.0.smithi019.stderr: Importing image: 100% complete...done. 2023-09-30T09:54:43.662 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:43.665 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse 2023-09-30T09:54:43.665 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse 2023-09-30T09:54:43.666 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse 2023-09-30T09:54:43.666 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:54:43.666 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:54:43.682 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:43.708 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.b9fca72bc72a 2023-09-30T09:54:43.709 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.b9fca72bc72a 2023-09-30T09:54:43.709 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse 2023-09-30T09:54:43.709 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse 2023-09-30T09:54:43.709 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:54:43.710 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.b9fca72bc72a\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:54:43.710 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse 2023-09-30T09:54:43.710 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:54:43.710 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:54:43.747 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:54:43.747 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:54:43.747 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:54:43.748 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:54:43.748 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:54:43.781 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects= 2023-09-30T09:54:43.781 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 2023-09-30T09:54:43.781 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-30T09:54:43.782 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse 2023-09-30T09:54:43.874 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:43.878 INFO:tasks.workunit.client.0.smithi019.stderr:+ echo 'zeros export to sparse file' 2023-09-30T09:54:43.879 INFO:tasks.workunit.client.0.smithi019.stdout:zeros export to sparse file 2023-09-30T09:54:43.879 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd create --new-format sparse --size 4 2023-09-30T09:54:43.916 INFO:tasks.workunit.client.0.smithi019.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-30T09:54:43.918 INFO:tasks.workunit.client.0.smithi019.stderr:++ rbd info sparse 2023-09-30T09:54:43.918 INFO:tasks.workunit.client.0.smithi019.stderr:++ grep block_name_prefix 2023-09-30T09:54:43.919 INFO:tasks.workunit.client.0.smithi019.stderr:++ awk '{print $NF;}' 2023-09-30T09:54:43.957 INFO:tasks.workunit.client.0.smithi019.stderr:+ prefix=rbd_data.9324817447ae 2023-09-30T09:54:43.957 INFO:tasks.workunit.client.0.smithi019.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-30T09:54:43.957 INFO:tasks.workunit.client.0.smithi019.stderr:++ get_image_data_pool sparse 2023-09-30T09:54:43.958 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse 2023-09-30T09:54:43.958 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse 2023-09-30T09:54:43.958 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep 'data_pool: ' 2023-09-30T09:54:43.958 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk -F: '{ print $NF }' 2023-09-30T09:54:43.997 INFO:tasks.workunit.client.0.smithi019.stderr:++ data_pool= 2023-09-30T09:54:43.997 INFO:tasks.workunit.client.0.smithi019.stderr:++ '[' -z ']' 2023-09-30T09:54:43.997 INFO:tasks.workunit.client.0.smithi019.stderr:++ data_pool=rbd 2023-09-30T09:54:43.997 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo rbd 2023-09-30T09:54:43.997 INFO:tasks.workunit.client.0.smithi019.stderr:+ rados -p rbd put rbd_data.9324817447ae.000000000000 - 2023-09-30T09:54:44.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:54:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:54:43] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2023-09-30T09:54:44.031 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records in 2023-09-30T09:54:44.031 INFO:tasks.workunit.client.0.smithi019.stderr:1+0 records out 2023-09-30T09:54:44.031 INFO:tasks.workunit.client.0.smithi019.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.073825 s, 56.8 MB/s 2023-09-30T09:54:44.052 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects sparse 2023-09-30T09:54:44.052 INFO:tasks.workunit.client.0.smithi019.stderr:++ image=sparse 2023-09-30T09:54:44.052 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rbd info sparse 2023-09-30T09:54:44.053 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep block_name_prefix 2023-09-30T09:54:44.053 INFO:tasks.workunit.client.0.smithi019.stderr:+++ awk '{print $NF;}' 2023-09-30T09:54:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-30T09:54:44.077 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-30T09:54:44.086 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:44.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:44.097 INFO:tasks.workunit.client.0.smithi019.stderr:++ prefix=rbd_data.9324817447ae 2023-09-30T09:54:44.097 INFO:tasks.workunit.client.0.smithi019.stderr:+++ grep rbd_data.9324817447ae 2023-09-30T09:54:44.097 INFO:tasks.workunit.client.0.smithi019.stderr:++++ get_image_data_pool sparse 2023-09-30T09:54:44.098 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sed -e 's/rbd_data.9324817447ae\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-30T09:54:44.098 INFO:tasks.workunit.client.0.smithi019.stderr:++++ image=sparse 2023-09-30T09:54:44.098 INFO:tasks.workunit.client.0.smithi019.stderr:+++ sort -u 2023-09-30T09:54:44.098 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ rbd info sparse 2023-09-30T09:54:44.098 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ grep 'data_pool: ' 2023-09-30T09:54:44.098 INFO:tasks.workunit.client.0.smithi019.stderr:+++++ awk -F: '{ print $NF }' 2023-09-30T09:54:44.136 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool= 2023-09-30T09:54:44.136 INFO:tasks.workunit.client.0.smithi019.stderr:++++ '[' -z ']' 2023-09-30T09:54:44.137 INFO:tasks.workunit.client.0.smithi019.stderr:++++ data_pool=rbd 2023-09-30T09:54:44.137 INFO:tasks.workunit.client.0.smithi019.stderr:++++ echo rbd 2023-09-30T09:54:44.137 INFO:tasks.workunit.client.0.smithi019.stderr:+++ rados ls -p rbd 2023-09-30T09:54:44.170 INFO:tasks.workunit.client.0.smithi019.stderr:++ objects=0 2023-09-30T09:54:44.170 INFO:tasks.workunit.client.0.smithi019.stderr:++ echo 0 2023-09-30T09:54:44.170 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-30T09:54:44.170 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/sparse 2023-09-30T09:54:44.171 INFO:tasks.workunit.client.0.smithi019.stderr:rm: cannot remove '/tmp/rbd_import_export_690681/sparse': No such file or directory 2023-09-30T09:54:44.171 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:54:44.171 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd export sparse /tmp/rbd_import_export_690681/sparse 2023-09-30T09:54:44.190 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:44.210 INFO:tasks.workunit.client.0.smithi019.stderr: Exporting image: 100% complete...done. 2023-09-30T09:54:44.213 INFO:tasks.workunit.client.0.smithi019.stderr:++ stat /tmp/rbd_import_export_690681/sparse --format=%b 2023-09-30T09:54:44.214 INFO:tasks.workunit.client.0.smithi019.stderr:+ '[' 0 = 0 ']' 2023-09-30T09:54:44.214 INFO:tasks.workunit.client.0.smithi019.stderr:+ rbd rm sparse 2023-09-30T09:54:44.222 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:44.267 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:44.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:44.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:44.311 INFO:tasks.workunit.client.0.smithi019.stderr: Removing image: 100% complete...done. 2023-09-30T09:54:44.314 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm /tmp/rbd_import_export_690681/sparse /tmp/rbd_import_export_690681/sparse1 /tmp/rbd_import_export_690681/sparse2 /tmp/rbd_import_export_690681/sparse3 2023-09-30T09:54:44.316 INFO:tasks.workunit.client.0.smithi019.stderr:rm: cannot remove '/tmp/rbd_import_export_690681/sparse3': No such file or directory 2023-09-30T09:54:44.316 INFO:tasks.workunit.client.0.smithi019.stderr:+ true 2023-09-30T09:54:44.317 INFO:tasks.workunit.client.0.smithi019.stdout:OK 2023-09-30T09:54:44.318 INFO:tasks.workunit.client.0.smithi019.stderr:+ echo OK 2023-09-30T09:54:44.318 INFO:tasks.workunit.client.0.smithi019.stderr:+ rm -rf /tmp/rbd_import_export_690681 2023-09-30T09:54:44.318 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-30T09:54:44.318 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-30T09:54:44.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:44 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 4/111 objects degraded (3.604%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:44.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:44 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 4/111 objects degraded (3.604%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:44.363 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-30T09:54:44.363 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-30T09:54:44.370 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:44.397 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:44.439 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:44.441 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:44 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 4/111 objects degraded (3.604%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:44.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:44.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:44.543 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:44.570 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:44.617 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:44.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:44.717 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:54:44.720 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 2023-09-30T09:54:44.746 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:44.788 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-30T09:54:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:44.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.5' 2023-09-30T09:54:44.891 DEBUG:teuthology.orchestra.run.smithi019:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 2023-09-30T09:54:44.920 INFO:teuthology.orchestra.run.smithi019.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:45.057 INFO:teuthology.orchestra.run.smithi019.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-30T09:54:45.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:45.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi019 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.3' 2023-09-30T09:54:45.161 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 2023-09-30T09:54:45.188 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:45.236 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-30T09:54:45.238 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:45.238 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:45 smithi019 ceph-mon[224043]: pgmap v1717: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 113 KiB/s rd, 8.0 KiB/s wr, 148 op/s; 4/108 objects degraded (3.704%); 1/108 objects misplaced (0.926%) 2023-09-30T09:54:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:45.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.6' 2023-09-30T09:54:45.239 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:45 smithi019 ceph-mon[216815]: pgmap v1717: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 113 KiB/s rd, 8.0 KiB/s wr, 148 op/s; 4/108 objects degraded (3.704%); 1/108 objects misplaced (0.926%) 2023-09-30T09:54:45.340 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 2023-09-30T09:54:45.351 DEBUG:teuthology.parallel:result is None 2023-09-30T09:54:45.352 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:54:45.369 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:45.382 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:54:45.383 DEBUG:teuthology.orchestra.run.smithi019:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:54:45.440 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T09:54:45.440 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-30T09:54:45.441 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-30T09:54:45.441 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-30T09:54:45.441 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-30T09:54:45.441 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-30T09:54:45.442 DEBUG:teuthology.orchestra.run.smithi019:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:54:45.445 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-30T09:54:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:45.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.7' 2023-09-30T09:54:45.448 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:45 smithi167 ceph-mon[192880]: pgmap v1717: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 4.8 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 113 KiB/s rd, 8.0 KiB/s wr, 148 op/s; 4/108 objects degraded (3.704%); 1/108 objects misplaced (0.926%) 2023-09-30T09:54:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T09:54:45.500 INFO:teuthology.orchestra.run.smithi019.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-30T09:54:45.501 DEBUG:teuthology.orchestra.run.smithi019:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-30T09:54:45.549 DEBUG:teuthology.orchestra.run.smithi167:> sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 2023-09-30T09:54:45.560 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-30T09:54:45.560 DEBUG:teuthology.orchestra.run.smithi019:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-30T09:54:45.576 INFO:teuthology.orchestra.run.smithi167.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-30T09:54:45.618 INFO:tasks.thrashosds:joining thrashosds 2023-09-30T09:54:45.621 INFO:teuthology.orchestra.run.smithi167.stderr:Error: no container with name or ID ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-30T09:54:45.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-30T09:54:45.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi167 with status 125: 'sudo docker kill -s 1 ceph-608495ac-5f70-11ee-8db4-212e2dc638e7-osd.4' 2023-09-30T09:54:45.772 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:54:45.772 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:54:45.772 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:54:45.772 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:54:45.772 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.772 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101677 10.f 10:f6c8d05b:::benchmark_data_smithi019_617837_object8277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.505301+0000", 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 245.26362082, 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0377062750000001, 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.773 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101677 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505301+0000", 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.774 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505301+0000", 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505303+0000", 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.855e-06 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505311+0000", 2023-09-30T09:54:45.775 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2969999999999998e-06 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505330+0000", 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9848999999999999e-05 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505340+0000", 2023-09-30T09:54:45.776 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4909999999999994e-06 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505430+0000", 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0117000000000005e-05 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.505445+0000", 2023-09-30T09:54:45.777 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5268e-05 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380620+0000", 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87517486700000002 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380634+0000", 2023-09-30T09:54:45.778 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4201e-05 2023-09-30T09:54:45.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.779 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.471995+0000", 2023-09-30T09:54:45.779 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0913604770000001 2023-09-30T09:54:45.779 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.779 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.779 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.472756+0000", 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00076066000000000003 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.479103+0000", 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063471689999999997 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.780 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.479257+0000", 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000154597 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.479448+0000", 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00019083199999999999 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.781 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.536950+0000", 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.057502191000000001 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.536993+0000", 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2487999999999997e-05 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.782 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.541678+0000", 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00468558 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.541722+0000", 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3269000000000002e-05 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.783 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543008+0000", 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012860669999999999 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101679 10.f 10:f6ce7bfd:::benchmark_data_smithi019_617837_object8279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.508259+0000", 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 245.26066345699999, 2023-09-30T09:54:45.784 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0366561970000001, 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101679 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.785 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508259+0000", 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508259+0000", 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.786 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508260+0000", 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1519999999999999e-06 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508266+0000", 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5500000000000002e-06 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.787 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508281+0000", 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5359999999999999e-05 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508289+0000", 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.259e-06 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.788 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508347+0000", 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7509000000000001e-05 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.508355+0000", 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5329999999999994e-06 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.789 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380662+0000", 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.87230725399999998 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380671+0000", 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5939999999999994e-06 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.474961+0000", 2023-09-30T09:54:45.790 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0942902880000001 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.475717+0000", 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00075602500000000001 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.791 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.534287+0000", 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.058569953000000001 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.534371+0000", 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3350000000000007e-05 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.792 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.534471+0000", 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000100073 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543024+0000", 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0085535560000000004 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543089+0000", 2023-09-30T09:54:45.793 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4846999999999998e-05 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543967+0000", 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00087815400000000002 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.544119+0000", 2023-09-30T09:54:45.794 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015164900000000001 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.544915+0000", 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00079609099999999996 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.795 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101692 10.f 10:fff5665a:::benchmark_data_smithi019_617837_object8292:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:40.537296+0000", 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 245.231626748, 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0086183219999998, 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.796 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101692 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537296+0000", 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.797 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537296+0000", 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537297+0000", 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3599999999999999e-06 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.798 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537304+0000", 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6989999999999997e-06 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537322+0000", 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8817e-05 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.799 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537330+0000", 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8620000000000003e-06 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537420+0000", 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0041000000000001e-05 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:40.537438+0000", 2023-09-30T09:54:45.800 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7496999999999999e-05 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380817+0000", 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.843379557 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.380827+0000", 2023-09-30T09:54:45.801 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0027000000000001e-05 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.475791+0000", 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0949634210000001 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.476372+0000", 2023-09-30T09:54:45.802 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00058119900000000002 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.539164+0000", 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.062791945000000002 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.539256+0000", 2023-09-30T09:54:45.803 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1797000000000002e-05 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.539346+0000", 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9776000000000002e-05 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.804 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.543104+0000", 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003758094 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.544969+0000", 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001865589 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.805 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.545017+0000", 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7542e-05 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.545055+0000", 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8742999999999998e-05 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.806 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.545914+0000", 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00085835600000000001 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101699 10.f 10:fa38cb57:::benchmark_data_smithi019_617837_object8299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:54:45.807 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.503819+0000", 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 244.26510280599999, 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.078656871, 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101699 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.808 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503819+0000", 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503819+0000", 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.809 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503822+0000", 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.368e-06 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503837+0000", 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5077e-05 2023-09-30T09:54:45.810 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503864+0000", 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6801999999999999e-05 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503879+0000", 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5268e-05 2023-09-30T09:54:45.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.503992+0000", 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011311099999999999 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.504009+0000", 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6515e-05 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.812 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.545973+0000", 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.041964361 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.546650+0000", 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000676726 2023-09-30T09:54:45.813 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.558255+0000", 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011605284 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.558332+0000", 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6892999999999997e-05 2023-09-30T09:54:45.814 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.558427+0000", 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.5353999999999995e-05 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.568221+0000", 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0097933819999999998 2023-09-30T09:54:45.815 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.568276+0000", 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5772000000000001e-05 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.580557+0000", 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012280307000000001 2023-09-30T09:54:45.816 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.580634+0000", 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7471000000000004e-05 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.582476+0000", 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018421799999999999 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.817 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101748 10.f 10:feb50901:::benchmark_data_smithi019_617837_object8348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.594287+0000", 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 244.17463497200001, 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.99615996500000004, 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.818 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101748 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594287+0000", 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.819 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594287+0000", 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594289+0000", 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.9290000000000001e-06 2023-09-30T09:54:45.820 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594296+0000", 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3729999999999998e-06 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594322+0000", 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6910000000000002e-05 2023-09-30T09:54:45.821 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594335+0000", 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2669e-05 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594420+0000", 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4666000000000006e-05 2023-09-30T09:54:45.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.594432+0000", 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2429000000000001e-05 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.546692+0000", 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95225960600000004 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.823 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.548960+0000", 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0022686210000000002 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.565214+0000", 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016253648999999998 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.824 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.565301+0000", 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6386999999999998e-05 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.565390+0000", 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.9759999999999994e-05 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.825 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.574662+0000", 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0092722020000000002 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.574675+0000", 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2934e-05 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.826 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.588018+0000", 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.01334277 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.588094+0000", 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5696000000000002e-05 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.827 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.590447+0000", 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0023533640000000002 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101752 10.f 10:fe3db57c:::benchmark_data_smithi019_617837_object8352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.606755+0000", 2023-09-30T09:54:45.828 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 244.162166918, 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.98578958500000002, 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101752 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.829 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606755+0000", 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606755+0000", 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.830 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606757+0000", 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1160000000000002e-06 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606766+0000", 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1380000000000007e-06 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.831 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606788+0000", 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2617e-05 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606803+0000", 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4627000000000001e-05 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.832 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606893+0000", 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0177000000000001e-05 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.606908+0000", 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4789e-05 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.833 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.549029+0000", 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94212122499999995 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.552141+0000", 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031117340000000001 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.834 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.571101+0000", 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.018960685000000001 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.571183+0000", 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1176999999999999e-05 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.835 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.571305+0000", 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012193600000000001 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.584504+0000", 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.013199934 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.836 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.590561+0000", 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.006056797 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.590610+0000", 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8690999999999998e-05 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.590686+0000", 2023-09-30T09:54:45.837 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.5815999999999994e-05 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.592545+0000", 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001859126 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.838 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101772 10.f 10:fc2b4dbf:::benchmark_data_smithi019_617837_object8372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.642283+0000", 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 244.12663952899999, 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95144165599999997, 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.839 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101772 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642283+0000", 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.840 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642283+0000", 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642284+0000", 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6279999999999999e-06 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.841 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642290+0000", 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.981e-06 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642309+0000", 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8532000000000001e-05 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.842 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642319+0000", 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0508e-05 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642385+0000", 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5679999999999995e-05 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.843 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.642397+0000", 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1635e-05 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.552217+0000", 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.90982037199999999 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.844 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.553472+0000", 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012552240000000001 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.577416+0000", 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.023943760000000001 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.845 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.577539+0000", 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000123406 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.577711+0000", 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017168399999999999 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.846 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.587950+0000", 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010238939000000001 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.592604+0000", 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.004653768 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.847 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.592632+0000", 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8572e-05 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.592688+0000", 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5447e-05 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.848 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.593724+0000", 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00103652 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101807 10.f 10:f1360924:::benchmark_data_smithi019_617837_object8407:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.699381+0000", 2023-09-30T09:54:45.849 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 244.06954119700001, 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.97619895899999998, 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101807 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.850 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699381+0000", 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699381+0000", 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.851 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699383+0000", 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.3860000000000001e-06 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699392+0000", 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4109999999999996e-06 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.852 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699419+0000", 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.7382e-05 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699431+0000", 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1914e-05 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.853 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699514+0000", 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2646999999999997e-05 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.699527+0000", 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.295e-05 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.854 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.594872+0000", 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.89534542399999995 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.595403+0000", 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00053071599999999998 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.855 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.618246+0000", 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.022842713000000001 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.618340+0000", 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4481000000000006e-05 2023-09-30T09:54:45.856 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.618516+0000", 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000175687 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.630704+0000", 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.012188399000000001 2023-09-30T09:54:45.857 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.638907+0000", 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0082030609999999993 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.673663+0000", 2023-09-30T09:54:45.858 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.034755449000000001 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.673726+0000", 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3511999999999998e-05 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.675580+0000", 2023-09-30T09:54:45.859 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018538269999999999 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101822 10.f 10:f4bbe328:::benchmark_data_smithi019_617837_object8422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.731256+0000", 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 244.03766589700001, 2023-09-30T09:54:45.860 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.95043909299999996, 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101822 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.861 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731256+0000", 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731256+0000", 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.862 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731258+0000", 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.742e-06 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731269+0000", 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1353e-05 2023-09-30T09:54:45.863 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731290+0000", 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0508000000000001e-05 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731300+0000", 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.8369999999999992e-06 2023-09-30T09:54:45.864 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731386+0000", 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6588000000000005e-05 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.731404+0000", 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7808999999999998e-05 2023-09-30T09:54:45.865 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.595466+0000", 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86406184500000005 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.596751+0000", 2023-09-30T09:54:45.866 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00128535 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.627388+0000", 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.030636587 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.627535+0000", 2023-09-30T09:54:45.867 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000147374 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.627725+0000", 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00018972800000000001 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.638916+0000", 2023-09-30T09:54:45.868 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011191362999999999 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.680403+0000", 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.041486519999999999 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.869 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.680446+0000", 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3257999999999998e-05 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.680488+0000", 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.1591999999999998e-05 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.870 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681695+0000", 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0012076389999999999 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:101827 10.f 10:fcfbd7c6:::benchmark_data_smithi019_617837_object8427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:50:41.734887+0000", 2023-09-30T09:54:45.871 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 244.03403550300001, 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.94782315299999997, 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 101827 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.872 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734887+0000", 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734887+0000", 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.873 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734889+0000", 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.827e-06 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734895+0000", 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.635e-06 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.874 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734917+0000", 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1786000000000001e-05 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.734928+0000", 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0849999999999999e-05 2023-09-30T09:54:45.875 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.735006+0000", 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7921000000000002e-05 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:41.735017+0000", 2023-09-30T09:54:45.876 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1321000000000001e-05 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.596804+0000", 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.86178649900000004 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.597425+0000", 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00062158600000000004 2023-09-30T09:54:45.877 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.635761+0000", 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.038336108000000001 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.635872+0000", 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000111202 2023-09-30T09:54:45.878 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.636045+0000", 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017237099999999999 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.650267+0000", 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.014222088000000001 2023-09-30T09:54:45.879 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681746+0000", 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.031478610999999997 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681815+0000", 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9295000000000007e-05 2023-09-30T09:54:45.880 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.681851+0000", 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6275000000000001e-05 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:50:42.682710+0000", 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00085877800000000004 2023-09-30T09:54:45.881 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127770 10.7 10:ea5acc8c:::benchmark_data_smithi019_617837_object34370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:56.740170+0000", 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 169.028752008, 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.661720278, 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.882 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127770 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740170+0000", 2023-09-30T09:54:45.883 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740170+0000", 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740172+0000", 2023-09-30T09:54:45.884 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4699999999999999e-06 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740179+0000", 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8120000000000002e-06 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740205+0000", 2023-09-30T09:54:45.885 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6679999999999999e-05 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740213+0000", 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2670000000000006e-06 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740304+0000", 2023-09-30T09:54:45.886 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0814000000000002e-05 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.740321+0000", 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6646e-05 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.141529+0000", 2023-09-30T09:54:45.887 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.40120852600000001 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.141543+0000", 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3888e-05 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.327671+0000", 2023-09-30T09:54:45.888 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1861271870000001 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.328554+0000", 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00088332799999999996 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.389968+0000", 2023-09-30T09:54:45.889 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.061414174000000002 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.390100+0000", 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013192299999999999 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4,6", 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.390266+0000", 2023-09-30T09:54:45.890 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016573099999999999 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.396122+0000", 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058566570000000004 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.396244+0000", 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012189300000000001 2023-09-30T09:54:45.891 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.396265+0000", 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0639999999999999e-05 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.399991+0000", 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0037262939999999998 2023-09-30T09:54:45.892 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.400077+0000", 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5991999999999999e-05 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.401891+0000", 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018133559999999999 2023-09-30T09:54:45.893 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127773 10.7 10:e8a99f2e:::benchmark_data_smithi019_617837_object34373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:56.832885+0000", 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 168.93603767299999, 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6041828979999999, 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.894 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127773 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832885+0000", 2023-09-30T09:54:45.895 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832885+0000", 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832886+0000", 2023-09-30T09:54:45.896 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0690000000000001e-06 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832890+0000", 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9190000000000001e-06 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832898+0000", 2023-09-30T09:54:45.897 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5690000000000005e-06 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832904+0000", 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.772e-06 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832961+0000", 2023-09-30T09:54:45.898 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7107000000000001e-05 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:56.832969+0000", 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3939999999999996e-06 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.141571+0000", 2023-09-30T09:54:45.899 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30860205000000002 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.141582+0000", 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0254999999999999e-05 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.393765+0000", 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2521827939999999 2023-09-30T09:54:45.900 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.394470+0000", 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00070521500000000003 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.410662+0000", 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016191855000000002 2023-09-30T09:54:45.901 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.410797+0000", 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013578400000000001 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,4,6", 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.410983+0000", 2023-09-30T09:54:45.902 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000185649 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.431634+0000", 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.020651051 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.435059+0000", 2023-09-30T09:54:45.903 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.003424584 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.435089+0000", 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0303000000000001e-05 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.435121+0000", 2023-09-30T09:54:45.904 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2403999999999997e-05 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.435192+0000", 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1141000000000002e-05 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.437067+0000", 2023-09-30T09:54:45.905 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001874983 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129749 10.7 10:e073f11d:::benchmark_data_smithi019_617837_object36349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e595)", 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:05.214651+0000", 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 160.55427157599999, 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2038800529999998, 2023-09-30T09:54:45.906 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129749 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.907 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.214651+0000", 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.214651+0000", 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.908 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.214653+0000", 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.84e-06 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.214660+0000", 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4780000000000002e-06 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.909 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.214682+0000", 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.2325999999999999e-05 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.214693+0000", 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0497e-05 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.910 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.214757+0000", 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4610000000000007e-05 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:05.214767+0000", 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7450000000000007e-06 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.911 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.283043+0000", 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.068276255 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.283054+0000", 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0105999999999999e-05 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.912 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.339305+0000", 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0562514270000001 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.340117+0000", 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00081157699999999996 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.913 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.403643+0000", 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.063526735000000001 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.403751+0000", 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010773299999999999 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:54:45.914 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.403907+0000", 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00015620900000000001 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.410759+0000", 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0068518720000000002 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.915 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.415179+0000", 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0044194259999999997 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.416877+0000", 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001698055 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.916 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.416942+0000", 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5432000000000006e-05 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.418531+0000", 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00158873 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.917 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:129753 10.7 10:ec379fbb:::benchmark_data_smithi019_617837_object36353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:06.485716+0000", 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 159.28320602100001, 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.96383679099999997, 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.918 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 129753 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485716+0000", 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.919 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485716+0000", 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485718+0000", 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0779999999999998e-06 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.920 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485728+0000", 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.6169999999999996e-06 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485749+0000", 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1039999999999998e-05 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.921 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485761+0000", 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2330999999999999e-05 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485849+0000", 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7869999999999997e-05 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:54:45.922 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:06.485862+0000", 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3019e-05 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.406891+0000", 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.92102907700000003 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for degraded object", 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.409963+0000", 2023-09-30T09:54:45.923 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0030714290000000001 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.421527+0000", 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011564295 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.924 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.421649+0000", 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000122194 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 5,6", 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.421828+0000", 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000178782 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.425475+0000", 2023-09-30T09:54:45.925 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0036475040000000002 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.425520+0000", 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4860000000000001e-05 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.446876+0000", 2023-09-30T09:54:45.926 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.021356152 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.446935+0000", 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8597000000000002e-05 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:07.449553+0000", 2023-09-30T09:54:45.927 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026179459999999999 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:186737 10.f 10:fd519d04:::benchmark_data_smithi019_617837_object93337:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.526064+0000", 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 68.242858409999997, 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31276836299999999, 2023-09-30T09:54:45.928 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 186737 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.929 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526064+0000", 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526064+0000", 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.930 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526065+0000", 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2890000000000001e-06 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526070+0000", 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0100000000000003e-06 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526081+0000", 2023-09-30T09:54:45.931 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0553e-05 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526087+0000", 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1480000000000003e-06 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526145+0000", 2023-09-30T09:54:45.932 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8260999999999997e-05 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526273+0000", 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012802199999999999 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.526391+0000", 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011767700000000001 2023-09-30T09:54:45.933 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.532522+0000", 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061307879999999999 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.533734+0000", 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001212776 2023-09-30T09:54:45.934 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838726+0000", 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30499163099999999 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838797+0000", 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0970999999999995e-05 2023-09-30T09:54:45.935 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838832+0000", 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5237000000000002e-05 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.936 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:186738 10.f 10:fdcc3767:::benchmark_data_smithi019_617837_object93338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.528904+0000", 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 68.240018742000004, 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31335640300000001, 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 186738 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.937 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.528904+0000", 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.528904+0000", 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.938 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.528905+0000", 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2359999999999999e-06 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.528910+0000", 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7039999999999999e-06 2023-09-30T09:54:45.939 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.528924+0000", 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3504e-05 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.528944+0000", 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0106e-05 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.940 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.529420+0000", 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000476043 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.529514+0000", 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3505999999999995e-05 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.941 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.529637+0000", 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012383000000000001 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.532616+0000", 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029780649999999998 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.942 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.533918+0000", 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001302295 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.842144+0000", 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30822658200000003 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.943 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.842216+0000", 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1676999999999998e-05 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.842260+0000", 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3854999999999999e-05 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.944 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:186729 10.3 e621/618 10:c070af46:::benchmark_data_smithi019_617837_object93329:head v 621'11788, mlcod=0'0)", 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.535546+0000", 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 68.233375953000007, 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.303073019, 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.945 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.535546+0000", 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.535546+0000", 2023-09-30T09:54:45.946 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.535547+0000", 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3e-07 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.535553+0000", 2023-09-30T09:54:45.947 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1029999999999998e-06 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.535563+0000", 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0361e-05 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.535570+0000", 2023-09-30T09:54:45.948 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8410000000000002e-06 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.535614+0000", 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.3343000000000002e-05 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.535630+0000", 2023-09-30T09:54:45.949 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6500000000000001e-05 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838545+0000", 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30291463200000002 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838619+0000", 2023-09-30T09:54:45.950 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4509000000000001e-05 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:186740 10.d e621/596 10:bd33435d:::benchmark_data_smithi019_617837_object93340:head v 621'11622, mlcod=562'5915)", 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.536698+0000", 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 68.232224736000006, 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32179569899999999, 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.951 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.536698+0000", 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.536698+0000", 2023-09-30T09:54:45.952 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.536698+0000", 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0200000000000001e-07 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.536704+0000", 2023-09-30T09:54:45.953 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7289999999999997e-06 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.536715+0000", 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.082e-05 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.536722+0000", 2023-09-30T09:54:45.954 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6490000000000002e-06 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838607+0000", 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.30188595600000001 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838640+0000", 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2251e-05 2023-09-30T09:54:45.955 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.858448+0000", 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.019808101000000002 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.858493+0000", 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5391e-05 2023-09-30T09:54:45.956 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:186751 10.5 e621/618 10:a524a895:::benchmark_data_smithi019_617837_object93351:head v 621'11632, mlcod=621'11630)", 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:37.540125+0000", 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 68.228796877999997, 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.31488255100000001, 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.957 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.540125+0000", 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.540125+0000", 2023-09-30T09:54:45.958 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.540126+0000", 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0999999999999998e-07 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.540131+0000", 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.7090000000000001e-06 2023-09-30T09:54:45.959 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838500+0000", 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.298369247 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838530+0000", 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.9586000000000001e-05 2023-09-30T09:54:45.960 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838621+0000", 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1407000000000001e-05 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.838647+0000", 2023-09-30T09:54:45.961 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5724999999999998e-05 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.854954+0000", 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.016307642000000001 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:37.855008+0000", 2023-09-30T09:54:45.962 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3625000000000002e-05 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.47267.0:49 2.1 2:8c45e075:::rbd_header.b891b7e7ff6f:head [call lock.unlock in=10b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e636)", 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:54:32.465028+0000", 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 13.303894544, 2023-09-30T09:54:45.963 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.011420115, 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.47267", 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/103261076", 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 49 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:54:45.964 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465028+0000", 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465028+0000", 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:54:45.965 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465028+0000", 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.99e-07 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465033+0000", 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4150000000000001e-06 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:54:45.966 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465046+0000", 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3045e-05 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465054+0000", 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2060000000000007e-06 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:54:45.967 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465112+0000", 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7531000000000002e-05 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465179+0000", 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7817000000000005e-05 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:54:45.968 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.465329+0000", 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014951799999999999 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.470651+0000", 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0053224240000000001 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.470690+0000", 2023-09-30T09:54:45.969 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.8541000000000003e-05 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.476286+0000", 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.005595633 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.476338+0000", 2023-09-30T09:54:45.970 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2077000000000003e-05 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:32.476448+0000", 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000110209 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:54:45.971 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:54:45.972 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:54:45.972 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:54:45.972 INFO:teuthology.orchestra.run.smithi167.stdout: "success": "" 2023-09-30T09:54:45.972 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:54:47.007 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-30T09:54:47.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:47 smithi019 ceph-mon[216815]: pgmap v1718: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 384 op/s; 6/93 objects degraded (6.452%); 1/93 objects misplaced (1.075%) 2023-09-30T09:54:47.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:47 smithi019 ceph-mon[224043]: pgmap v1718: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 384 op/s; 6/93 objects degraded (6.452%); 1/93 objects misplaced (1.075%) 2023-09-30T09:54:47.357 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:54:47.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:47 smithi167 ceph-mon[192880]: pgmap v1718: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 384 op/s; 6/93 objects degraded (6.452%); 1/93 objects misplaced (1.075%) 2023-09-30T09:54:48.249 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:54:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:48.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1356854891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:48.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1356854891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:54:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1356854891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:49.002 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:54:49.002 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:54:49.002 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:54:49.003 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:54:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:49 smithi167 ceph-mon[192880]: pgmap v1719: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 938 KiB/s rd, 1.3 MiB/s wr, 328 op/s; 6/90 objects degraded (6.667%) 2023-09-30T09:54:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:49 smithi167 ceph-mon[192880]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:49 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1356854891' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:49 smithi167 ceph-mon[192880]: failed to encode map e645 with expected crc 2023-09-30T09:54:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:49 smithi167 ceph-mon[192880]: failed to encode map e645 with expected crc 2023-09-30T09:54:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:49 smithi167 ceph-mon[192880]: osdmap e645: 8 total, 7 up, 8 in 2023-09-30T09:54:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:49 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 6/90 objects degraded (6.667%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:49.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[216815]: pgmap v1719: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 938 KiB/s rd, 1.3 MiB/s wr, 328 op/s; 6/90 objects degraded (6.667%) 2023-09-30T09:54:49.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[216815]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:49.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1356854891' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:49.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[216815]: failed to encode map e645 with expected crc 2023-09-30T09:54:49.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[216815]: failed to encode map e645 with expected crc 2023-09-30T09:54:49.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[216815]: osdmap e645: 8 total, 7 up, 8 in 2023-09-30T09:54:49.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 6/90 objects degraded (6.667%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:49.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[224043]: pgmap v1719: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 938 KiB/s rd, 1.3 MiB/s wr, 328 op/s; 6/90 objects degraded (6.667%) 2023-09-30T09:54:49.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[224043]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-30T09:54:49.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1356854891' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:49.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[224043]: failed to encode map e645 with expected crc 2023-09-30T09:54:49.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[224043]: failed to encode map e645 with expected crc 2023-09-30T09:54:49.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[224043]: osdmap e645: 8 total, 7 up, 8 in 2023-09-30T09:54:49.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:49 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 6/90 objects degraded (6.667%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:49.719 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-30T09:54:50.066 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:54:50.958 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-30T09:54:51.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:51 smithi019 ceph-mon[224043]: pgmap v1721: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 823 KiB/s rd, 1.1 MiB/s wr, 267 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:51.339 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:51.389 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:51 smithi167 ceph-mon[192880]: pgmap v1721: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 823 KiB/s rd, 1.1 MiB/s wr, 267 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:51.554 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:51 smithi019 ceph-mon[216815]: pgmap v1721: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 823 KiB/s rd, 1.1 MiB/s wr, 267 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:51.587 INFO:tasks.thrashosds.thrasher:reviving osd 2023-09-30T09:54:51.587 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-09-30T09:54:51.588 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-09-30T09:54:51.588 DEBUG:teuthology.orchestra.run.smithi019:osd.0> sudo journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-30T09:54:51.590 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl start ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.0 2023-09-30T09:54:51.596 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:54:51.597 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:54:51.597 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:54:51.597 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:54:51.597 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:54:51.627 INFO:journalctl@ceph.osd.0.smithi019.stdout:-- Logs begin at Sat 2023-09-30 08:48:54 UTC. -- 2023-09-30T09:54:51.834 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:51 smithi019 systemd[1]: Starting Ceph osd.0 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:54:52.262 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-30T09:54:52.610 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:54:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:53 smithi167 ceph-mon[192880]: pgmap v1722: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:53.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:53 smithi019 ceph-mon[216815]: pgmap v1722: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:53.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:53 smithi019 ceph-mon[224043]: pgmap v1722: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:54.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:54:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:54:53] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2023-09-30T09:54:54.086 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:53 smithi019 bash[698990]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-30T09:54:54.086 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:53 smithi019 bash[698990]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-30T09:54:54.086 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:53 smithi019 bash[698990]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-30T09:54:54.086 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:53 smithi019 bash[698990]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-30T09:54:54.086 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:53 smithi019 bash[698990]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-30T09:54:54.086 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:53 smithi019 bash[698990]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-30T09:54:54.086 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:53 smithi019 bash[698990]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-30T09:54:54.087 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:53 smithi019 bash[698990]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-30T09:54:54.268 INFO:teuthology.orchestra.run.smithi019.stderr:noscrub is unset 2023-09-30T09:54:54.280 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:54:54.280 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:54:54.280 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:54:54.280 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:54:54.280 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.280 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101654 10.e 10:7e27f7a0:::benchmark_data_smithi019_617837_object8254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:54:54.281 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.458430+0000", 2023-09-30T09:54:54.281 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 253.81857475699999, 2023-09-30T09:54:54.281 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0722781880000001, 2023-09-30T09:54:54.281 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.281 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.281 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.281 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.281 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101654 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458430+0000", 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458430+0000", 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.282 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458429+0000", 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458434+0000", 2023-09-30T09:54:54.283 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6629999999999998e-06 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458451+0000", 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.643e-05 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458456+0000", 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2240000000000003e-06 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.284 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458512+0000", 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5869999999999999e-05 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.458523+0000", 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0805000000000001e-05 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.285 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.457801+0000", 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99927842300000003 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.457958+0000", 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015693099999999999 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.286 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.458122+0000", 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016425700000000001 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.472646+0000", 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01452355 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.287 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.473651+0000", 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001005288 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.529695+0000", 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.056044163000000001 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.288 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.529758+0000", 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3034000000000007e-05 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.530708+0000", 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00094970799999999995 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.289 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101664 10.e 10:7d1aed30:::benchmark_data_smithi019_617837_object8264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.484162+0000", 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 253.79284199700001, 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.057487034, 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.290 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101664 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484162+0000", 2023-09-30T09:54:54.291 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484162+0000", 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484161+0000", 2023-09-30T09:54:54.292 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484167+0000", 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7599999999999999e-06 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484180+0000", 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2488e-05 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.293 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484185+0000", 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1989999999999997e-06 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484244+0000", 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8811000000000003e-05 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.294 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.295 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.295 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.484254+0000", 2023-09-30T09:54:54.295 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9310000000000003e-06 2023-09-30T09:54:54.295 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.295 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.295 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.295 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462696+0000", 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97844258799999995 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462826+0000", 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000130183 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462966+0000", 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013949200000000001 2023-09-30T09:54:54.296 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.472671+0000", 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097053090000000005 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.473685+0000", 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0010137290000000001 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.297 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.540844+0000", 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.067158849000000007 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.540902+0000", 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8048999999999999e-05 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.298 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.541649+0000", 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00074773600000000004 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127781 10.4 10:25ddb007:::benchmark_data_smithi019_617837_object34381:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:57.175490+0000", 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 177.10151469100001, 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1234206959999999, 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.299 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127781 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175490+0000", 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.300 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175490+0000", 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175488+0000", 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.301 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175495+0000", 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8619999999999997e-06 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175508+0000", 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2673e-05 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175515+0000", 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2049999999999996e-06 2023-09-30T09:54:54.302 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175913+0000", 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00039778900000000001 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.175926+0000", 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3489000000000001e-05 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.303 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.279219+0000", 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1032926949999999 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.281411+0000", 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021921359999999999 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,3", 2023-09-30T09:54:54.304 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.281685+0000", 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027379999999999999 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.292203+0000", 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010517581999999999 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.305 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.293803+0000", 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016001679999999999 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298752+0000", 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049493999999999996 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298803+0000", 2023-09-30T09:54:54.306 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.083e-05 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298843+0000", 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0145000000000003e-05 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298910+0000", 2023-09-30T09:54:54.307 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7354000000000006e-05 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127783 10.e 10:75bd171f:::benchmark_data_smithi019_617837_object34383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:57.262645+0000", 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 177.01435929799999, 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92964328500000004, 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.308 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127783 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262645+0000", 2023-09-30T09:54:54.309 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262645+0000", 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262644+0000", 2023-09-30T09:54:54.310 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262651+0000", 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5349999999999999e-06 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.311 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262667+0000", 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5947000000000002e-05 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262681+0000", 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3978999999999999e-05 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262734+0000", 2023-09-30T09:54:54.312 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2907e-05 2023-09-30T09:54:54.313 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.313 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.313 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.313 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.262741+0000", 2023-09-30T09:54:54.313 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7719999999999997e-06 2023-09-30T09:54:54.313 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.313 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.313 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.186184+0000", 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92344351300000005 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.186337+0000", 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015267500000000001 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.314 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.186484+0000", 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014657599999999999 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.189850+0000", 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0033660320000000001 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.315 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.191036+0000", 2023-09-30T09:54:54.316 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011866800000000001 2023-09-30T09:54:54.316 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.316 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.316 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.316 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.192173+0000", 2023-09-30T09:54:54.316 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011366900000000001 2023-09-30T09:54:54.316 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.316 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.192234+0000", 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1278999999999994e-05 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.192288+0000", 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4039999999999998e-05 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.317 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127784 10.4 10:218c61d4:::benchmark_data_smithi019_617837_object34384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:57.336285+0000", 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 176.94071944000001, 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.962762848, 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.318 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127784 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336285+0000", 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.319 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336285+0000", 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336284+0000", 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336291+0000", 2023-09-30T09:54:54.320 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9399999999999996e-06 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336303+0000", 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2449999999999999e-05 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336308+0000", 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0900000000000004e-06 2023-09-30T09:54:54.321 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336368+0000", 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9815000000000002e-05 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.336383+0000", 2023-09-30T09:54:54.322 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5102e-05 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.289022+0000", 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95263867800000002 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.289175+0000", 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015364900000000001 2023-09-30T09:54:54.323 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,3", 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.289322+0000", 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014690699999999999 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.292213+0000", 2023-09-30T09:54:54.324 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002890882 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.296819+0000", 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004605441 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298780+0000", 2023-09-30T09:54:54.325 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019614049999999998 2023-09-30T09:54:54.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.326 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.326 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.326 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298940+0000", 2023-09-30T09:54:54.326 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016015800000000001 2023-09-30T09:54:54.326 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.298971+0000", 2023-09-30T09:54:54.327 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0460999999999999e-05 2023-09-30T09:54:54.327 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.327 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.327 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.327 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.299048+0000", 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7195999999999997e-05 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129750 10.d 10:b52c4f95:::benchmark_data_smithi019_617837_object36350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e595)", 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:05.215988+0000", 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 169.06101599600001, 2023-09-30T09:54:54.328 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3956790290000001, 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129750 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.215988+0000", 2023-09-30T09:54:54.329 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.215988+0000", 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.215987+0000", 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.330 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.215995+0000", 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2190000000000004e-06 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.216012+0000", 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7005999999999999e-05 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.331 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.216019+0000", 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6830000000000002e-06 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.216073+0000", 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3575999999999997e-05 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:05.216080+0000", 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4329999999999998e-06 2023-09-30T09:54:54.332 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.213695+0000", 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99761513400000001 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.213707+0000", 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1734e-05 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.333 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.510717+0000", 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.297010123 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.510818+0000", 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100617 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.510914+0000", 2023-09-30T09:54:54.334 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6471999999999994e-05 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.519218+0000", 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0083036840000000004 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532043+0000", 2023-09-30T09:54:54.335 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012824797000000001 2023-09-30T09:54:54.336 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.336 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.609411+0000", 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.07736809 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.609473+0000", 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2677999999999999e-05 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.337 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.611667+0000", 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021942200000000002 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129758 10.e 10:7ba26817:::benchmark_data_smithi019_617837_object36358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.493951+0000", 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 167.78305296799999, 2023-09-30T09:54:54.338 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.85264479900000001, 2023-09-30T09:54:54.339 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.339 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.339 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.339 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.339 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.339 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129758 2023-09-30T09:54:54.339 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.339 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493951+0000", 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493951+0000", 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.340 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493950+0000", 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493956+0000", 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4110000000000004e-06 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.341 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493969+0000", 2023-09-30T09:54:54.342 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3861e-05 2023-09-30T09:54:54.342 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.342 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.342 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.493975+0000", 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7579999999999998e-06 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.494043+0000", 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7724000000000004e-05 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.343 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.494051+0000", 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2160000000000006e-06 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.336428+0000", 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.84237695599999995 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.344 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.336555+0000", 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012707500000000001 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5,6", 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.336652+0000", 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6813999999999998e-05 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339699+0000", 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030473219999999999 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.345 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339768+0000", 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8276999999999997e-05 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339779+0000", 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0873000000000001e-05 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.346 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346487+0000", 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0067086469999999999 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346522+0000", 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5327000000000002e-05 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346596+0000", 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3813999999999995e-05 2023-09-30T09:54:54.347 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129760 10.d 10:b92f266b:::benchmark_data_smithi019_617837_object36360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.503079+0000", 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 167.77392512700001, 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1105047159999999, 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.348 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129760 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503079+0000", 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503079+0000", 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503078+0000", 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.350 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503083+0000", 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1549999999999998e-06 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503096+0000", 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3030000000000001e-05 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.352 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503102+0000", 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1839999999999998e-06 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503147+0000", 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.549e-05 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.503156+0000", 2023-09-30T09:54:54.353 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5199999999999997e-06 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.513645+0000", 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.010489609 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.513875+0000", 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00022947700000000001 2023-09-30T09:54:54.354 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.514017+0000", 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014177600000000001 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.526800+0000", 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012783216 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.355 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532094+0000", 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005293865 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.611739+0000", 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.079645117000000001 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.611805+0000", 2023-09-30T09:54:54.356 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6489999999999995e-05 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.613584+0000", 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017788280000000001 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129774 10.d 10:b44bed1b:::benchmark_data_smithi019_617837_object36374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:54:54.357 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.525695+0000", 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 167.751309773, 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.089412802, 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129774 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.358 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525695+0000", 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525695+0000", 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525693+0000", 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.359 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525713+0000", 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9320000000000001e-05 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525726+0000", 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3645000000000001e-05 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.360 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525733+0000", 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8419999999999999e-06 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525799+0000", 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6038999999999996e-05 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.525810+0000", 2023-09-30T09:54:54.361 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1438e-05 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.516754+0000", 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99094344400000001 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.516870+0000", 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0001158 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.362 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.516996+0000", 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012621099999999999 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.529302+0000", 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012305732999999999 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532146+0000", 2023-09-30T09:54:54.363 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028442839999999999 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.613698+0000", 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.081551636999999996 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.613755+0000", 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7737e-05 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.364 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.615107+0000", 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00135217 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129775 10.e 10:77c928e6:::benchmark_data_smithi019_617837_object36375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.528831+0000", 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 167.74817383199999, 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82096436800000006, 2023-09-30T09:54:54.365 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129775 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528831+0000", 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.366 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528831+0000", 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528829+0000", 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.367 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528835+0000", 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9660000000000001e-06 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528853+0000", 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7550999999999998e-05 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528860+0000", 2023-09-30T09:54:54.368 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6669999999999999e-06 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528925+0000", 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5370999999999995e-05 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.528937+0000", 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2218000000000001e-05 2023-09-30T09:54:54.369 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339805+0000", 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81086787900000001 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.339934+0000", 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012879300000000001 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.370 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5,6", 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.340067+0000", 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013331899999999999 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.342961+0000", 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028942180000000001 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.345879+0000", 2023-09-30T09:54:54.371 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029180859999999999 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.345952+0000", 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2434000000000005e-05 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349615+0000", 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.003662733 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.372 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349674+0000", 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9312999999999999e-05 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349795+0000", 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012102700000000001 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.373 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129782 10.e 10:7bfbc37c:::benchmark_data_smithi019_617837_object36382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.537734+0000", 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 167.73927083500001, 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81226853300000001, 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.374 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129782 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537734+0000", 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537734+0000", 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.375 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537732+0000", 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537738+0000", 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1369999999999999e-06 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.376 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537752+0000", 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4039000000000001e-05 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537759+0000", 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2829999999999998e-06 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537814+0000", 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4897e-05 2023-09-30T09:54:54.377 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.537822+0000", 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2439999999999993e-06 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.342981+0000", 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.80515920500000004 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.378 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.343129+0000", 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000147885 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,5,6", 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.343267+0000", 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000137775 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.345989+0000", 2023-09-30T09:54:54.379 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027219599999999998 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346026+0000", 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7230000000000001e-05 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.346034+0000", 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9869999999999999e-06 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.380 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349847+0000", 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0038129589999999999 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349899+0000", 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1651999999999999e-05 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.381 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.350002+0000", 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000103626 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129787 10.d 10:b2aeaf7e:::benchmark_data_smithi019_617837_object36387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.544982+0000", 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 167.73202210900001, 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0717242520000001, 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.382 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129787 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.544982+0000", 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.383 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.544982+0000", 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.544981+0000", 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.384 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.544987+0000", 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5820000000000001e-06 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.545000+0000", 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2938999999999999e-05 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.385 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.545005+0000", 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0379999999999999e-06 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.545065+0000", 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9932999999999997e-05 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.545076+0000", 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1488000000000001e-05 2023-09-30T09:54:54.386 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.519270+0000", 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97419379699999997 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.519425+0000", 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015525099999999999 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.387 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.519549+0000", 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012434600000000001 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.529308+0000", 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097589800000000004 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.388 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532180+0000", 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028711100000000001 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.615153+0000", 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.082973619999999998 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.389 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.615192+0000", 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9079000000000003e-05 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.616707+0000", 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015143489999999999 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.390 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129794 10.d 10:b60cc818:::benchmark_data_smithi019_617837_object36394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.566027+0000", 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 167.71097717699999, 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0527564389999999, 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.391 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129794 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566027+0000", 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566027+0000", 2023-09-30T09:54:54.392 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566026+0000", 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566032+0000", 2023-09-30T09:54:54.393 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0100000000000001e-06 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566045+0000", 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2456999999999999e-05 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566050+0000", 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9529999999999998e-06 2023-09-30T09:54:54.394 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566098+0000", 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8383000000000001e-05 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.566105+0000", 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5290000000000003e-06 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.395 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.521542+0000", 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95543642299999998 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.521665+0000", 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012298800000000001 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.521787+0000", 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000121971 2023-09-30T09:54:54.396 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.529314+0000", 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0075275400000000001 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.532211+0000", 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028969 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.397 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.616756+0000", 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.084544362999999997 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.616805+0000", 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8987999999999999e-05 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.398 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.618784+0000", 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019790889999999998 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:174213 10.4 10:20d5688b:::benchmark_data_smithi019_617837_object80813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:18.692268+0000", 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 95.584736038000003, 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28092419600000001, 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.399 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 174213 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.692268+0000", 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.400 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.692268+0000", 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.692268+0000", 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.692270+0000", 2023-09-30T09:54:54.401 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0059999999999999e-06 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.692275+0000", 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1599999999999997e-06 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.692279+0000", 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2619999999999999e-06 2023-09-30T09:54:54.402 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.695167+0000", 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002888837 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.695319+0000", 2023-09-30T09:54:54.403 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015136899999999999 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,3", 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.695488+0000", 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000169338 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.702553+0000", 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0070651129999999996 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.404 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.705665+0000", 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0031119759999999998 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.706075+0000", 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000409539 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.405 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.973004+0000", 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.26692954000000002 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.973066+0000", 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1334999999999995e-05 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:18.973193+0000", 2023-09-30T09:54:54.406 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012697899999999999 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186740 10.d 10:bd33435d:::benchmark_data_smithi019_617837_object93340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.528978+0000", 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 76.748026792000005, 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33083675899999998, 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.407 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186740 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.528978+0000", 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.408 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.528978+0000", 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.528977+0000", 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.409 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.528982+0000", 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7729999999999996e-06 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.529004+0000", 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.1795000000000001e-05 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.529010+0000", 2023-09-30T09:54:54.410 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8220000000000003e-06 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.531118+0000", 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021077489999999999 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.531252+0000", 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013427799999999999 2023-09-30T09:54:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.531402+0000", 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015030100000000001 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.547621+0000", 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.016218825999999999 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.412 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.547673+0000", 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.198e-05 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.858488+0000", 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.31081487899999999 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.413 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.858544+0000", 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6119999999999998e-05 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.859814+0000", 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001270382 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186744 10.d 10:b5a21e09:::benchmark_data_smithi019_617837_object93344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.536416+0000", 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 76.740588836000001, 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34087392700000002, 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186744 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.415 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.536416+0000", 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.536416+0000", 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.416 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.536414+0000", 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.536422+0000", 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5669999999999997e-06 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.536434+0000", 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2255999999999999e-05 2023-09-30T09:54:54.417 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.536441+0000", 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7000000000000002e-06 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.544174+0000", 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0077330630000000001 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.418 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.544304+0000", 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012967600000000001 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.544453+0000", 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014937899999999999 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.551655+0000", 2023-09-30T09:54:54.419 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0072021969999999996 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.553015+0000", 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001359498 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.876053+0000", 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.323038626 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.420 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.876090+0000", 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6378999999999998e-05 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.877290+0000", 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0011997850000000001 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.421 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186752 10.d 10:b2e5d8ae:::benchmark_data_smithi019_617837_object93352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.545696+0000", 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 76.731308665, 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33249263099999998, 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.422 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186752 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.545696+0000", 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.545696+0000", 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.423 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.545695+0000", 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.545701+0000", 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3779999999999998e-06 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.424 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.545716+0000", 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5038999999999999e-05 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.545722+0000", 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2609999999999999e-06 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.547701+0000", 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019784870000000001 2023-09-30T09:54:54.425 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.547835+0000", 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013393300000000001 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,6", 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.547970+0000", 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013537299999999999 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.426 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.551664+0000", 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036938829999999998 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.553230+0000", 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015657889999999999 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.877349+0000", 2023-09-30T09:54:54.427 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32411954599999998 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.877394+0000", 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4490999999999998e-05 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.878188+0000", 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000794668 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.428 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186763 10.4 10:24f1c4e3:::benchmark_data_smithi019_617837_object93363:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.560921+0000", 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 76.716083440999995, 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28734732400000002, 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.429 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186763 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.560921+0000", 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.560921+0000", 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.430 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.560920+0000", 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.560927+0000", 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.126e-06 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.431 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.560947+0000", 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.0479000000000002e-05 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.560955+0000", 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8080000000000005e-06 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.432 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.561017+0000", 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1903999999999995e-05 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.561155+0000", 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000138042 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.561273+0000", 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000118235 2023-09-30T09:54:54.433 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565007+0000", 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0037341259999999999 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565022+0000", 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4409e-05 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.434 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.848165+0000", 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28314325200000001 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.848203+0000", 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7731999999999997e-05 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.435 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.848268+0000", 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5607999999999999e-05 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186765 10.4 10:253401d3:::benchmark_data_smithi019_617837_object93365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.564984+0000", 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 76.712020937000005, 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28769687399999999, 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.436 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186765 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.564984+0000", 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.437 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.564984+0000", 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.564982+0000", 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.438 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.564988+0000", 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9880000000000001e-06 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565000+0000", 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1172000000000001e-05 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565006+0000", 2023-09-30T09:54:54.439 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3949999999999998e-06 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565040+0000", 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4434000000000003e-05 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565181+0000", 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000140433 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.440 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565329+0000", 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000148448 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.570395+0000", 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050657870000000004 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.441 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.570428+0000", 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2907000000000002e-05 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.852573+0000", 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28214497700000002 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.852608+0000", 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4955e-05 2023-09-30T09:54:54.442 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.852680+0000", 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2547999999999997e-05 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:186766 10.4 10:20682171:::benchmark_data_smithi019_617837_object93366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:53:37.565167+0000", 2023-09-30T09:54:54.443 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 76.711837355, 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28765144500000001, 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 186766 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565167+0000", 2023-09-30T09:54:54.444 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565167+0000", 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565167+0000", 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.445 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565171+0000", 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4649999999999996e-06 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565178+0000", 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0029999999999997e-06 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:54:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.565182+0000", 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1969999999999998e-06 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.568031+0000", 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028482529999999998 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.568162+0000", 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013166799999999999 2023-09-30T09:54:54.447 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,3", 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.568320+0000", 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000157823 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.570423+0000", 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002102853 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.570495+0000", 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2410000000000006e-05 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.852719+0000", 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28222326599999997 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.852748+0000", 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9694999999999999e-05 2023-09-30T09:54:54.449 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:53:37.852819+0000", 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0312e-05 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:54:54.450 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:54:54.451 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:54 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2051990856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:54.451 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:54 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 5/87 objects degraded (5.747%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:54.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:54 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2051990856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:54.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:54 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 5/87 objects degraded (5.747%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:54.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:54 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2051990856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-30T09:54:54.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:54 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 5/87 objects degraded (5.747%), 4 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-30T09:54:55.036 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-30T09:54:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:55 smithi167 ceph-mon[192880]: pgmap v1723: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 792 KiB/s rd, 1.1 MiB/s wr, 239 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2051990856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:55 smithi167 ceph-mon[192880]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:55 smithi167 ceph-mon[192880]: osdmap e646: 8 total, 7 up, 8 in 2023-09-30T09:54:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:55 smithi167 ceph-mon[192880]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:55 smithi167 ceph-mon[192880]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:55 smithi167 ceph-mon[192880]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.338 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-30T09:54:55.388 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:54:55.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[216815]: pgmap v1723: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 792 KiB/s rd, 1.1 MiB/s wr, 239 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2051990856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[216815]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[216815]: osdmap e646: 8 total, 7 up, 8 in 2023-09-30T09:54:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[216815]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[216815]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[216815]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[224043]: pgmap v1723: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 792 KiB/s rd, 1.1 MiB/s wr, 239 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:55.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2051990856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-30T09:54:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[224043]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[224043]: osdmap e646: 8 total, 7 up, 8 in 2023-09-30T09:54:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[224043]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[224043]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-mon[224043]: failed to encode map e646 with expected crc 2023-09-30T09:54:55.908 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:54:55.977 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:56.195 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:55 smithi019 ceph-osd[699657]: -- 172.21.15.19:0/2718485705 <== mon.0 v2:172.21.15.19:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b8324304e0 con 0x55b832402c00 2023-09-30T09:54:56.195 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:55 smithi019 bash[699513]: 8d4c99f376a7b9825399ee13a23699b9f59f15059280bc9472efcbb77a269c53 2023-09-30T09:54:56.195 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:55 smithi019 systemd[1]: Started Ceph osd.0 for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:54:56.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:56 smithi167 ceph-mon[192880]: failed to encode map e646 with expected crc 2023-09-30T09:54:56.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:56 smithi019 ceph-mon[216815]: failed to encode map e646 with expected crc 2023-09-30T09:54:56.584 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:56 smithi019 ceph-mon[224043]: failed to encode map e646 with expected crc 2023-09-30T09:54:56.585 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:54:56 smithi019 conmon[699606]: 2023-09-30T09:54:56.419+0000 7f20c7b78640 -1 Falling back to public interface 2023-09-30T09:54:57.011 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:54:57.011 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:54:57.011 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:54:57.012 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:54:57.341 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:54:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:57 smithi167 ceph-mon[192880]: pgmap v1725: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 895 B/s rd, 0 B/s wr, 1 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:57.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:57 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:54:57.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:57 smithi019 ceph-mon[224043]: pgmap v1725: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 895 B/s rd, 0 B/s wr, 1 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:57.579 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:57 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:54:57.580 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:57 smithi019 ceph-mon[216815]: pgmap v1725: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 895 B/s rd, 0 B/s wr, 1 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:57.580 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:57 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:54:57.704 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-30T09:54:58.275 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:54:59.287 INFO:teuthology.orchestra.run.smithi019.stderr:nodeep-scrub is unset 2023-09-30T09:54:59.516 INFO:teuthology.orchestra.run.smithi019.stderr:no valid command found; 10 closest matches: 2023-09-30T09:54:59.516 INFO:teuthology.orchestra.run.smithi019.stderr:0 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:1 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:2 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:abort 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:assert 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:bluefs debug_inject_read_zeros 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:bluefs files list 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:bluefs stats 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:bluestore bluefs device info [] 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:config diff 2023-09-30T09:54:59.517 INFO:teuthology.orchestra.run.smithi019.stderr:admin_socket: invalid command 2023-09-30T09:54:59.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:59 smithi167 ceph-mon[192880]: pgmap v1726: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 727 B/s rd, 0 B/s wr, 1 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:59.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:54:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2879370700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:54:59.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:59 smithi019 ceph-mon[216815]: pgmap v1726: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 727 B/s rd, 0 B/s wr, 1 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:59.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:54:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2879370700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:54:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:59 smithi019 ceph-mon[224043]: pgmap v1726: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 727 B/s rd, 0 B/s wr, 1 op/s; 5/87 objects degraded (5.747%) 2023-09-30T09:54:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:54:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2879370700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-30T09:55:00.436 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:55:00.437 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:55:00.437 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:55:00.437 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:55:00.437 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:55:00.456 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:00 smithi167 ceph-mon[192880]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:55:00.456 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:00 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2879370700' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:55:00.456 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:00 smithi167 ceph-mon[192880]: failed to encode map e647 with expected crc 2023-09-30T09:55:00.456 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:00 smithi167 ceph-mon[192880]: failed to encode map e647 with expected crc 2023-09-30T09:55:00.457 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:00 smithi167 ceph-mon[192880]: osdmap e647: 8 total, 7 up, 8 in 2023-09-30T09:55:00.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[224043]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:55:00.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2879370700' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:55:00.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[224043]: failed to encode map e647 with expected crc 2023-09-30T09:55:00.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[224043]: failed to encode map e647 with expected crc 2023-09-30T09:55:00.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[224043]: osdmap e647: 8 total, 7 up, 8 in 2023-09-30T09:55:00.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[216815]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-30T09:55:00.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2879370700' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-30T09:55:00.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[216815]: failed to encode map e647 with expected crc 2023-09-30T09:55:00.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[216815]: failed to encode map e647 with expected crc 2023-09-30T09:55:00.526 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:00 smithi019 ceph-mon[216815]: osdmap e647: 8 total, 7 up, 8 in 2023-09-30T09:55:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-30T09:55:00.780 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-0, ['dump_ops_in_flight'] 2023-09-30T09:55:01.494 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-30T09:55:01.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:01 smithi019 ceph-mon[216815]: pgmap v1728: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:01.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:01 smithi019 ceph-mon[224043]: pgmap v1728: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:01.772 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:01 smithi167 ceph-mon[192880]: pgmap v1728: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:02.016 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:55:03.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[216815]: pgmap v1729: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:03.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[224043]: pgmap v1729: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:03.453 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.453 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.453 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:03 smithi167 ceph-mon[192880]: pgmap v1729: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:03.453 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.454 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.454 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:03.454 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:03.678 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79004 9.0 9:0d8925f6:::benchmark_data_smithi019_560486_object79003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 573.140818008, 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.377661042, 2023-09-30T09:55:03.679 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79004 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.680 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533492+0000", 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533491+0000", 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533548+0000", 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7012999999999998e-05 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533561+0000", 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2352e-05 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533565+0000", 2023-09-30T09:55:03.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8060000000000002e-06 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533608+0000", 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2279e-05 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533742+0000", 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013385 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533885+0000", 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014328400000000001 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.538697+0000", 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004812584 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.540380+0000", 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001682944 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911056+0000", 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.37067566800000001 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911112+0000", 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6187999999999998e-05 2023-09-30T09:55:03.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911153+0000", 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0515000000000001e-05 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79005 9.0 9:01cc776d:::benchmark_data_smithi019_560486_object79004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:55:03.686 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 573.14073162900002, 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.377676597, 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79005 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.687 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533578+0000", 2023-09-30T09:55:03.688 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533627+0000", 2023-09-30T09:55:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9091000000000001e-05 2023-09-30T09:55:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533634+0000", 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.3649999999999998e-06 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.533638+0000", 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2409999999999997e-06 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536549+0000", 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0029111250000000001 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536664+0000", 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011423500000000001 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.536750+0000", 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5932000000000003e-05 2023-09-30T09:55:03.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.538813+0000", 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0020632480000000002 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.544406+0000", 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0055927779999999996 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911189+0000", 2023-09-30T09:55:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.366783097 2023-09-30T09:55:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911227+0000", 2023-09-30T09:55:03.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8250999999999997e-05 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.911255+0000", 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7742e-05 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.694 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:79017 9.0 9:0d388969:::benchmark_data_smithi019_560486_object79016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 573.125631086, 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.37037067400000001, 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79017 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.695 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548679+0000", 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.696 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548677+0000", 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548770+0000", 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.2744999999999994e-05 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548786+0000", 2023-09-30T09:55:03.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6557000000000001e-05 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548793+0000", 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4880000000000004e-06 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548838+0000", 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5550000000000003e-05 2023-09-30T09:55:03.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.548961+0000", 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012252199999999999 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.549114+0000", 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015329499999999999 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.554511+0000", 2023-09-30T09:55:03.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0053968749999999998 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.560437+0000", 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0059257829999999996 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.918969+0000", 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.35853247300000002 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.919009+0000", 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9582999999999997e-05 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:45:30.919049+0000", 2023-09-30T09:55:03.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0509999999999997e-05 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:172370 9.1 9:8087334a:::benchmark_data_smithi019_560486_object75625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 440.67519332099999, 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28865046500000002, 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.704 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 172370 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.705 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999116+0000", 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999119+0000", 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.807e-06 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999125+0000", 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.959e-06 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999129+0000", 2023-09-30T09:55:03.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4599999999999996e-06 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999170+0000", 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0299999999999997e-05 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999273+0000", 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010336999999999999 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999396+0000", 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012268500000000001 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.005183+0000", 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0057871479999999998 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.007181+0000", 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019979009999999998 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287675+0000", 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28049398800000003 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287725+0000", 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0720999999999998e-05 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.287767+0000", 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1523999999999997e-05 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.711 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:172371 9.1 9:8d82cc29:::benchmark_data_smithi019_560486_object75626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 440.67516759099999, 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.29074772500000001, 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 172371 2023-09-30T09:55:03.712 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999142+0000", 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999144+0000", 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.424e-06 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.714 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999150+0000", 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8680000000000004e-06 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:42.999154+0000", 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7950000000000001e-06 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003063+0000", 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0039091020000000002 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003142+0000", 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9460000000000002e-05 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.003210+0000", 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7366000000000005e-05 2023-09-30T09:55:03.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.005317+0000", 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0021074150000000001 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.007204+0000", 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018863300000000001 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289803+0000", 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.28259912500000001 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289853+0000", 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0512e-05 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:47:43.289890+0000", 2023-09-30T09:55:03.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6699999999999998e-05 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187021 9.1 9:8424d638:::benchmark_data_smithi019_560486_object90276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 419.50819767299998, 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34549460300000001, 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.719 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187021 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.720 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166112+0000", 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166111+0000", 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166115+0000", 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2400000000000001e-06 2023-09-30T09:55:03.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166124+0000", 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4719999999999995e-06 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166128+0000", 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3390000000000003e-06 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166156+0000", 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7353999999999999e-05 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166240+0000", 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4368000000000003e-05 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.166360+0000", 2023-09-30T09:55:03.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119819 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.174444+0000", 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0080842840000000006 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.174458+0000", 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.397e-05 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511510+0000", 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33705242400000002 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511561+0000", 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0382e-05 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511607+0000", 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5847000000000003e-05 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187029 9.1 9:8b8b6a3e:::benchmark_data_smithi019_560486_object90284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 419.50712270100001, 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34455740800000001, 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.726 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187029 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:55:03.727 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.728 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.728 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.728 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.728 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167187+0000", 2023-09-30T09:55:03.728 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.728 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.728 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.728 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167186+0000", 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167191+0000", 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2349999999999999e-06 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167203+0000", 2023-09-30T09:55:03.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1552999999999999e-05 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.167209+0000", 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0039999999999996e-06 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170626+0000", 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0034176219999999999 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170739+0000", 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000112713 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.170847+0000", 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000107892 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.175569+0000", 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0047219749999999998 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511470+0000", 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33590098299999999 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511656+0000", 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018579499999999999 2023-09-30T09:55:03.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511698+0000", 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2017000000000001e-05 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511744+0000", 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6542999999999998e-05 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.733 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187032 9.0 9:01af9635:::benchmark_data_smithi019_560486_object90287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 419.50181779100001, 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33105306600000001, 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.734 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187032 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172492+0000", 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172491+0000", 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172496+0000", 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4219999999999999e-06 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.736 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172506+0000", 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8460000000000003e-06 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172511+0000", 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6589999999999998e-06 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172557+0000", 2023-09-30T09:55:03.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6353999999999997e-05 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172666+0000", 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109045 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,2", 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.172769+0000", 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010234900000000001 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176428+0000", 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036596440000000001 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176537+0000", 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108589 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503453+0000", 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32691612199999998 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503501+0000", 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8403999999999999e-05 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503545+0000", 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3650999999999999e-05 2023-09-30T09:55:03.740 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187034 9.1 9:85c6d82b:::benchmark_data_smithi019_560486_object90289:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 419.49764265699997, 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34325798600000001, 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.741 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187034 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176667+0000", 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176666+0000", 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176672+0000", 2023-09-30T09:55:03.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.0789999999999997e-06 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176683+0000", 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1401e-05 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176689+0000", 2023-09-30T09:55:03.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8710000000000002e-06 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503342+0000", 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32665281600000001 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503509+0000", 2023-09-30T09:55:03.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016727300000000001 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503648+0000", 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013894800000000001 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511477+0000", 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0078290040000000005 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.511790+0000", 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031249499999999998 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519837+0000", 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.008047874 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.747 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519883+0000", 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5791999999999998e-05 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519925+0000", 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1763e-05 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.748 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46214.0:187035 9.1 9:8a5829f5:::benchmark_data_smithi019_560486_object90290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 419.49760724499998, 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.34335321800000002, 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46214", 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1907764730", 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 187035 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176702+0000", 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176705+0000", 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5059999999999998e-06 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176710+0000", 2023-09-30T09:55:03.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7910000000000001e-06 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.176714+0000", 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7469999999999999e-06 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507398+0000", 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.33068423200000002 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507518+0000", 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000119772 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,7", 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.507653+0000", 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000134466 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:03.753 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519800+0000", 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012147956999999999 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519964+0000", 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016390900000000001 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.519994+0000", 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9394999999999999e-05 2023-09-30T09:55:03.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.520026+0000", 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2098999999999999e-05 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.520056+0000", 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9793000000000001e-05 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.755 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop_reply(client.46214.0:187032 9.0 e526/483 ondisk, result = 0)", 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 419.496279018, 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32552107499999999, 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.756 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178031+0000", 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178030+0000", 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.178036+0000", 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.468e-06 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503359+0000", 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.32532316900000002 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503369+0000", 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0635999999999999e-05 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503435+0000", 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6155000000000006e-05 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503444+0000", 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5150000000000006e-06 2023-09-30T09:55:03.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:48:04.503552+0000", 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010775100000000001 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGPush(10.a 574/560 [PushOp(10:50caf5ec:::benchmark_data_smithi019_617837_object16072:head, version: 540'1030, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:50caf5ec:::benchmark_data_smithi019_617837_object16072:head@540'1030, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:55.580297+0000", 2023-09-30T09:55:03.760 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 248.094012897, 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.173555193, 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580297+0000", 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580297+0000", 2023-09-30T09:55:03.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580297+0000", 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967296 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580324+0000", 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7742999999999998e-05 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.763 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580331+0000", 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4250000000000003e-06 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580335+0000", 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5290000000000002e-06 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.582974+0000", 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026384960000000002 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.582981+0000", 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8190000000000003e-06 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.753852+0000", 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.170871355 2023-09-30T09:55:03.765 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGPush(10.b 574/560 [PushOp(10:d9182ae5:::benchmark_data_smithi019_617837_object16071:head, version: 540'1015, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:d9182ae5:::benchmark_data_smithi019_617837_object16071:head@540'1015, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:55.580343+0000", 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 248.09396685799999, 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.17126106999999999, 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580343+0000", 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580343+0000", 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580343+0000", 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967296 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580371+0000", 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8549999999999999e-05 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580382+0000", 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1122e-05 2023-09-30T09:55:03.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.580387+0000", 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6779999999999999e-06 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.582336+0000", 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019489679999999999 2023-09-30T09:55:03.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.582341+0000", 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8600000000000001e-06 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751604+0000", 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16926305 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.770 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "MOSDPGPush(10.f 574/570 [PushOp(10:f620aa59:::benchmark_data_smithi019_617837_object16955:head, version: 540'1067, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:f620aa59:::benchmark_data_smithi019_617837_object16955:head@540'1067, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:55.584876+0000", 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 248.089433748, 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16918993399999999, 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.771 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.584876+0000", 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.584876+0000", 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.584875+0000", 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.584927+0000", 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1319000000000001e-05 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751568+0000", 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.16664183299999999 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751608+0000", 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9156000000000003e-05 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751782+0000", 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000173981 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.751810+0000", 2023-09-30T09:55:03.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8518e-05 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:55.754066+0000", 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0022557950000000001 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:141187 10.f e607/583 10:f5fa8c0b:::benchmark_data_smithi019_617837_object47787:head v 607'8750, mlcod=607'8748)", 2023-09-30T09:55:03.775 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:28.809320+0000", 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 154.86498926499999, 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.15343026900000001, 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809320+0000", 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809320+0000", 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809319+0000", 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809329+0000", 2023-09-30T09:55:03.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4239999999999999e-06 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809343+0000", 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3906999999999999e-05 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809350+0000", 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9e-06 2023-09-30T09:55:03.778 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809432+0000", 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2191e-05 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.809451+0000", 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9449e-05 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.962711+0000", 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.15325945099999999 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.962751+0000", 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9928999999999997e-05 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.780 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:141190 10.7 e607/596 10:e09c7f56:::benchmark_data_smithi019_617837_object47790:head v 607'8911, mlcod=586'7848)", 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:28.813334+0000", 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 154.86097551099999, 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.15209569100000001, 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813334+0000", 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813334+0000", 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813333+0000", 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813342+0000", 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9979999999999992e-06 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813353+0000", 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1445e-05 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.813360+0000", 2023-09-30T09:55:03.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2650000000000002e-06 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.818744+0000", 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005384598 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.818767+0000", 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2548000000000002e-05 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.965390+0000", 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.14662345600000001 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.965430+0000", 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9539000000000002e-05 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.46583.0:159832 10.d e613/596 10:b2dfef1c:::benchmark_data_smithi019_617837_object66432:head v 613'9923, mlcod=548'3686)", 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:58.741044+0000", 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 124.933265889, 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.14674000600000001, 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.741044+0000", 2023-09-30T09:55:03.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.741044+0000", 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.741043+0000", 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.741055+0000", 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2097e-05 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.741072+0000", 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7268000000000001e-05 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.741080+0000", 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8399999999999995e-06 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.878310+0000", 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.13722984799999999 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.878349+0000", 2023-09-30T09:55:03.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8921999999999998e-05 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.887755+0000", 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0094063129999999995 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:58.887784+0000", 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8915999999999999e-05 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.47240.0:33 2.6 e635/559 2:63807385:::rbd_data.b888bffab72a.0000000000000019:head v 635'258, mlcod=635'256)", 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.180008+0000", 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 31.494301975999999, 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0025519789999999998, 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.791 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180008+0000", 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180008+0000", 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180007+0000", 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180032+0000", 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4984e-05 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180045+0000", 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.273e-05 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180057+0000", 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2588e-05 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180107+0000", 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0018999999999997e-05 2023-09-30T09:55:03.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180129+0000", 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2087e-05 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182518+0000", 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023890180000000001 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182560+0000", 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1301000000000003e-05 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.47240.0:34 2.2 e635/559 2:52c99878:::rbd_data.b888bffab72a.000000000000001a:head v 635'12, mlcod=635'10)", 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.180444+0000", 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 31.493865206999999, 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046336279999999999, 2023-09-30T09:55:03.796 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180444+0000", 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180444+0000", 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180444+0000", 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180452+0000", 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8709999999999997e-06 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180460+0000", 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.909e-06 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180464+0000", 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9389999999999999e-06 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180509+0000", 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5710000000000001e-05 2023-09-30T09:55:03.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180537+0000", 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7274999999999999e-05 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.185038+0000", 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0045012769999999997 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.185078+0000", 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0238000000000003e-05 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_repop(client.47240.0:35 2.6 e635/559 2:65d19774:::rbd_data.b888bffab72a.000000000000001b:head v 635'259, mlcod=635'256)", 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.182835+0000", 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 31.491474464, 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048096859999999996, 2023-09-30T09:55:03.801 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182835+0000", 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182835+0000", 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:03.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182835+0000", 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182853+0000", 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8037e-05 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182868+0000", 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.52e-05 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182874+0000", 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2809999999999997e-06 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182920+0000", 2023-09-30T09:55:03.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6186000000000001e-05 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182939+0000", 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8703999999999999e-05 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.187570+0000", 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0046311319999999996 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:03.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.187645+0000", 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4802000000000007e-05 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:03.806 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:55:04.068 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:55:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:55:03] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2023-09-30T09:55:04.069 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:55:03 smithi019 conmon[699606]: 2023-09-30T09:55:03.820+0000 7f20c7b78640 -1 osd.0 617 log_to_monitors true 2023-09-30T09:55:04.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:55:04.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:55:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:04.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:04 smithi167 ceph-mon[192880]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:55:04.334 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:55:04 smithi019 conmon[699606]: 2023-09-30T09:55:04.182+0000 7f20b6fe0700 -1 osd.0 617 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-30T09:55:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:55:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:55:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:04 smithi019 ceph-mon[224043]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:55:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:55:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:55:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:04 smithi019 ceph-mon[216815]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-30T09:55:04.383 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-30T09:55:04.736 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:55:05.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:05 smithi167 ceph-mon[192880]: pgmap v1730: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:05.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:05 smithi167 ceph-mon[192880]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:55:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:05 smithi167 ceph-mon[192880]: osdmap e648: 8 total, 7 up, 8 in 2023-09-30T09:55:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:05 smithi167 ceph-mon[192880]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:55:05.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:05 smithi167 ceph-mon[192880]: failed to encode map e648 with expected crc 2023-09-30T09:55:05.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[216815]: pgmap v1730: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[216815]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[216815]: osdmap e648: 8 total, 7 up, 8 in 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[216815]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[216815]: failed to encode map e648 with expected crc 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[224043]: pgmap v1730: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[224043]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[224043]: osdmap e648: 8 total, 7 up, 8 in 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[224043]: from='osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2023-09-30T09:55:05.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:05 smithi019 ceph-mon[224043]: failed to encode map e648 with expected crc 2023-09-30T09:55:05.781 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:55:06.157 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:06.290 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:55:06.291 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:55:06.291 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:55:06.291 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:55:06.313 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:55:06.314 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: failed to encode map e648 with expected crc 2023-09-30T09:55:06.314 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: failed to encode map e649 with expected crc 2023-09-30T09:55:06.314 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971] boot 2023-09-30T09:55:06.314 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: osdmap e649: 8 total, 8 up, 8 in 2023-09-30T09:55:06.314 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:55:06.314 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: failed to encode map e649 with expected crc 2023-09-30T09:55:06.314 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: failed to encode map e648 with expected crc 2023-09-30T09:55:06.314 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:06 smithi167 ceph-mon[192880]: osdmap e650: 8 total, 8 up, 8 in 2023-09-30T09:55:06.450 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:55:06.450 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: failed to encode map e648 with expected crc 2023-09-30T09:55:06.450 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: failed to encode map e649 with expected crc 2023-09-30T09:55:06.450 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971] boot 2023-09-30T09:55:06.450 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: osdmap e649: 8 total, 8 up, 8 in 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: failed to encode map e649 with expected crc 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: failed to encode map e648 with expected crc 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[216815]: osdmap e650: 8 total, 8 up, 8 in 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: failed to encode map e648 with expected crc 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: failed to encode map e649 with expected crc 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: osd.0 [v2:172.21.15.19:6802/2441438971,v1:172.21.15.19:6803/2441438971] boot 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: osdmap e649: 8 total, 8 up, 8 in 2023-09-30T09:55:06.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-30T09:55:06.452 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: failed to encode map e649 with expected crc 2023-09-30T09:55:06.452 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: failed to encode map e648 with expected crc 2023-09-30T09:55:06.452 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:06 smithi019 ceph-mon[224043]: osdmap e650: 8 total, 8 up, 8 in 2023-09-30T09:55:07.063 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-30T09:55:07.417 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:55:07.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:07 smithi167 ceph-mon[192880]: pgmap v1733: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:07 smithi167 ceph-mon[192880]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:07 smithi167 ceph-mon[192880]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:07 smithi167 ceph-mon[192880]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:07 smithi167 ceph-mon[192880]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:07 smithi167 ceph-mon[192880]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:07 smithi167 ceph-mon[192880]: osdmap e651: 8 total, 8 up, 8 in 2023-09-30T09:55:07.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[216815]: pgmap v1733: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[216815]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[216815]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[216815]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[216815]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[216815]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[216815]: osdmap e651: 8 total, 8 up, 8 in 2023-09-30T09:55:07.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[224043]: pgmap v1733: 103 pgs: 1 active+clean+remapped, 10 active+undersized, 4 active+undersized+degraded, 88 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5/87 objects degraded (5.747%) 2023-09-30T09:55:07.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[224043]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[224043]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[224043]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[224043]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[224043]: failed to encode map e650 with expected crc 2023-09-30T09:55:07.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:07 smithi019 ceph-mon[224043]: osdmap e651: 8 total, 8 up, 8 in 2023-09-30T09:55:08.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:08 smithi167 ceph-mon[192880]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:08 smithi167 ceph-mon[192880]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:08 smithi167 ceph-mon[192880]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:08 smithi167 ceph-mon[192880]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.302 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:55:08.302 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:55:08.302 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 0 2023-09-30T09:55:08.302 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:55:08.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:08 smithi019 ceph-mon[224043]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:08 smithi019 ceph-mon[224043]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:08 smithi019 ceph-mon[224043]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:08 smithi019 ceph-mon[224043]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:08 smithi019 ceph-mon[216815]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:08 smithi019 ceph-mon[216815]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:08 smithi019 ceph-mon[216815]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:08 smithi019 ceph-mon[216815]: failed to encode map e651 with expected crc 2023-09-30T09:55:08.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:55:08 smithi167 conmon[172457]: logger=cleanup t=2023-09-30T09:55:08.398748417Z level=info msg="Completed cleanup jobs" duration=91.692691ms 2023-09-30T09:55:09.007 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:55:09.007 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:55:09.007 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:55:09.007 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:55:09.007 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:55:09.228 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-30T09:55:09.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: failed to encode map e651 with expected crc 2023-09-30T09:55:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: pgmap v1736: 103 pgs: 1 activating+undersized+degraded+remapped, 9 active+undersized, 3 active+undersized+degraded, 90 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 15/87 objects degraded (17.241%) 2023-09-30T09:55:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: Health check update: Degraded data redundancy: 15/87 objects degraded (17.241%), 4 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:55:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: osdmap e652: 8 total, 8 up, 8 in 2023-09-30T09:55:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:09 smithi167 ceph-mon[192880]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: failed to encode map e651 with expected crc 2023-09-30T09:55:09.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: pgmap v1736: 103 pgs: 1 activating+undersized+degraded+remapped, 9 active+undersized, 3 active+undersized+degraded, 90 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 15/87 objects degraded (17.241%) 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: Health check update: Degraded data redundancy: 15/87 objects degraded (17.241%), 4 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: osdmap e652: 8 total, 8 up, 8 in 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[216815]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: failed to encode map e651 with expected crc 2023-09-30T09:55:09.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: pgmap v1736: 103 pgs: 1 activating+undersized+degraded+remapped, 9 active+undersized, 3 active+undersized+degraded, 90 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 15/87 objects degraded (17.241%) 2023-09-30T09:55:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: Health check update: Degraded data redundancy: 15/87 objects degraded (17.241%), 4 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-30T09:55:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: osdmap e652: 8 total, 8 up, 8 in 2023-09-30T09:55:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:09 smithi019 ceph-mon[224043]: failed to encode map e652 with expected crc 2023-09-30T09:55:09.594 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:09.815 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-30T09:55:10.167 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:55:10.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:10 smithi167 ceph-mon[192880]: failed to encode map e652 with expected crc 2023-09-30T09:55:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:10 smithi167 ceph-mon[192880]: failed to encode map e653 with expected crc 2023-09-30T09:55:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:10 smithi167 ceph-mon[192880]: osdmap e653: 8 total, 8 up, 8 in 2023-09-30T09:55:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:10 smithi167 ceph-mon[192880]: failed to encode map e653 with expected crc 2023-09-30T09:55:10.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:10 smithi167 ceph-mon[192880]: failed to encode map e653 with expected crc 2023-09-30T09:55:10.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[224043]: failed to encode map e652 with expected crc 2023-09-30T09:55:10.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[224043]: failed to encode map e653 with expected crc 2023-09-30T09:55:10.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[224043]: osdmap e653: 8 total, 8 up, 8 in 2023-09-30T09:55:10.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[224043]: failed to encode map e653 with expected crc 2023-09-30T09:55:10.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[224043]: failed to encode map e653 with expected crc 2023-09-30T09:55:10.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[216815]: failed to encode map e652 with expected crc 2023-09-30T09:55:10.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[216815]: failed to encode map e653 with expected crc 2023-09-30T09:55:10.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[216815]: osdmap e653: 8 total, 8 up, 8 in 2023-09-30T09:55:10.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[216815]: failed to encode map e653 with expected crc 2023-09-30T09:55:10.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:10 smithi019 ceph-mon[216815]: failed to encode map e653 with expected crc 2023-09-30T09:55:11.343 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a: {} 2023-09-30T09:55:11.352 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b: {} 2023-09-30T09:55:11.362 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c: {} 2023-09-30T09:55:11.363 INFO:teuthology.orchestra.run.smithi019.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-30T09:55:11.363 INFO:teuthology.orchestra.run.smithi019.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-30T09:55:11.363 INFO:teuthology.orchestra.run.smithi019.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-30T09:55:11.394 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:11 smithi019 ceph-mon[224043]: failed to encode map e653 with expected crc 2023-09-30T09:55:11.394 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:11 smithi019 ceph-mon[224043]: pgmap v1739: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1024 KiB/s, 2 objects/s recovering 2023-09-30T09:55:11.394 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:11 smithi019 ceph-mon[224043]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/87 objects degraded (17.241%), 4 pgs degraded, 12 pgs undersized) 2023-09-30T09:55:11.394 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:11 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:55:11.394 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:11 smithi019 ceph-mon[216815]: failed to encode map e653 with expected crc 2023-09-30T09:55:11.394 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:11 smithi019 ceph-mon[216815]: pgmap v1739: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1024 KiB/s, 2 objects/s recovering 2023-09-30T09:55:11.394 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:11 smithi019 ceph-mon[216815]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/87 objects degraded (17.241%), 4 pgs degraded, 12 pgs undersized) 2023-09-30T09:55:11.394 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:11 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:55:11.443 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:11 smithi167 ceph-mon[192880]: failed to encode map e653 with expected crc 2023-09-30T09:55:11.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:11 smithi167 ceph-mon[192880]: pgmap v1739: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1024 KiB/s, 2 objects/s recovering 2023-09-30T09:55:11.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:11 smithi167 ceph-mon[192880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/87 objects degraded (17.241%), 4 pgs degraded, 12 pgs undersized) 2023-09-30T09:55:11.444 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:11 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:55:11.654 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:55:11.654 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129752 10.1 10:8e848273:::benchmark_data_smithi019_617837_object36352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.454555+0000", 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 185.19680458299999, 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.90929194899999999, 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.655 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129752 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454555+0000", 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.656 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454555+0000", 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454555+0000", 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454559+0000", 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4240000000000003e-06 2023-09-30T09:55:11.657 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454572+0000", 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2964e-05 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.454582+0000", 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0463e-05 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.658 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.455021+0000", 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000438212 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.455034+0000", 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3044000000000001e-05 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.349334+0000", 2023-09-30T09:55:11.659 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.89429994800000001 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.351250+0000", 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001916474 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,4", 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.351397+0000", 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014671000000000001 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.660 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.358565+0000", 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0071678640000000004 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363661+0000", 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0050959519999999999 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363688+0000", 2023-09-30T09:55:11.661 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7625000000000001e-05 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363746+0000", 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7918999999999999e-05 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363781+0000", 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4502999999999998e-05 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.662 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363847+0000", 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6691000000000002e-05 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129776 10.1 10:81a28789:::benchmark_data_smithi019_617837_object36376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.530436+0000", 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 185.120923605, 2023-09-30T09:55:11.663 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83958421100000002, 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129776 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530436+0000", 2023-09-30T09:55:11.664 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530436+0000", 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530435+0000", 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.665 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530443+0000", 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0320000000000003e-06 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530462+0000", 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8967e-05 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.666 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530470+0000", 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.7770000000000002e-06 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530534+0000", 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4591000000000006e-05 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.530547+0000", 2023-09-30T09:55:11.667 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2830000000000001e-05 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.353679+0000", 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82313198899999995 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.353769+0000", 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0099999999999995e-05 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.668 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,4", 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.353857+0000", 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7779000000000001e-05 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.361026+0000", 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0071691879999999999 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363721+0000", 2023-09-30T09:55:11.669 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026950239999999999 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363881+0000", 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000160094 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.369883+0000", 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0060018609999999998 2023-09-30T09:55:11.670 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.369918+0000", 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4863999999999997e-05 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.370021+0000", 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000102556 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.671 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:129780 10.1 10:800ff011:::benchmark_data_smithi019_617837_object36380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:06.533905+0000", 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 185.11745464200001, 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83700672499999995, 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 129780 2023-09-30T09:55:11.672 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533905+0000", 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533905+0000", 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.673 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533904+0000", 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533916+0000", 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2052000000000001e-05 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.674 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533934+0000", 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7849000000000001e-05 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.533941+0000", 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0700000000000001e-06 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.534007+0000", 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5928999999999998e-05 2023-09-30T09:55:11.675 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:06.534018+0000", 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0937e-05 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.356020+0000", 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82200270900000005 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.676 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.356101+0000", 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0483999999999997e-05 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 0,1,4", 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.356191+0000", 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0212999999999998e-05 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363617+0000", 2023-09-30T09:55:11.677 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0074258939999999997 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.363911+0000", 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00029366400000000001 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.369585+0000", 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.005674241 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.678 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.370811+0000", 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001225627 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.370849+0000", 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8702999999999999e-05 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:07.370912+0000", 2023-09-30T09:55:11.679 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2793999999999995e-05 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140289 10.0 10:01361017:::benchmark_data_smithi019_617837_object46889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:26.932245+0000", 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.71911481800001, 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.181625616, 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.680 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140289 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932245+0000", 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.681 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932245+0000", 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932244+0000", 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932251+0000", 2023-09-30T09:55:11.682 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2239999999999998e-06 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932264+0000", 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2996999999999999e-05 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.932276+0000", 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1749999999999999e-05 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.683 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.934603+0000", 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0023272969999999999 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.934611+0000", 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2600000000000005e-06 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.073563+0000", 2023-09-30T09:55:11.684 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1389515109999999 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.073729+0000", 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016571399999999999 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.073879+0000", 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015019499999999999 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.685 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.082090+0000", 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0082116629999999993 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113709+0000", 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.031618269999999997 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.686 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113772+0000", 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3044000000000002e-05 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113829+0000", 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6829000000000001e-05 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113871+0000", 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.2313e-05 2023-09-30T09:55:11.687 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140309 10.0 10:0c5e3186:::benchmark_data_smithi019_617837_object46909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:26.964625+0000", 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.68673480499999, 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1494339469999999, 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.688 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140309 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964625+0000", 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.689 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964625+0000", 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964624+0000", 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964630+0000", 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3720000000000001e-06 2023-09-30T09:55:11.690 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964642+0000", 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1508999999999999e-05 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964648+0000", 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9290000000000003e-06 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.691 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964895+0000", 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00024736300000000001 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.964904+0000", 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7190000000000007e-06 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.076507+0000", 2023-09-30T09:55:11.692 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1116031259999999 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.076607+0000", 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9716999999999994e-05 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.076693+0000", 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.5878999999999995e-05 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.693 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.087652+0000", 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010959652 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113941+0000", 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.026288270999999998 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.113976+0000", 2023-09-30T09:55:11.694 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5824999999999997e-05 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114021+0000", 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4641999999999997e-05 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114059+0000", 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8155999999999999e-05 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.695 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140310 10.0 10:0b99e2fd:::benchmark_data_smithi019_617837_object46910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:26.970537+0000", 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.680822812, 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.143803983, 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.696 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140310 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970537+0000", 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970537+0000", 2023-09-30T09:55:11.697 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970536+0000", 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970543+0000", 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5050000000000001e-06 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.698 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970562+0000", 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8515000000000001e-05 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970569+0000", 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4039999999999998e-06 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970659+0000", 2023-09-30T09:55:11.699 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0309000000000006e-05 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.970667+0000", 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9610000000000005e-06 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.078368+0000", 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1077000889999999 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.700 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.078473+0000", 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105572 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.078544+0000", 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0735000000000006e-05 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.701 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.087661+0000", 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0091170669999999995 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114213+0000", 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.026552503000000002 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114250+0000", 2023-09-30T09:55:11.702 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6606999999999997e-05 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114299+0000", 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9483e-05 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114341+0000", 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1709000000000003e-05 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.703 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140330 10.0 10:03ecfbaf:::benchmark_data_smithi019_617837_object46930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:26.992195+0000", 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.659164756, 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.122355701, 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.704 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140330 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992195+0000", 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992195+0000", 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.705 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992194+0000", 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992201+0000", 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2529999999999998e-06 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.706 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992216+0000", 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5355000000000001e-05 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992223+0000", 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8759999999999999e-06 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992277+0000", 2023-09-30T09:55:11.707 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4326999999999997e-05 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:26.992286+0000", 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9099999999999994e-06 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.080080+0000", 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.087793515 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.708 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.080175+0000", 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4795000000000003e-05 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.080240+0000", 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5370999999999995e-05 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093529+0000", 2023-09-30T09:55:11.709 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013288659 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114402+0000", 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.020873769 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114459+0000", 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7014e-05 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.710 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114512+0000", 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2544999999999999e-05 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114551+0000", 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8980000000000003e-05 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.711 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140355 10.0 10:03e17385:::benchmark_data_smithi019_617837_object46955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.121630+0000", 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.529730468, 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.993256892, 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140355 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.712 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121630+0000", 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121630+0000", 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.713 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121629+0000", 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121632+0000", 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8480000000000001e-06 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121694+0000", 2023-09-30T09:55:11.714 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1736000000000007e-05 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121700+0000", 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3439999999999997e-06 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121748+0000", 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7710000000000002e-05 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.715 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.121755+0000", 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9670000000000004e-06 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.082150+0000", 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.960395675 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.716 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.082315+0000", 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016461500000000001 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.082488+0000", 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000172957 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093537+0000", 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011049392 2023-09-30T09:55:11.717 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114730+0000", 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.021192175000000001 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114794+0000", 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4325000000000005e-05 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.718 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114850+0000", 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6282000000000001e-05 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114886+0000", 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6229999999999997e-05 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.719 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140381 10.0 10:07b9ea47:::benchmark_data_smithi019_617837_object46981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.155244+0000", 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.49611644300001, 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.95981043799999999, 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140381 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155244+0000", 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155244+0000", 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.721 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155242+0000", 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155248+0000", 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2890000000000003e-06 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155263+0000", 2023-09-30T09:55:11.722 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4634000000000001e-05 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155270+0000", 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7700000000000004e-06 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155315+0000", 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.494e-05 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.723 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.155322+0000", 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1439999999999997e-06 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.084861+0000", 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.92953928600000002 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.724 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.085003+0000", 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014227 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.085141+0000", 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013766399999999999 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.099313+0000", 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014172298999999999 2023-09-30T09:55:11.725 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114944+0000", 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015630511 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.114978+0000", 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3821e-05 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.726 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115019+0000", 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1462999999999999e-05 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115054+0000", 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4782e-05 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.727 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140433 10.0 10:0f44e937:::benchmark_data_smithi019_617837_object47033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.277027+0000", 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.37433307699999, 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83815227199999998, 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140433 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.728 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277027+0000", 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277027+0000", 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277025+0000", 2023-09-30T09:55:11.729 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277033+0000", 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.678e-06 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277047+0000", 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3487e-05 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.730 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277053+0000", 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.8739999999999997e-06 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277098+0000", 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4750999999999999e-05 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.731 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.277108+0000", 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4970000000000007e-06 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.087718+0000", 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81061056200000003 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.087886+0000", 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016749500000000001 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.732 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.088030+0000", 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000144036 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.099319+0000", 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011288815000000001 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.733 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115090+0000", 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015771866999999998 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115117+0000", 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6359e-05 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.734 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115149+0000", 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2673999999999999e-05 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115179+0000", 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9756000000000001e-05 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.735 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140434 10.0 10:0622d357:::benchmark_data_smithi019_617837_object47034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.279733+0000", 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.37162682900001, 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.83556097699999998, 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140434 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.736 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279733+0000", 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279733+0000", 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.737 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279731+0000", 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279741+0000", 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0037999999999999e-05 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279766+0000", 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5111000000000001e-05 2023-09-30T09:55:11.738 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.279776+0000", 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.3179999999999996e-06 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.280109+0000", 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00033358599999999997 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.739 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.280119+0000", 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9350000000000006e-06 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.090718+0000", 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81059848499999998 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.090857+0000", 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000139507 2023-09-30T09:55:11.740 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.090983+0000", 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000125809 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.101562+0000", 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010579313999999999 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.741 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115208+0000", 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.01364548 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115234+0000", 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5853000000000001e-05 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115265+0000", 2023-09-30T09:55:11.742 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1569000000000002e-05 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115294+0000", 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8830999999999999e-05 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140442 10.0 10:0c0950c7:::benchmark_data_smithi019_617837_object47042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.743 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.294201+0000", 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.35715869200001, 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.82121004799999997, 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140442 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.744 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294201+0000", 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294201+0000", 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.745 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294200+0000", 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294209+0000", 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5149999999999995e-06 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.746 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294234+0000", 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4958999999999999e-05 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294242+0000", 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.028e-06 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294296+0000", 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3514999999999998e-05 2023-09-30T09:55:11.747 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.294305+0000", 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1300000000000007e-06 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093585+0000", 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.79928037600000001 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.748 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093731+0000", 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000145909 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.093883+0000", 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015131899999999999 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.749 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.101573+0000", 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0076901340000000004 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115322+0000", 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.013749014 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.750 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115347+0000", 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5230999999999999e-05 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115384+0000", 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6678999999999999e-05 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115411+0000", 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7801999999999999e-05 2023-09-30T09:55:11.751 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:140444 10.0 10:02b09ef4:::benchmark_data_smithi019_617837_object47044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:52:27.297861+0000", 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 164.353499129, 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.81766610399999995, 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.752 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 140444 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297861+0000", 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.753 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297861+0000", 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297860+0000", 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297865+0000", 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7089999999999999e-06 2023-09-30T09:55:11.754 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297878+0000", 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3018e-05 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297884+0000", 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2100000000000001e-06 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.755 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297941+0000", 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7741999999999997e-05 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:27.297948+0000", 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9149999999999999e-06 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.096567+0000", 2023-09-30T09:55:11.756 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.79861880299999999 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.096706+0000", 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013912899999999999 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.096839+0000", 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000133022 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.757 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.106542+0000", 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0097024299999999997 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115440+0000", 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088988499999999998 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.758 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115466+0000", 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5695999999999999e-05 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115498+0000", 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1962999999999999e-05 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:52:28.115527+0000", 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.8861999999999999e-05 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.759 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37074.0:22 2.7 2:fe326f4d:::rbd_header.90cc420e04da:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e632)", 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:28.158955+0000", 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 43.492404622999999, 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0084326760000000001, 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37074", 2023-09-30T09:55:11.760 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/4020943958", 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 22 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.158955+0000", 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.158955+0000", 2023-09-30T09:55:11.761 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.158953+0000", 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.158965+0000", 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1965e-05 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.762 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.158989+0000", 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3428e-05 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.159000+0000", 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1117e-05 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.159088+0000", 2023-09-30T09:55:11.763 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7731000000000004e-05 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.159098+0000", 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0112000000000001e-05 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.165137+0000", 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0060394159999999997 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.764 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.165216+0000", 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8529999999999995e-05 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.165467+0000", 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00025148299999999999 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.765 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.166009+0000", 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00054139200000000002 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.166392+0000", 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00038396300000000002 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.766 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.167289+0000", 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00089654899999999996 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.167328+0000", 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9138999999999999e-05 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.167388+0000", 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9907000000000001e-05 2023-09-30T09:55:11.767 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47084.0:49 2.0 2:0fa80f97:::rbd_header.b7ec5ad5e03e:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e633)", 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:29.161274+0000", 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 42.490086429999998, 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0070313509999999999, 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.768 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47084", 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2534816667", 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 49 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.161274+0000", 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.769 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.161274+0000", 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.161272+0000", 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.161280+0000", 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9079999999999995e-06 2023-09-30T09:55:11.770 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.161297+0000", 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7028999999999999e-05 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.161306+0000", 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0720000000000006e-06 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.771 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.161364+0000", 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7726999999999998e-05 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.161369+0000", 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.3739999999999997e-06 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.166702+0000", 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0053332409999999998 2023-09-30T09:55:11.772 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.166835+0000", 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013228 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.167242+0000", 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00040749299999999998 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.773 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.167764+0000", 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052202200000000002 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.168192+0000", 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00042814600000000002 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.168245+0000", 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2942999999999998e-05 2023-09-30T09:55:11.774 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:29.168305+0000", 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9580999999999998e-05 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37146.0:26 2.0 2:0fa80f97:::rbd_header.b7ec5ad5e03e:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e634)", 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:30.179677+0000", 2023-09-30T09:55:11.775 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 41.471683378999998, 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0078744339999999996, 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37146", 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2344245568", 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 26 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.776 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.179677+0000", 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.179677+0000", 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.179675+0000", 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.777 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.778 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.778 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.179687+0000", 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1941e-05 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.179709+0000", 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2007000000000001e-05 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.779 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.179720+0000", 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1168e-05 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.179810+0000", 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.0418000000000001e-05 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.179824+0000", 2023-09-30T09:55:11.780 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4064e-05 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.184782+0000", 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0049580140000000002 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.185050+0000", 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00026800700000000001 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.781 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.185498+0000", 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00044783699999999997 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.187402+0000", 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00190358 2023-09-30T09:55:11.782 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.187445+0000", 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3038999999999999e-05 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.187488+0000", 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3099000000000002e-05 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.783 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:30.187551+0000", 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3323000000000003e-05 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37158.0:26 2.7 2:fe326f4d:::rbd_header.90cc420e04da:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e635)", 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:31.178515+0000", 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 40.472845456999998, 2023-09-30T09:55:11.784 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0064826390000000001, 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37158", 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1881488559", 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 26 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.178515+0000", 2023-09-30T09:55:11.785 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.178515+0000", 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.178513+0000", 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999981 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.786 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.178526+0000", 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2685e-05 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.178553+0000", 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7699e-05 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.787 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.178566+0000", 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3159999999999999e-05 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.178663+0000", 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6314e-05 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op must wait for map", 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.178672+0000", 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.6639999999999993e-06 2023-09-30T09:55:11.788 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.183519+0000", 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.004846688 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.183599+0000", 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.9616000000000005e-05 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.789 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.183912+0000", 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031316800000000001 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.184513+0000", 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00060080699999999997 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.790 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.184901+0000", 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00038814900000000001 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.184930+0000", 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9342000000000001e-05 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.184970+0000", 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9582999999999997e-05 2023-09-30T09:55:11.791 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.184997+0000", 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.7503000000000001e-05 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47240.0:31 2.4 2:281fe4ea:::rbd_object_map.b888bffab72a:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e635)", 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.177330+0000", 2023-09-30T09:55:11.792 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 39.474029852999998, 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0048444509999999996, 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47240", 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1961398279", 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 31 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.793 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.177330+0000", 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.177330+0000", 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.177330+0000", 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:11.794 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.177332+0000", 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6079999999999998e-06 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.177338+0000", 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6169999999999998e-06 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.795 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.177341+0000", 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2490000000000001e-06 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.178161+0000", 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00082016399999999996 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for rw locks", 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.178249+0000", 2023-09-30T09:55:11.796 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7861000000000005e-05 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.179414+0000", 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001164923 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for rw locks", 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.179460+0000", 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5343000000000003e-05 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.797 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.179906+0000", 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000446513 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.179973+0000", 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6629000000000002e-05 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.798 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180292+0000", 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031903699999999999 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180661+0000", 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00036948200000000002 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182096+0000", 2023-09-30T09:55:11.799 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014347229999999999 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182127+0000", 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0836000000000001e-05 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.182175+0000", 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.7817e-05 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.800 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37296.0:25 2.5 2:a2f9dc83:::rbd_object_map.91b0d12c6068:head [call] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e636)", 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.670633+0000", 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 38.980727156999997, 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0028029280000000001, 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37296", 2023-09-30T09:55:11.801 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1235285789", 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 25 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.670633+0000", 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.670633+0000", 2023-09-30T09:55:11.802 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.670632+0000", 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.670636+0000", 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3469999999999999e-06 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.803 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.670644+0000", 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4090000000000003e-06 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.670648+0000", 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.6720000000000002e-06 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.804 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.670678+0000", 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9988000000000002e-05 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.670731+0000", 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2816999999999999e-05 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.673436+0000", 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0027052140000000001 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.805 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47666.0:1 2.0 2:08141c09:::rbd_data.9324817447ae.000000000000:head [writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:44.032854+0000", 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 27.618505944999999, 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.015735118999999999, 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47666", 2023-09-30T09:55:11.806 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2612929794", 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 1 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.032854+0000", 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.032854+0000", 2023-09-30T09:55:11.807 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.032853+0000", 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.038247+0000", 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0053941600000000003 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.808 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.038280+0000", 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2858000000000003e-05 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.038288+0000", 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8769999999999993e-06 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:11.809 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.038346+0000", 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7899999999999998e-05 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.038466+0000", 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012023000000000001 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1", 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.038575+0000", 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000108928 2023-09-30T09:55:11.810 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.044672+0000", 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0060967219999999997 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.048357+0000", 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0036851639999999999 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.811 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.048483+0000", 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012655699999999999 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:44.048589+0000", 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105779 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:11.812 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:11.813 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:55:12.127 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-30T09:55:12.290 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-30T09:55:12.502 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:12.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/862745437' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:55:12.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/862745437' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:55:12.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/862745437' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-30T09:55:13.029 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:13 smithi167 ceph-mon[192880]: pgmap v1740: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 690 KiB/s, 1 objects/s recovering 2023-09-30T09:55:13.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:13 smithi019 ceph-mon[216815]: pgmap v1740: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 690 KiB/s, 1 objects/s recovering 2023-09-30T09:55:13.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:13 smithi019 ceph-mon[224043]: pgmap v1740: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 690 KiB/s, 1 objects/s recovering 2023-09-30T09:55:14.066 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:55:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:55:13] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2023-09-30T09:55:15.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:14 smithi019 ceph-mon[216815]: pgmap v1741: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 590 KiB/s, 1 objects/s recovering 2023-09-30T09:55:15.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:14 smithi019 ceph-mon[224043]: pgmap v1741: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 590 KiB/s, 1 objects/s recovering 2023-09-30T09:55:15.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:15 smithi167 ceph-mon[192880]: pgmap v1741: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 590 KiB/s, 1 objects/s recovering 2023-09-30T09:55:15.504 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:55:15.505 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:55:15.505 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 0 2023-09-30T09:55:15.505 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:55:16.017 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.2 to 1 (10000) 2023-09-30T09:55:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4236402604' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:16.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:16 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:16.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4236402604' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:16.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:16 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4236402604' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:16 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:16.724 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-30T09:55:17.069 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-30T09:55:17.140 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:17.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:17 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:55:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:17 smithi167 ceph-mon[192880]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:17 smithi167 ceph-mon[192880]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:17 smithi167 ceph-mon[192880]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:17 smithi167 ceph-mon[192880]: osdmap e654: 8 total, 8 up, 8 in 2023-09-30T09:55:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:17 smithi167 ceph-mon[192880]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:17 smithi167 ceph-mon[192880]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:17 smithi167 ceph-mon[192880]: pgmap v1743: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 517 KiB/s, 2 objects/s recovering 2023-09-30T09:55:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:55:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[216815]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[216815]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[216815]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[216815]: osdmap e654: 8 total, 8 up, 8 in 2023-09-30T09:55:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[216815]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[216815]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[216815]: pgmap v1743: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 517 KiB/s, 2 objects/s recovering 2023-09-30T09:55:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:55:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[224043]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[224043]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[224043]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[224043]: osdmap e654: 8 total, 8 up, 8 in 2023-09-30T09:55:17.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[224043]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[224043]: failed to encode map e654 with expected crc 2023-09-30T09:55:17.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:17 smithi019 ceph-mon[224043]: pgmap v1743: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 517 KiB/s, 2 objects/s recovering 2023-09-30T09:55:18.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[216815]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[216815]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[216815]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[216815]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[216815]: osdmap e655: 8 total, 8 up, 8 in 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[216815]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[216815]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[224043]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[224043]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[224043]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[224043]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[224043]: osdmap e655: 8 total, 8 up, 8 in 2023-09-30T09:55:18.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[224043]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:18 smithi019 ceph-mon[224043]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:18 smithi167 ceph-mon[192880]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:18 smithi167 ceph-mon[192880]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:18 smithi167 ceph-mon[192880]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:18 smithi167 ceph-mon[192880]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:18 smithi167 ceph-mon[192880]: osdmap e655: 8 total, 8 up, 8 in 2023-09-30T09:55:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:18 smithi167 ceph-mon[192880]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:18 smithi167 ceph-mon[192880]: failed to encode map e655 with expected crc 2023-09-30T09:55:18.439 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:19.159 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:55:19.159 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:55:19.159 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:55:19.160 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:55:19.160 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:55:19.179 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: osdmap e656: 8 total, 8 up, 8 in 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:19.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:19.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: pgmap v1746: 103 pgs: 1 remapped+peering, 102 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:55:19.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[224043]: osdmap e657: 8 total, 8 up, 8 in 2023-09-30T09:55:19.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.181 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: osdmap e656: 8 total, 8 up, 8 in 2023-09-30T09:55:19.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:19.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:19.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: pgmap v1746: 103 pgs: 1 remapped+peering, 102 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:55:19.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:19 smithi019 ceph-mon[216815]: osdmap e657: 8 total, 8 up, 8 in 2023-09-30T09:55:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: osdmap e656: 8 total, 8 up, 8 in 2023-09-30T09:55:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: failed to encode map e656 with expected crc 2023-09-30T09:55:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: pgmap v1746: 103 pgs: 1 remapped+peering, 102 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-30T09:55:19.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:19 smithi167 ceph-mon[192880]: osdmap e657: 8 total, 8 up, 8 in 2023-09-30T09:55:20.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:55:20.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:20 smithi167 ceph-mon[192880]: osdmap e658: 8 total, 8 up, 8 in 2023-09-30T09:55:20.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:55:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[224043]: osdmap e658: 8 total, 8 up, 8 in 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:55:20.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:20 smithi019 ceph-mon[216815]: osdmap e658: 8 total, 8 up, 8 in 2023-09-30T09:55:20.477 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-30T09:55:21.054 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:21.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e657 with expected crc 2023-09-30T09:55:21.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[224043]: pgmap v1749: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 64 KiB/s, 5 objects/s recovering 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e657 with expected crc 2023-09-30T09:55:21.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:21 smithi019 ceph-mon[216815]: pgmap v1749: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 64 KiB/s, 5 objects/s recovering 2023-09-30T09:55:21.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e657 with expected crc 2023-09-30T09:55:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: failed to encode map e658 with expected crc 2023-09-30T09:55:21.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:21 smithi167 ceph-mon[192880]: pgmap v1749: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 64 KiB/s, 5 objects/s recovering 2023-09-30T09:55:22.039 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:22 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:55:22.039 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:22 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:55:22.039 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:22 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3234698051' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:22.039 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:22 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:22.040 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-30T09:55:22.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:22 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:55:22.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:22 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:55:22.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:22 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3234698051' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:22.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:22 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:22 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:55:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:22 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:55:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:22 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3234698051' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:22.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:22 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-30T09:55:22.992 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-30T09:55:23.314 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:55:23.314 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127752 10.0 10:0aff3a70:::benchmark_data_smithi019_617837_object34352:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:57.146004+0000", 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 206.16581033899999, 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.136793248, 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.315 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127752 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146004+0000", 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.316 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146004+0000", 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146005+0000", 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3120000000000001e-06 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.317 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146008+0000", 2023-09-30T09:55:23.318 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.0750000000000002e-06 2023-09-30T09:55:23.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.318 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.318 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.318 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146019+0000", 2023-09-30T09:55:23.318 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0814e-05 2023-09-30T09:55:23.318 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.318 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146029+0000", 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9920000000000002e-06 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.147573+0000", 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015432379999999999 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:55:23.319 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.147585+0000", 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.257e-05 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.268716+0000", 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1211314100000001 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.268829+0000", 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000112135 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.320 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6,7", 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.268909+0000", 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.0795000000000001e-05 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.276607+0000", 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.007697092 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.321 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.276651+0000", 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4756999999999999e-05 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.276689+0000", 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7339000000000003e-05 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.282582+0000", 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0058938549999999999 2023-09-30T09:55:23.322 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.282661+0000", 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8351999999999998e-05 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.282797+0000", 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013651200000000001 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.323 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127762 10.0 10:07f08312:::benchmark_data_smithi019_617837_object34362:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:57.146044+0000", 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 206.16577049899999, 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1378637279999999, 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:23.324 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127762 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146044+0000", 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146044+0000", 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.325 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146044+0000", 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9999999999999998e-07 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146049+0000", 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0860000000000001e-06 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.326 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146057+0000", 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.7069999999999992e-06 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146065+0000", 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6869999999999994e-06 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.327 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.147615+0000", 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001550177 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.147625+0000", 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0152e-05 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.271503+0000", 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1238782380000001 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.328 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.271661+0000", 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000157665 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6,7", 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.271828+0000", 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016659699999999999 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:55:23.329 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.276660+0000", 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0048323929999999999 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.282844+0000", 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0061836449999999998 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.282888+0000", 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4076999999999998e-05 2023-09-30T09:55:23.330 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.283718+0000", 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00082990500000000003 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.283791+0000", 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.2985999999999995e-05 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.331 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.283908+0000", 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000116913 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127764 10.0 10:0c0e5c26:::benchmark_data_smithi019_617837_object34364:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:57.146075+0000", 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 206.16573981799999, 2023-09-30T09:55:23.332 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.146665533, 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127764 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146075+0000", 2023-09-30T09:55:23.333 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146075+0000", 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146075+0000", 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4000000000000002e-07 2023-09-30T09:55:23.334 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146077+0000", 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4619999999999999e-06 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146084+0000", 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9979999999999997e-06 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.335 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146093+0000", 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7900000000000005e-06 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.147653+0000", 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0015597370000000001 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.147663+0000", 2023-09-30T09:55:23.336 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7899999999999994e-06 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.274284+0000", 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1266211230000001 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.274442+0000", 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000157567 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.337 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6,7", 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.274530+0000", 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8216999999999999e-05 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.283662+0000", 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091327349999999995 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.338 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.283958+0000", 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000295894 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.284089+0000", 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013101000000000001 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292552+0000", 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0084628829999999992 2023-09-30T09:55:23.339 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292621+0000", 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8405000000000007e-05 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292740+0000", 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011948200000000001 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.340 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127769 10.0 10:0953ee38:::benchmark_data_smithi019_617837_object34369:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:57.146103+0000", 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 206.16571129299999, 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.146800431, 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:23.341 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127769 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146103+0000", 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146103+0000", 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.342 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146104+0000", 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9000000000000002e-07 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146106+0000", 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5639999999999999e-06 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.343 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146114+0000", 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4359999999999996e-06 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.146121+0000", 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0439999999999998e-06 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.147690+0000", 2023-09-30T09:55:23.344 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001569209 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.147700+0000", 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7599999999999997e-06 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.276722+0000", 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.129022862 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.345 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.276841+0000", 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000119052 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6,7", 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.276925+0000", 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3555999999999998e-05 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.346 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.284002+0000", 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0070771810000000001 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.284133+0000", 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013121899999999999 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.285333+0000", 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011995669999999999 2023-09-30T09:55:23.347 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292757+0000", 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0074236670000000001 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292803+0000", 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6125999999999998e-05 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.348 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292904+0000", 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00010079800000000001 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:127787 10.0 10:05c411bc:::benchmark_data_smithi019_617837_object34387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:51:57.446861+0000", 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 205.86495344599999, 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.846191578, 2023-09-30T09:55:23.349 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 127787 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.446861+0000", 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.446861+0000", 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.446863+0000", 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.717e-06 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.351 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.446870+0000", 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6590000000000007e-06 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.446889+0000", 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8943000000000001e-05 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.446898+0000", 2023-09-30T09:55:23.352 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.6670000000000002e-06 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.446964+0000", 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5811999999999999e-05 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for peered", 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:57.446974+0000", 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7399999999999999e-06 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.353 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.279360+0000", 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.83238657299999996 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.279537+0000", 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017705600000000001 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6,7", 2023-09-30T09:55:23.354 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.279717+0000", 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000179617 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.284048+0000", 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0043312170000000001 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.284874+0000", 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00082579599999999995 2023-09-30T09:55:23.355 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292531+0000", 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0076568210000000003 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292911+0000", 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00038006900000000002 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.356 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.292955+0000", 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4388000000000003e-05 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:51:58.293053+0000", 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7503000000000005e-05 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.357 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:138187 10.0 10:07485bc0:::benchmark_data_smithi019_617837_object44787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:23.066839+0000", 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 180.24497571699999, 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.360792102, 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 138187 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.358 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.066839+0000", 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.066839+0000", 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.359 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.066840+0000", 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.491e-06 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.066847+0000", 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5370000000000003e-06 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.066860+0000", 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3662999999999999e-05 2023-09-30T09:55:23.360 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.066870+0000", 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0023999999999999e-05 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.066942+0000", 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1254999999999995e-05 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.361 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.067077+0000", 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000135749 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6,7", 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.067222+0000", 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014490799999999999 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.073562+0000", 2023-09-30T09:55:23.362 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0063392639999999998 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.082747+0000", 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0091850760000000004 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.087858+0000", 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0051114530000000002 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.363 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427427+0000", 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33956867499999999 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427497+0000", 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0080000000000007e-05 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.364 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427631+0000", 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013392699999999999 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_op(client.46583.0:138195 10.0 10:0241b613:::benchmark_data_smithi019_617837_object44795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:23.074362+0000", 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 180.23745262200001, 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.353496898, 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.365 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "client_info": { 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "client": "client.46583", 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "tid": 138195 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074362+0000", 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.366 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074362+0000", 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074364+0000", 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0990000000000001e-06 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074369+0000", 2023-09-30T09:55:23.367 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5550000000000002e-06 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074383+0000", 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3831e-05 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074394+0000", 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0462000000000001e-05 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.368 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074462+0000", 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7782999999999998e-05 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074606+0000", 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000144623 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "waiting for subops from 4,6,7", 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.074760+0000", 2023-09-30T09:55:23.369 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000153855 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "op_commit", 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.077661+0000", 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0029006560000000002 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.087913+0000", 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.010252466 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.370 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.093167+0000", 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0052533980000000003 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427677+0000", 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.33451034499999999 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.371 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427743+0000", 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5773999999999997e-05 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427859+0000", 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000116051 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop_reply(client.46583.0:138187 10.0 e604/590 ondisk, result = 0)", 2023-09-30T09:55:23.372 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:52:23.097726+0000", 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 180.21408804500001, 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32991221399999998, 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.097726+0000", 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.373 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.097726+0000", 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.097727+0000", 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3399999999999998e-07 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.097734+0000", 2023-09-30T09:55:23.374 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8249999999999999e-06 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427291+0000", 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.32955750499999997 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427304+0000", 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2673e-05 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.375 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427410+0000", 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000105998 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427420+0000", 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4870000000000008e-06 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:52:23.427639+0000", 2023-09-30T09:55:23.376 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021899199999999999 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:162654 10.0 e615/606 10:03d45a2a:::benchmark_data_smithi019_617837_object69254:head v 615'10273, mlcod=615'10271)", 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:03.019811+0000", 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 140.29200365899999, 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.039076937999999999, 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.377 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.019811+0000", 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.019811+0000", 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.378 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.019812+0000", 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.308e-06 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.019822+0000", 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0312999999999999e-05 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.019849+0000", 2023-09-30T09:55:23.379 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6247000000000001e-05 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.019859+0000", 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0209e-05 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.022982+0000", 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0031230780000000001 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.380 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.023006+0000", 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4239999999999998e-05 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.058836+0000", 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.035829883999999999 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:03.058888+0000", 2023-09-30T09:55:23.381 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1659000000000001e-05 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:177274 10.0 e617/606 10:04fb6ed3:::benchmark_data_smithi019_617837_object83874:head v 617'11163, mlcod=617'11161)", 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:23.046664+0000", 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 120.265150336, 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.039449107999999997, 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.382 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.046664+0000", 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.046664+0000", 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.383 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.046665+0000", 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1200000000000002e-07 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.046671+0000", 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1979999999999998e-06 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.076942+0000", 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.030270505 2023-09-30T09:55:23.384 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.076978+0000", 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.6128999999999999e-05 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.077142+0000", 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00016428600000000001 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.385 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.077225+0000", 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2836000000000004e-05 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.086057+0000", 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0088325320000000006 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.386 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:23.086113+0000", 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5810000000000003e-05 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.46583.0:185185 10.4 e621/618 10:203db848:::benchmark_data_smithi019_617837_object91785:head v 621'11820, mlcod=621'11817)", 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:35.571936+0000", 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 107.739878123, 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15052022500000001, 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.387 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.571936+0000", 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.571936+0000", 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.388 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.571937+0000", 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.1100000000000005e-07 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.571945+0000", 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.763e-06 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.571954+0000", 2023-09-30T09:55:23.389 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3749999999999992e-06 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.571961+0000", 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5819999999999998e-06 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.572029+0000", 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8194000000000005e-05 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.390 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.572055+0000", 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.6118999999999998e-05 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.722398+0000", 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.15034317 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.391 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:35.722457+0000", 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8211000000000002e-05 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.36972.0:1338 11.6 e627/625 11:67cae2ed:::rbd_mirroring:head v 627'844, mlcod=627'842)", 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:56.798826+0000", 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 86.512988450999998, 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.067053493000000006, 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.392 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.798826+0000", 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.798826+0000", 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.393 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.798826+0000", 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.4700000000000002e-07 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.798830+0000", 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.89e-06 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.394 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.798839+0000", 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7839999999999992e-06 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.798843+0000", 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2529999999999998e-06 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.798884+0000", 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.0589000000000001e-05 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.395 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.798895+0000", 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1317e-05 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.799244+0000", 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000349043 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.396 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:56.865879+0000", 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.066635169999999994 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.36972.0:1452 11.6 e627/625 11:67cae2ed:::rbd_mirroring:head v 627'958, mlcod=627'956)", 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:57.000089+0000", 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 86.311724988999998, 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.045337802000000003, 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.397 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000089+0000", 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000089+0000", 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.398 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000091+0000", 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1599999999999999e-06 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000100+0000", 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.3100000000000006e-06 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000119+0000", 2023-09-30T09:55:23.399 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8614000000000002e-05 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000130+0000", 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1195999999999999e-05 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000201+0000", 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1579e-05 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.400 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000213+0000", 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1639e-05 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.000613+0000", 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00040045899999999998 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.401 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.045427+0000", 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.044813844999999998 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.36972.0:1981 11.6 e627/625 11:67cae2ed:::rbd_mirroring:head v 627'1487, mlcod=627'1485)", 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:53:57.664987+0000", 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 85.646827185000006, 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.138453257, 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.402 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.664987+0000", 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.664987+0000", 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.403 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.664988+0000", 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2199999999999999e-07 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.664992+0000", 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.2420000000000002e-06 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.664999+0000", 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3679999999999996e-06 2023-09-30T09:55:23.404 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.665003+0000", 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9539999999999998e-06 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.665056+0000", 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.3023999999999999e-05 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.405 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.665068+0000", 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1652e-05 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.665545+0000", 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00047697400000000002 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:53:57.803440+0000", 2023-09-30T09:55:23.406 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.137895621 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.37068.0:27 2.4 e631/618 2:37c57d42:::rbd_object_map.90cc420e04da:head v 631'108, mlcod=631'106)", 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:54:27.636894+0000", 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 55.674920078, 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0034063129999999998, 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.407 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.636894+0000", 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.636894+0000", 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.408 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.636895+0000", 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.3900000000000004e-07 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.636902+0000", 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.9759999999999998e-06 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.636912+0000", 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0077e-05 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.409 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.636917+0000", 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9760000000000003e-06 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.636962+0000", 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5204000000000003e-05 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.410 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.636980+0000", 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7756000000000001e-05 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.640272+0000", 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0032922189999999999 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:27.640301+0000", 2023-09-30T09:55:23.411 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8466000000000002e-05 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.37557.0:29 2.4 e644/618 2:20f2bc66:::rbd_data.92b547bbad77.0000000000000000:head v 644'191, mlcod=644'189)", 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:54:42.963222+0000", 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 40.348592686000003, 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0033606320000000001, 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.412 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.963222+0000", 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.963222+0000", 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.413 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.963223+0000", 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.514e-06 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.964795+0000", 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001571734 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.414 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.964827+0000", 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1680999999999997e-05 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.964842+0000", 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5724000000000001e-05 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.964930+0000", 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7181000000000004e-05 2023-09-30T09:55:23.415 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.964962+0000", 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2314000000000003e-05 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.966517+0000", 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001555601 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.416 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.966582+0000", 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4882999999999995e-05 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.37557.0:51 2.2 e644/559 2:4f1735dd:::rbd_data.92b547bbad77.0000000000000003:head v 644'33, mlcod=644'31)", 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:54:42.985271+0000", 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 40.326543297999997, 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.002789831, 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.417 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.985271+0000", 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.985271+0000", 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.418 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.985272+0000", 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.1100000000000004e-07 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.986707+0000", 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001435277 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.986724+0000", 2023-09-30T09:55:23.419 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7031999999999999e-05 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.986734+0000", 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.9559999999999992e-06 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.986809+0000", 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.4902999999999998e-05 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.420 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.986838+0000", 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8928000000000002e-05 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.988022+0000", 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011838090000000001 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.421 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:42.988061+0000", 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.9014999999999998e-05 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "osd_repop(client.47666.0:1 2.0 e644/618 2:08141c09:::rbd_data.9324817447ae.000000000000:head v 644'256, mlcod=644'254)", 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:54:44.038881+0000", 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 39.272933786000003, 2023-09-30T09:55:23.422 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0092247749999999993, 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.038881+0000", 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.423 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.424 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.038881+0000", 2023-09-30T09:55:23.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.424 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.424 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.424 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.038883+0000", 2023-09-30T09:55:23.424 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.136e-06 2023-09-30T09:55:23.424 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.042810+0000", 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0039273559999999999 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.042835+0000", 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5143e-05 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.425 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.042848+0000", 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2978e-05 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.042924+0000", 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6018000000000003e-05 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.426 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.042956+0000", 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1803e-05 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "commit_sent", 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.048054+0000", 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0050982140000000002 2023-09-30T09:55:23.427 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.428 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.428 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.428 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:54:44.048105+0000", 2023-09-30T09:55:23.428 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1127e-05 2023-09-30T09:55:23.428 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.428 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.428 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.428 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.4 e651,650 2:388a7c0f:::rbd_data.909cbf420c93.0000000000000000:head,631'101)", 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.739725+0000", 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 15.572089653999999, 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027314400000000001, 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.429 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.739725+0000", 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.739725+0000", 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.739726+0000", 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.16e-07 2023-09-30T09:55:23.430 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.431 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.431 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.431 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.739733+0000", 2023-09-30T09:55:23.431 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.1629999999999999e-06 2023-09-30T09:55:23.431 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.431 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.431 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.432 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.739747+0000", 2023-09-30T09:55:23.432 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3818000000000001e-05 2023-09-30T09:55:23.432 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.432 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.432 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.432 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.739755+0000", 2023-09-30T09:55:23.432 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.8130000000000002e-06 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.739826+0000", 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0568000000000006e-05 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.433 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.739998+0000", 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017186600000000001 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.4 e651,650 2:20f2bc66:::rbd_data.92b547bbad77.0000000000000000:head,644'193)", 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.746936+0000", 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 15.56487819, 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00026126200000000001, 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:23.434 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.746936+0000", 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.746936+0000", 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.435 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.746937+0000", 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6499999999999998e-07 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.746942+0000", 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2240000000000003e-06 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:23.436 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.746957+0000", 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4863e-05 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.746965+0000", 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6939999999999995e-06 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.747030+0000", 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.5234999999999995e-05 2023-09-30T09:55:23.437 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.747197+0000", 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000167481 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:23.438 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:55:23.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:55:23.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[216815]: failed to encode map e659 with expected crc 2023-09-30T09:55:23.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[216815]: osdmap e659: 8 total, 8 up, 8 in 2023-09-30T09:55:23.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[216815]: pgmap v1751: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 63 KiB/s, 5 objects/s recovering 2023-09-30T09:55:23.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[216815]: failed to encode map e659 with expected crc 2023-09-30T09:55:23.439 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:55:23.440 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[224043]: failed to encode map e659 with expected crc 2023-09-30T09:55:23.440 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[224043]: osdmap e659: 8 total, 8 up, 8 in 2023-09-30T09:55:23.440 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[224043]: pgmap v1751: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 63 KiB/s, 5 objects/s recovering 2023-09-30T09:55:23.440 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:23 smithi019 ceph-mon[224043]: failed to encode map e659 with expected crc 2023-09-30T09:55:23.440 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:23.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:23 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-30T09:55:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:23 smithi167 ceph-mon[192880]: failed to encode map e659 with expected crc 2023-09-30T09:55:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:23 smithi167 ceph-mon[192880]: osdmap e659: 8 total, 8 up, 8 in 2023-09-30T09:55:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:23 smithi167 ceph-mon[192880]: pgmap v1751: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 63 KiB/s, 5 objects/s recovering 2023-09-30T09:55:23.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:23 smithi167 ceph-mon[192880]: failed to encode map e659 with expected crc 2023-09-30T09:55:24.044 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:55:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:55:23] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.43.0" 2023-09-30T09:55:24.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:24 smithi019 ceph-mon[216815]: failed to encode map e659 with expected crc 2023-09-30T09:55:24.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:24 smithi019 ceph-mon[216815]: failed to encode map e659 with expected crc 2023-09-30T09:55:24.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:24 smithi019 ceph-mon[224043]: failed to encode map e659 with expected crc 2023-09-30T09:55:24.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:24 smithi019 ceph-mon[224043]: failed to encode map e659 with expected crc 2023-09-30T09:55:24.374 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-30T09:55:24.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:24 smithi167 ceph-mon[192880]: failed to encode map e659 with expected crc 2023-09-30T09:55:24.407 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:24 smithi167 ceph-mon[192880]: failed to encode map e659 with expected crc 2023-09-30T09:55:24.737 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:55:25.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:25 smithi167 ceph-mon[192880]: pgmap v1752: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s, 3 objects/s recovering 2023-09-30T09:55:25.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:25 smithi019 ceph-mon[216815]: pgmap v1752: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s, 3 objects/s recovering 2023-09-30T09:55:25.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:25 smithi019 ceph-mon[224043]: pgmap v1752: 103 pgs: 6 peering, 97 active+clean; 8.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s, 3 objects/s recovering 2023-09-30T09:55:26.058 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.3 to 1 (10000) 2023-09-30T09:55:26.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3771538577' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:26.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3771538577' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:26.317 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3771538577' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:26.538 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:55:26.539 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:55:26.539 INFO:teuthology.orchestra.run.smithi167.stdout: "num_ops": 0 2023-09-30T09:55:26.539 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:55:26.811 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-30T09:55:27.180 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:27.255 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-30T09:55:27.322 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:27 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3771538577' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:55:27.322 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:27 smithi167 ceph-mon[192880]: failed to encode map e660 with expected crc 2023-09-30T09:55:27.322 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:27 smithi167 ceph-mon[192880]: failed to encode map e660 with expected crc 2023-09-30T09:55:27.322 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:27 smithi167 ceph-mon[192880]: osdmap e660: 8 total, 8 up, 8 in 2023-09-30T09:55:27.322 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:27 smithi167 ceph-mon[192880]: pgmap v1754: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:27.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3771538577' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:55:27.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[216815]: failed to encode map e660 with expected crc 2023-09-30T09:55:27.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[216815]: failed to encode map e660 with expected crc 2023-09-30T09:55:27.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[216815]: osdmap e660: 8 total, 8 up, 8 in 2023-09-30T09:55:27.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[216815]: pgmap v1754: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3771538577' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:55:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[224043]: failed to encode map e660 with expected crc 2023-09-30T09:55:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[224043]: failed to encode map e660 with expected crc 2023-09-30T09:55:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[224043]: osdmap e660: 8 total, 8 up, 8 in 2023-09-30T09:55:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:27 smithi019 ceph-mon[224043]: pgmap v1754: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:27.639 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:55:29.243 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:55:29.244 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [], 2023-09-30T09:55:29.244 INFO:teuthology.orchestra.run.smithi167.stdout: "complaint_time": 30, 2023-09-30T09:55:29.244 INFO:teuthology.orchestra.run.smithi167.stdout: "num_blocked_ops": 0 2023-09-30T09:55:29.244 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:55:29.430 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:29 smithi167 ceph-mon[192880]: pgmap v1755: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:29.431 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:29 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/362959265' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:29.431 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:29 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:29.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:29 smithi019 ceph-mon[216815]: pgmap v1755: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:29.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:29 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/362959265' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:29.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:29 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:29.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:29 smithi019 ceph-mon[224043]: pgmap v1755: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:29.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:29 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/362959265' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:29.589 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:29 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-30T09:55:29.906 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-30T09:55:30.015 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-30T09:55:30.299 INFO:teuthology.orchestra.run.smithi167.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-30T09:55:30.972 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-30T09:55:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:31 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:55:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:31 smithi167 ceph-mon[192880]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:31 smithi167 ceph-mon[192880]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:31 smithi167 ceph-mon[192880]: osdmap e661: 8 total, 8 up, 8 in 2023-09-30T09:55:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:31 smithi167 ceph-mon[192880]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:31 smithi167 ceph-mon[192880]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:31 smithi167 ceph-mon[192880]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:31 smithi167 ceph-mon[192880]: pgmap v1757: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:31.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:55:31.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[216815]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[216815]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[216815]: osdmap e661: 8 total, 8 up, 8 in 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[216815]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[216815]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[216815]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[216815]: pgmap v1757: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[224043]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[224043]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[224043]: osdmap e661: 8 total, 8 up, 8 in 2023-09-30T09:55:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[224043]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[224043]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[224043]: failed to encode map e661 with expected crc 2023-09-30T09:55:31.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:31 smithi019 ceph-mon[224043]: pgmap v1757: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:31.358 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:31.810 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-09-30T09:55:31.810 INFO:teuthology.orchestra.run.smithi167.stdout: "size": 20, 2023-09-30T09:55:31.810 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 600, 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "ops": [ 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101653 10.9 10:9320278f:::benchmark_data_smithi019_617837_object8253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.437960+0000", 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 291.36972554599998, 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1004105500000001, 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:31.811 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101653 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437960+0000", 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437960+0000", 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.812 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437958+0000", 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437966+0000", 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.5440000000000001e-06 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.813 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437980+0000", 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4278e-05 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.437987+0000", 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7479999999999996e-06 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.438042+0000", 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4951999999999998e-05 2023-09-30T09:55:31.814 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.438054+0000", 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1787000000000001e-05 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462424+0000", 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.02437044 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.815 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462572+0000", 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000147677 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.462700+0000", 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000128238 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538056+0000", 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.075356266000000005 2023-09-30T09:55:31.816 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538088+0000", 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1581000000000001e-05 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538219+0000", 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013155900000000001 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.817 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538270+0000", 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.0521e-05 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538370+0000", 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000100263 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.818 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101657 10.3 10:ce53bcc2:::benchmark_data_smithi019_617837_object8257:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.470044+0000", 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 291.33764112900002, 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0323604369999999, 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101657 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.819 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470044+0000", 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470044+0000", 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470043+0000", 2023-09-30T09:55:31.820 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470050+0000", 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6620000000000005e-06 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470068+0000", 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.7569999999999999e-05 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.821 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470075+0000", 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7159999999999997e-06 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470126+0000", 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.109e-05 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.822 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.470136+0000", 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0567000000000001e-05 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.448589+0000", 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.97845242099999996 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.448733+0000", 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014411799999999999 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.823 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,3", 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.448876+0000", 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014285599999999999 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502174+0000", 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.053298092999999998 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502182+0000", 2023-09-30T09:55:31.824 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.3720000000000005e-06 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502244+0000", 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1347999999999996e-05 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502300+0000", 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.6619000000000001e-05 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.825 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.502404+0000", 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00010429699999999999 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101659 10.9 10:9101f776:::benchmark_data_smithi019_617837_object8259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.471630+0000", 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 291.33605480599999, 2023-09-30T09:55:31.826 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.066891738, 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101659 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471630+0000", 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.827 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471630+0000", 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471629+0000", 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.828 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471636+0000", 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5359999999999998e-06 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471649+0000", 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.331e-05 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.471656+0000", 2023-09-30T09:55:31.829 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2690000000000002e-06 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.472458+0000", 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00080184699999999995 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.472474+0000", 2023-09-30T09:55:31.830 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5597999999999999e-05 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.468844+0000", 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.99637029499999996 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.468934+0000", 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9730999999999999e-05 2023-09-30T09:55:31.831 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.469028+0000", 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4293e-05 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538118+0000", 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.069089388000000002 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.832 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538176+0000", 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.8721999999999998e-05 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538383+0000", 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00020698299999999999 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538428+0000", 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4286000000000003e-05 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.833 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538522+0000", 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.4582999999999999e-05 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:101667 10.9 10:92ad1a50:::benchmark_data_smithi019_617837_object8267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:50:40.489225+0000", 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 291.31846061900001, 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.049433718, 2023-09-30T09:55:31.834 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 101667 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489225+0000", 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.835 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489225+0000", 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489223+0000", 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.836 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489230+0000", 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5749999999999997e-06 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489243+0000", 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2921999999999999e-05 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489250+0000", 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6529999999999997e-06 2023-09-30T09:55:31.837 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489314+0000", 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4048999999999996e-05 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:40.489323+0000", 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.1260000000000004e-06 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.838 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.471681+0000", 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.982358697 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.471824+0000", 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00014217400000000001 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.839 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.471949+0000", 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000125208 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538147+0000", 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.066198513 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538210+0000", 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.3071000000000007e-05 2023-09-30T09:55:31.840 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538529+0000", 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00031860800000000001 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538569+0000", 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9814999999999997e-05 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.841 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:50:41.538658+0000", 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9487000000000006e-05 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127765 10.3 10:c1bc492e:::benchmark_data_smithi019_617837_object34365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:56.655995+0000", 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 215.15169065399999, 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6746205079999998, 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127765 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.655995+0000", 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.655995+0000", 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.655993+0000", 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.656000+0000", 2023-09-30T09:55:31.844 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.2400000000000001e-06 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.656015+0000", 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4610000000000001e-05 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.656022+0000", 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6760000000000001e-06 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.845 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.846 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.656075+0000", 2023-09-30T09:55:31.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.2957999999999997e-05 2023-09-30T09:55:31.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.846 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.846 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.846 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.656083+0000", 2023-09-30T09:55:31.846 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8639999999999996e-06 2023-09-30T09:55:31.846 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.148779+0000", 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.492696105 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.148787+0000", 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8760000000000005e-06 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.847 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.295249+0000", 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.146462321 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.295375+0000", 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012576699999999999 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 3,4", 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.295499+0000", 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012480400000000001 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.848 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.301507+0000", 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0060079560000000001 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.310352+0000", 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0088441089999999993 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.327890+0000", 2023-09-30T09:55:31.849 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.017538583 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.327955+0000", 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4455000000000005e-05 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.850 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.330615+0000", 2023-09-30T09:55:31.851 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0026604559999999998 2023-09-30T09:55:31.851 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.851 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.851 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.852 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.852 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.852 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127775 10.a 10:5a03dd27:::benchmark_data_smithi019_617837_object34375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-30T09:55:31.852 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:56.946546+0000", 2023-09-30T09:55:31.852 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 214.861138926, 2023-09-30T09:55:31.852 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4332348509999999, 2023-09-30T09:55:31.852 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.852 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.853 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.853 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:31.853 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:31.853 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127775 2023-09-30T09:55:31.853 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.853 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.853 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.853 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.946546+0000", 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.946546+0000", 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.854 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.946545+0000", 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.946553+0000", 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.0339999999999997e-06 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.946572+0000", 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9320000000000001e-05 2023-09-30T09:55:31.855 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.946580+0000", 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.622e-06 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.946638+0000", 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.7957000000000001e-05 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.856 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:56.946649+0000", 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0841e-05 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.156812+0000", 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.21016367799999999 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.156821+0000", 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4190000000000002e-06 2023-09-30T09:55:31.857 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.352253+0000", 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1954325640000001 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.352372+0000", 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00011895099999999999 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.858 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.352472+0000", 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.9853999999999996e-05 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.363638+0000", 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.011165757 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.365606+0000", 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0019678489999999998 2023-09-30T09:55:31.859 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.378234+0000", 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.012628026000000001 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.378299+0000", 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.5251000000000003e-05 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.860 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.379781+0000", 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014821809999999999 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127777 10.a 10:5d7b97e2:::benchmark_data_smithi019_617837_object34377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:57.017671+0000", 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 214.79001388500001, 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.362996535, 2023-09-30T09:55:31.861 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127777 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.017671+0000", 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.862 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.017671+0000", 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.017670+0000", 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.017678+0000", 2023-09-30T09:55:31.863 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.4009999999999997e-06 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.017697+0000", 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8776000000000001e-05 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.017704+0000", 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6590000000000001e-06 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.864 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.017770+0000", 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6773000000000006e-05 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.017783+0000", 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.29e-05 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.865 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.156842+0000", 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.13905901600000001 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.156849+0000", 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1969999999999999e-06 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.354820+0000", 2023-09-30T09:55:31.866 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1979707479999999 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.354959+0000", 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013928599999999999 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.355057+0000", 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7281000000000006e-05 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.867 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.365516+0000", 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.010459467 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.365678+0000", 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000161688 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.379830+0000", 2023-09-30T09:55:31.868 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.014152092 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.379870+0000", 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0247999999999998e-05 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.380668+0000", 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00079759199999999996 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.869 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.46583.0:127778 10.a 10:5ec2b67f:::benchmark_data_smithi019_617837_object34378:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:51:57.032869+0000", 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 214.774816063, 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.3577379030000001, 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.46583", 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2059060853", 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 127778 2023-09-30T09:55:31.870 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.032869+0000", 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.032869+0000", 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.871 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.032868+0000", 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.032873+0000", 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4320000000000001e-06 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.872 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.873 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.032886+0000", 2023-09-30T09:55:31.873 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2887e-05 2023-09-30T09:55:31.873 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.032891+0000", 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9620000000000001e-06 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.032931+0000", 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.9923999999999999e-05 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.874 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.032938+0000", 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.79e-06 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.156869+0000", 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.12393088400000001 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for peered", 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:57.156876+0000", 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.0859999999999996e-06 2023-09-30T09:55:31.875 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.357878+0000", 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2010018060000001 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.357994+0000", 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000115451 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.876 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,6", 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.358089+0000", 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.5570999999999997e-05 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.365528+0000", 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.007438638 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.368774+0000", 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0032458420000000001 2023-09-30T09:55:31.877 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.388998+0000", 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.020223984 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.389043+0000", 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5073e-05 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.878 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:51:58.390607+0000", 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015642869999999999 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47084.0:47 2.3 2:c464f6fb:::rbd_data.b7ec5ad5e03e.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 102400~10240 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e632)", 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:28.399003+0000", 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 63.408682558999999, 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0013781139999999999, 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.879 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47084", 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2534816667", 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 47 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399003+0000", 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.880 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399003+0000", 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399002+0000", 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399017+0000", 2023-09-30T09:55:31.881 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.5886999999999999e-05 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399037+0000", 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.9253999999999999e-05 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399044+0000", 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4390000000000003e-06 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.882 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399093+0000", 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8986000000000002e-05 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399192+0000", 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.8691999999999996e-05 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399289+0000", 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.7597999999999996e-05 2023-09-30T09:55:31.883 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.399904+0000", 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00061417799999999995 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.400151+0000", 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00024716499999999998 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.884 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.400316+0000", 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016493500000000001 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.400351+0000", 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5624000000000003e-05 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:28.400381+0000", 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9437e-05 2023-09-30T09:55:31.885 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37212.0:20 2.2 2:4b91fa85:::rbd_data.915c45ca5577.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 102400~10240 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e635)", 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:31.786722+0000", 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 60.020962859000001, 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0018119200000000001, 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.886 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37212", 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1629046623", 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 20 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.786722+0000", 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.786722+0000", 2023-09-30T09:55:31.887 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.786722+0000", 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.786758+0000", 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5979999999999998e-05 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.888 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.786767+0000", 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.55e-06 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.786771+0000", 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.0409999999999999e-06 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.786820+0000", 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8511999999999999e-05 2023-09-30T09:55:31.889 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.786936+0000", 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000116443 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.787095+0000", 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015877 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.890 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.787576+0000", 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00048080099999999999 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.787818+0000", 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000242492 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.788479+0000", 2023-09-30T09:55:31.891 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00066054299999999998 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.788511+0000", 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2592999999999998e-05 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:31.788534+0000", 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.2745999999999999e-05 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.892 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47234.0:13 2.3 2:c98ceba9:::rbd_header.915c45ca5577:head [watch watch cookie 140506432554176] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e635)", 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.080282+0000", 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 59.727403664999997, 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001396771, 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47234", 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/961595316", 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 13 2023-09-30T09:55:31.893 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080282+0000", 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080282+0000", 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080281+0000", 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080283+0000", 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3910000000000001e-06 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080291+0000", 2023-09-30T09:55:31.895 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.1439999999999997e-06 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080295+0000", 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4889999999999997e-06 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080336+0000", 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1152000000000002e-05 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.896 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080397+0000", 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1047e-05 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080487+0000", 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.9659000000000003e-05 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.080836+0000", 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00034879399999999998 2023-09-30T09:55:31.897 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.081079+0000", 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00024362099999999999 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.081608+0000", 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052893199999999997 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.898 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.081640+0000", 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.1569000000000002e-05 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.081678+0000", 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.8427000000000003e-05 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.899 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47234.0:55 2.3 2:c98ceba9:::rbd_header.915c45ca5577:head [watch unwatch cookie 140506566836928] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e635)", 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.110661+0000", 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 59.697024642999999, 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.002026131, 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47234", 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/961595316", 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 55 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.900 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110661+0000", 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110661+0000", 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110660+0000", 2023-09-30T09:55:31.901 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110664+0000", 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3109999999999999e-06 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110674+0000", 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.0226e-05 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.902 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110679+0000", 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4329999999999997e-06 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110724+0000", 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.5463999999999997e-05 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.903 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110811+0000", 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.7182000000000007e-05 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.110963+0000", 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00015160600000000001 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.111407+0000", 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00044430700000000001 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.904 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.111631+0000", 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00022320700000000001 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.112619+0000", 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00098842000000000005 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.112649+0000", 2023-09-30T09:55:31.905 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.0287999999999999e-05 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.112687+0000", 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.7342999999999998e-05 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47240.0:34 2.2 2:52c99878:::rbd_data.b888bffab72a.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 0~4096 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e635)", 2023-09-30T09:55:31.906 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.179971+0000", 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 59.62771412, 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0053132960000000003, 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47240", 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1961398279", 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 34 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.907 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.179971+0000", 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.179971+0000", 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.179970+0000", 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:31.908 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.179982+0000", 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2167999999999999e-05 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180002+0000", 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.037e-05 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.909 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180010+0000", 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.8369999999999997e-06 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180084+0000", 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.4010000000000005e-05 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180213+0000", 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012893099999999999 2023-09-30T09:55:31.910 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 1,6", 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.180339+0000", 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012659800000000001 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.181953+0000", 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0016132449999999999 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.911 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.185045+0000", 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0030928290000000001 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.185224+0000", 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017844399999999999 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.185259+0000", 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.4953000000000002e-05 2023-09-30T09:55:31.912 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.185284+0000", 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.5463e-05 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47249.0:42 2.3 2:d8d8c787:::rbd_data.b891b7e7ff6f.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 0~4096 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e636)", 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.308908+0000", 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 59.498777156999999, 2023-09-30T09:55:31.913 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0015250559999999999, 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47249", 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/3854307421", 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 42 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.308908+0000", 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.914 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.308908+0000", 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.308906+0000", 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999986 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.915 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.308919+0000", 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2228e-05 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.308937+0000", 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.8474999999999998e-05 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.308945+0000", 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 8.2910000000000007e-06 2023-09-30T09:55:31.916 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.309015+0000", 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.9383999999999998e-05 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.309159+0000", 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000143812 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.917 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.309336+0000", 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00017746299999999999 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.310000+0000", 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00066385100000000002 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.310320+0000", 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00032012499999999999 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.918 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.310356+0000", 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5933000000000003e-05 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.310407+0000", 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1065e-05 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.310433+0000", 2023-09-30T09:55:31.919 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.6047999999999999e-05 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.47267.0:84 2.3 2:d8d8c787:::rbd_data.b891b7e7ff6f.000000000000001a:head [delete] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e636)", 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:32.501750+0000", 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 59.305935484000003, 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0017871160000000001, 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.920 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.47267", 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/103261076", 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 84 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501750+0000", 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.921 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501750+0000", 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501749+0000", 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.9999995 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501753+0000", 2023-09-30T09:55:31.922 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.4259999999999996e-06 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501783+0000", 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.9417e-05 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501788+0000", 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9459999999999997e-06 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.923 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501823+0000", 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.5457000000000003e-05 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501870+0000", 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6999999999999997e-05 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.501933+0000", 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.2564999999999995e-05 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.924 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.502258+0000", 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000325397 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.502550+0000", 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00029174700000000002 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.503465+0000", 2023-09-30T09:55:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00091507600000000002 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.503514+0000", 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.8529999999999998e-05 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:32.503537+0000", 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3260000000000001e-05 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.926 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.57265.0:15 2.2 2:4f1735dd:::rbd_data.92b547bbad77.0000000000000003:head [sparse-read 0~1048576 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e644)", 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:43.206167+0000", 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 48.601517897000001, 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001408723, 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "started", 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.57265", 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/1540310276", 2023-09-30T09:55:31.927 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 15 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.206167+0000", 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.206167+0000", 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.928 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.206167+0000", 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.206171+0000", 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.6500000000000004e-06 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.929 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.206183+0000", 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1960999999999999e-05 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.206190+0000", 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7229999999999998e-06 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.206241+0000", 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1258000000000003e-05 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.930 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.206336+0000", 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 9.446e-05 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.207576+0000", 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0012404029999999999 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.931 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37602.0:9 2.3 2:d46db87e:::rbd_object_map.92e233479586:head [call rbd.object_map_resize in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:43.359194+0000", 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 48.448490935999999, 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014536110000000001, 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37602", 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2982768084", 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 9 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.932 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359194+0000", 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359194+0000", 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359193+0000", 2023-09-30T09:55:31.933 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359199+0000", 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.5079999999999996e-06 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359212+0000", 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.2717e-05 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.934 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359216+0000", 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9690000000000002e-06 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359284+0000", 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.7915000000000003e-05 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359453+0000", 2023-09-30T09:55:31.935 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016852800000000001 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.359692+0000", 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00023870799999999999 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.360275+0000", 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00058293600000000004 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.936 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.360410+0000", 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00013516799999999999 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.360590+0000", 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00018062300000000001 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.360624+0000", 2023-09-30T09:55:31.937 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.3719e-05 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.360648+0000", 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.3785e-05 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37602.0:27 2.3 2:d46db87e:::rbd_object_map.92e233479586:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:43.375156+0000", 2023-09-30T09:55:31.938 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 48.432529608000003, 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001642131, 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37602", 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2982768084", 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 27 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.939 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375156+0000", 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375156+0000", 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375155+0000", 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.940 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375160+0000", 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9740000000000001e-06 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375171+0000", 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.1141999999999999e-05 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.941 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375178+0000", 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.6030000000000001e-06 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375229+0000", 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1273999999999997e-05 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375334+0000", 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000105529 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.942 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.375752+0000", 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00041763500000000002 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.376282+0000", 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00052988000000000004 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.376448+0000", 2023-09-30T09:55:31.943 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016639899999999999 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.376705+0000", 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00025701899999999999 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.376757+0000", 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.1269e-05 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.944 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.376798+0000", 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.1167000000000001e-05 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37629.0:46 2.3 2:d46db87e:::rbd_object_map.92e233479586:head [call lock.unlock in=10b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e644)", 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:43.559560+0000", 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 48.248125375999997, 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.001472416, 2023-09-30T09:55:31.945 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37629", 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2699470591", 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 46 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559560+0000", 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.946 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559560+0000", 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559559+0000", 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559565+0000", 2023-09-30T09:55:31.947 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9619999999999998e-06 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559581+0000", 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.6041000000000001e-05 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559589+0000", 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 7.6839999999999996e-06 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.948 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559628+0000", 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.977e-05 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559693+0000", 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.4703999999999996e-05 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.559858+0000", 2023-09-30T09:55:31.949 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00016511400000000001 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.560395+0000", 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00053702399999999999 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.560849+0000", 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00045367200000000002 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.950 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.560975+0000", 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00012571699999999999 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.561007+0000", 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 3.2977e-05 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.561032+0000", 2023-09-30T09:55:31.951 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 2.4732e-05 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: "description": "osd_op(client.37629.0:79 2.3 2:d46db87e:::rbd_object_map.92e233479586:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e644)", 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: "initiated_at": "2023-09-30T09:54:43.582919+0000", 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: "age": 48.224766508000002, 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.0014166210000000001, 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: "type_data": { 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: "client_info": { 2023-09-30T09:55:31.952 INFO:teuthology.orchestra.run.smithi167.stdout: "client": "client.37629", 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: "client_addr": "172.21.15.19:0/2699470591", 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: "tid": 79 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: "events": [ 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "initiated", 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.582919+0000", 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "throttled", 2023-09-30T09:55:31.953 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.582919+0000", 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "header_read", 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.582918+0000", 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4294967295.999999 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "all_read", 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.582924+0000", 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.9229999999999999e-06 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.954 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "dispatched", 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.582938+0000", 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 1.4476e-05 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "queued_for_pg", 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.582944+0000", 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 6.1709999999999999e-06 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "reached_pg", 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.582994+0000", 2023-09-30T09:55:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.9196999999999997e-05 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "started", 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.583095+0000", 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000101549 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "waiting for subops from 2,7", 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.583460+0000", 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00036442700000000001 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.956 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "op_commit", 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.583858+0000", 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00039807500000000001 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.584128+0000", 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.00027015299999999997 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "sub_op_commit_rec", 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.584237+0000", 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 0.000109249 2023-09-30T09:55:31.957 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "commit_sent", 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.584291+0000", 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 5.4388999999999997e-05 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: }, 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: { 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: "event": "done", 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: "time": "2023-09-30T09:54:43.584335+0000", 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: "duration": 4.3878999999999998e-05 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.958 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.959 INFO:teuthology.orchestra.run.smithi167.stdout: } 2023-09-30T09:55:31.959 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-09-30T09:55:31.959 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-09-30T09:55:32.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:32 smithi167 ceph-mon[192880]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:32 smithi167 ceph-mon[192880]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:32 smithi167 ceph-mon[192880]: osdmap e662: 8 total, 8 up, 8 in 2023-09-30T09:55:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:32 smithi167 ceph-mon[192880]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:32 smithi167 ceph-mon[192880]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:32 smithi167 ceph-mon[192880]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[216815]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[216815]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[216815]: osdmap e662: 8 total, 8 up, 8 in 2023-09-30T09:55:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[216815]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[216815]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[216815]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[224043]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[224043]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[224043]: osdmap e662: 8 total, 8 up, 8 in 2023-09-30T09:55:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[224043]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[224043]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:32 smithi019 ceph-mon[224043]: failed to encode map e662 with expected crc 2023-09-30T09:55:32.554 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-30T09:55:33.159 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:33.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:33 smithi167 ceph-mon[192880]: pgmap v1759: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:33.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:33.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:33 smithi019 ceph-mon[216815]: pgmap v1759: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:33 smithi019 ceph-mon[224043]: pgmap v1759: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:34.020 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:55:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:55:33] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.43.0" 2023-09-30T09:55:34.039 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.4 to 1 (10000) 2023-09-30T09:55:34.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:34 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3898221929' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:34.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:34 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:34.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:34 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3898221929' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:34.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:34 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:34 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3898221929' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:34.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:34 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:35.011 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-30T09:55:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3898221929' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:55:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[216815]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[216815]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[216815]: osdmap e663: 8 total, 8 up, 8 in 2023-09-30T09:55:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[216815]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[216815]: pgmap v1761: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-30T09:55:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3898221929' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:55:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[224043]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[224043]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[224043]: osdmap e663: 8 total, 8 up, 8 in 2023-09-30T09:55:35.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[224043]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.337 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:35 smithi019 ceph-mon[224043]: pgmap v1761: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-30T09:55:35.450 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:35.455 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:55:35.455 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:55:35.455 INFO:teuthology.orchestra.run.smithi019.stdout: "num_ops": 0 2023-09-30T09:55:35.455 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:55:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:35 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3898221929' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:55:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:35 smithi167 ceph-mon[192880]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:35 smithi167 ceph-mon[192880]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:35 smithi167 ceph-mon[192880]: osdmap e663: 8 total, 8 up, 8 in 2023-09-30T09:55:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:35 smithi167 ceph-mon[192880]: failed to encode map e663 with expected crc 2023-09-30T09:55:35.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:35 smithi167 ceph-mon[192880]: pgmap v1761: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-30T09:55:36.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: osdmap e664: 8 total, 8 up, 8 in 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[224043]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: osdmap e664: 8 total, 8 up, 8 in 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:36 smithi019 ceph-mon[216815]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: osdmap e664: 8 total, 8 up, 8 in 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:36 smithi167 ceph-mon[192880]: failed to encode map e664 with expected crc 2023-09-30T09:55:36.546 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-30T09:55:37.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: osdmap e665: 8 total, 8 up, 8 in 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[216815]: pgmap v1764: 103 pgs: 2 remapped+peering, 15 peering, 86 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 4 op/s; 14 KiB/s, 0 objects/s recovering 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: osdmap e665: 8 total, 8 up, 8 in 2023-09-30T09:55:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:37 smithi019 ceph-mon[224043]: pgmap v1764: 103 pgs: 2 remapped+peering, 15 peering, 86 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 4 op/s; 14 KiB/s, 0 objects/s recovering 2023-09-30T09:55:37.434 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: osdmap e665: 8 total, 8 up, 8 in 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: failed to encode map e665 with expected crc 2023-09-30T09:55:37.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:37 smithi167 ceph-mon[192880]: pgmap v1764: 103 pgs: 2 remapped+peering, 15 peering, 86 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 4 op/s; 14 KiB/s, 0 objects/s recovering 2023-09-30T09:55:38.049 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: failed to encode map e665 with expected crc 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: osdmap e666: 8 total, 8 up, 8 in 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.050 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[224043]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: failed to encode map e665 with expected crc 2023-09-30T09:55:38.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:55:38.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: osdmap e666: 8 total, 8 up, 8 in 2023-09-30T09:55:38.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.298 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:38 smithi019 ceph-mon[216815]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: failed to encode map e665 with expected crc 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: osdmap e666: 8 total, 8 up, 8 in 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: failed to encode map e666 with expected crc 2023-09-30T09:55:38.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:38 smithi167 ceph-mon[192880]: failed to encode map e666 with expected crc 2023-09-30T09:55:39.067 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-30T09:55:39.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: osdmap e667: 8 total, 8 up, 8 in 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: pgmap v1767: 103 pgs: 2 remapped+peering, 17 peering, 84 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 5 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-30T09:55:39.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3307474307' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: osdmap e667: 8 total, 8 up, 8 in 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: pgmap v1767: 103 pgs: 2 remapped+peering, 17 peering, 84 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 5 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-30T09:55:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3307474307' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:39.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:39 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:39.436 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:55:39.437 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [], 2023-09-30T09:55:39.437 INFO:teuthology.orchestra.run.smithi019.stdout: "complaint_time": 30, 2023-09-30T09:55:39.437 INFO:teuthology.orchestra.run.smithi019.stdout: "num_blocked_ops": 0 2023-09-30T09:55:39.437 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: osdmap e667: 8 total, 8 up, 8 in 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: failed to encode map e667 with expected crc 2023-09-30T09:55:39.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: pgmap v1767: 103 pgs: 2 remapped+peering, 17 peering, 84 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 5 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-30T09:55:39.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3307474307' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:39.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:39 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-30T09:55:40.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:40 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:55:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:40 smithi019 ceph-mon[216815]: failed to encode map e668 with expected crc 2023-09-30T09:55:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:40 smithi019 ceph-mon[216815]: osdmap e668: 8 total, 8 up, 8 in 2023-09-30T09:55:40.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:40 smithi019 ceph-mon[216815]: failed to encode map e668 with expected crc 2023-09-30T09:55:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:40 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:55:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:40 smithi019 ceph-mon[224043]: failed to encode map e668 with expected crc 2023-09-30T09:55:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:40 smithi019 ceph-mon[224043]: osdmap e668: 8 total, 8 up, 8 in 2023-09-30T09:55:40.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:40 smithi019 ceph-mon[224043]: failed to encode map e668 with expected crc 2023-09-30T09:55:40.369 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-30T09:55:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:40 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-30T09:55:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:40 smithi167 ceph-mon[192880]: failed to encode map e668 with expected crc 2023-09-30T09:55:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:40 smithi167 ceph-mon[192880]: osdmap e668: 8 total, 8 up, 8 in 2023-09-30T09:55:40.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:40 smithi167 ceph-mon[192880]: failed to encode map e668 with expected crc 2023-09-30T09:55:40.608 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-30T09:55:40.766 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:41 smithi019 ceph-mon[224043]: failed to encode map e668 with expected crc 2023-09-30T09:55:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:41 smithi019 ceph-mon[224043]: failed to encode map e668 with expected crc 2023-09-30T09:55:41.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:41 smithi019 ceph-mon[224043]: pgmap v1769: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.7 KiB/s wr, 5 op/s; 2.3 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-30T09:55:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:41 smithi019 ceph-mon[216815]: failed to encode map e668 with expected crc 2023-09-30T09:55:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:41 smithi019 ceph-mon[216815]: failed to encode map e668 with expected crc 2023-09-30T09:55:41.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:41 smithi019 ceph-mon[216815]: pgmap v1769: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.7 KiB/s wr, 5 op/s; 2.3 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-30T09:55:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:41 smithi167 ceph-mon[192880]: failed to encode map e668 with expected crc 2023-09-30T09:55:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:41 smithi167 ceph-mon[192880]: failed to encode map e668 with expected crc 2023-09-30T09:55:41.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:41 smithi167 ceph-mon[192880]: pgmap v1769: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.7 KiB/s wr, 5 op/s; 2.3 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-30T09:55:41.986 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:42.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:42 smithi019 ceph-mon[224043]: 1.1a scrub starts 2023-09-30T09:55:42.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:42 smithi019 ceph-mon[224043]: 1.1a scrub ok 2023-09-30T09:55:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:42 smithi019 ceph-mon[224043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:55:42.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:42 smithi019 ceph-mon[224043]: Cluster is now healthy 2023-09-30T09:55:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:42 smithi019 ceph-mon[216815]: 1.1a scrub starts 2023-09-30T09:55:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:42 smithi019 ceph-mon[216815]: 1.1a scrub ok 2023-09-30T09:55:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:42 smithi019 ceph-mon[216815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:55:42.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:42 smithi019 ceph-mon[216815]: Cluster is now healthy 2023-09-30T09:55:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:42 smithi167 ceph-mon[192880]: 1.1a scrub starts 2023-09-30T09:55:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:42 smithi167 ceph-mon[192880]: 1.1a scrub ok 2023-09-30T09:55:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:42 smithi167 ceph-mon[192880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-30T09:55:42.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:42 smithi167 ceph-mon[192880]: Cluster is now healthy 2023-09-30T09:55:43.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:43 smithi019 ceph-mon[216815]: pgmap v1770: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.2 KiB/s wr, 3 op/s; 1.5 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:43.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:43 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/649947007' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:43.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:43 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:43.077 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.6 to 1 (10000) 2023-09-30T09:55:43.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:43 smithi019 ceph-mon[224043]: pgmap v1770: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.2 KiB/s wr, 3 op/s; 1.5 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:43.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:43 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/649947007' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:43.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:43 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:43 smithi167 ceph-mon[192880]: pgmap v1770: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.2 KiB/s wr, 3 op/s; 1.5 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:43 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/649947007' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:43 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:44.076 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:55:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:55:43] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.43.0" 2023-09-30T09:55:44.104 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-30T09:55:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:44 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:55:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:44 smithi019 ceph-mon[216815]: failed to encode map e669 with expected crc 2023-09-30T09:55:44.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:44 smithi019 ceph-mon[216815]: osdmap e669: 8 total, 8 up, 8 in 2023-09-30T09:55:44.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:44 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:55:44.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:44 smithi019 ceph-mon[224043]: failed to encode map e669 with expected crc 2023-09-30T09:55:44.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:44 smithi019 ceph-mon[224043]: osdmap e669: 8 total, 8 up, 8 in 2023-09-30T09:55:44.453 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2023-09-30T09:55:44.454 INFO:teuthology.orchestra.run.smithi019.stdout: "size": 20, 2023-09-30T09:55:44.454 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 600, 2023-09-30T09:55:44.454 INFO:teuthology.orchestra.run.smithi019.stdout: "ops": [ 2023-09-30T09:55:44.454 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.454 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDelete(2.4 e651,650 2:388a7c0f:::rbd_data.909cbf420c93.0000000000000000:head,631'101)", 2023-09-30T09:55:44.454 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.738569+0000", 2023-09-30T09:55:44.455 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.712258685000002, 2023-09-30T09:55:44.455 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000198178, 2023-09-30T09:55:44.455 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.455 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.455 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.455 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.455 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.455 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738569+0000", 2023-09-30T09:55:44.456 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.456 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.456 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.456 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.456 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738569+0000", 2023-09-30T09:55:44.456 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.456 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.457 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.457 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.457 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738569+0000", 2023-09-30T09:55:44.457 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4300000000000003e-07 2023-09-30T09:55:44.457 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.457 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.457 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.457 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738576+0000", 2023-09-30T09:55:44.458 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1469999999999998e-06 2023-09-30T09:55:44.458 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.458 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.458 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.458 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738585+0000", 2023-09-30T09:55:44.458 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.4939999999999992e-06 2023-09-30T09:55:44.459 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.459 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.459 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.459 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738595+0000", 2023-09-30T09:55:44.459 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0193999999999999e-05 2023-09-30T09:55:44.459 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.459 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.460 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.460 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738678+0000", 2023-09-30T09:55:44.460 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.2709999999999999e-05 2023-09-30T09:55:44.460 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.460 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.460 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.460 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738688+0000", 2023-09-30T09:55:44.461 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0036e-05 2023-09-30T09:55:44.461 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.461 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.461 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.461 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.738767+0000", 2023-09-30T09:55:44.461 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8954000000000003e-05 2023-09-30T09:55:44.461 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.462 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.462 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.462 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.462 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.462 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(2.4 651/650 [PushOp(2:3831950c:::rbd_directory:head, version: 644'215, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(2:3831950c:::rbd_directory:head@644'215, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: true), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 0, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:55:44.462 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.749491+0000", 2023-09-30T09:55:44.462 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.701336427999998, 2023-09-30T09:55:44.463 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000303307, 2023-09-30T09:55:44.463 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.463 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.463 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.463 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.463 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.463 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749491+0000", 2023-09-30T09:55:44.464 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.464 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.464 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.464 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.464 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749491+0000", 2023-09-30T09:55:44.464 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.464 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.464 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.465 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.465 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749492+0000", 2023-09-30T09:55:44.465 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.7599999999999996e-07 2023-09-30T09:55:44.465 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.465 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.465 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.465 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749506+0000", 2023-09-30T09:55:44.466 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4107999999999999e-05 2023-09-30T09:55:44.466 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.466 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.466 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.466 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749528+0000", 2023-09-30T09:55:44.466 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1701000000000001e-05 2023-09-30T09:55:44.466 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.467 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.467 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.467 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749550+0000", 2023-09-30T09:55:44.467 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1894000000000001e-05 2023-09-30T09:55:44.467 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.467 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.467 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.467 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749635+0000", 2023-09-30T09:55:44.468 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.4865999999999997e-05 2023-09-30T09:55:44.468 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.468 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.468 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.468 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749655+0000", 2023-09-30T09:55:44.468 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.0027e-05 2023-09-30T09:55:44.469 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.469 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.469 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.469 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.749794+0000", 2023-09-30T09:55:44.469 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013983499999999999 2023-09-30T09:55:44.469 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.470 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.470 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.471 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.471 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.471 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDelete(2.0 e651,650 2:0fa80f97:::rbd_header.b7ec5ad5e03e:head,634'177)", 2023-09-30T09:55:44.471 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.754940+0000", 2023-09-30T09:55:44.471 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.695887532999997, 2023-09-30T09:55:44.471 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000413247, 2023-09-30T09:55:44.471 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.471 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.754940+0000", 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.754940+0000", 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.472 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.754941+0000", 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3e-06 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.754957+0000", 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5586e-05 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.473 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.755006+0000", 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9480000000000001e-05 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.755039+0000", 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.2233000000000001e-05 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.755209+0000", 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00017042500000000001 2023-09-30T09:55:44.474 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.755233+0000", 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.4239999999999998e-05 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.755353+0000", 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011998300000000001 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.475 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.476 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.476 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.476 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDelete(2.0 e651,650 2:12fe5ab6:::rbd_object_map.b846f6d52223:head,635'196)", 2023-09-30T09:55:44.476 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.756336+0000", 2023-09-30T09:55:44.476 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.694491229999997, 2023-09-30T09:55:44.476 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022144800000000001, 2023-09-30T09:55:44.476 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.476 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756336+0000", 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756336+0000", 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.477 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756338+0000", 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4869999999999999e-06 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756354+0000", 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.5786000000000001e-05 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.478 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756383+0000", 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.898e-05 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756400+0000", 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7520000000000002e-05 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756491+0000", 2023-09-30T09:55:44.479 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.0476999999999994e-05 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756501+0000", 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0572e-05 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.756558+0000", 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6626000000000003e-05 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.480 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDelete(2.0 e651,650 2:177f3300:::rbd_object_map.b8d3b6aca525.0000000000000016:head,642'210)", 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.760011+0000", 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.690816626, 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021478600000000001, 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.481 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760011+0000", 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760011+0000", 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760012+0000", 2023-09-30T09:55:44.482 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.7599999999999997e-07 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760019+0000", 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6820000000000003e-06 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760034+0000", 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4659e-05 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.483 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760041+0000", 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0219999999999999e-06 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760104+0000", 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2550000000000003e-05 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.484 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760109+0000", 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.8699999999999997e-06 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.760226+0000", 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000116327 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.485 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(2.0 651/650 [PushOp(2:08141c09:::rbd_data.9324817447ae.000000000000:head, version: 644'256, data_included: [0~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(2:08141c09:::rbd_data.9324817447ae.000000000000:head@644'256, size: 4194304, copy_subset: [(0, 4194304)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 1048576, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2023-09-30T09:55:44.486 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.764380+0000", 2023-09-30T09:55:44.486 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.686448038000002, 2023-09-30T09:55:44.486 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0040975580000000003, 2023-09-30T09:55:44.486 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.486 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.486 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.486 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.764380+0000", 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.764380+0000", 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.487 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.764380+0000", 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.23e-07 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.767057+0000", 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0026767179999999998 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.767106+0000", 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.9172000000000003e-05 2023-09-30T09:55:44.488 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.767135+0000", 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.8778000000000001e-05 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.767305+0000", 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000170216 2023-09-30T09:55:44.489 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.767324+0000", 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8899e-05 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.768477+0000", 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0011529520000000001 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.490 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(2.0 651/650 [PushOp(2:08141c09:::rbd_data.9324817447ae.000000000000:head, version: 644'256, data_included: [1048576~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(2:08141c09:::rbd_data.9324817447ae.000000000000:head@644'256, size: 4194304, copy_subset: [(0, 4194304)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2097152, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to: 1048576, data_complete: false, omap_recovered_to: , omap_complete: true, error: false))])", 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.770725+0000", 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.680102632999997, 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0028605010000000001, 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.491 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.770725+0000", 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.770725+0000", 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.770727+0000", 2023-09-30T09:55:44.492 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.8470000000000001e-06 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.772366+0000", 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0016389449999999999 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.772411+0000", 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5025999999999999e-05 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.493 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.772442+0000", 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.1000999999999997e-05 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.772591+0000", 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00014929799999999999 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.494 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.772613+0000", 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1812999999999999e-05 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.773585+0000", 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00097257099999999998 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.495 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(2.0 651/650 [PushOp(2:08141c09:::rbd_data.9324817447ae.000000000000:head, version: 644'256, data_included: [2097152~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(2:08141c09:::rbd_data.9324817447ae.000000000000:head@644'256, size: 4194304, copy_subset: [(0, 4194304)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 3145728, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2097152, data_complete: false, omap_recovered_to: , omap_complete: true, error: false))])", 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.775370+0000", 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.675457373, 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.0018493450000000001, 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.775370+0000", 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.496 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.775370+0000", 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.775372+0000", 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.46e-06 2023-09-30T09:55:44.497 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.776386+0000", 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001014586 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.776412+0000", 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.5721e-05 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.498 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.776427+0000", 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.4752999999999999e-05 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.776542+0000", 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000115479 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.776560+0000", 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7728e-05 2023-09-30T09:55:44.499 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.777220+0000", 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00065961800000000005 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.500 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGPush(2.0 651/650 [PushOp(2:08141c09:::rbd_data.9324817447ae.000000000000:head, version: 644'256, data_included: [3145728~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(2:08141c09:::rbd_data.9324817447ae.000000000000:head@644'256, size: 4194304, copy_subset: [(0, 4194304)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 4194304, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to: 3145728, data_complete: false, omap_recovered_to: , omap_complete: true, error: false))])", 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:07.778588+0000", 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 36.672240035999998, 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.001932334, 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "started", 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.778588+0000", 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.501 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.778588+0000", 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.778589+0000", 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1370000000000001e-06 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.502 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.779568+0000", 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00097937900000000006 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.779589+0000", 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 2.1220999999999999e-05 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.779603+0000", 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3403e-05 2023-09-30T09:55:44.503 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.779704+0000", 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000101029 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "started", 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.779721+0000", 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7728999999999999e-05 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.504 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:07.780520+0000", 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00079843600000000002 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:9f0aabf8:::rbd_data.39f6d346161b.0000000000000002:head,72'6)", 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.024872+0000", 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.425955844000001, 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00035488700000000002, 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.505 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.024872+0000", 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.024872+0000", 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.506 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.507 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.507 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.024872+0000", 2023-09-30T09:55:44.508 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.6400000000000001e-07 2023-09-30T09:55:44.508 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.508 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.508 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.508 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.024877+0000", 2023-09-30T09:55:44.508 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.8740000000000003e-06 2023-09-30T09:55:44.508 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.508 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.024894+0000", 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.7019000000000001e-05 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.024905+0000", 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0380999999999999e-05 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.024990+0000", 2023-09-30T09:55:44.509 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5655999999999994e-05 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.025227+0000", 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00023619299999999999 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:978407d8:::rbd_data.39f6d346161b.0000000000000018:head,72'7)", 2023-09-30T09:55:44.510 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.027420+0000", 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.423407708999999, 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033754999999999999, 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.027420+0000", 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.511 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.027420+0000", 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.027421+0000", 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.7100000000000011e-07 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.027427+0000", 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6710000000000002e-06 2023-09-30T09:55:44.512 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.027444+0000", 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.6464999999999999e-05 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.027455+0000", 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.1212999999999999e-05 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.513 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.027529+0000", 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.3460999999999993e-05 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.027757+0000", 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000228769 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.514 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:92108d80:::rbd_object_map.3a142dee05d:head,73'34)", 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.028665+0000", 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.422162391000001, 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000224866, 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.515 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.028665+0000", 2023-09-30T09:55:44.516 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.516 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.516 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.516 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.516 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.028665+0000", 2023-09-30T09:55:44.516 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.516 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.516 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.028666+0000", 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8899999999999999e-07 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.028671+0000", 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.4759999999999997e-06 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.517 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.028682+0000", 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0764999999999999e-05 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.028689+0000", 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0940000000000002e-06 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.028735+0000", 2023-09-30T09:55:44.518 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.5834999999999997e-05 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.028890+0000", 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000155007 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:8b616052:::rbd_header.61888ecf613c:head,77'76)", 2023-09-30T09:55:44.519 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.029650+0000", 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.42117708, 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033911899999999998, 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.029650+0000", 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.520 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.029650+0000", 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.029651+0000", 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.0699999999999996e-07 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.029656+0000", 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 4.6879999999999998e-06 2023-09-30T09:55:44.521 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.029670+0000", 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.367e-05 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.029678+0000", 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.5350000000000004e-06 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.522 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.029731+0000", 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.2979000000000002e-05 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.029990+0000", 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00025853999999999997 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.523 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:8461c864:::rbd_object_map.61888ecf613c:head,77'78)", 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.031195+0000", 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.419632373999999, 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00033245099999999999, 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.031195+0000", 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.524 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.031195+0000", 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.031196+0000", 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7800000000000001e-07 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.525 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.031200+0000", 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.777e-06 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.031212+0000", 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.199e-05 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.031218+0000", 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.8680000000000001e-06 2023-09-30T09:55:44.526 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.031276+0000", 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7317e-05 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.031528+0000", 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000251921 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.527 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:93eaab03:::rbd_object_map.3b0467cc6f56.0000000000000008:head,81'89)", 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.032419+0000", 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.418408102000001, 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00022486000000000001, 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.032419+0000", 2023-09-30T09:55:44.528 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.032419+0000", 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.032420+0000", 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4300000000000003e-07 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.529 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.032424+0000", 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7479999999999999e-06 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.032437+0000", 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.2955999999999999e-05 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.032443+0000", 2023-09-30T09:55:44.530 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.0730000000000001e-06 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.032522+0000", 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 7.8991000000000003e-05 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.032644+0000", 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012244900000000001 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.531 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:8491f9f3:::rbd_object_map.3b2297cc5bf7:head,85'102)", 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.033455+0000", 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.41737268, 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020406299999999999, 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.532 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.033455+0000", 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.033455+0000", 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.033455+0000", 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2500000000000005e-07 2023-09-30T09:55:44.533 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.033459+0000", 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.5939999999999998e-06 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.033470+0000", 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0643e-05 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.534 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.033475+0000", 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.6559999999999997e-06 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.033540+0000", 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.4554999999999994e-05 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.535 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.033659+0000", 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00011899 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:8cb50e11:::rbd_header.90b4791e4b4:head,631'165)", 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.035448+0000", 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.415379161000001, 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00020357700000000001, 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.536 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.035448+0000", 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.035448+0000", 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.537 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.035449+0000", 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2699999999999999e-07 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.035453+0000", 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.703e-06 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.035464+0000", 2023-09-30T09:55:44.538 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.151e-05 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.035473+0000", 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 9.2450000000000004e-06 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.035524+0000", 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0250000000000002e-05 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.539 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.035652+0000", 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00012824200000000001 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:947c8f87:::rbd_header.b846f6d52223:head,635'190)", 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.037305+0000", 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.413522486000002, 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00034921399999999998, 2023-09-30T09:55:44.540 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.037305+0000", 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.037305+0000", 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.541 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.037306+0000", 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2699999999999999e-07 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.037309+0000", 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.715e-06 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.542 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.037320+0000", 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0499000000000001e-05 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.037327+0000", 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.6359999999999997e-06 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.037384+0000", 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.7340000000000003e-05 2023-09-30T09:55:44.543 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.037654+0000", 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00027039700000000002 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:8c45e075:::rbd_header.b891b7e7ff6f:head,636'218)", 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.040498+0000", 2023-09-30T09:55:44.544 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.410329821000001, 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00021240300000000001, 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.040498+0000", 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.040498+0000", 2023-09-30T09:55:44.545 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.040498+0000", 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.2300000000000001e-07 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.040502+0000", 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.7809999999999999e-06 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.546 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.040516+0000", 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.3892e-05 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.040524+0000", 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 8.3550000000000005e-06 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.547 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.040575+0000", 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.0262000000000001e-05 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.040710+0000", 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.00013548999999999999 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "description": "MOSDPGRecoveryDeleteReply(2.1 e656,655 2:978ab7ea:::rbd_id.testimg:head,644'225)", 2023-09-30T09:55:44.548 INFO:teuthology.orchestra.run.smithi019.stdout: "initiated_at": "2023-09-30T09:55:18.042534+0000", 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "age": 26.408293570000001, 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000224348, 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "type_data": { 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "flag_point": "reached pg", 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "events": [ 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "initiated", 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.042534+0000", 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.549 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "header_read", 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.042534+0000", 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "throttled", 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.042535+0000", 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 6.1900000000000002e-07 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "all_read", 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.042538+0000", 2023-09-30T09:55:44.550 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 3.4989999999999999e-06 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "dispatched", 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.042548+0000", 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 1.0271e-05 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "queued_for_pg", 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.042554+0000", 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.1739999999999999e-06 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.551 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "reached_pg", 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.042609+0000", 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 5.5272999999999997e-05 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: { 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "event": "done", 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "time": "2023-09-30T09:55:18.042758+0000", 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: "duration": 0.000149512 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: } 2023-09-30T09:55:44.552 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2023-09-30T09:55:44.553 INFO:teuthology.orchestra.run.smithi019.stdout:} 2023-09-30T09:55:44.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:44 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:55:44.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:44 smithi167 ceph-mon[192880]: failed to encode map e669 with expected crc 2023-09-30T09:55:44.553 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:44 smithi167 ceph-mon[192880]: osdmap e669: 8 total, 8 up, 8 in 2023-09-30T09:55:45.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[224043]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[224043]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[224043]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[224043]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[224043]: pgmap v1772: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.1 KiB/s wr, 3 op/s; 1.5 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[216815]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[216815]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[216815]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[216815]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:45 smithi019 ceph-mon[216815]: pgmap v1772: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.1 KiB/s wr, 3 op/s; 1.5 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:45.336 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:45.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:45 smithi167 ceph-mon[192880]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:45 smithi167 ceph-mon[192880]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:45 smithi167 ceph-mon[192880]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:45 smithi167 ceph-mon[192880]: failed to encode map e669 with expected crc 2023-09-30T09:55:45.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:45 smithi167 ceph-mon[192880]: pgmap v1772: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.1 KiB/s wr, 3 op/s; 1.5 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:46.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:46 smithi019 ceph-mon[216815]: failed to encode map e669 with expected crc 2023-09-30T09:55:46.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:46 smithi019 ceph-mon[224043]: failed to encode map e669 with expected crc 2023-09-30T09:55:46.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:46 smithi167 ceph-mon[192880]: failed to encode map e669 with expected crc 2023-09-30T09:55:47.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:47 smithi019 ceph-mon[216815]: pgmap v1773: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:47.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:47 smithi019 ceph-mon[224043]: pgmap v1773: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:47.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:47 smithi167 ceph-mon[192880]: pgmap v1773: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-30T09:55:48.106 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-30T09:55:48.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/479951691' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:48 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/479951691' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:48 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:48.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/479951691' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:48 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-30T09:55:48.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:55:48.908 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-30T09:55:49.284 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:49.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[216815]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[216815]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[216815]: osdmap e670: 8 total, 8 up, 8 in 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[216815]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[216815]: pgmap v1775: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[224043]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[224043]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[224043]: osdmap e670: 8 total, 8 up, 8 in 2023-09-30T09:55:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[224043]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[224043]: pgmap v1775: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:49.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:49 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:49 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-30T09:55:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:49 smithi167 ceph-mon[192880]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:49 smithi167 ceph-mon[192880]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:49 smithi167 ceph-mon[192880]: osdmap e670: 8 total, 8 up, 8 in 2023-09-30T09:55:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:49 smithi167 ceph-mon[192880]: failed to encode map e670 with expected crc 2023-09-30T09:55:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:49 smithi167 ceph-mon[192880]: pgmap v1775: 103 pgs: 103 active+clean; 8.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:49.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:49 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:55:50.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:50 smithi019 ceph-mon[216815]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:50 smithi019 ceph-mon[216815]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:50 smithi019 ceph-mon[216815]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:50 smithi019 ceph-mon[216815]: osdmap e671: 8 total, 8 up, 8 in 2023-09-30T09:55:50.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:50 smithi019 ceph-mon[224043]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:50 smithi019 ceph-mon[224043]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:50 smithi019 ceph-mon[224043]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:50 smithi019 ceph-mon[224043]: osdmap e671: 8 total, 8 up, 8 in 2023-09-30T09:55:50.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:50 smithi167 ceph-mon[192880]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:50 smithi167 ceph-mon[192880]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:50 smithi167 ceph-mon[192880]: failed to encode map e671 with expected crc 2023-09-30T09:55:50.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:50 smithi167 ceph-mon[192880]: osdmap e671: 8 total, 8 up, 8 in 2023-09-30T09:55:51.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:51 smithi019 ceph-mon[216815]: pgmap v1777: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:51.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:51 smithi019 ceph-mon[224043]: pgmap v1777: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:51.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:51 smithi167 ceph-mon[192880]: pgmap v1777: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:52.130 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.7 to 1 (10000) 2023-09-30T09:55:52.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3852649988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:52 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3852649988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:52.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:52 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3852649988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:52.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:52 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:52.992 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-30T09:55:53.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[216815]: pgmap v1778: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[216815]: failed to encode map e672 with expected crc 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[216815]: osdmap e672: 8 total, 8 up, 8 in 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[216815]: failed to encode map e672 with expected crc 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[224043]: pgmap v1778: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[224043]: failed to encode map e672 with expected crc 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[224043]: osdmap e672: 8 total, 8 up, 8 in 2023-09-30T09:55:53.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:53 smithi019 ceph-mon[224043]: failed to encode map e672 with expected crc 2023-09-30T09:55:53.366 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:53 smithi167 ceph-mon[192880]: pgmap v1778: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:53 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:55:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:53 smithi167 ceph-mon[192880]: failed to encode map e672 with expected crc 2023-09-30T09:55:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:53 smithi167 ceph-mon[192880]: osdmap e672: 8 total, 8 up, 8 in 2023-09-30T09:55:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:53 smithi167 ceph-mon[192880]: failed to encode map e672 with expected crc 2023-09-30T09:55:54.029 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:55:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:55:53] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.43.0" 2023-09-30T09:55:55.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:55 smithi167 ceph-mon[192880]: pgmap v1780: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:55.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:55 smithi019 ceph-mon[216815]: pgmap v1780: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:55.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:55 smithi019 ceph-mon[224043]: pgmap v1780: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:56.019 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-30T09:55:56.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3665565073' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:56.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3665565073' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:56.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3665565073' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-30T09:55:56.768 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-30T09:55:57.021 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3665565073' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:55:57.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[216815]: failed to encode map e673 with expected crc 2023-09-30T09:55:57.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[216815]: failed to encode map e673 with expected crc 2023-09-30T09:55:57.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[216815]: osdmap e673: 8 total, 8 up, 8 in 2023-09-30T09:55:57.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[216815]: pgmap v1782: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:57.140 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:55:57.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:57 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3665565073' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:55:57.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:57 smithi167 ceph-mon[192880]: failed to encode map e673 with expected crc 2023-09-30T09:55:57.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:57 smithi167 ceph-mon[192880]: failed to encode map e673 with expected crc 2023-09-30T09:55:57.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:57 smithi167 ceph-mon[192880]: osdmap e673: 8 total, 8 up, 8 in 2023-09-30T09:55:57.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:57 smithi167 ceph-mon[192880]: pgmap v1782: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:57.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3665565073' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-30T09:55:57.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[224043]: failed to encode map e673 with expected crc 2023-09-30T09:55:57.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[224043]: failed to encode map e673 with expected crc 2023-09-30T09:55:57.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[224043]: osdmap e673: 8 total, 8 up, 8 in 2023-09-30T09:55:57.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:57 smithi019 ceph-mon[224043]: pgmap v1782: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:58.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:58 smithi019 ceph-mon[216815]: failed to encode map e673 with expected crc 2023-09-30T09:55:58.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:58 smithi019 ceph-mon[224043]: failed to encode map e673 with expected crc 2023-09-30T09:55:58.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:58 smithi167 ceph-mon[192880]: failed to encode map e673 with expected crc 2023-09-30T09:55:59.044 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.1 to 1 (10000) 2023-09-30T09:55:59.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:59 smithi019 ceph-mon[216815]: pgmap v1783: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:59.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:59 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/1945829123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:55:59.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:55:59 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:55:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:59 smithi019 ceph-mon[224043]: pgmap v1783: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:59 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/1945829123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:55:59.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:55:59 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:55:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:59 smithi167 ceph-mon[192880]: pgmap v1783: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:55:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:59 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/1945829123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:55:59.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:55:59 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:55:59.836 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-30T09:56:00.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:00 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:56:00.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:00 smithi019 ceph-mon[224043]: failed to encode map e674 with expected crc 2023-09-30T09:56:00.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:00 smithi019 ceph-mon[224043]: failed to encode map e674 with expected crc 2023-09-30T09:56:00.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:00 smithi019 ceph-mon[224043]: osdmap e674: 8 total, 8 up, 8 in 2023-09-30T09:56:00.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:00 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:56:00.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:00 smithi019 ceph-mon[216815]: failed to encode map e674 with expected crc 2023-09-30T09:56:00.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:00 smithi019 ceph-mon[216815]: failed to encode map e674 with expected crc 2023-09-30T09:56:00.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:00 smithi019 ceph-mon[216815]: osdmap e674: 8 total, 8 up, 8 in 2023-09-30T09:56:00.204 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:00.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:00 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:56:00.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:00 smithi167 ceph-mon[192880]: failed to encode map e674 with expected crc 2023-09-30T09:56:00.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:00 smithi167 ceph-mon[192880]: failed to encode map e674 with expected crc 2023-09-30T09:56:00.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:00 smithi167 ceph-mon[192880]: osdmap e674: 8 total, 8 up, 8 in 2023-09-30T09:56:01.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:01 smithi019 ceph-mon[216815]: pgmap v1785: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:01.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:01 smithi019 ceph-mon[224043]: pgmap v1785: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:01.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:01 smithi167 ceph-mon[192880]: pgmap v1785: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:03.185 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-30T09:56:03.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:03 smithi019 ceph-mon[224043]: pgmap v1786: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:03.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:03 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2742811479' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:56:03.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:03.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:03 smithi019 ceph-mon[216815]: pgmap v1786: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:03.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:03 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2742811479' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:56:03.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:03.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:03 smithi167 ceph-mon[192880]: pgmap v1786: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:03.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:03 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2742811479' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-30T09:56:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:04.064 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:56:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:56:03] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2023-09-30T09:56:04.065 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-30T09:56:04.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2742811479' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:56:04.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[224043]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[224043]: osdmap e675: 8 total, 8 up, 8 in 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[224043]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[224043]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[224043]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2742811479' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[216815]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[216815]: osdmap e675: 8 total, 8 up, 8 in 2023-09-30T09:56:04.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[216815]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[216815]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[216815]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:04 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:56:04.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:04 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2742811479' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-30T09:56:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:04 smithi167 ceph-mon[192880]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:04 smithi167 ceph-mon[192880]: osdmap e675: 8 total, 8 up, 8 in 2023-09-30T09:56:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:04 smithi167 ceph-mon[192880]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:04 smithi167 ceph-mon[192880]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:04 smithi167 ceph-mon[192880]: failed to encode map e675 with expected crc 2023-09-30T09:56:04.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:04 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-30T09:56:04.796 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:05.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[216815]: pgmap v1788: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[216815]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[216815]: osdmap e676: 8 total, 8 up, 8 in 2023-09-30T09:56:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[216815]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[216815]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[216815]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-30T09:56:05.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-30T09:56:05.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[224043]: pgmap v1788: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[224043]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[224043]: osdmap e676: 8 total, 8 up, 8 in 2023-09-30T09:56:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[224043]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[224043]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[224043]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-30T09:56:05.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:05 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-30T09:56:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:05 smithi167 ceph-mon[192880]: pgmap v1788: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:05 smithi167 ceph-mon[192880]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:05 smithi167 ceph-mon[192880]: osdmap e676: 8 total, 8 up, 8 in 2023-09-30T09:56:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:05 smithi167 ceph-mon[192880]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:05 smithi167 ceph-mon[192880]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:05 smithi167 ceph-mon[192880]: failed to encode map e676 with expected crc 2023-09-30T09:56:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-30T09:56:05.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:05 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-30T09:56:07.023 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.5 to 1 (10000) 2023-09-30T09:56:07.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-30T09:56:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-30T09:56:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: osdmap e677: 8 total, 8 up, 8 in 2023-09-30T09:56:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: pgmap v1791: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:07.273 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/848616192' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:07.274 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:07 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:07.302 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-30T09:56:07.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-30T09:56:07.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: osdmap e677: 8 total, 8 up, 8 in 2023-09-30T09:56:07.303 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: pgmap v1791: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:07.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:07.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:07.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/848616192' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:07.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:07.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-30T09:56:07.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-30T09:56:07.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: osdmap e677: 8 total, 8 up, 8 in 2023-09-30T09:56:07.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: failed to encode map e677 with expected crc 2023-09-30T09:56:07.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: pgmap v1791: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:07.306 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:07.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:07.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/848616192' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:07.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:07 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:08.017 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-30T09:56:08.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:08 smithi167 ceph-mon[192880]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:56:08.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:08 smithi167 ceph-mon[192880]: failed to encode map e678 with expected crc 2023-09-30T09:56:08.272 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:08 smithi167 ceph-mon[192880]: osdmap e678: 8 total, 8 up, 8 in 2023-09-30T09:56:08.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:08 smithi019 ceph-mon[216815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:56:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:08 smithi019 ceph-mon[216815]: failed to encode map e678 with expected crc 2023-09-30T09:56:08.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:08 smithi019 ceph-mon[216815]: osdmap e678: 8 total, 8 up, 8 in 2023-09-30T09:56:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:08 smithi019 ceph-mon[224043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:56:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:08 smithi019 ceph-mon[224043]: failed to encode map e678 with expected crc 2023-09-30T09:56:08.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:08 smithi019 ceph-mon[224043]: osdmap e678: 8 total, 8 up, 8 in 2023-09-30T09:56:08.507 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:09.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[216815]: failed to encode map e678 with expected crc 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[216815]: pgmap v1793: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[216815]: failed to encode map e678 with expected crc 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[224043]: failed to encode map e678 with expected crc 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[224043]: pgmap v1793: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:09.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[224043]: failed to encode map e678 with expected crc 2023-09-30T09:56:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:56:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:56:09.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:09 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:09.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:09 smithi167 ceph-mon[192880]: failed to encode map e678 with expected crc 2023-09-30T09:56:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:09 smithi167 ceph-mon[192880]: pgmap v1793: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:09 smithi167 ceph-mon[192880]: failed to encode map e678 with expected crc 2023-09-30T09:56:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-30T09:56:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-30T09:56:09.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:09 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:11.173 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-30T09:56:11.451 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:11 smithi019 ceph-mon[224043]: pgmap v1794: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:11.452 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:11 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4142014778' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:11.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:11 smithi019 ceph-mon[216815]: pgmap v1794: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:11.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:11 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4142014778' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:11 smithi167 ceph-mon[192880]: pgmap v1794: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:11.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:11 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4142014778' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-30T09:56:12.039 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-30T09:56:12.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4142014778' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:56:12.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[216815]: failed to encode map e679 with expected crc 2023-09-30T09:56:12.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[216815]: failed to encode map e679 with expected crc 2023-09-30T09:56:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[216815]: osdmap e679: 8 total, 8 up, 8 in 2023-09-30T09:56:12.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[216815]: failed to encode map e679 with expected crc 2023-09-30T09:56:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4142014778' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:56:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[224043]: failed to encode map e679 with expected crc 2023-09-30T09:56:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[224043]: failed to encode map e679 with expected crc 2023-09-30T09:56:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[224043]: osdmap e679: 8 total, 8 up, 8 in 2023-09-30T09:56:12.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:12 smithi019 ceph-mon[224043]: failed to encode map e679 with expected crc 2023-09-30T09:56:12.416 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:12.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:12 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4142014778' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-30T09:56:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:12 smithi167 ceph-mon[192880]: failed to encode map e679 with expected crc 2023-09-30T09:56:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:12 smithi167 ceph-mon[192880]: failed to encode map e679 with expected crc 2023-09-30T09:56:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:12 smithi167 ceph-mon[192880]: osdmap e679: 8 total, 8 up, 8 in 2023-09-30T09:56:12.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:12 smithi167 ceph-mon[192880]: failed to encode map e679 with expected crc 2023-09-30T09:56:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:13 smithi167 ceph-mon[192880]: pgmap v1796: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:13 smithi167 ceph-mon[192880]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:13 smithi167 ceph-mon[192880]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:13 smithi167 ceph-mon[192880]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:13 smithi167 ceph-mon[192880]: osdmap e680: 8 total, 8 up, 8 in 2023-09-30T09:56:13.584 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[216815]: pgmap v1796: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[216815]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[216815]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[216815]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[216815]: osdmap e680: 8 total, 8 up, 8 in 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[224043]: pgmap v1796: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[224043]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[224043]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[224043]: failed to encode map e680 with expected crc 2023-09-30T09:56:13.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:13 smithi019 ceph-mon[224043]: osdmap e680: 8 total, 8 up, 8 in 2023-09-30T09:56:14.054 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:56:13 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:56:13] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2023-09-30T09:56:15.025 INFO:teuthology.orchestra.run.smithi019.stderr:reweighted osd.0 to 1 (10000) 2023-09-30T09:56:15.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:15 smithi019 ceph-mon[216815]: pgmap v1798: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:15.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:15 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/953566519' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:56:15.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:15 smithi019 ceph-mon[224043]: pgmap v1798: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:15.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:15 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/953566519' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:56:15.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:15 smithi167 ceph-mon[192880]: pgmap v1798: 103 pgs: 103 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:15.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:15 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/953566519' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:56:15.863 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-30T09:56:16.252 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:16.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/953566519' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:56:16.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: osdmap e681: 8 total, 8 up, 8 in 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[216815]: osdmap e682: 8 total, 8 up, 8 in 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/953566519' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: osdmap e681: 8 total, 8 up, 8 in 2023-09-30T09:56:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:16 smithi019 ceph-mon[224043]: osdmap e682: 8 total, 8 up, 8 in 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/953566519' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: osdmap e681: 8 total, 8 up, 8 in 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: failed to encode map e681 with expected crc 2023-09-30T09:56:16.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:16 smithi167 ceph-mon[192880]: osdmap e682: 8 total, 8 up, 8 in 2023-09-30T09:56:17.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: pgmap v1801: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 6 op/s; 26 KiB/s, 0 objects/s recovering 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: 1.a deep-scrub starts 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: 1.a deep-scrub ok 2023-09-30T09:56:17.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[224043]: osdmap e683: 8 total, 8 up, 8 in 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: pgmap v1801: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 6 op/s; 26 KiB/s, 0 objects/s recovering 2023-09-30T09:56:17.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: 1.a deep-scrub starts 2023-09-30T09:56:17.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: 1.a deep-scrub ok 2023-09-30T09:56:17.337 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:17 smithi019 ceph-mon[216815]: osdmap e683: 8 total, 8 up, 8 in 2023-09-30T09:56:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: failed to encode map e682 with expected crc 2023-09-30T09:56:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: pgmap v1801: 103 pgs: 2 peering, 101 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 6 op/s; 26 KiB/s, 0 objects/s recovering 2023-09-30T09:56:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: 1.a deep-scrub starts 2023-09-30T09:56:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: 1.a deep-scrub ok 2023-09-30T09:56:17.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:17 smithi167 ceph-mon[192880]: osdmap e683: 8 total, 8 up, 8 in 2023-09-30T09:56:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:18.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[216815]: osdmap e684: 8 total, 8 up, 8 in 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:18.088 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:18 smithi019 ceph-mon[224043]: osdmap e684: 8 total, 8 up, 8 in 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: failed to encode map e683 with expected crc 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:18.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:18 smithi167 ceph-mon[192880]: osdmap e684: 8 total, 8 up, 8 in 2023-09-30T09:56:19.028 INFO:teuthology.orchestra.run.smithi019.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-30T09:56:19.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: pgmap v1804: 103 pgs: 1 active+undersized+remapped, 2 peering, 100 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 7 op/s; 33 KiB/s, 0 objects/s recovering 2023-09-30T09:56:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3847848825' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:56:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3847848825' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:56:19.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[224043]: osdmap e685: 8 total, 8 up, 8 in 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: pgmap v1804: 103 pgs: 1 active+undersized+remapped, 2 peering, 100 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 7 op/s; 33 KiB/s, 0 objects/s recovering 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3847848825' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3847848825' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:56:19.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:19 smithi019 ceph-mon[216815]: osdmap e685: 8 total, 8 up, 8 in 2023-09-30T09:56:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: failed to encode map e684 with expected crc 2023-09-30T09:56:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: pgmap v1804: 103 pgs: 1 active+undersized+remapped, 2 peering, 100 active+clean; 8.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 7 op/s; 33 KiB/s, 0 objects/s recovering 2023-09-30T09:56:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3847848825' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-30T09:56:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3847848825' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-30T09:56:19.523 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:19 smithi167 ceph-mon[192880]: osdmap e685: 8 total, 8 up, 8 in 2023-09-30T09:56:19.936 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-30T09:56:19.937 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-30T09:56:19.937 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-30T09:56:19.937 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-30T09:56:19.937 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-30T09:56:19.937 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:56:20.314 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:20.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[216815]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[216815]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[216815]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[216815]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[216815]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[216815]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[216815]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[224043]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[224043]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[224043]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[224043]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[224043]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[224043]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:20 smithi019 ceph-mon[224043]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:20 smithi167 ceph-mon[192880]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:20 smithi167 ceph-mon[192880]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:20 smithi167 ceph-mon[192880]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:20 smithi167 ceph-mon[192880]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:20 smithi167 ceph-mon[192880]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:20 smithi167 ceph-mon[192880]: failed to encode map e685 with expected crc 2023-09-30T09:56:20.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:20 smithi167 ceph-mon[192880]: failed to encode map e685 with expected crc 2023-09-30T09:56:21.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:21 smithi019 ceph-mon[216815]: pgmap v1806: 103 pgs: 1 active+undersized+remapped, 2 peering, 100 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2023-09-30T09:56:21.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:21 smithi019 ceph-mon[224043]: pgmap v1806: 103 pgs: 1 active+undersized+remapped, 2 peering, 100 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2023-09-30T09:56:21.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:21 smithi167 ceph-mon[192880]: pgmap v1806: 103 pgs: 1 active+undersized+remapped, 2 peering, 100 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2023-09-30T09:56:22.183 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:56:22.184 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":685,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:56:19.015407+0000","last_up_change":"2023-09-30T09:55:05.155866+0000","last_in_change":"2023-09-30T09:50:40.358426+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"276","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"260","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-30T09:11:07.846610+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"644","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":644,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-30T09:12:06.877159+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-30T09:12:07.385950+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":94,"last_clean_end":617,"up_from":649,"up_thru":684,"down_at":618,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":2441438971},{"type":"v1","addr":"172.21.15.19:6803","nonce":2441438971}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":2441438971},{"type":"v1","addr":"172.21.15.19:6805","nonce":2441438971}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":2441438971},{"type":"v1","addr":"172.21.15.19:6809","nonce":2441438971}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":2441438971},{"type":"v1","addr":"172.21.15.19:6807","nonce":2441438971}]},"public_addr":"172.21.15.19:6803/2441438971","cluster_addr":"172.21.15.19:6805/2441438971","heartbeat_back_addr":"172.21.15.19:6809/2441438971","heartbeat_front_addr":"172.21.15.19:6807/2441438971","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":99,"last_clean_end":286,"up_from":301,"up_thru":684,"down_at":287,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":2442769377},{"type":"v1","addr":"172.21.15.19:6811","nonce":2442769377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":2442769377},{"type":"v1","addr":"172.21.15.19:6813","nonce":2442769377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":2442769377},{"type":"v1","addr":"172.21.15.19:6817","nonce":2442769377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":2442769377},{"type":"v1","addr":"172.21.15.19:6815","nonce":2442769377}]},"public_addr":"172.21.15.19:6811/2442769377","cluster_addr":"172.21.15.19:6813/2442769377","heartbeat_back_addr":"172.21.15.19:6817/2442769377","heartbeat_front_addr":"172.21.15.19:6815/2442769377","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":677,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6819","nonce":2623064465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6821","nonce":2623064465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6825","nonce":2623064465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6823","nonce":2623064465}]},"public_addr":"172.21.15.19:6819/2623064465","cluster_addr":"172.21.15.19:6821/2623064465","heartbeat_back_addr":"172.21.15.19:6825/2623064465","heartbeat_front_addr":"172.21.15.19:6823/2623064465","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":684,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6827","nonce":3712654409}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6829","nonce":3712654409}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6833","nonce":3712654409}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6831","nonce":3712654409}]},"public_addr":"172.21.15.19:6827/3712654409","cluster_addr":"172.21.15.19:6829/3712654409","heartbeat_back_addr":"172.21.15.19:6833/3712654409","heartbeat_front_addr":"172.21.15.19:6831/3712654409","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":681,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":427,"up_from":489,"up_thru":684,"down_at":428,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":3203515215},{"type":"v1","addr":"172.21.15.167:6809","nonce":3203515215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":3203515215},{"type":"v1","addr":"172.21.15.167:6811","nonce":3203515215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":3203515215},{"type":"v1","addr":"172.21.15.167:6815","nonce":3203515215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":3203515215},{"type":"v1","addr":"172.21.15.167:6813","nonce":3203515215}]},"public_addr":"172.21.15.167:6809/3203515215","cluster_addr":"172.21.15.167:6811/3203515215","heartbeat_back_addr":"172.21.15.167:6815/3203515215","heartbeat_front_addr":"172.21.15.167:6813/3203515215","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":684,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":684,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":617},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":286},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":427},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.10","mappings":[{"from":7,"to":1}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/4266586334":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/1275272106":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.167:0/625642582":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/956155":"2023-10-01T09:14:41.945226+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.167:6835/1123065418":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/861352263":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6800/605197989":"2023-10-01T09:16:02.836873+0000","172.21.15.167:6834/1123065418":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/3324367156":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/1856357891":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3152350528":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/1638946842":"2023-10-01T09:14:41.945226+0000","172.21.15.167:0/798457358":"2023-10-01T09:15:21.453516+0000","172.21.15.167:0/320118286":"2023-10-01T09:15:21.453516+0000","172.21.15.19:6800/2650629233":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/238111816":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6800/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3931489393":"2023-10-01T09:14:41.945226+0000","172.21.15.19:6801/2650629233":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/936416645":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/2022890892":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/1679148187":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/309324931":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/2218856933":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3289282363":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/554243229":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6801/605197989":"2023-10-01T09:16:02.836873+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:56:22.892 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-30T09:56:22.892 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-30T09:56:23.258 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:23.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:23 smithi019 ceph-mon[216815]: pgmap v1807: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-09-30T09:56:23.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:23 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/869069247' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:56:23.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:23 smithi019 ceph-mon[224043]: pgmap v1807: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-09-30T09:56:23.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:23 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/869069247' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:56:23.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:23 smithi167 ceph-mon[192880]: pgmap v1807: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-09-30T09:56:23.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:23 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/869069247' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:56:24.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:56:23 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:56:23] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.43.0" 2023-09-30T09:56:25.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:25 smithi019 ceph-mon[216815]: pgmap v1808: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-30T09:56:25.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:25 smithi019 ceph-mon[224043]: pgmap v1808: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-30T09:56:25.253 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:56:25.253 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":685,"fsid":"608495ac-5f70-11ee-8db4-212e2dc638e7","created":"2023-09-30T09:05:29.547224+0000","modified":"2023-09-30T09:56:19.015407+0000","last_up_change":"2023-09-30T09:55:05.155866+0000","last_in_change":"2023-09-30T09:50:40.358426+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-30T09:08:36.585315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"276","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"260","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-30T09:11:07.846610+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"644","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":644,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-30T09:12:06.877159+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-30T09:12:07.385950+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"91b26362-fc1a-4b53-a717-f8d48720a017","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":94,"last_clean_end":617,"up_from":649,"up_thru":684,"down_at":618,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":2441438971},{"type":"v1","addr":"172.21.15.19:6803","nonce":2441438971}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":2441438971},{"type":"v1","addr":"172.21.15.19:6805","nonce":2441438971}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":2441438971},{"type":"v1","addr":"172.21.15.19:6809","nonce":2441438971}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":2441438971},{"type":"v1","addr":"172.21.15.19:6807","nonce":2441438971}]},"public_addr":"172.21.15.19:6803/2441438971","cluster_addr":"172.21.15.19:6805/2441438971","heartbeat_back_addr":"172.21.15.19:6809/2441438971","heartbeat_front_addr":"172.21.15.19:6807/2441438971","state":["exists","up"]},{"osd":1,"uuid":"64e9dc26-b495-4d5f-83f7-a38a9466d999","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":99,"last_clean_end":286,"up_from":301,"up_thru":684,"down_at":287,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":2442769377},{"type":"v1","addr":"172.21.15.19:6811","nonce":2442769377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":2442769377},{"type":"v1","addr":"172.21.15.19:6813","nonce":2442769377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":2442769377},{"type":"v1","addr":"172.21.15.19:6817","nonce":2442769377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":2442769377},{"type":"v1","addr":"172.21.15.19:6815","nonce":2442769377}]},"public_addr":"172.21.15.19:6811/2442769377","cluster_addr":"172.21.15.19:6813/2442769377","heartbeat_back_addr":"172.21.15.19:6817/2442769377","heartbeat_front_addr":"172.21.15.19:6815/2442769377","state":["exists","up"]},{"osd":2,"uuid":"c5256ec9-304c-4028-8256-970c6bd03bb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":677,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6819","nonce":2623064465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6821","nonce":2623064465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6825","nonce":2623064465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":2623064465},{"type":"v1","addr":"172.21.15.19:6823","nonce":2623064465}]},"public_addr":"172.21.15.19:6819/2623064465","cluster_addr":"172.21.15.19:6821/2623064465","heartbeat_back_addr":"172.21.15.19:6825/2623064465","heartbeat_front_addr":"172.21.15.19:6823/2623064465","state":["exists","up"]},{"osd":3,"uuid":"1f9fc59d-faab-4c30-938c-f244c7f972d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":684,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6827","nonce":3712654409}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6829","nonce":3712654409}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6833","nonce":3712654409}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":3712654409},{"type":"v1","addr":"172.21.15.19:6831","nonce":3712654409}]},"public_addr":"172.21.15.19:6827/3712654409","cluster_addr":"172.21.15.19:6829/3712654409","heartbeat_back_addr":"172.21.15.19:6833/3712654409","heartbeat_front_addr":"172.21.15.19:6831/3712654409","state":["exists","up"]},{"osd":4,"uuid":"cb4cb89f-3933-48cd-a588-95a2f2bf8d13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":681,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6801","nonce":3844930573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6803","nonce":3844930573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6807","nonce":3844930573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3844930573},{"type":"v1","addr":"172.21.15.167:6805","nonce":3844930573}]},"public_addr":"172.21.15.167:6801/3844930573","cluster_addr":"172.21.15.167:6803/3844930573","heartbeat_back_addr":"172.21.15.167:6807/3844930573","heartbeat_front_addr":"172.21.15.167:6805/3844930573","state":["exists","up"]},{"osd":5,"uuid":"1bc3ed47-225a-429a-a825-36b3357cf271","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":427,"up_from":489,"up_thru":684,"down_at":428,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":3203515215},{"type":"v1","addr":"172.21.15.167:6809","nonce":3203515215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":3203515215},{"type":"v1","addr":"172.21.15.167:6811","nonce":3203515215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":3203515215},{"type":"v1","addr":"172.21.15.167:6815","nonce":3203515215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":3203515215},{"type":"v1","addr":"172.21.15.167:6813","nonce":3203515215}]},"public_addr":"172.21.15.167:6809/3203515215","cluster_addr":"172.21.15.167:6811/3203515215","heartbeat_back_addr":"172.21.15.167:6815/3203515215","heartbeat_front_addr":"172.21.15.167:6813/3203515215","state":["exists","up"]},{"osd":6,"uuid":"3a4720ae-d64a-4bda-b7d3-478ec2742819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":684,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6817","nonce":1453229455}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6819","nonce":1453229455}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6823","nonce":1453229455}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":1453229455},{"type":"v1","addr":"172.21.15.167:6821","nonce":1453229455}]},"public_addr":"172.21.15.167:6817/1453229455","cluster_addr":"172.21.15.167:6819/1453229455","heartbeat_back_addr":"172.21.15.167:6823/1453229455","heartbeat_front_addr":"172.21.15.167:6821/1453229455","state":["exists","up"]},{"osd":7,"uuid":"88b4cb7b-99e4-46e9-a9aa-24f886b0174b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":684,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6825","nonce":1838064506}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6827","nonce":1838064506}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6831","nonce":1838064506}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":1838064506},{"type":"v1","addr":"172.21.15.167:6829","nonce":1838064506}]},"public_addr":"172.21.15.167:6825/1838064506","cluster_addr":"172.21.15.167:6827/1838064506","heartbeat_back_addr":"172.21.15.167:6831/1838064506","heartbeat_front_addr":"172.21.15.167:6829/1838064506","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:07:45.376854+0000","dead_epoch":617},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:10.184576+0000","dead_epoch":286},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:33.944545+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:08:59.144092+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:20.619613+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:09:42.660492+0000","dead_epoch":427},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:04.803900+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-30T09:10:26.902886+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.10","mappings":[{"from":7,"to":1}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/956155":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/2702007278":"2023-10-01T09:06:26.045265+0000","172.21.15.167:0/625642582":"2023-10-01T09:15:21.453516+0000","172.21.15.19:6800/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6801/1837817935":"2023-10-01T09:06:26.045265+0000","172.21.15.167:6835/1123065418":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/861352263":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/3001919590":"2023-10-01T09:06:26.045265+0000","172.21.15.19:6800/605197989":"2023-10-01T09:16:02.836873+0000","172.21.15.167:6834/1123065418":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/4266586334":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/3289282363":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/3324367156":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/1856357891":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/2559536428":"2023-10-01T09:06:26.045265+0000","172.21.15.19:0/3152350528":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/1638946842":"2023-10-01T09:14:41.945226+0000","172.21.15.167:0/798457358":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/1275272106":"2023-10-01T09:16:02.836873+0000","172.21.15.167:0/320118286":"2023-10-01T09:15:21.453516+0000","172.21.15.19:6800/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/238111816":"2023-10-01T09:14:41.945226+0000","172.21.15.19:6800/2650629233":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/3793098865":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6800/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3931489393":"2023-10-01T09:14:41.945226+0000","172.21.15.19:6801/2650629233":"2023-10-01T09:14:41.945226+0000","172.21.15.19:0/936416645":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/2022890892":"2023-10-01T09:15:21.453516+0000","172.21.15.19:0/1679148187":"2023-10-01T09:11:06.498249+0000","172.21.15.19:0/3546350991":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/4099317090":"2023-10-01T09:05:58.980233+0000","172.21.15.19:0/554243229":"2023-10-01T09:16:02.836873+0000","172.21.15.19:6801/605197989":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/309324931":"2023-10-01T09:16:02.836873+0000","172.21.15.19:0/572446726":"2023-10-01T09:05:58.980233+0000","172.21.15.19:6801/1814390438":"2023-10-01T09:11:06.498249+0000","172.21.15.167:0/2218856933":"2023-10-01T09:15:21.453516+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-30T09:56:25.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:25 smithi167 ceph-mon[192880]: pgmap v1808: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-30T09:56:26.111 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-30T09:56:26.111 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-30T09:56:26.112 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-30T09:56:26.112 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-30T09:56:26.112 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-30T09:56:26.113 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-30T09:56:26.113 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-30T09:56:26.114 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-30T09:56:26.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:26 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2278130769' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:56:26.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:26 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2278130769' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:56:26.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:26 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2278130769' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-30T09:56:27.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:27 smithi019 ceph-mon[216815]: pgmap v1809: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-30T09:56:27.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:27 smithi019 ceph-mon[224043]: pgmap v1809: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-30T09:56:27.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:27 smithi167 ceph-mon[192880]: pgmap v1809: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-30T09:56:27.682 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:27.734 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:27.799 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:27.935 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:28.078 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:28.080 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:28.290 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:28.816 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:29.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:29 smithi167 ceph-mon[192880]: pgmap v1810: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-30T09:56:29.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:29 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:29.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:29 smithi019 ceph-mon[224043]: pgmap v1810: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-30T09:56:29.563 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:29 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:29.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:29 smithi019 ceph-mon[216815]: pgmap v1810: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-30T09:56:29.563 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:29 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' 2023-09-30T09:56:31.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:31 smithi019 ceph-mon[216815]: pgmap v1811: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-30T09:56:31.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:31 smithi019 ceph-mon[224043]: pgmap v1811: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-30T09:56:31.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:31 smithi167 ceph-mon[192880]: pgmap v1811: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-30T09:56:33.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:33 smithi019 ceph-mon[216815]: pgmap v1812: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:33.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:33 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:33 smithi019 ceph-mon[224043]: pgmap v1812: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:33.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:33 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:33.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:33 smithi167 ceph-mon[192880]: pgmap v1812: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:33.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:33 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:34.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:56:33 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:56:33] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.43.0" 2023-09-30T09:56:35.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:35 smithi167 ceph-mon[192880]: pgmap v1813: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:35.325 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:35 smithi019 ceph-mon[216815]: pgmap v1813: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:35.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:35 smithi019 ceph-mon[224043]: pgmap v1813: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:35.728 INFO:teuthology.orchestra.run.smithi019.stdout:1292785156374 2023-09-30T09:56:35.729 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-30T09:56:37.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:37 smithi167 ceph-mon[192880]: pgmap v1814: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:37.566 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:37 smithi019 ceph-mon[216815]: pgmap v1814: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:37.566 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:37 smithi019 ceph-mon[224043]: pgmap v1814: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:39.223 INFO:teuthology.orchestra.run.smithi019.stdout:2787433775124 2023-09-30T09:56:39.223 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-30T09:56:39.341 INFO:teuthology.orchestra.run.smithi019.stdout:2100239007892 2023-09-30T09:56:39.342 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-30T09:56:39.434 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:39 smithi019 ceph-mon[216815]: pgmap v1815: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:39.434 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:39 smithi019 ceph-mon[224043]: pgmap v1815: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:39.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:39 smithi167 ceph-mon[192880]: pgmap v1815: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:39.745 INFO:teuthology.orchestra.run.smithi019.stdout:231928234538 2023-09-30T09:56:39.746 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-30T09:56:41.295 INFO:teuthology.orchestra.run.smithi019.stdout:128849019450 2023-09-30T09:56:41.295 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-30T09:56:41.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:41 smithi167 ceph-mon[192880]: pgmap v1816: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:41.585 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:41 smithi019 ceph-mon[216815]: pgmap v1816: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:41.585 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:41 smithi019 ceph-mon[224043]: pgmap v1816: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:41.749 INFO:teuthology.orchestra.run.smithi019.stdout:468151435688 2023-09-30T09:56:41.750 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-30T09:56:41.935 INFO:teuthology.orchestra.run.smithi019.stdout:446676599212 2023-09-30T09:56:41.936 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-30T09:56:41.991 INFO:teuthology.orchestra.run.smithi019.stdout:197568496177 2023-09-30T09:56:41.991 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-30T09:56:42.186 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:42.437 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:42.662 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:42.845 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:42.941 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:43.184 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:43.212 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:43 smithi019 ceph-mon[224043]: pgmap v1817: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:43.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:43 smithi019 ceph-mon[216815]: pgmap v1817: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:43.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:43 smithi167 ceph-mon[192880]: pgmap v1817: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:43.993 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:56:43 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:56:43] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.43.0" 2023-09-30T09:56:45.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:45 smithi019 ceph-mon[216815]: pgmap v1818: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:45.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:45 smithi019 ceph-mon[224043]: pgmap v1818: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:45.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:45 smithi167 ceph-mon[192880]: pgmap v1818: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:47.060 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:47.213 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:47.390 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:47 smithi019 ceph-mon[216815]: pgmap v1819: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:47.390 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:47 smithi019 ceph-mon[224043]: pgmap v1819: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:47.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:47 smithi167 ceph-mon[192880]: pgmap v1819: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:47.607 INFO:teuthology.orchestra.run.smithi019.stdout:2787433775126 2023-09-30T09:56:47.985 INFO:teuthology.orchestra.run.smithi019.stdout:1292785156377 2023-09-30T09:56:48.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/641798826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:56:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:48 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/13934524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:56:48.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:48 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/641798826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:56:48.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:48 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/13934524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:56:48.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:48 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:48.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/641798826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-30T09:56:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:48 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/13934524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-30T09:56:48.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:48 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:56:49.456 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:49 smithi019 ceph-mon[216815]: pgmap v1820: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:49.456 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:49 smithi019 ceph-mon[224043]: pgmap v1820: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:49.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:49 smithi167 ceph-mon[192880]: pgmap v1820: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:50.102 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1292785156374 got 1292785156377 for osd.1 2023-09-30T09:56:50.102 DEBUG:teuthology.parallel:result is None 2023-09-30T09:56:50.588 INFO:teuthology.orchestra.run.smithi019.stdout:128849019453 2023-09-30T09:56:51.001 INFO:teuthology.orchestra.run.smithi019.stdout:197568496180 2023-09-30T09:56:51.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:51 smithi019 ceph-mon[216815]: pgmap v1821: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:51.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/4011186454' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:56:51.470 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:51 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3275323931' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:56:51.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:51 smithi019 ceph-mon[224043]: pgmap v1821: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:51.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/4011186454' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:56:51.471 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:51 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3275323931' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:56:51.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:51 smithi167 ceph-mon[192880]: pgmap v1821: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:51.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/4011186454' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-30T09:56:51.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:51 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3275323931' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-30T09:56:51.972 INFO:teuthology.orchestra.run.smithi019.stdout:231928234542 2023-09-30T09:56:52.151 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019450 got 128849019453 for osd.4 2023-09-30T09:56:52.152 DEBUG:teuthology.parallel:result is None 2023-09-30T09:56:52.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:52 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/2501520388' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:56:52.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:52 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/2501520388' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:56:52.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:52 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/2501520388' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-30T09:56:52.743 INFO:teuthology.orchestra.run.smithi019.stdout:2100239007895 2023-09-30T09:56:52.993 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496177 got 197568496180 for osd.6 2023-09-30T09:56:52.993 DEBUG:teuthology.parallel:result is None 2023-09-30T09:56:53.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:53 smithi167 ceph-mon[192880]: pgmap v1822: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:53.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:53 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3712323407' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:56:53.530 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:53 smithi019 ceph-mon[216815]: pgmap v1822: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:53.530 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:53 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3712323407' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:56:53.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:53 smithi019 ceph-mon[224043]: pgmap v1822: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:53.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:53 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3712323407' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-30T09:56:53.800 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2787433775124 got 2787433775126 for osd.0 2023-09-30T09:56:53.800 DEBUG:teuthology.parallel:result is None 2023-09-30T09:56:54.082 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:56:53 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:56:53] "GET /metrics HTTP/1.1" 200 34077 "" "Prometheus/2.43.0" 2023-09-30T09:56:54.083 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234538 got 231928234542 for osd.7 2023-09-30T09:56:54.083 DEBUG:teuthology.parallel:result is None 2023-09-30T09:56:54.840 INFO:teuthology.orchestra.run.smithi019.stdout:468151435690 2023-09-30T09:56:55.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:55 smithi019 ceph-mon[216815]: pgmap v1823: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:55.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:55 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/543514844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:56:55.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:55 smithi019 ceph-mon[224043]: pgmap v1823: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:55.265 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:55 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/543514844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:56:55.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:55 smithi167 ceph-mon[192880]: pgmap v1823: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:55.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:55 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/543514844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-30T09:56:55.760 INFO:teuthology.orchestra.run.smithi019.stdout:446676599215 2023-09-30T09:56:55.812 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2100239007892 got 2100239007895 for osd.5 2023-09-30T09:56:55.812 DEBUG:teuthology.parallel:result is None 2023-09-30T09:56:56.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:56 smithi019 ceph-mon[216815]: from='client.? 172.21.15.19:0/3301776223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:56:56.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:56 smithi019 ceph-mon[224043]: from='client.? 172.21.15.19:0/3301776223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:56:56.271 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:56 smithi167 ceph-mon[192880]: from='client.? 172.21.15.19:0/3301776223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-30T09:56:56.561 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435688 got 468151435690 for osd.3 2023-09-30T09:56:56.561 DEBUG:teuthology.parallel:result is None 2023-09-30T09:56:56.778 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676599212 got 446676599215 for osd.2 2023-09-30T09:56:56.779 DEBUG:teuthology.parallel:result is None 2023-09-30T09:56:56.779 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-30T09:56:56.779 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:56:57.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:57 smithi019 ceph-mon[216815]: pgmap v1824: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:57.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:57 smithi019 ceph-mon[224043]: pgmap v1824: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:57.144 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:56:57.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:57 smithi167 ceph-mon[192880]: pgmap v1824: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:58.984 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:56:58.986 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:56:59.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:56:59 smithi019 ceph-mon[216815]: pgmap v1825: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:59.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:56:59 smithi019 ceph-mon[224043]: pgmap v1825: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:59.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:56:59 smithi167 ceph-mon[192880]: pgmap v1825: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:56:59.872 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1825,"stamp":"2023-09-30T09:56:58.126115+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984487,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11978,"num_read_kb":26422,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":441,"num_bytes_recovered":17972206,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5051364,"kb_used_data":142280,"kb_used_omap":35,"kb_used_meta":4908956,"kb_avail":744909852,"statfs":{"total":767960285184,"available":762787688448,"internally_reserved":0,"allocated":145694720,"data_stored":122021999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36380,"internal_metadata":5026771428},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003610"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":1178,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024957+0000","last_change":"2023-09-30T09:55:20.016692+0000","last_active":"2023-09-30T09:56:18.024957+0000","last_peered":"2023-09-30T09:56:18.024957+0000","last_clean":"2023-09-30T09:56:18.024957+0000","last_became_active":"2023-09-30T09:55:20.016204+0000","last_became_peered":"2023-09-30T09:55:20.016204+0000","last_unstale":"2023-09-30T09:56:18.024957+0000","last_undegraded":"2023-09-30T09:56:18.024957+0000","last_fullsized":"2023-09-30T09:56:18.024957+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:56:35.749055+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":721,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028158+0000","last_change":"2023-09-30T09:50:41.402505+0000","last_active":"2023-09-30T09:56:18.028158+0000","last_peered":"2023-09-30T09:56:18.028158+0000","last_clean":"2023-09-30T09:56:18.028158+0000","last_became_active":"2023-09-30T09:50:41.402099+0000","last_became_peered":"2023-09-30T09:50:41.402099+0000","last_unstale":"2023-09-30T09:56:18.028158+0000","last_undegraded":"2023-09-30T09:56:18.028158+0000","last_fullsized":"2023-09-30T09:56:18.028158+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:22:16.978324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":414,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.023114+0000","last_change":"2023-09-30T09:55:31.011863+0000","last_active":"2023-09-30T09:56:18.023114+0000","last_peered":"2023-09-30T09:56:18.023114+0000","last_clean":"2023-09-30T09:56:18.023114+0000","last_became_active":"2023-09-30T09:55:31.011503+0000","last_became_peered":"2023-09-30T09:55:31.011503+0000","last_unstale":"2023-09-30T09:56:18.023114+0000","last_undegraded":"2023-09-30T09:56:18.023114+0000","last_fullsized":"2023-09-30T09:56:18.023114+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:08:04.366818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":1150,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.034683+0000","last_change":"2023-09-30T09:56:19.034683+0000","last_active":"2023-09-30T09:56:19.034683+0000","last_peered":"2023-09-30T09:56:19.034683+0000","last_clean":"2023-09-30T09:56:19.034683+0000","last_became_active":"2023-09-30T09:56:19.034285+0000","last_became_peered":"2023-09-30T09:56:19.034285+0000","last_unstale":"2023-09-30T09:56:19.034683+0000","last_undegraded":"2023-09-30T09:56:19.034683+0000","last_fullsized":"2023-09-30T09:56:19.034683+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:03:31.719439+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":260,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031416+0000","last_change":"2023-09-30T09:56:04.199026+0000","last_active":"2023-09-30T09:56:18.031416+0000","last_peered":"2023-09-30T09:56:18.031416+0000","last_clean":"2023-09-30T09:56:18.031416+0000","last_became_active":"2023-09-30T09:56:04.198742+0000","last_became_peered":"2023-09-30T09:56:04.198742+0000","last_unstale":"2023-09-30T09:56:18.031416+0000","last_undegraded":"2023-09-30T09:56:18.031416+0000","last_fullsized":"2023-09-30T09:56:18.031416+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:04:30.898770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026082+0000","last_change":"2023-09-30T09:56:16.034010+0000","last_active":"2023-09-30T09:56:19.026082+0000","last_peered":"2023-09-30T09:56:19.026082+0000","last_clean":"2023-09-30T09:56:19.026082+0000","last_became_active":"2023-09-30T09:56:16.033459+0000","last_became_peered":"2023-09-30T09:56:16.033459+0000","last_unstale":"2023-09-30T09:56:19.026082+0000","last_undegraded":"2023-09-30T09:56:19.026082+0000","last_fullsized":"2023-09-30T09:56:19.026082+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:40:58.995167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":472,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.027887+0000","last_change":"2023-09-30T09:55:40.611251+0000","last_active":"2023-09-30T09:56:18.027887+0000","last_peered":"2023-09-30T09:56:18.027887+0000","last_clean":"2023-09-30T09:56:18.027887+0000","last_became_active":"2023-09-30T09:55:38.063330+0000","last_became_peered":"2023-09-30T09:55:38.063330+0000","last_unstale":"2023-09-30T09:56:18.027887+0000","last_undegraded":"2023-09-30T09:56:18.027887+0000","last_fullsized":"2023-09-30T09:56:18.027887+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:55:40.611145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:55:40.611145+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:44:34.231149+0000","scrub_duration":0.0017040180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":473,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022434+0000","last_change":"2023-09-30T09:56:16.026563+0000","last_active":"2023-09-30T09:56:18.022434+0000","last_peered":"2023-09-30T09:56:18.022434+0000","last_clean":"2023-09-30T09:56:18.022434+0000","last_became_active":"2023-09-30T09:56:16.026274+0000","last_became_peered":"2023-09-30T09:56:16.026274+0000","last_unstale":"2023-09-30T09:56:18.022434+0000","last_undegraded":"2023-09-30T09:56:18.022434+0000","last_fullsized":"2023-09-30T09:56:18.022434+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:43:02.853772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":144,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031301+0000","last_change":"2023-09-30T09:49:31.310014+0000","last_active":"2023-09-30T09:56:18.031301+0000","last_peered":"2023-09-30T09:56:18.031301+0000","last_clean":"2023-09-30T09:56:18.031301+0000","last_became_active":"2023-09-30T09:49:31.309121+0000","last_became_peered":"2023-09-30T09:49:31.309121+0000","last_unstale":"2023-09-30T09:56:18.031301+0000","last_undegraded":"2023-09-30T09:56:18.031301+0000","last_fullsized":"2023-09-30T09:56:18.031301+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:46.598805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":1086,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034597+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:56:18.034597+0000","last_peered":"2023-09-30T09:56:18.034597+0000","last_clean":"2023-09-30T09:56:18.034597+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:56:18.034597+0000","last_undegraded":"2023-09-30T09:56:18.034597+0000","last_fullsized":"2023-09-30T09:56:18.034597+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026279+0000","last_change":"2023-09-30T09:56:16.028635+0000","last_active":"2023-09-30T09:56:19.026279+0000","last_peered":"2023-09-30T09:56:19.026279+0000","last_clean":"2023-09-30T09:56:19.026279+0000","last_became_active":"2023-09-30T09:56:16.028298+0000","last_became_peered":"2023-09-30T09:56:16.028298+0000","last_unstale":"2023-09-30T09:56:19.026279+0000","last_undegraded":"2023-09-30T09:56:19.026279+0000","last_fullsized":"2023-09-30T09:56:19.026279+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:59:32.064635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":286,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.023083+0000","last_change":"2023-09-30T09:49:31.295632+0000","last_active":"2023-09-30T09:56:19.023083+0000","last_peered":"2023-09-30T09:56:19.023083+0000","last_clean":"2023-09-30T09:56:19.023083+0000","last_became_active":"2023-09-30T09:49:31.295269+0000","last_became_peered":"2023-09-30T09:49:31.295269+0000","last_unstale":"2023-09-30T09:56:19.023083+0000","last_undegraded":"2023-09-30T09:56:19.023083+0000","last_fullsized":"2023-09-30T09:56:19.023083+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:32.754585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":702,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032792+0000","last_change":"2023-09-30T09:55:38.066782+0000","last_active":"2023-09-30T09:56:18.032792+0000","last_peered":"2023-09-30T09:56:18.032792+0000","last_clean":"2023-09-30T09:56:18.032792+0000","last_became_active":"2023-09-30T09:55:38.066366+0000","last_became_peered":"2023-09-30T09:55:38.066366+0000","last_unstale":"2023-09-30T09:56:18.032792+0000","last_undegraded":"2023-09-30T09:56:18.032792+0000","last_fullsized":"2023-09-30T09:56:18.032792+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:15:07.468106+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":105,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025290+0000","last_change":"2023-09-30T09:51:57.160642+0000","last_active":"2023-09-30T09:56:18.025290+0000","last_peered":"2023-09-30T09:56:18.025290+0000","last_clean":"2023-09-30T09:56:18.025290+0000","last_became_active":"2023-09-30T09:51:57.160357+0000","last_became_peered":"2023-09-30T09:51:57.160357+0000","last_unstale":"2023-09-30T09:56:18.025290+0000","last_undegraded":"2023-09-30T09:56:18.025290+0000","last_fullsized":"2023-09-30T09:56:18.025290+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:40.417904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":552,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024650+0000","last_change":"2023-09-30T09:55:31.024533+0000","last_active":"2023-09-30T09:56:18.024650+0000","last_peered":"2023-09-30T09:56:18.024650+0000","last_clean":"2023-09-30T09:56:18.024650+0000","last_became_active":"2023-09-30T09:55:31.024098+0000","last_became_peered":"2023-09-30T09:55:31.024098+0000","last_unstale":"2023-09-30T09:56:18.024650+0000","last_undegraded":"2023-09-30T09:56:18.024650+0000","last_fullsized":"2023-09-30T09:56:18.024650+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:52:06.951507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":894,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.035750+0000","last_change":"2023-09-30T09:56:19.035750+0000","last_active":"2023-09-30T09:56:19.035750+0000","last_peered":"2023-09-30T09:56:19.035750+0000","last_clean":"2023-09-30T09:56:19.035750+0000","last_became_active":"2023-09-30T09:56:19.035369+0000","last_became_peered":"2023-09-30T09:56:19.035369+0000","last_unstale":"2023-09-30T09:56:19.035750+0000","last_undegraded":"2023-09-30T09:56:19.035750+0000","last_fullsized":"2023-09-30T09:56:19.035750+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:56:23.317181+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":399,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034787+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:56:18.034787+0000","last_peered":"2023-09-30T09:56:18.034787+0000","last_clean":"2023-09-30T09:56:18.034787+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:56:18.034787+0000","last_undegraded":"2023-09-30T09:56:18.034787+0000","last_fullsized":"2023-09-30T09:56:18.034787+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":357,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028752+0000","last_change":"2023-09-30T09:56:04.201468+0000","last_active":"2023-09-30T09:56:18.028752+0000","last_peered":"2023-09-30T09:56:18.028752+0000","last_clean":"2023-09-30T09:56:18.028752+0000","last_became_active":"2023-09-30T09:56:04.200792+0000","last_became_peered":"2023-09-30T09:56:04.200792+0000","last_unstale":"2023-09-30T09:56:18.028752+0000","last_undegraded":"2023-09-30T09:56:18.028752+0000","last_fullsized":"2023-09-30T09:56:18.028752+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:36.404355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":514,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032117+0000","last_change":"2023-09-30T09:56:04.194412+0000","last_active":"2023-09-30T09:56:18.032117+0000","last_peered":"2023-09-30T09:56:18.032117+0000","last_clean":"2023-09-30T09:56:18.032117+0000","last_became_active":"2023-09-30T09:56:04.194022+0000","last_became_peered":"2023-09-30T09:56:04.194022+0000","last_unstale":"2023-09-30T09:56:18.032117+0000","last_undegraded":"2023-09-30T09:56:18.032117+0000","last_fullsized":"2023-09-30T09:56:18.032117+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:42:18.764126+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":41,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028091+0000","last_change":"2023-09-30T09:55:35.062278+0000","last_active":"2023-09-30T09:56:18.028091+0000","last_peered":"2023-09-30T09:56:18.028091+0000","last_clean":"2023-09-30T09:56:18.028091+0000","last_became_active":"2023-09-30T09:55:35.055015+0000","last_became_peered":"2023-09-30T09:55:35.055015+0000","last_unstale":"2023-09-30T09:56:18.028091+0000","last_undegraded":"2023-09-30T09:56:18.028091+0000","last_fullsized":"2023-09-30T09:56:18.028091+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:35:52.637156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9444,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":725,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030527+0000","last_change":"2023-09-30T09:49:31.304624+0000","last_active":"2023-09-30T09:56:18.030527+0000","last_peered":"2023-09-30T09:56:18.030527+0000","last_clean":"2023-09-30T09:56:18.030527+0000","last_became_active":"2023-09-30T09:49:31.304106+0000","last_became_peered":"2023-09-30T09:49:31.304106+0000","last_unstale":"2023-09-30T09:56:18.030527+0000","last_undegraded":"2023-09-30T09:56:18.030527+0000","last_fullsized":"2023-09-30T09:56:18.030527+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:42.343641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":848,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.031051+0000","last_change":"2023-09-30T09:56:19.030331+0000","last_active":"2023-09-30T09:56:19.031051+0000","last_peered":"2023-09-30T09:56:19.031051+0000","last_clean":"2023-09-30T09:56:19.031051+0000","last_became_active":"2023-09-30T09:56:19.029784+0000","last_became_peered":"2023-09-30T09:56:19.029784+0000","last_unstale":"2023-09-30T09:56:19.031051+0000","last_undegraded":"2023-09-30T09:56:19.031051+0000","last_fullsized":"2023-09-30T09:56:19.031051+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:50:13.869560+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":399,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.033100+0000","last_change":"2023-09-30T09:55:35.058374+0000","last_active":"2023-09-30T09:56:18.033100+0000","last_peered":"2023-09-30T09:56:18.033100+0000","last_clean":"2023-09-30T09:56:18.033100+0000","last_became_active":"2023-09-30T09:55:35.051488+0000","last_became_peered":"2023-09-30T09:55:35.051488+0000","last_unstale":"2023-09-30T09:56:18.033100+0000","last_undegraded":"2023-09-30T09:56:18.033100+0000","last_fullsized":"2023-09-30T09:56:18.033100+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:59:40.904205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":476,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022626+0000","last_change":"2023-09-30T09:55:31.025249+0000","last_active":"2023-09-30T09:56:18.022626+0000","last_peered":"2023-09-30T09:56:18.022626+0000","last_clean":"2023-09-30T09:56:18.022626+0000","last_became_active":"2023-09-30T09:55:31.024871+0000","last_became_peered":"2023-09-30T09:55:31.024871+0000","last_unstale":"2023-09-30T09:56:18.022626+0000","last_undegraded":"2023-09-30T09:56:18.022626+0000","last_fullsized":"2023-09-30T09:56:18.022626+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:59.192864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":1046,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024837+0000","last_change":"2023-09-30T09:49:34.332543+0000","last_active":"2023-09-30T09:56:18.024837+0000","last_peered":"2023-09-30T09:56:18.024837+0000","last_clean":"2023-09-30T09:56:18.024837+0000","last_became_active":"2023-09-30T09:49:34.332215+0000","last_became_peered":"2023-09-30T09:49:34.332215+0000","last_unstale":"2023-09-30T09:56:18.024837+0000","last_undegraded":"2023-09-30T09:56:18.024837+0000","last_fullsized":"2023-09-30T09:56:18.024837+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.827671+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026680+0000","last_change":"2023-09-30T09:56:16.032853+0000","last_active":"2023-09-30T09:56:19.026680+0000","last_peered":"2023-09-30T09:56:19.026680+0000","last_clean":"2023-09-30T09:56:19.026680+0000","last_became_active":"2023-09-30T09:56:16.032523+0000","last_became_peered":"2023-09-30T09:56:16.032523+0000","last_unstale":"2023-09-30T09:56:19.026680+0000","last_undegraded":"2023-09-30T09:56:19.026680+0000","last_fullsized":"2023-09-30T09:56:19.026680+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:24.924923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025134+0000","last_change":"2023-09-30T09:51:57.157155+0000","last_active":"2023-09-30T09:56:18.025134+0000","last_peered":"2023-09-30T09:56:18.025134+0000","last_clean":"2023-09-30T09:56:18.025134+0000","last_became_active":"2023-09-30T09:51:57.156843+0000","last_became_peered":"2023-09-30T09:51:57.156843+0000","last_unstale":"2023-09-30T09:56:18.025134+0000","last_undegraded":"2023-09-30T09:56:18.025134+0000","last_fullsized":"2023-09-30T09:56:18.025134+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:12:00.333473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":742,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.033445+0000","last_change":"2023-09-30T09:56:04.201313+0000","last_active":"2023-09-30T09:56:18.033445+0000","last_peered":"2023-09-30T09:56:18.033445+0000","last_clean":"2023-09-30T09:56:18.033445+0000","last_became_active":"2023-09-30T09:56:04.201004+0000","last_became_peered":"2023-09-30T09:56:04.201004+0000","last_unstale":"2023-09-30T09:56:18.033445+0000","last_undegraded":"2023-09-30T09:56:18.033445+0000","last_fullsized":"2023-09-30T09:56:18.033445+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:32.950192+0000","scrub_duration":0.010869449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":389,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034400+0000","last_change":"2023-09-30T09:49:31.309022+0000","last_active":"2023-09-30T09:56:18.034400+0000","last_peered":"2023-09-30T09:56:18.034400+0000","last_clean":"2023-09-30T09:56:18.034400+0000","last_became_active":"2023-09-30T09:49:31.308236+0000","last_became_peered":"2023-09-30T09:49:31.308236+0000","last_unstale":"2023-09-30T09:56:18.034400+0000","last_undegraded":"2023-09-30T09:56:18.034400+0000","last_fullsized":"2023-09-30T09:56:18.034400+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:39.779954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026842+0000","last_change":"2023-09-30T09:56:16.033776+0000","last_active":"2023-09-30T09:56:19.026842+0000","last_peered":"2023-09-30T09:56:19.026842+0000","last_clean":"2023-09-30T09:56:19.026842+0000","last_became_active":"2023-09-30T09:56:16.033010+0000","last_became_peered":"2023-09-30T09:56:16.033010+0000","last_unstale":"2023-09-30T09:56:19.026842+0000","last_undegraded":"2023-09-30T09:56:19.026842+0000","last_fullsized":"2023-09-30T09:56:19.026842+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:46.366804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":1135,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025139+0000","last_change":"2023-09-30T09:49:34.332561+0000","last_active":"2023-09-30T09:56:18.025139+0000","last_peered":"2023-09-30T09:56:18.025139+0000","last_clean":"2023-09-30T09:56:18.025139+0000","last_became_active":"2023-09-30T09:49:34.332170+0000","last_became_peered":"2023-09-30T09:49:34.332170+0000","last_unstale":"2023-09-30T09:56:18.025139+0000","last_undegraded":"2023-09-30T09:56:18.025139+0000","last_fullsized":"2023-09-30T09:56:18.025139+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:22.548068+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":74,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030987+0000","last_change":"2023-09-30T09:56:16.044871+0000","last_active":"2023-09-30T09:56:18.030987+0000","last_peered":"2023-09-30T09:56:18.030987+0000","last_clean":"2023-09-30T09:56:18.030987+0000","last_became_active":"2023-09-30T09:56:16.026570+0000","last_became_peered":"2023-09-30T09:56:16.026570+0000","last_unstale":"2023-09-30T09:56:18.030987+0000","last_undegraded":"2023-09-30T09:56:18.030987+0000","last_fullsized":"2023-09-30T09:56:18.030987+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:42:25.744814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":261,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.023272+0000","last_change":"2023-09-30T09:56:07.282221+0000","last_active":"2023-09-30T09:56:18.023272+0000","last_peered":"2023-09-30T09:56:18.023272+0000","last_clean":"2023-09-30T09:56:18.023272+0000","last_became_active":"2023-09-30T09:56:07.281802+0000","last_became_peered":"2023-09-30T09:56:07.281802+0000","last_unstale":"2023-09-30T09:56:18.023272+0000","last_undegraded":"2023-09-30T09:56:18.023272+0000","last_fullsized":"2023-09-30T09:56:18.023272+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:34:05.315437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1101,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034695+0000","last_change":"2023-09-30T09:55:38.064282+0000","last_active":"2023-09-30T09:56:18.034695+0000","last_peered":"2023-09-30T09:56:18.034695+0000","last_clean":"2023-09-30T09:56:18.034695+0000","last_became_active":"2023-09-30T09:55:38.063909+0000","last_became_peered":"2023-09-30T09:55:38.063909+0000","last_unstale":"2023-09-30T09:56:18.034695+0000","last_undegraded":"2023-09-30T09:56:18.034695+0000","last_fullsized":"2023-09-30T09:56:18.034695+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:22.387239+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":389,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.033462+0000","last_change":"2023-09-30T09:56:16.049380+0000","last_active":"2023-09-30T09:56:18.033462+0000","last_peered":"2023-09-30T09:56:18.033462+0000","last_clean":"2023-09-30T09:56:18.033462+0000","last_became_active":"2023-09-30T09:56:16.039822+0000","last_became_peered":"2023-09-30T09:56:16.039822+0000","last_unstale":"2023-09-30T09:56:18.033462+0000","last_undegraded":"2023-09-30T09:56:18.033462+0000","last_fullsized":"2023-09-30T09:56:18.033462+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:13:02.626062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024746+0000","last_change":"2023-09-30T09:51:57.167306+0000","last_active":"2023-09-30T09:56:18.024746+0000","last_peered":"2023-09-30T09:56:18.024746+0000","last_clean":"2023-09-30T09:56:18.024746+0000","last_became_active":"2023-09-30T09:51:57.167040+0000","last_became_peered":"2023-09-30T09:51:57.167040+0000","last_unstale":"2023-09-30T09:56:18.024746+0000","last_undegraded":"2023-09-30T09:56:18.024746+0000","last_fullsized":"2023-09-30T09:56:18.024746+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:27:26.793530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":1132,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.024359+0000","last_change":"2023-09-30T09:55:06.180049+0000","last_active":"2023-09-30T09:56:19.024359+0000","last_peered":"2023-09-30T09:56:19.024359+0000","last_clean":"2023-09-30T09:56:19.024359+0000","last_became_active":"2023-09-30T09:55:06.179542+0000","last_became_peered":"2023-09-30T09:55:06.179542+0000","last_unstale":"2023-09-30T09:56:19.024359+0000","last_undegraded":"2023-09-30T09:56:19.024359+0000","last_fullsized":"2023-09-30T09:56:19.024359+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:28:32.391624+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":359,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032917+0000","last_change":"2023-09-30T09:55:35.060477+0000","last_active":"2023-09-30T09:56:18.032917+0000","last_peered":"2023-09-30T09:56:18.032917+0000","last_clean":"2023-09-30T09:56:18.032917+0000","last_became_active":"2023-09-30T09:55:35.052062+0000","last_became_peered":"2023-09-30T09:55:35.052062+0000","last_unstale":"2023-09-30T09:56:18.032917+0000","last_undegraded":"2023-09-30T09:56:18.032917+0000","last_fullsized":"2023-09-30T09:56:18.032917+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:50:59.125368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":744,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029064+0000","last_change":"2023-09-30T09:50:26.903683+0000","last_active":"2023-09-30T09:56:18.029064+0000","last_peered":"2023-09-30T09:56:18.029064+0000","last_clean":"2023-09-30T09:56:18.029064+0000","last_became_active":"2023-09-30T09:50:26.903262+0000","last_became_peered":"2023-09-30T09:50:26.903262+0000","last_unstale":"2023-09-30T09:56:18.029064+0000","last_undegraded":"2023-09-30T09:56:18.029064+0000","last_fullsized":"2023-09-30T09:56:18.029064+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:40.724403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":640,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031492+0000","last_change":"2023-09-30T09:55:49.124176+0000","last_active":"2023-09-30T09:56:18.031492+0000","last_peered":"2023-09-30T09:56:18.031492+0000","last_clean":"2023-09-30T09:56:18.031492+0000","last_became_active":"2023-09-30T09:55:49.123676+0000","last_became_peered":"2023-09-30T09:55:49.123676+0000","last_unstale":"2023-09-30T09:56:18.031492+0000","last_undegraded":"2023-09-30T09:56:18.031492+0000","last_fullsized":"2023-09-30T09:56:18.031492+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:16.116795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:16.116795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:25:26.020663+0000","scrub_duration":0.045979904000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":454,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034153+0000","last_change":"2023-09-30T09:51:57.192521+0000","last_active":"2023-09-30T09:56:18.034153+0000","last_peered":"2023-09-30T09:56:18.034153+0000","last_clean":"2023-09-30T09:56:18.034153+0000","last_became_active":"2023-09-30T09:51:57.170876+0000","last_became_peered":"2023-09-30T09:51:57.170876+0000","last_unstale":"2023-09-30T09:56:18.034153+0000","last_undegraded":"2023-09-30T09:56:18.034153+0000","last_fullsized":"2023-09-30T09:56:18.034153+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:44:49.877225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":463,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029192+0000","last_change":"2023-09-30T09:55:49.121505+0000","last_active":"2023-09-30T09:56:18.029192+0000","last_peered":"2023-09-30T09:56:18.029192+0000","last_clean":"2023-09-30T09:56:18.029192+0000","last_became_active":"2023-09-30T09:55:49.121169+0000","last_became_peered":"2023-09-30T09:55:49.121169+0000","last_unstale":"2023-09-30T09:56:18.029192+0000","last_undegraded":"2023-09-30T09:56:18.029192+0000","last_fullsized":"2023-09-30T09:56:18.029192+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:34.361732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":586,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029945+0000","last_change":"2023-09-30T09:55:38.064129+0000","last_active":"2023-09-30T09:56:18.029945+0000","last_peered":"2023-09-30T09:56:18.029945+0000","last_clean":"2023-09-30T09:56:18.029945+0000","last_became_active":"2023-09-30T09:55:38.063727+0000","last_became_peered":"2023-09-30T09:55:38.063727+0000","last_unstale":"2023-09-30T09:56:18.029945+0000","last_undegraded":"2023-09-30T09:56:18.029945+0000","last_fullsized":"2023-09-30T09:56:18.029945+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:17.971207+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":640,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025606+0000","last_change":"2023-09-30T09:49:31.301460+0000","last_active":"2023-09-30T09:56:18.025606+0000","last_peered":"2023-09-30T09:56:18.025606+0000","last_clean":"2023-09-30T09:56:18.025606+0000","last_became_active":"2023-09-30T09:49:31.301114+0000","last_became_peered":"2023-09-30T09:49:31.301114+0000","last_unstale":"2023-09-30T09:56:18.025606+0000","last_undegraded":"2023-09-30T09:56:18.025606+0000","last_fullsized":"2023-09-30T09:56:18.025606+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:42:01.066974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":228,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028067+0000","last_change":"2023-09-30T09:55:06.212302+0000","last_active":"2023-09-30T09:56:18.028067+0000","last_peered":"2023-09-30T09:56:18.028067+0000","last_clean":"2023-09-30T09:56:18.028067+0000","last_became_active":"2023-09-30T09:55:06.211890+0000","last_became_peered":"2023-09-30T09:55:06.211890+0000","last_unstale":"2023-09-30T09:56:18.028067+0000","last_undegraded":"2023-09-30T09:56:18.028067+0000","last_fullsized":"2023-09-30T09:56:18.028067+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:38:14.958331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1247,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034392+0000","last_change":"2023-09-30T09:55:20.013357+0000","last_active":"2023-09-30T09:56:18.034392+0000","last_peered":"2023-09-30T09:56:18.034392+0000","last_clean":"2023-09-30T09:56:18.034392+0000","last_became_active":"2023-09-30T09:55:20.013015+0000","last_became_peered":"2023-09-30T09:55:20.013015+0000","last_unstale":"2023-09-30T09:56:18.034392+0000","last_undegraded":"2023-09-30T09:56:18.034392+0000","last_fullsized":"2023-09-30T09:56:18.034392+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:30:47.162310+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":84,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024035+0000","last_change":"2023-09-30T09:56:16.047280+0000","last_active":"2023-09-30T09:56:18.024035+0000","last_peered":"2023-09-30T09:56:18.024035+0000","last_clean":"2023-09-30T09:56:18.024035+0000","last_became_active":"2023-09-30T09:56:16.033154+0000","last_became_peered":"2023-09-30T09:56:16.033154+0000","last_unstale":"2023-09-30T09:56:18.024035+0000","last_undegraded":"2023-09-30T09:56:18.024035+0000","last_fullsized":"2023-09-30T09:56:18.024035+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:00.373555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":701,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028177+0000","last_change":"2023-09-30T09:49:31.300551+0000","last_active":"2023-09-30T09:56:18.028177+0000","last_peered":"2023-09-30T09:56:18.028177+0000","last_clean":"2023-09-30T09:56:18.028177+0000","last_became_active":"2023-09-30T09:49:31.300231+0000","last_became_peered":"2023-09-30T09:49:31.300231+0000","last_unstale":"2023-09-30T09:56:18.028177+0000","last_undegraded":"2023-09-30T09:56:18.028177+0000","last_fullsized":"2023-09-30T09:56:18.028177+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:45.832564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":975,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024759+0000","last_change":"2023-09-30T09:56:12.198931+0000","last_active":"2023-09-30T09:56:18.024759+0000","last_peered":"2023-09-30T09:56:18.024759+0000","last_clean":"2023-09-30T09:56:18.024759+0000","last_became_active":"2023-09-30T09:56:12.198479+0000","last_became_peered":"2023-09-30T09:56:12.198479+0000","last_unstale":"2023-09-30T09:56:18.024759+0000","last_undegraded":"2023-09-30T09:56:18.024759+0000","last_fullsized":"2023-09-30T09:56:18.024759+0000","mapping_epoch":679,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:40:05.730878+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":420,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029273+0000","last_change":"2023-09-30T09:55:49.125117+0000","last_active":"2023-09-30T09:56:18.029273+0000","last_peered":"2023-09-30T09:56:18.029273+0000","last_clean":"2023-09-30T09:56:18.029273+0000","last_became_active":"2023-09-30T09:55:49.124787+0000","last_became_peered":"2023-09-30T09:55:49.124787+0000","last_unstale":"2023-09-30T09:56:18.029273+0000","last_undegraded":"2023-09-30T09:56:18.029273+0000","last_fullsized":"2023-09-30T09:56:18.029273+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:02.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028406+0000","last_change":"2023-09-30T09:55:35.053238+0000","last_active":"2023-09-30T09:56:18.028406+0000","last_peered":"2023-09-30T09:56:18.028406+0000","last_clean":"2023-09-30T09:56:18.028406+0000","last_became_active":"2023-09-30T09:55:35.052746+0000","last_became_peered":"2023-09-30T09:55:35.052746+0000","last_unstale":"2023-09-30T09:56:18.028406+0000","last_undegraded":"2023-09-30T09:56:18.028406+0000","last_fullsized":"2023-09-30T09:56:18.028406+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:51:23.636286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":769,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.029857+0000","last_change":"2023-09-30T09:56:19.029857+0000","last_active":"2023-09-30T09:56:19.029857+0000","last_peered":"2023-09-30T09:56:19.029857+0000","last_clean":"2023-09-30T09:56:19.029857+0000","last_became_active":"2023-09-30T09:56:19.029372+0000","last_became_peered":"2023-09-30T09:56:19.029372+0000","last_unstale":"2023-09-30T09:56:19.029857+0000","last_undegraded":"2023-09-30T09:56:19.029857+0000","last_fullsized":"2023-09-30T09:56:19.029857+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:43:41.661080+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028982+0000","last_change":"2023-09-30T09:55:35.051277+0000","last_active":"2023-09-30T09:56:18.028982+0000","last_peered":"2023-09-30T09:56:18.028982+0000","last_clean":"2023-09-30T09:56:18.028982+0000","last_became_active":"2023-09-30T09:55:35.050773+0000","last_became_peered":"2023-09-30T09:55:35.050773+0000","last_unstale":"2023-09-30T09:56:18.028982+0000","last_undegraded":"2023-09-30T09:56:18.028982+0000","last_fullsized":"2023-09-30T09:56:18.028982+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:39:07.565803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":715,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030184+0000","last_change":"2023-09-30T09:49:31.311818+0000","last_active":"2023-09-30T09:56:18.030184+0000","last_peered":"2023-09-30T09:56:18.030184+0000","last_clean":"2023-09-30T09:56:18.030184+0000","last_became_active":"2023-09-30T09:49:31.311481+0000","last_became_peered":"2023-09-30T09:49:31.311481+0000","last_unstale":"2023-09-30T09:56:18.030184+0000","last_undegraded":"2023-09-30T09:56:18.030184+0000","last_fullsized":"2023-09-30T09:56:18.030184+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:54.941072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":611,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025406+0000","last_change":"2023-09-30T09:56:16.692506+0000","last_active":"2023-09-30T09:56:18.025406+0000","last_peered":"2023-09-30T09:56:18.025406+0000","last_clean":"2023-09-30T09:56:18.025406+0000","last_became_active":"2023-09-30T09:56:12.198024+0000","last_became_peered":"2023-09-30T09:56:12.198024+0000","last_unstale":"2023-09-30T09:56:18.025406+0000","last_undegraded":"2023-09-30T09:56:18.025406+0000","last_fullsized":"2023-09-30T09:56:18.025406+0000","mapping_epoch":679,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:56:16.692428+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:56:16.692428+0000","last_clean_scrub_stamp":"2023-09-30T09:56:16.692428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:04:40.230998+0000","scrub_duration":0.0016202269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":254,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022275+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:56:18.022275+0000","last_peered":"2023-09-30T09:56:18.022275+0000","last_clean":"2023-09-30T09:56:18.022275+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:56:18.022275+0000","last_undegraded":"2023-09-30T09:56:18.022275+0000","last_fullsized":"2023-09-30T09:56:18.022275+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":237,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030718+0000","last_change":"2023-09-30T09:56:04.200329+0000","last_active":"2023-09-30T09:56:18.030718+0000","last_peered":"2023-09-30T09:56:18.030718+0000","last_clean":"2023-09-30T09:56:18.030718+0000","last_became_active":"2023-09-30T09:56:04.200012+0000","last_became_peered":"2023-09-30T09:56:04.200012+0000","last_unstale":"2023-09-30T09:56:18.030718+0000","last_undegraded":"2023-09-30T09:56:18.030718+0000","last_fullsized":"2023-09-30T09:56:18.030718+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:01:44.617024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":747,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031400+0000","last_change":"2023-09-30T09:55:49.124036+0000","last_active":"2023-09-30T09:56:18.031400+0000","last_peered":"2023-09-30T09:56:18.031400+0000","last_clean":"2023-09-30T09:56:18.031400+0000","last_became_active":"2023-09-30T09:55:49.122904+0000","last_became_peered":"2023-09-30T09:55:49.122904+0000","last_unstale":"2023-09-30T09:56:18.031400+0000","last_undegraded":"2023-09-30T09:56:18.031400+0000","last_fullsized":"2023-09-30T09:56:18.031400+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:33:29.088907+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"644'215","reported_seq":908,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029584+0000","last_change":"2023-09-30T09:56:04.199590+0000","last_active":"2023-09-30T09:56:18.029584+0000","last_peered":"2023-09-30T09:56:18.029584+0000","last_clean":"2023-09-30T09:56:18.029584+0000","last_became_active":"2023-09-30T09:56:04.198968+0000","last_became_peered":"2023-09-30T09:56:04.198968+0000","last_unstale":"2023-09-30T09:56:18.029584+0000","last_undegraded":"2023-09-30T09:56:18.029584+0000","last_fullsized":"2023-09-30T09:56:18.029584+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":215,"log_dups_size":0,"ondisk_log_size":215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:46:16.237810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":735,"num_read_kb":2210,"num_write":243,"num_write_kb":1428,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":633,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025372+0000","last_change":"2023-09-30T09:50:41.396047+0000","last_active":"2023-09-30T09:56:18.025372+0000","last_peered":"2023-09-30T09:56:18.025372+0000","last_clean":"2023-09-30T09:56:18.025372+0000","last_became_active":"2023-09-30T09:50:41.395627+0000","last_became_peered":"2023-09-30T09:50:41.395627+0000","last_unstale":"2023-09-30T09:56:18.025372+0000","last_undegraded":"2023-09-30T09:56:18.025372+0000","last_fullsized":"2023-09-30T09:56:18.025372+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:16.050977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032902+0000","last_change":"2023-09-30T09:56:07.684173+0000","last_active":"2023-09-30T09:56:18.032902+0000","last_peered":"2023-09-30T09:56:18.032902+0000","last_clean":"2023-09-30T09:56:18.032902+0000","last_became_active":"2023-09-30T09:56:07.683845+0000","last_became_peered":"2023-09-30T09:56:07.683845+0000","last_unstale":"2023-09-30T09:56:18.032902+0000","last_undegraded":"2023-09-30T09:56:18.032902+0000","last_fullsized":"2023-09-30T09:56:18.032902+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:15:39.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":1206,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.029693+0000","last_change":"2023-09-30T09:56:19.029693+0000","last_active":"2023-09-30T09:56:19.029693+0000","last_peered":"2023-09-30T09:56:19.029693+0000","last_clean":"2023-09-30T09:56:19.029693+0000","last_became_active":"2023-09-30T09:56:19.029203+0000","last_became_peered":"2023-09-30T09:56:19.029203+0000","last_unstale":"2023-09-30T09:56:19.029693+0000","last_undegraded":"2023-09-30T09:56:19.029693+0000","last_fullsized":"2023-09-30T09:56:19.029693+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:21:07.406569+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"644'42","reported_seq":857,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025009+0000","last_change":"2023-09-30T09:56:12.197041+0000","last_active":"2023-09-30T09:56:18.025009+0000","last_peered":"2023-09-30T09:56:18.025009+0000","last_clean":"2023-09-30T09:56:18.025009+0000","last_became_active":"2023-09-30T09:56:12.196550+0000","last_became_peered":"2023-09-30T09:56:12.196550+0000","last_unstale":"2023-09-30T09:56:18.025009+0000","last_undegraded":"2023-09-30T09:56:18.025009+0000","last_fullsized":"2023-09-30T09:56:18.025009+0000","mapping_epoch":679,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":42,"log_dups_size":0,"ondisk_log_size":42,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:27:54.946400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079558799999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":665,"num_read_kb":3718,"num_write":32,"num_write_kb":3092,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":742,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030539+0000","last_change":"2023-09-30T09:56:16.031416+0000","last_active":"2023-09-30T09:56:18.030539+0000","last_peered":"2023-09-30T09:56:18.030539+0000","last_clean":"2023-09-30T09:56:18.030539+0000","last_became_active":"2023-09-30T09:56:16.030763+0000","last_became_peered":"2023-09-30T09:56:16.030763+0000","last_unstale":"2023-09-30T09:56:18.030539+0000","last_undegraded":"2023-09-30T09:56:18.030539+0000","last_fullsized":"2023-09-30T09:56:18.030539+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:51:47.913934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":338,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025322+0000","last_change":"2023-09-30T09:56:12.198890+0000","last_active":"2023-09-30T09:56:18.025322+0000","last_peered":"2023-09-30T09:56:18.025322+0000","last_clean":"2023-09-30T09:56:18.025322+0000","last_became_active":"2023-09-30T09:56:12.198494+0000","last_became_peered":"2023-09-30T09:56:12.198494+0000","last_unstale":"2023-09-30T09:56:18.025322+0000","last_undegraded":"2023-09-30T09:56:18.025322+0000","last_fullsized":"2023-09-30T09:56:18.025322+0000","mapping_epoch":679,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:17:12.142363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":1201,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032503+0000","last_change":"2023-09-30T09:55:38.065400+0000","last_active":"2023-09-30T09:56:18.032503+0000","last_peered":"2023-09-30T09:56:18.032503+0000","last_clean":"2023-09-30T09:56:18.032503+0000","last_became_active":"2023-09-30T09:55:38.065017+0000","last_became_peered":"2023-09-30T09:55:38.065017+0000","last_unstale":"2023-09-30T09:56:18.032503+0000","last_undegraded":"2023-09-30T09:56:18.032503+0000","last_fullsized":"2023-09-30T09:56:18.032503+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:35:09.877057+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"644'264","reported_seq":1527,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032435+0000","last_change":"2023-09-30T09:55:20.009001+0000","last_active":"2023-09-30T09:56:18.032435+0000","last_peered":"2023-09-30T09:56:18.032435+0000","last_clean":"2023-09-30T09:56:18.032435+0000","last_became_active":"2023-09-30T09:55:20.008454+0000","last_became_peered":"2023-09-30T09:55:20.008454+0000","last_unstale":"2023-09-30T09:56:18.032435+0000","last_undegraded":"2023-09-30T09:56:18.032435+0000","last_fullsized":"2023-09-30T09:56:18.032435+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:59:08.571424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018706759999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1813,"num_read_kb":2868,"num_write":229,"num_write_kb":1359,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":342,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028463+0000","last_change":"2023-09-30T09:56:04.198792+0000","last_active":"2023-09-30T09:56:18.028463+0000","last_peered":"2023-09-30T09:56:18.028463+0000","last_clean":"2023-09-30T09:56:18.028463+0000","last_became_active":"2023-09-30T09:56:04.198468+0000","last_became_peered":"2023-09-30T09:56:04.198468+0000","last_unstale":"2023-09-30T09:56:18.028463+0000","last_undegraded":"2023-09-30T09:56:18.028463+0000","last_fullsized":"2023-09-30T09:56:18.028463+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:37.457966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":469,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030639+0000","last_change":"2023-09-30T09:56:16.028292+0000","last_active":"2023-09-30T09:56:18.030639+0000","last_peered":"2023-09-30T09:56:18.030639+0000","last_clean":"2023-09-30T09:56:18.030639+0000","last_became_active":"2023-09-30T09:56:16.027938+0000","last_became_peered":"2023-09-30T09:56:16.027938+0000","last_unstale":"2023-09-30T09:56:18.030639+0000","last_undegraded":"2023-09-30T09:56:18.030639+0000","last_fullsized":"2023-09-30T09:56:18.030639+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:46:54.999972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"644'350","reported_seq":1662,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.026084+0000","last_change":"2023-09-30T09:54:40.236180+0000","last_active":"2023-09-30T09:56:18.026084+0000","last_peered":"2023-09-30T09:56:18.026084+0000","last_clean":"2023-09-30T09:56:18.026084+0000","last_became_active":"2023-09-30T09:50:42.387121+0000","last_became_peered":"2023-09-30T09:50:42.387121+0000","last_unstale":"2023-09-30T09:56:18.026084+0000","last_undegraded":"2023-09-30T09:56:18.026084+0000","last_fullsized":"2023-09-30T09:56:18.026084+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":350,"log_dups_size":0,"ondisk_log_size":350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:34:05.192090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00090046199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1079,"num_read_kb":2584,"num_write":339,"num_write_kb":1551,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1127,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.034046+0000","last_change":"2023-09-30T09:56:19.034046+0000","last_active":"2023-09-30T09:56:19.034046+0000","last_peered":"2023-09-30T09:56:19.034046+0000","last_clean":"2023-09-30T09:56:19.034046+0000","last_became_active":"2023-09-30T09:56:19.033659+0000","last_became_peered":"2023-09-30T09:56:19.033659+0000","last_unstale":"2023-09-30T09:56:19.034046+0000","last_undegraded":"2023-09-30T09:56:19.034046+0000","last_fullsized":"2023-09-30T09:56:19.034046+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:32:25.427476+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":172,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022412+0000","last_change":"2023-09-30T09:55:31.023445+0000","last_active":"2023-09-30T09:56:18.022412+0000","last_peered":"2023-09-30T09:56:18.022412+0000","last_clean":"2023-09-30T09:56:18.022412+0000","last_became_active":"2023-09-30T09:55:31.023061+0000","last_became_peered":"2023-09-30T09:55:31.023061+0000","last_unstale":"2023-09-30T09:56:18.022412+0000","last_undegraded":"2023-09-30T09:56:18.022412+0000","last_fullsized":"2023-09-30T09:56:18.022412+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:49:21.250674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":542,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028617+0000","last_change":"2023-09-30T09:55:49.123422+0000","last_active":"2023-09-30T09:56:18.028617+0000","last_peered":"2023-09-30T09:56:18.028617+0000","last_clean":"2023-09-30T09:56:18.028617+0000","last_became_active":"2023-09-30T09:55:49.123110+0000","last_became_peered":"2023-09-30T09:55:49.123110+0000","last_unstale":"2023-09-30T09:56:18.028617+0000","last_undegraded":"2023-09-30T09:56:18.028617+0000","last_fullsized":"2023-09-30T09:56:18.028617+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:40.848966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":832,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.033740+0000","last_change":"2023-09-30T09:56:04.203765+0000","last_active":"2023-09-30T09:56:18.033740+0000","last_peered":"2023-09-30T09:56:18.033740+0000","last_clean":"2023-09-30T09:56:18.033740+0000","last_became_active":"2023-09-30T09:56:04.203571+0000","last_became_peered":"2023-09-30T09:56:04.203571+0000","last_unstale":"2023-09-30T09:56:18.033740+0000","last_undegraded":"2023-09-30T09:56:18.033740+0000","last_fullsized":"2023-09-30T09:56:18.033740+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:39.230227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.230227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:28:51.856227+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"644'256","reported_seq":1742,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034945+0000","last_change":"2023-09-30T09:55:09.210709+0000","last_active":"2023-09-30T09:56:18.034945+0000","last_peered":"2023-09-30T09:56:18.034945+0000","last_clean":"2023-09-30T09:56:18.034945+0000","last_became_active":"2023-09-30T09:55:09.208764+0000","last_became_peered":"2023-09-30T09:55:09.208764+0000","last_unstale":"2023-09-30T09:56:18.034945+0000","last_undegraded":"2023-09-30T09:56:18.034945+0000","last_fullsized":"2023-09-30T09:56:18.034945+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":256,"log_dups_size":0,"ondisk_log_size":256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:19:55.206358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001530228,"stat_sum":{"num_bytes":8388608,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2028,"num_read_kb":2464,"num_write":188,"num_write_kb":8612,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":269,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029117+0000","last_change":"2023-09-30T09:55:35.050416+0000","last_active":"2023-09-30T09:56:18.029117+0000","last_peered":"2023-09-30T09:56:18.029117+0000","last_clean":"2023-09-30T09:56:18.029117+0000","last_became_active":"2023-09-30T09:55:35.050060+0000","last_became_peered":"2023-09-30T09:55:35.050060+0000","last_unstale":"2023-09-30T09:56:18.029117+0000","last_undegraded":"2023-09-30T09:56:18.029117+0000","last_fullsized":"2023-09-30T09:56:18.029117+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:03.272812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026873+0000","last_change":"2023-09-30T09:56:16.024455+0000","last_active":"2023-09-30T09:56:19.026873+0000","last_peered":"2023-09-30T09:56:19.026873+0000","last_clean":"2023-09-30T09:56:19.026873+0000","last_became_active":"2023-09-30T09:56:16.024013+0000","last_became_peered":"2023-09-30T09:56:16.024013+0000","last_unstale":"2023-09-30T09:56:19.026873+0000","last_undegraded":"2023-09-30T09:56:19.026873+0000","last_fullsized":"2023-09-30T09:56:19.026873+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:11.515103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":802,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.033910+0000","last_change":"2023-09-30T09:56:19.033910+0000","last_active":"2023-09-30T09:56:19.033910+0000","last_peered":"2023-09-30T09:56:19.033910+0000","last_clean":"2023-09-30T09:56:19.033910+0000","last_became_active":"2023-09-30T09:56:19.033507+0000","last_became_peered":"2023-09-30T09:56:19.033507+0000","last_unstale":"2023-09-30T09:56:19.033910+0000","last_undegraded":"2023-09-30T09:56:19.033910+0000","last_fullsized":"2023-09-30T09:56:19.033910+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:31:02.045496+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"644'253","reported_seq":1737,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.036312+0000","last_change":"2023-09-30T09:56:19.036312+0000","last_active":"2023-09-30T09:56:19.036312+0000","last_peered":"2023-09-30T09:56:19.036312+0000","last_clean":"2023-09-30T09:56:19.036312+0000","last_became_active":"2023-09-30T09:56:19.035567+0000","last_became_peered":"2023-09-30T09:56:19.035567+0000","last_unstale":"2023-09-30T09:56:19.036312+0000","last_undegraded":"2023-09-30T09:56:19.036312+0000","last_fullsized":"2023-09-30T09:56:19.036312+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":253,"log_dups_size":0,"ondisk_log_size":253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:32:48.808343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00122043,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1727,"num_read_kb":3650,"num_write":210,"num_write_kb":2245,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":256,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029344+0000","last_change":"2023-09-30T09:56:04.189586+0000","last_active":"2023-09-30T09:56:18.029344+0000","last_peered":"2023-09-30T09:56:18.029344+0000","last_clean":"2023-09-30T09:56:18.029344+0000","last_became_active":"2023-09-30T09:56:04.189213+0000","last_became_peered":"2023-09-30T09:56:04.189213+0000","last_unstale":"2023-09-30T09:56:18.029344+0000","last_undegraded":"2023-09-30T09:56:18.029344+0000","last_fullsized":"2023-09-30T09:56:18.029344+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:19:41.345044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":277,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028266+0000","last_change":"2023-09-30T09:55:06.172278+0000","last_active":"2023-09-30T09:56:18.028266+0000","last_peered":"2023-09-30T09:56:18.028266+0000","last_clean":"2023-09-30T09:56:18.028266+0000","last_became_active":"2023-09-30T09:55:06.171955+0000","last_became_peered":"2023-09-30T09:55:06.171955+0000","last_unstale":"2023-09-30T09:56:18.028266+0000","last_undegraded":"2023-09-30T09:56:18.028266+0000","last_fullsized":"2023-09-30T09:56:18.028266+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:47:33.173930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1252,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028572+0000","last_change":"2023-09-30T09:55:38.068371+0000","last_active":"2023-09-30T09:56:18.028572+0000","last_peered":"2023-09-30T09:56:18.028572+0000","last_clean":"2023-09-30T09:56:18.028572+0000","last_became_active":"2023-09-30T09:55:38.067940+0000","last_became_peered":"2023-09-30T09:55:38.067940+0000","last_unstale":"2023-09-30T09:56:18.028572+0000","last_undegraded":"2023-09-30T09:56:18.028572+0000","last_fullsized":"2023-09-30T09:56:18.028572+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:28:09.245875+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":72,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.023420+0000","last_change":"2023-09-30T09:55:31.024573+0000","last_active":"2023-09-30T09:56:18.023420+0000","last_peered":"2023-09-30T09:56:18.023420+0000","last_clean":"2023-09-30T09:56:18.023420+0000","last_became_active":"2023-09-30T09:55:31.024173+0000","last_became_peered":"2023-09-30T09:55:31.024173+0000","last_unstale":"2023-09-30T09:56:18.023420+0000","last_undegraded":"2023-09-30T09:56:18.023420+0000","last_fullsized":"2023-09-30T09:56:18.023420+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:47.224041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"644'380","reported_seq":1846,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031849+0000","last_change":"2023-09-30T09:56:04.200828+0000","last_active":"2023-09-30T09:56:18.031849+0000","last_peered":"2023-09-30T09:56:18.031849+0000","last_clean":"2023-09-30T09:56:18.031849+0000","last_became_active":"2023-09-30T09:56:04.200034+0000","last_became_peered":"2023-09-30T09:56:04.200034+0000","last_unstale":"2023-09-30T09:56:18.031849+0000","last_undegraded":"2023-09-30T09:56:18.031849+0000","last_fullsized":"2023-09-30T09:56:18.031849+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":380,"log_dups_size":0,"ondisk_log_size":380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:07.235547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001189673,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2651,"num_read_kb":5320,"num_write":283,"num_write_kb":4761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":669,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022132+0000","last_change":"2023-09-30T09:56:16.029529+0000","last_active":"2023-09-30T09:56:18.022132+0000","last_peered":"2023-09-30T09:56:18.022132+0000","last_clean":"2023-09-30T09:56:18.022132+0000","last_became_active":"2023-09-30T09:56:16.029162+0000","last_became_peered":"2023-09-30T09:56:16.029162+0000","last_unstale":"2023-09-30T09:56:18.022132+0000","last_undegraded":"2023-09-30T09:56:18.022132+0000","last_fullsized":"2023-09-30T09:56:18.022132+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:54.078754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":1114,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022698+0000","last_change":"2023-09-30T09:51:59.873401+0000","last_active":"2023-09-30T09:56:18.022698+0000","last_peered":"2023-09-30T09:56:18.022698+0000","last_clean":"2023-09-30T09:56:18.022698+0000","last_became_active":"2023-09-30T09:51:59.873066+0000","last_became_peered":"2023-09-30T09:51:59.873066+0000","last_unstale":"2023-09-30T09:56:18.022698+0000","last_undegraded":"2023-09-30T09:56:18.022698+0000","last_fullsized":"2023-09-30T09:56:18.022698+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:48:11.565720+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"644'190","reported_seq":1414,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031601+0000","last_change":"2023-09-30T09:55:49.123303+0000","last_active":"2023-09-30T09:56:18.031601+0000","last_peered":"2023-09-30T09:56:18.031601+0000","last_clean":"2023-09-30T09:56:18.031601+0000","last_became_active":"2023-09-30T09:55:49.122113+0000","last_became_peered":"2023-09-30T09:55:49.122113+0000","last_unstale":"2023-09-30T09:56:18.031601+0000","last_undegraded":"2023-09-30T09:56:18.031601+0000","last_fullsized":"2023-09-30T09:56:18.031601+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:05:29.519408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094274099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":3608,"num_write":166,"num_write_kb":2342,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":18,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026024+0000","last_change":"2023-09-30T09:56:16.051225+0000","last_active":"2023-09-30T09:56:19.026024+0000","last_peered":"2023-09-30T09:56:19.026024+0000","last_clean":"2023-09-30T09:56:19.026024+0000","last_became_active":"2023-09-30T09:56:16.031575+0000","last_became_peered":"2023-09-30T09:56:16.031575+0000","last_unstale":"2023-09-30T09:56:19.026024+0000","last_undegraded":"2023-09-30T09:56:19.026024+0000","last_fullsized":"2023-09-30T09:56:19.026024+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:20:02.019985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028903+0000","last_change":"2023-09-30T09:55:35.053458+0000","last_active":"2023-09-30T09:56:18.028903+0000","last_peered":"2023-09-30T09:56:18.028903+0000","last_clean":"2023-09-30T09:56:18.028903+0000","last_became_active":"2023-09-30T09:55:35.053063+0000","last_became_peered":"2023-09-30T09:55:35.053063+0000","last_unstale":"2023-09-30T09:56:18.028903+0000","last_undegraded":"2023-09-30T09:56:18.028903+0000","last_fullsized":"2023-09-30T09:56:18.028903+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:44.710137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":870,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.033963+0000","last_change":"2023-09-30T09:56:19.033963+0000","last_active":"2023-09-30T09:56:19.033963+0000","last_peered":"2023-09-30T09:56:19.033963+0000","last_clean":"2023-09-30T09:56:19.033963+0000","last_became_active":"2023-09-30T09:56:19.033614+0000","last_became_peered":"2023-09-30T09:56:19.033614+0000","last_unstale":"2023-09-30T09:56:19.033963+0000","last_undegraded":"2023-09-30T09:56:19.033963+0000","last_fullsized":"2023-09-30T09:56:19.033963+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:13:51.692768+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.025897+0000","last_change":"2023-09-30T09:56:16.025946+0000","last_active":"2023-09-30T09:56:19.025897+0000","last_peered":"2023-09-30T09:56:19.025897+0000","last_clean":"2023-09-30T09:56:19.025897+0000","last_became_active":"2023-09-30T09:56:16.025593+0000","last_became_peered":"2023-09-30T09:56:16.025593+0000","last_unstale":"2023-09-30T09:56:19.025897+0000","last_undegraded":"2023-09-30T09:56:19.025897+0000","last_fullsized":"2023-09-30T09:56:19.025897+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:56:59.679977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":41,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032702+0000","last_change":"2023-09-30T09:55:17.024355+0000","last_active":"2023-09-30T09:56:18.032702+0000","last_peered":"2023-09-30T09:56:18.032702+0000","last_clean":"2023-09-30T09:56:18.032702+0000","last_became_active":"2023-09-30T09:55:17.023998+0000","last_became_peered":"2023-09-30T09:55:17.023998+0000","last_unstale":"2023-09-30T09:56:18.032702+0000","last_undegraded":"2023-09-30T09:56:18.032702+0000","last_fullsized":"2023-09-30T09:56:18.032702+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:42.172168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":677,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.023860+0000","last_change":"2023-09-30T09:54:02.680525+0000","last_active":"2023-09-30T09:56:18.023860+0000","last_peered":"2023-09-30T09:56:18.023860+0000","last_clean":"2023-09-30T09:56:18.023860+0000","last_became_active":"2023-09-30T09:54:02.200078+0000","last_became_peered":"2023-09-30T09:54:02.200078+0000","last_unstale":"2023-09-30T09:56:18.023860+0000","last_undegraded":"2023-09-30T09:56:18.023860+0000","last_fullsized":"2023-09-30T09:56:18.023860+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:54:02.680294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:54:02.680294+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:00:33.759005+0000","scrub_duration":0.0025974169999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":613,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025209+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:56:18.025209+0000","last_peered":"2023-09-30T09:56:18.025209+0000","last_clean":"2023-09-30T09:56:18.025209+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:56:18.025209+0000","last_undegraded":"2023-09-30T09:56:18.025209+0000","last_fullsized":"2023-09-30T09:56:18.025209+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030212+0000","last_change":"2023-09-30T09:55:35.052986+0000","last_active":"2023-09-30T09:56:18.030212+0000","last_peered":"2023-09-30T09:56:18.030212+0000","last_clean":"2023-09-30T09:56:18.030212+0000","last_became_active":"2023-09-30T09:55:35.052418+0000","last_became_peered":"2023-09-30T09:55:35.052418+0000","last_unstale":"2023-09-30T09:56:18.030212+0000","last_undegraded":"2023-09-30T09:56:18.030212+0000","last_fullsized":"2023-09-30T09:56:18.030212+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:47.726885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":583,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025555+0000","last_change":"2023-09-30T09:49:31.313229+0000","last_active":"2023-09-30T09:56:18.025555+0000","last_peered":"2023-09-30T09:56:18.025555+0000","last_clean":"2023-09-30T09:56:18.025555+0000","last_became_active":"2023-09-30T09:49:31.301137+0000","last_became_peered":"2023-09-30T09:49:31.301137+0000","last_unstale":"2023-09-30T09:56:18.025555+0000","last_undegraded":"2023-09-30T09:56:18.025555+0000","last_fullsized":"2023-09-30T09:56:18.025555+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:57.362379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028001+0000","last_change":"2023-09-30T09:55:35.051703+0000","last_active":"2023-09-30T09:56:18.028001+0000","last_peered":"2023-09-30T09:56:18.028001+0000","last_clean":"2023-09-30T09:56:18.028001+0000","last_became_active":"2023-09-30T09:55:35.051349+0000","last_became_peered":"2023-09-30T09:55:35.051349+0000","last_unstale":"2023-09-30T09:56:18.028001+0000","last_undegraded":"2023-09-30T09:56:18.028001+0000","last_fullsized":"2023-09-30T09:56:18.028001+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:13:27.965290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":861,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032900+0000","last_change":"2023-09-30T09:54:03.339280+0000","last_active":"2023-09-30T09:56:18.032900+0000","last_peered":"2023-09-30T09:56:18.032900+0000","last_clean":"2023-09-30T09:56:18.032900+0000","last_became_active":"2023-09-30T09:50:29.883457+0000","last_became_peered":"2023-09-30T09:50:29.883457+0000","last_unstale":"2023-09-30T09:56:18.032900+0000","last_undegraded":"2023-09-30T09:56:18.032900+0000","last_fullsized":"2023-09-30T09:56:18.032900+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:54:03.339198+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:54:03.339198+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:05.265754+0000","scrub_duration":0.0015674910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":160,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.024208+0000","last_change":"2023-09-30T09:55:31.023790+0000","last_active":"2023-09-30T09:56:19.024208+0000","last_peered":"2023-09-30T09:56:19.024208+0000","last_clean":"2023-09-30T09:56:19.024208+0000","last_became_active":"2023-09-30T09:55:31.023424+0000","last_became_peered":"2023-09-30T09:55:31.023424+0000","last_unstale":"2023-09-30T09:56:19.024208+0000","last_undegraded":"2023-09-30T09:56:19.024208+0000","last_fullsized":"2023-09-30T09:56:19.024208+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:48:10.845081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":450,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024720+0000","last_change":"2023-09-30T09:55:31.016117+0000","last_active":"2023-09-30T09:56:18.024720+0000","last_peered":"2023-09-30T09:56:18.024720+0000","last_clean":"2023-09-30T09:56:18.024720+0000","last_became_active":"2023-09-30T09:55:31.015808+0000","last_became_peered":"2023-09-30T09:55:31.015808+0000","last_unstale":"2023-09-30T09:56:18.024720+0000","last_undegraded":"2023-09-30T09:56:18.024720+0000","last_fullsized":"2023-09-30T09:56:18.024720+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:54:52.567443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":775,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030316+0000","last_change":"2023-09-30T09:56:04.203349+0000","last_active":"2023-09-30T09:56:18.030316+0000","last_peered":"2023-09-30T09:56:18.030316+0000","last_clean":"2023-09-30T09:56:18.030316+0000","last_became_active":"2023-09-30T09:56:04.203000+0000","last_became_peered":"2023-09-30T09:56:04.203000+0000","last_unstale":"2023-09-30T09:56:18.030316+0000","last_undegraded":"2023-09-30T09:56:18.030316+0000","last_fullsized":"2023-09-30T09:56:18.030316+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:28.264439+0000","scrub_duration":0.004695744,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":20,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026378+0000","last_change":"2023-09-30T09:56:16.048318+0000","last_active":"2023-09-30T09:56:19.026378+0000","last_peered":"2023-09-30T09:56:19.026378+0000","last_clean":"2023-09-30T09:56:19.026378+0000","last_became_active":"2023-09-30T09:56:16.028434+0000","last_became_peered":"2023-09-30T09:56:16.028434+0000","last_unstale":"2023-09-30T09:56:19.026378+0000","last_undegraded":"2023-09-30T09:56:19.026378+0000","last_fullsized":"2023-09-30T09:56:19.026378+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:28:50.558440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":728,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.027642+0000","last_change":"2023-09-30T09:50:26.905427+0000","last_active":"2023-09-30T09:56:18.027642+0000","last_peered":"2023-09-30T09:56:18.027642+0000","last_clean":"2023-09-30T09:56:18.027642+0000","last_became_active":"2023-09-30T09:50:26.905100+0000","last_became_peered":"2023-09-30T09:50:26.905100+0000","last_unstale":"2023-09-30T09:56:18.027642+0000","last_undegraded":"2023-09-30T09:56:18.027642+0000","last_fullsized":"2023-09-30T09:56:18.027642+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:38.036208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":123,"num_bytes_recovered":14254,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33222,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11978,"num_read_kb":26422,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":314,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234543,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":643488,"kb_used_data":15832,"kb_used_omap":0,"kb_used_meta":627648,"kb_avail":93101664,"statfs":{"total":95995035648,"available":95336103936,"internally_reserved":0,"allocated":16211968,"data_stored":14109791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":642711552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.63500000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.96599999999999997,"15min":3.4100000000000001},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.79800000000000004,"5min":6.3479999999999999,"15min":157.578},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.97299999999999998,"15min":3.4159999999999999},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.83699999999999997,"5min":7.2919999999999998,"15min":157.589},"last":0.46700000000000003}]},{"osd":2,"last update":"Sat Sep 30 09:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.1899999999999999,"15min":3.6030000000000002},"min":{"1min":0.313,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":0.78400000000000003,"5min":11.467000000000001,"15min":187.74600000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.403,"15min":3.621},"min":{"1min":0.41899999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.79300000000000004,"5min":10.949,"15min":187.583},"last":0.52900000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":2.036,"15min":4.0430000000000001},"min":{"1min":0.39300000000000002,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":0.82599999999999996,"5min":102.139,"15min":154.06700000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.8839999999999999,"15min":4.202},"min":{"1min":0.46400000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.86799999999999999,"5min":99.713999999999999,"15min":154.24799999999999},"last":0.58799999999999997}]},{"osd":4,"last update":"Sat Sep 30 09:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":1.1140000000000001,"15min":2.0299999999999998},"min":{"1min":0.30499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.85699999999999998,"5min":7.1660000000000004,"15min":238.511},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":1.6839999999999999,"15min":2.2000000000000002},"min":{"1min":0.33100000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.90900000000000003,"5min":11,"15min":238.387},"last":0.67400000000000004}]},{"osd":5,"last update":"Sat Sep 30 09:56:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.327,"15min":5.0209999999999999},"min":{"1min":0.38,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.80600000000000005,"5min":16.314,"15min":197.75700000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.1679999999999999,"15min":4.9199999999999999},"min":{"1min":0.441,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.86699999999999999,"5min":11.863,"15min":197.63300000000001},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.242,"15min":1.1459999999999999},"min":{"1min":0.41199999999999998,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":0.76700000000000002,"5min":9.4730000000000008,"15min":9.4730000000000008},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.425,"15min":1.1830000000000001},"min":{"1min":0.42099999999999999,"5min":0.32700000000000001,"15min":0.29599999999999999},"max":{"1min":0.96799999999999997,"5min":11.960000000000001,"15min":12.058},"last":0.69999999999999996}]}]},{"osd":6,"up_from":46,"seq":197568496181,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670348,"kb_used_data":7684,"kb_used_omap":0,"kb_used_meta":662656,"kb_avail":93074804,"statfs":{"total":95995035648,"available":95308599296,"internally_reserved":0,"allocated":7868416,"data_stored":5785460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":678559744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.73599999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":1.278,"15min":1.8839999999999999},"min":{"1min":0.502,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.85099999999999998,"5min":10.954000000000001,"15min":103.038},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.4059999999999999,"15min":1.972},"min":{"1min":0.379,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.92800000000000005,"5min":25.446999999999999,"15min":103.05},"last":0.58299999999999996}]},{"osd":2,"last update":"Sat Sep 30 09:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":1.1279999999999999,"15min":2.9670000000000001},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":0.879,"5min":12.044,"15min":207.02500000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.286,"15min":2.9460000000000002},"min":{"1min":0.43099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.76700000000000002,"5min":10.923999999999999,"15min":207.114},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.284,"15min":2.5379999999999998},"min":{"1min":0.47299999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.79300000000000004,"5min":10.894,"15min":107.11799999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.64800000000000002,"5min":1.403,"15min":2.673},"min":{"1min":0.51800000000000002,"5min":0.33500000000000002,"15min":0.33000000000000002},"max":{"1min":0.94299999999999995,"5min":10.625999999999999,"15min":106.982},"last":0.77700000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.048,"15min":1.657},"min":{"1min":0.36499999999999999,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":0.80400000000000005,"5min":7.2320000000000002,"15min":63.798000000000002},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.625,"5min":0.997,"15min":1.53},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":0.89800000000000002,"5min":10.724,"15min":63.133000000000003},"last":0.63400000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.49,"15min":1.78},"min":{"1min":0.375,"5min":0.27800000000000002,"15min":0.27100000000000002},"max":{"1min":0.83499999999999996,"5min":11.972,"15min":19.212},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.622,"5min":1.3620000000000001,"15min":1.8720000000000001},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.26900000000000002},"max":{"1min":0.92800000000000005,"5min":18.135999999999999,"15min":18.135999999999999},"last":0.79000000000000004}]},{"osd":7,"last update":"Sat Sep 30 09:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.833,"15min":1.417},"min":{"1min":0.44700000000000001,"5min":0.32800000000000001,"15min":0.26500000000000001},"max":{"1min":0.91500000000000004,"5min":39.868000000000002,"15min":39.868000000000002},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.3140000000000001,"15min":1.2749999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.27500000000000002},"max":{"1min":0.96099999999999997,"5min":10.843,"15min":12.625},"last":0.75600000000000001}]}]},{"osd":1,"up_from":301,"seq":1292785156379,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":771112,"kb_used_data":17432,"kb_used_omap":9,"kb_used_meta":753654,"kb_avail":92974040,"statfs":{"total":95995035648,"available":95205416960,"internally_reserved":0,"allocated":17850368,"data_stored":14093852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":771741714},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:09 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.89800000000000002,"15min":3.1030000000000002},"min":{"1min":0.316,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":0.52000000000000002,"5min":16.443000000000001,"15min":189.14500000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.94799999999999995,"15min":3.1160000000000001},"min":{"1min":0.32500000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":0.53700000000000003,"5min":11.973000000000001,"15min":189.333},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":1.4850000000000001,"15min":3.5129999999999999},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.23400000000000001},"max":{"1min":0.443,"5min":31.151,"15min":199.41},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.2909999999999999,"15min":3.262},"min":{"1min":0.31900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.60199999999999998,"5min":31.727,"15min":199.58099999999999},"last":0.45800000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.359,"15min":1.5449999999999999},"min":{"1min":0.42699999999999999,"5min":0.29499999999999998,"15min":0.26800000000000002},"max":{"1min":0.91000000000000003,"5min":47.332000000000001,"15min":48.604999999999997},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.831,"15min":1.839},"min":{"1min":0.44900000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":0.86099999999999999,"5min":47.606999999999999,"15min":47.606999999999999},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:56:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":2.714,"15min":2.407},"min":{"1min":0.51700000000000002,"5min":0.35399999999999998,"15min":0.32800000000000001},"max":{"1min":0.82499999999999996,"5min":165.88300000000001,"15min":165.88300000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.64700000000000002,"5min":2.734,"15min":1.944},"min":{"1min":0.50900000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84099999999999997,"5min":165.863,"15min":165.863},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":2.2999999999999998,"15min":1.851},"min":{"1min":0.45900000000000002,"5min":0.33800000000000002,"15min":0.29899999999999999},"max":{"1min":0.91500000000000004,"5min":97.281999999999996,"15min":97.281999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.71699999999999997,"5min":2.5329999999999999,"15min":1.956},"min":{"1min":0.45800000000000002,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":0.94799999999999995,"5min":97.242000000000004,"15min":97.242000000000004},"last":0.69699999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.109,"15min":1.393},"min":{"1min":0.48799999999999999,"5min":0.36899999999999999,"15min":0.28299999999999997},"max":{"1min":0.83899999999999997,"5min":9.8390000000000004,"15min":9.8390000000000004},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.1459999999999999,"15min":1.4299999999999999},"min":{"1min":0.53600000000000003,"5min":0.35699999999999998,"15min":0.316},"max":{"1min":0.89300000000000002,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":0.68700000000000006}]}]},{"osd":0,"up_from":649,"seq":2787433775128,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":117736,"kb_used_data":67232,"kb_used_omap":10,"kb_used_meta":50485,"kb_avail":93627416,"statfs":{"total":95995035648,"available":95874473984,"internally_reserved":0,"allocated":68845568,"data_stored":64853766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10881,"internal_metadata":51697023},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.33200000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.433}]},{"osd":4,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.69399999999999995}]},{"osd":6,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.80100000000000005}]},{"osd":7,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.73999999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599216,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":938032,"kb_used_data":9436,"kb_used_omap":5,"kb_used_meta":928570,"kb_avail":92807120,"statfs":{"total":95995035648,"available":95034490880,"internally_reserved":0,"allocated":9662464,"data_stored":5960119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":950856333},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.48499999999999999}]},{"osd":1,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":1.9299999999999999,"15min":3.7530000000000001},"min":{"1min":0.26000000000000001,"5min":0.20499999999999999,"15min":0.20399999999999999},"max":{"1min":0.58699999999999997,"5min":80.828999999999994,"15min":202.369},"last":0.374},{"interface":"front","average":{"1min":0.46400000000000002,"5min":2,"15min":3.8570000000000002},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.21099999999999999},"max":{"1min":0.66500000000000004,"5min":83.114999999999995,"15min":202.54499999999999},"last":0.32200000000000001}]},{"osd":3,"last update":"Sat Sep 30 09:56:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":3.6760000000000002,"15min":4.3570000000000002},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.23699999999999999},"max":{"1min":0.66400000000000003,"5min":231.00700000000001,"15min":231.00700000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":3.6000000000000001,"15min":4.2670000000000003},"min":{"1min":0.35999999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.64900000000000002,"5min":230.91499999999999,"15min":230.91499999999999},"last":0.434}]},{"osd":4,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.97099999999999997,"15min":2.8300000000000001},"min":{"1min":0.42199999999999999,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":0.90200000000000002,"5min":4.96,"15min":247.655},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.627,"5min":0.86899999999999999,"15min":2.7330000000000001},"min":{"1min":0.40500000000000003,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":0.86199999999999999,"5min":5.7770000000000001,"15min":248.012},"last":0.57599999999999996}]},{"osd":5,"last update":"Sat Sep 30 09:56:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":1.1259999999999999,"15min":2.9449999999999998},"min":{"1min":0.44500000000000001,"5min":0.36399999999999999,"15min":0.30099999999999999},"max":{"1min":0.879,"5min":10.762,"15min":243.86500000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.4079999999999999,"15min":3.194},"min":{"1min":0.49099999999999999,"5min":0.36199999999999999,"15min":0.32100000000000001},"max":{"1min":0.94799999999999995,"5min":15.164999999999999,"15min":244.02199999999999},"last":0.71099999999999997}]},{"osd":6,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.6080000000000001,"15min":2.141},"min":{"1min":0.442,"5min":0.34699999999999998,"15min":0.29399999999999998},"max":{"1min":0.82699999999999996,"5min":46.732999999999997,"15min":202.70599999999999},"last":0.629},{"interface":"front","average":{"1min":0.68600000000000005,"5min":1.946,"15min":2.4569999999999999},"min":{"1min":0.52600000000000002,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":0.84399999999999997,"5min":46.68,"15min":202.916},"last":0.68700000000000006}]},{"osd":7,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.2190000000000001,"15min":3.407},"min":{"1min":0.51300000000000001,"5min":0.39500000000000002,"15min":0.32000000000000001},"max":{"1min":0.83499999999999996,"5min":9.1950000000000003,"15min":244.911},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.70999999999999996,"5min":1.6200000000000001,"15min":3.7000000000000002},"min":{"1min":0.55600000000000005,"5min":0.40999999999999998,"15min":0.36599999999999999},"max":{"1min":0.88500000000000001,"5min":10.782,"15min":244.66},"last":0.73099999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435692,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1020828,"kb_used_data":9676,"kb_used_omap":2,"kb_used_meta":1011133,"kb_avail":92724324,"statfs":{"total":95995035648,"available":94949707776,"internally_reserved":0,"allocated":9908224,"data_stored":6044551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":1035400551},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.45100000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:56:13 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":1.6699999999999999,"15min":2.5179999999999998},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.19400000000000001},"max":{"1min":0.54000000000000004,"5min":63.744999999999997,"15min":142.434},"last":0.378},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.708,"15min":2.4449999999999998},"min":{"1min":0.33100000000000002,"5min":0.24399999999999999,"15min":0.20699999999999999},"max":{"1min":0.55500000000000005,"5min":63.762,"15min":142.44900000000001},"last":0.41999999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.91700000000000004,"15min":3.085},"min":{"1min":0.28499999999999998,"5min":0.218,"15min":0.218},"max":{"1min":0.79900000000000004,"5min":8.7040000000000006,"15min":147.87},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.98599999999999999,"15min":3.1499999999999999},"min":{"1min":0.29299999999999998,"5min":0.22,"15min":0.22},"max":{"1min":0.82699999999999996,"5min":13.064,"15min":148.04900000000001},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.1830000000000001,"15min":2.3300000000000001},"min":{"1min":0.44600000000000001,"5min":0.28499999999999998,"15min":0.26800000000000002},"max":{"1min":5.6829999999999998,"5min":7.4729999999999999,"15min":172.298},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.629,"5min":1.462,"15min":2.6080000000000001},"min":{"1min":0.436,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":0.95399999999999996,"5min":9.8409999999999993,"15min":172.45099999999999},"last":0.66200000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:56:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":2.2629999999999999,"15min":1.958},"min":{"1min":0.40500000000000003,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":1.0309999999999999,"5min":61.426000000000002,"15min":61.426000000000002},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.67000000000000004,"5min":2.4980000000000002,"15min":2.052},"min":{"1min":0.39600000000000002,"5min":0.373,"15min":0.311},"max":{"1min":1.0529999999999999,"5min":61.469000000000001,"15min":61.469000000000001},"last":0.72199999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":2.4489999999999998,"15min":1.9330000000000001},"min":{"1min":0.44700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":5.7839999999999998,"5min":75.781000000000006,"15min":75.781000000000006},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":2.5019999999999998,"15min":2.173},"min":{"1min":0.47699999999999998,"5min":0.40899999999999997,"15min":0.35899999999999999},"max":{"1min":1.016,"5min":75.754000000000005,"15min":75.754000000000005},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.6080000000000001,"15min":2.645},"min":{"1min":0.437,"5min":0.36299999999999999,"15min":0.32900000000000001},"max":{"1min":0.92800000000000005,"5min":90.855999999999995,"15min":213.548},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.92200000000000004,"5min":2.641,"15min":2.649},"min":{"1min":0.51100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.7460000000000004,"5min":91.108000000000004,"15min":213.583},"last":0.79000000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019454,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":627088,"kb_used_data":7368,"kb_used_omap":0,"kb_used_meta":619712,"kb_avail":93118064,"statfs":{"total":95995035648,"available":95352897536,"internally_reserved":0,"allocated":7544832,"data_stored":5457895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":634585088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.47399999999999998}]},{"osd":1,"last update":"Sat Sep 30 09:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.522,"15min":2.3119999999999998},"min":{"1min":0.436,"5min":0.224,"15min":0.224},"max":{"1min":0.752,"5min":32.536000000000001,"15min":108.551},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.5349999999999999,"15min":2.4420000000000002},"min":{"1min":0.36299999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.83199999999999996,"5min":32.637,"15min":108.884},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 30 09:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":4.8680000000000003,"15min":3.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71699999999999997,"5min":139.53899999999999,"15min":142.88999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":4.4050000000000002,"15min":3.8399999999999999},"min":{"1min":0.49399999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.82899999999999996,"5min":139.56399999999999,"15min":142.90700000000001},"last":0.68200000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.5960000000000001,"15min":2.415},"min":{"1min":0.46200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":0.81499999999999995,"5min":29.675999999999998,"15min":105.142},"last":0.751},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.4259999999999999,"15min":2.2879999999999998},"min":{"1min":0.441,"5min":0.309,"15min":0.29299999999999998},"max":{"1min":0.76800000000000002,"5min":29.664999999999999,"15min":105.157},"last":0.69399999999999995}]},{"osd":5,"last update":"Sat Sep 30 09:56:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":1.0600000000000001,"15min":4.0300000000000002},"min":{"1min":0.38900000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":0.64500000000000002,"5min":8.4329999999999998,"15min":140.11099999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.5920000000000001,"15min":4.109},"min":{"1min":0.35599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.70199999999999996,"5min":22.248000000000001,"15min":140.065},"last":0.66600000000000004}]},{"osd":6,"last update":"Sat Sep 30 09:56:41 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.732,"15min":1.3140000000000001},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":0.79000000000000004,"5min":71.864999999999995,"15min":71.864999999999995},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.9550000000000001,"15min":1.403},"min":{"1min":0.40899999999999997,"5min":0.29099999999999998,"15min":0.253},"max":{"1min":0.71999999999999997,"5min":72.018000000000001,"15min":72.018000000000001},"last":0.624}]},{"osd":7,"last update":"Sat Sep 30 09:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.7949999999999999,"15min":1.9350000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27600000000000002},"max":{"1min":1.069,"5min":16.858000000000001,"15min":161.72999999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.2290000000000001,"15min":1.655},"min":{"1min":0.44500000000000001,"5min":0.32400000000000001,"15min":0.27000000000000002},"max":{"1min":1.117,"5min":14.018000000000001,"15min":162.60900000000001},"last":0.77900000000000003}]}]},{"osd":5,"up_from":489,"seq":2100239007896,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":262732,"kb_used_data":7620,"kb_used_omap":6,"kb_used_meta":255097,"kb_avail":93482420,"statfs":{"total":95995035648,"available":95725998080,"internally_reserved":0,"allocated":7802880,"data_stored":5716565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":261219423},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.375}]},{"osd":1,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.1020000000000001,"15min":5.8470000000000004},"min":{"1min":0.35899999999999999,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":1.083,"5min":11.234999999999999,"15min":273.48200000000003},"last":0.503},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.0620000000000001,"15min":5.5430000000000001},"min":{"1min":0.316,"5min":0.28899999999999998,"15min":0.255},"max":{"1min":0.80700000000000005,"5min":8.0820000000000007,"15min":273.50200000000001},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.266,"15min":3.363},"min":{"1min":0.39700000000000002,"5min":0.28899999999999998,"15min":0.26400000000000001},"max":{"1min":1.071,"5min":13.625999999999999,"15min":194.14400000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.629,"5min":1.4430000000000001,"15min":3.0979999999999999},"min":{"1min":0.41499999999999998,"5min":0.313,"15min":0.27800000000000002},"max":{"1min":0.95499999999999996,"5min":11.337999999999999,"15min":198.083},"last":0.621}]},{"osd":3,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.972,"15min":6.6200000000000001},"min":{"1min":0.44600000000000001,"5min":0.377,"15min":0.30299999999999999},"max":{"1min":1.0329999999999999,"5min":41.872,"15min":274.01799999999997},"last":0.435},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.8069999999999999,"15min":3.0670000000000002},"min":{"1min":0.54900000000000004,"5min":0.29799999999999999,"15min":0.29499999999999998},"max":{"1min":1.139,"5min":41.829000000000001,"15min":130.07400000000001},"last":0.65300000000000002}]},{"osd":4,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.997,"15min":2.1509999999999998},"min":{"1min":0.30599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.68100000000000005,"5min":13.601000000000001,"15min":132.94399999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.0209999999999999,"15min":2.2749999999999999},"min":{"1min":0.41599999999999998,"5min":0.29399999999999998,"15min":0.28899999999999998},"max":{"1min":0.89000000000000001,"5min":14.223000000000001,"15min":136.083},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.5189999999999999,"15min":2.0270000000000001},"min":{"1min":0.41399999999999998,"5min":0.30399999999999999,"15min":0.30199999999999999},"max":{"1min":0.75,"5min":66.123000000000005,"15min":130.09800000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.7549999999999999,"15min":2.044},"min":{"1min":0.49099999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.999,"5min":66.102999999999994,"15min":130.03399999999999},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":1.155,"15min":1.3899999999999999},"min":{"1min":0.51800000000000002,"5min":0.33800000000000002,"15min":0.26700000000000002},"max":{"1min":0.92600000000000005,"5min":32.578000000000003,"15min":32.578000000000003},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.4159999999999999,"15min":1.367},"min":{"1min":0.46700000000000003,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":0.86499999999999999,"5min":32.420999999999999,"15min":32.420999999999999},"last":0.57899999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10881,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:56:59.881 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-30T09:57:00.252 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-30T09:57:00.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:57:00 smithi019 ceph-mon[224043]: from='client.47939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:57:00.478 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:00 smithi019 ceph-mon[216815]: from='client.47939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:57:00.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:00 smithi167 ceph-mon[192880]: from='client.47939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:57:01.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:01 smithi019 ceph-mon[216815]: pgmap v1826: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:57:01.496 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:57:01 smithi019 ceph-mon[224043]: pgmap v1826: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:57:01.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:01 smithi167 ceph-mon[192880]: pgmap v1826: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:57:02.212 INFO:teuthology.orchestra.run.smithi019.stdout: 2023-09-30T09:57:02.214 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2023-09-30T09:57:03.132 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":1827,"stamp":"2023-09-30T09:57:02.127252+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984487,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11978,"num_read_kb":26422,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":441,"num_bytes_recovered":17972206,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5032164,"kb_used_data":123080,"kb_used_omap":35,"kb_used_meta":4908956,"kb_avail":744929052,"statfs":{"total":767960285184,"available":762807349248,"internally_reserved":0,"allocated":126033920,"data_stored":102361199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36380,"internal_metadata":5026771428},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003484"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":1178,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024957+0000","last_change":"2023-09-30T09:55:20.016692+0000","last_active":"2023-09-30T09:56:18.024957+0000","last_peered":"2023-09-30T09:56:18.024957+0000","last_clean":"2023-09-30T09:56:18.024957+0000","last_became_active":"2023-09-30T09:55:20.016204+0000","last_became_peered":"2023-09-30T09:55:20.016204+0000","last_unstale":"2023-09-30T09:56:18.024957+0000","last_undegraded":"2023-09-30T09:56:18.024957+0000","last_fullsized":"2023-09-30T09:56:18.024957+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.542660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.542660+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:56:35.749055+0000","scrub_duration":0.003453481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":721,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028158+0000","last_change":"2023-09-30T09:50:41.402505+0000","last_active":"2023-09-30T09:56:18.028158+0000","last_peered":"2023-09-30T09:56:18.028158+0000","last_clean":"2023-09-30T09:56:18.028158+0000","last_became_active":"2023-09-30T09:50:41.402099+0000","last_became_peered":"2023-09-30T09:50:41.402099+0000","last_unstale":"2023-09-30T09:56:18.028158+0000","last_undegraded":"2023-09-30T09:56:18.028158+0000","last_fullsized":"2023-09-30T09:56:18.028158+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:22:16.978324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":414,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.023114+0000","last_change":"2023-09-30T09:55:31.011863+0000","last_active":"2023-09-30T09:56:18.023114+0000","last_peered":"2023-09-30T09:56:18.023114+0000","last_clean":"2023-09-30T09:56:18.023114+0000","last_became_active":"2023-09-30T09:55:31.011503+0000","last_became_peered":"2023-09-30T09:55:31.011503+0000","last_unstale":"2023-09-30T09:56:18.023114+0000","last_undegraded":"2023-09-30T09:56:18.023114+0000","last_fullsized":"2023-09-30T09:56:18.023114+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:08:04.366818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":1150,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.034683+0000","last_change":"2023-09-30T09:56:19.034683+0000","last_active":"2023-09-30T09:56:19.034683+0000","last_peered":"2023-09-30T09:56:19.034683+0000","last_clean":"2023-09-30T09:56:19.034683+0000","last_became_active":"2023-09-30T09:56:19.034285+0000","last_became_peered":"2023-09-30T09:56:19.034285+0000","last_unstale":"2023-09-30T09:56:19.034683+0000","last_undegraded":"2023-09-30T09:56:19.034683+0000","last_fullsized":"2023-09-30T09:56:19.034683+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:01.514088+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:01.514088+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T21:03:31.719439+0000","scrub_duration":0.0022103969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":260,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031416+0000","last_change":"2023-09-30T09:56:04.199026+0000","last_active":"2023-09-30T09:56:18.031416+0000","last_peered":"2023-09-30T09:56:18.031416+0000","last_clean":"2023-09-30T09:56:18.031416+0000","last_became_active":"2023-09-30T09:56:04.198742+0000","last_became_peered":"2023-09-30T09:56:04.198742+0000","last_unstale":"2023-09-30T09:56:18.031416+0000","last_undegraded":"2023-09-30T09:56:18.031416+0000","last_fullsized":"2023-09-30T09:56:18.031416+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:04:30.898770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026082+0000","last_change":"2023-09-30T09:56:16.034010+0000","last_active":"2023-09-30T09:56:19.026082+0000","last_peered":"2023-09-30T09:56:19.026082+0000","last_clean":"2023-09-30T09:56:19.026082+0000","last_became_active":"2023-09-30T09:56:16.033459+0000","last_became_peered":"2023-09-30T09:56:16.033459+0000","last_unstale":"2023-09-30T09:56:19.026082+0000","last_undegraded":"2023-09-30T09:56:19.026082+0000","last_fullsized":"2023-09-30T09:56:19.026082+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:40:58.995167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":472,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.027887+0000","last_change":"2023-09-30T09:55:40.611251+0000","last_active":"2023-09-30T09:56:18.027887+0000","last_peered":"2023-09-30T09:56:18.027887+0000","last_clean":"2023-09-30T09:56:18.027887+0000","last_became_active":"2023-09-30T09:55:38.063330+0000","last_became_peered":"2023-09-30T09:55:38.063330+0000","last_unstale":"2023-09-30T09:56:18.027887+0000","last_undegraded":"2023-09-30T09:56:18.027887+0000","last_fullsized":"2023-09-30T09:56:18.027887+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:55:40.611145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:55:40.611145+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:44:34.231149+0000","scrub_duration":0.0017040180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":473,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022434+0000","last_change":"2023-09-30T09:56:16.026563+0000","last_active":"2023-09-30T09:56:18.022434+0000","last_peered":"2023-09-30T09:56:18.022434+0000","last_clean":"2023-09-30T09:56:18.022434+0000","last_became_active":"2023-09-30T09:56:16.026274+0000","last_became_peered":"2023-09-30T09:56:16.026274+0000","last_unstale":"2023-09-30T09:56:18.022434+0000","last_undegraded":"2023-09-30T09:56:18.022434+0000","last_fullsized":"2023-09-30T09:56:18.022434+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:43:02.853772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":144,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031301+0000","last_change":"2023-09-30T09:49:31.310014+0000","last_active":"2023-09-30T09:56:18.031301+0000","last_peered":"2023-09-30T09:56:18.031301+0000","last_clean":"2023-09-30T09:56:18.031301+0000","last_became_active":"2023-09-30T09:49:31.309121+0000","last_became_peered":"2023-09-30T09:49:31.309121+0000","last_unstale":"2023-09-30T09:56:18.031301+0000","last_undegraded":"2023-09-30T09:56:18.031301+0000","last_fullsized":"2023-09-30T09:56:18.031301+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:25:46.598805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":1086,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034597+0000","last_change":"2023-09-30T09:44:42.516907+0000","last_active":"2023-09-30T09:56:18.034597+0000","last_peered":"2023-09-30T09:56:18.034597+0000","last_clean":"2023-09-30T09:56:18.034597+0000","last_became_active":"2023-09-30T09:44:26.067162+0000","last_became_peered":"2023-09-30T09:44:26.067162+0000","last_unstale":"2023-09-30T09:56:18.034597+0000","last_undegraded":"2023-09-30T09:56:18.034597+0000","last_fullsized":"2023-09-30T09:56:18.034597+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:42.516842+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:04.164029+0000","last_clean_scrub_stamp":"2023-09-30T09:44:42.516842+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:13:47.488287+0000","scrub_duration":0.0075485639999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026279+0000","last_change":"2023-09-30T09:56:16.028635+0000","last_active":"2023-09-30T09:56:19.026279+0000","last_peered":"2023-09-30T09:56:19.026279+0000","last_clean":"2023-09-30T09:56:19.026279+0000","last_became_active":"2023-09-30T09:56:16.028298+0000","last_became_peered":"2023-09-30T09:56:16.028298+0000","last_unstale":"2023-09-30T09:56:19.026279+0000","last_undegraded":"2023-09-30T09:56:19.026279+0000","last_fullsized":"2023-09-30T09:56:19.026279+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:59:32.064635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":286,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.023083+0000","last_change":"2023-09-30T09:49:31.295632+0000","last_active":"2023-09-30T09:56:19.023083+0000","last_peered":"2023-09-30T09:56:19.023083+0000","last_clean":"2023-09-30T09:56:19.023083+0000","last_became_active":"2023-09-30T09:49:31.295269+0000","last_became_peered":"2023-09-30T09:49:31.295269+0000","last_unstale":"2023-09-30T09:56:19.023083+0000","last_undegraded":"2023-09-30T09:56:19.023083+0000","last_fullsized":"2023-09-30T09:56:19.023083+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:32.754585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":702,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032792+0000","last_change":"2023-09-30T09:55:38.066782+0000","last_active":"2023-09-30T09:56:18.032792+0000","last_peered":"2023-09-30T09:56:18.032792+0000","last_clean":"2023-09-30T09:56:18.032792+0000","last_became_active":"2023-09-30T09:55:38.066366+0000","last_became_peered":"2023-09-30T09:55:38.066366+0000","last_unstale":"2023-09-30T09:56:18.032792+0000","last_undegraded":"2023-09-30T09:56:18.032792+0000","last_fullsized":"2023-09-30T09:56:18.032792+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.801717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.801717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:15:07.468106+0000","scrub_duration":0.001467587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":105,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025290+0000","last_change":"2023-09-30T09:51:57.160642+0000","last_active":"2023-09-30T09:56:18.025290+0000","last_peered":"2023-09-30T09:56:18.025290+0000","last_clean":"2023-09-30T09:56:18.025290+0000","last_became_active":"2023-09-30T09:51:57.160357+0000","last_became_peered":"2023-09-30T09:51:57.160357+0000","last_unstale":"2023-09-30T09:56:18.025290+0000","last_undegraded":"2023-09-30T09:56:18.025290+0000","last_fullsized":"2023-09-30T09:56:18.025290+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:25:40.417904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":552,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024650+0000","last_change":"2023-09-30T09:55:31.024533+0000","last_active":"2023-09-30T09:56:18.024650+0000","last_peered":"2023-09-30T09:56:18.024650+0000","last_clean":"2023-09-30T09:56:18.024650+0000","last_became_active":"2023-09-30T09:55:31.024098+0000","last_became_peered":"2023-09-30T09:55:31.024098+0000","last_unstale":"2023-09-30T09:56:18.024650+0000","last_undegraded":"2023-09-30T09:56:18.024650+0000","last_fullsized":"2023-09-30T09:56:18.024650+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:52:06.951507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":894,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.035750+0000","last_change":"2023-09-30T09:56:19.035750+0000","last_active":"2023-09-30T09:56:19.035750+0000","last_peered":"2023-09-30T09:56:19.035750+0000","last_clean":"2023-09-30T09:56:19.035750+0000","last_became_active":"2023-09-30T09:56:19.035369+0000","last_became_peered":"2023-09-30T09:56:19.035369+0000","last_unstale":"2023-09-30T09:56:19.035750+0000","last_undegraded":"2023-09-30T09:56:19.035750+0000","last_fullsized":"2023-09-30T09:56:19.035750+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:59.523496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:59.523496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:56:23.317181+0000","scrub_duration":0.0040625289999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":399,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034787+0000","last_change":"2023-09-30T09:42:51.318514+0000","last_active":"2023-09-30T09:56:18.034787+0000","last_peered":"2023-09-30T09:56:18.034787+0000","last_clean":"2023-09-30T09:56:18.034787+0000","last_became_active":"2023-09-30T09:42:48.640119+0000","last_became_peered":"2023-09-30T09:42:48.640119+0000","last_unstale":"2023-09-30T09:56:18.034787+0000","last_undegraded":"2023-09-30T09:56:18.034787+0000","last_fullsized":"2023-09-30T09:56:18.034787+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:06:50.596780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":357,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028752+0000","last_change":"2023-09-30T09:56:04.201468+0000","last_active":"2023-09-30T09:56:18.028752+0000","last_peered":"2023-09-30T09:56:18.028752+0000","last_clean":"2023-09-30T09:56:18.028752+0000","last_became_active":"2023-09-30T09:56:04.200792+0000","last_became_peered":"2023-09-30T09:56:04.200792+0000","last_unstale":"2023-09-30T09:56:18.028752+0000","last_undegraded":"2023-09-30T09:56:18.028752+0000","last_fullsized":"2023-09-30T09:56:18.028752+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:43:36.404355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":514,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032117+0000","last_change":"2023-09-30T09:56:04.194412+0000","last_active":"2023-09-30T09:56:18.032117+0000","last_peered":"2023-09-30T09:56:18.032117+0000","last_clean":"2023-09-30T09:56:18.032117+0000","last_became_active":"2023-09-30T09:56:04.194022+0000","last_became_peered":"2023-09-30T09:56:04.194022+0000","last_unstale":"2023-09-30T09:56:18.032117+0000","last_undegraded":"2023-09-30T09:56:18.032117+0000","last_fullsized":"2023-09-30T09:56:18.032117+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:13.197982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:13.197982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:42:18.764126+0000","scrub_duration":0.002929367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":41,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028091+0000","last_change":"2023-09-30T09:55:35.062278+0000","last_active":"2023-09-30T09:56:18.028091+0000","last_peered":"2023-09-30T09:56:18.028091+0000","last_clean":"2023-09-30T09:56:18.028091+0000","last_became_active":"2023-09-30T09:55:35.055015+0000","last_became_peered":"2023-09-30T09:55:35.055015+0000","last_unstale":"2023-09-30T09:56:18.028091+0000","last_undegraded":"2023-09-30T09:56:18.028091+0000","last_fullsized":"2023-09-30T09:56:18.028091+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:35:52.637156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4722,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9444,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":725,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030527+0000","last_change":"2023-09-30T09:49:31.304624+0000","last_active":"2023-09-30T09:56:18.030527+0000","last_peered":"2023-09-30T09:56:18.030527+0000","last_clean":"2023-09-30T09:56:18.030527+0000","last_became_active":"2023-09-30T09:49:31.304106+0000","last_became_peered":"2023-09-30T09:49:31.304106+0000","last_unstale":"2023-09-30T09:56:18.030527+0000","last_undegraded":"2023-09-30T09:56:18.030527+0000","last_fullsized":"2023-09-30T09:56:18.030527+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:50:42.343641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":848,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.031051+0000","last_change":"2023-09-30T09:56:19.030331+0000","last_active":"2023-09-30T09:56:19.031051+0000","last_peered":"2023-09-30T09:56:19.031051+0000","last_clean":"2023-09-30T09:56:19.031051+0000","last_became_active":"2023-09-30T09:56:19.029784+0000","last_became_peered":"2023-09-30T09:56:19.029784+0000","last_unstale":"2023-09-30T09:56:19.031051+0000","last_undegraded":"2023-09-30T09:56:19.031051+0000","last_fullsized":"2023-09-30T09:56:19.031051+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:04.211154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:04.211154+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:50:13.869560+0000","scrub_duration":0.001957295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":399,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.033100+0000","last_change":"2023-09-30T09:55:35.058374+0000","last_active":"2023-09-30T09:56:18.033100+0000","last_peered":"2023-09-30T09:56:18.033100+0000","last_clean":"2023-09-30T09:56:18.033100+0000","last_became_active":"2023-09-30T09:55:35.051488+0000","last_became_peered":"2023-09-30T09:55:35.051488+0000","last_unstale":"2023-09-30T09:56:18.033100+0000","last_undegraded":"2023-09-30T09:56:18.033100+0000","last_fullsized":"2023-09-30T09:56:18.033100+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:59:40.904205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":476,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022626+0000","last_change":"2023-09-30T09:55:31.025249+0000","last_active":"2023-09-30T09:56:18.022626+0000","last_peered":"2023-09-30T09:56:18.022626+0000","last_clean":"2023-09-30T09:56:18.022626+0000","last_became_active":"2023-09-30T09:55:31.024871+0000","last_became_peered":"2023-09-30T09:55:31.024871+0000","last_unstale":"2023-09-30T09:56:18.022626+0000","last_undegraded":"2023-09-30T09:56:18.022626+0000","last_fullsized":"2023-09-30T09:56:18.022626+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:14:59.192864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":1046,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024837+0000","last_change":"2023-09-30T09:49:34.332543+0000","last_active":"2023-09-30T09:56:18.024837+0000","last_peered":"2023-09-30T09:56:18.024837+0000","last_clean":"2023-09-30T09:56:18.024837+0000","last_became_active":"2023-09-30T09:49:34.332215+0000","last_became_peered":"2023-09-30T09:49:34.332215+0000","last_unstale":"2023-09-30T09:56:18.024837+0000","last_undegraded":"2023-09-30T09:56:18.024837+0000","last_fullsized":"2023-09-30T09:56:18.024837+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:58.499394+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:31:58.499394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:49:28.827671+0000","scrub_duration":0.001420342,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026680+0000","last_change":"2023-09-30T09:56:16.032853+0000","last_active":"2023-09-30T09:56:19.026680+0000","last_peered":"2023-09-30T09:56:19.026680+0000","last_clean":"2023-09-30T09:56:19.026680+0000","last_became_active":"2023-09-30T09:56:16.032523+0000","last_became_peered":"2023-09-30T09:56:16.032523+0000","last_unstale":"2023-09-30T09:56:19.026680+0000","last_undegraded":"2023-09-30T09:56:19.026680+0000","last_fullsized":"2023-09-30T09:56:19.026680+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:32:24.924923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025134+0000","last_change":"2023-09-30T09:51:57.157155+0000","last_active":"2023-09-30T09:56:18.025134+0000","last_peered":"2023-09-30T09:56:18.025134+0000","last_clean":"2023-09-30T09:56:18.025134+0000","last_became_active":"2023-09-30T09:51:57.156843+0000","last_became_peered":"2023-09-30T09:51:57.156843+0000","last_unstale":"2023-09-30T09:56:18.025134+0000","last_undegraded":"2023-09-30T09:56:18.025134+0000","last_fullsized":"2023-09-30T09:56:18.025134+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:12:00.333473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":742,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.033445+0000","last_change":"2023-09-30T09:56:04.201313+0000","last_active":"2023-09-30T09:56:18.033445+0000","last_peered":"2023-09-30T09:56:18.033445+0000","last_clean":"2023-09-30T09:56:18.033445+0000","last_became_active":"2023-09-30T09:56:04.201004+0000","last_became_peered":"2023-09-30T09:56:04.201004+0000","last_unstale":"2023-09-30T09:56:18.033445+0000","last_undegraded":"2023-09-30T09:56:18.033445+0000","last_fullsized":"2023-09-30T09:56:18.033445+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.154171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.154171+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:59:32.950192+0000","scrub_duration":0.010869449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":389,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034400+0000","last_change":"2023-09-30T09:49:31.309022+0000","last_active":"2023-09-30T09:56:18.034400+0000","last_peered":"2023-09-30T09:56:18.034400+0000","last_clean":"2023-09-30T09:56:18.034400+0000","last_became_active":"2023-09-30T09:49:31.308236+0000","last_became_peered":"2023-09-30T09:49:31.308236+0000","last_unstale":"2023-09-30T09:56:18.034400+0000","last_undegraded":"2023-09-30T09:56:18.034400+0000","last_fullsized":"2023-09-30T09:56:18.034400+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:51:39.779954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026842+0000","last_change":"2023-09-30T09:56:16.033776+0000","last_active":"2023-09-30T09:56:19.026842+0000","last_peered":"2023-09-30T09:56:19.026842+0000","last_clean":"2023-09-30T09:56:19.026842+0000","last_became_active":"2023-09-30T09:56:16.033010+0000","last_became_peered":"2023-09-30T09:56:16.033010+0000","last_unstale":"2023-09-30T09:56:19.026842+0000","last_undegraded":"2023-09-30T09:56:19.026842+0000","last_fullsized":"2023-09-30T09:56:19.026842+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:13:46.366804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":1135,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025139+0000","last_change":"2023-09-30T09:49:34.332561+0000","last_active":"2023-09-30T09:56:18.025139+0000","last_peered":"2023-09-30T09:56:18.025139+0000","last_clean":"2023-09-30T09:56:18.025139+0000","last_became_active":"2023-09-30T09:49:34.332170+0000","last_became_peered":"2023-09-30T09:49:34.332170+0000","last_unstale":"2023-09-30T09:56:18.025139+0000","last_undegraded":"2023-09-30T09:56:18.025139+0000","last_fullsized":"2023-09-30T09:56:18.025139+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:39.746150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:39.746150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:11:22.548068+0000","scrub_duration":0.039977371999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":74,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030987+0000","last_change":"2023-09-30T09:56:16.044871+0000","last_active":"2023-09-30T09:56:18.030987+0000","last_peered":"2023-09-30T09:56:18.030987+0000","last_clean":"2023-09-30T09:56:18.030987+0000","last_became_active":"2023-09-30T09:56:16.026570+0000","last_became_peered":"2023-09-30T09:56:16.026570+0000","last_unstale":"2023-09-30T09:56:18.030987+0000","last_undegraded":"2023-09-30T09:56:18.030987+0000","last_fullsized":"2023-09-30T09:56:18.030987+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:42:25.744814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":261,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.023272+0000","last_change":"2023-09-30T09:56:07.282221+0000","last_active":"2023-09-30T09:56:18.023272+0000","last_peered":"2023-09-30T09:56:18.023272+0000","last_clean":"2023-09-30T09:56:18.023272+0000","last_became_active":"2023-09-30T09:56:07.281802+0000","last_became_peered":"2023-09-30T09:56:07.281802+0000","last_unstale":"2023-09-30T09:56:18.023272+0000","last_undegraded":"2023-09-30T09:56:18.023272+0000","last_fullsized":"2023-09-30T09:56:18.023272+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:34:05.315437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1101,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034695+0000","last_change":"2023-09-30T09:55:38.064282+0000","last_active":"2023-09-30T09:56:18.034695+0000","last_peered":"2023-09-30T09:56:18.034695+0000","last_clean":"2023-09-30T09:56:18.034695+0000","last_became_active":"2023-09-30T09:55:38.063909+0000","last_became_peered":"2023-09-30T09:55:38.063909+0000","last_unstale":"2023-09-30T09:56:18.034695+0000","last_undegraded":"2023-09-30T09:56:18.034695+0000","last_fullsized":"2023-09-30T09:56:18.034695+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:13.360305+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:32:08.030550+0000","last_clean_scrub_stamp":"2023-09-30T09:44:13.360305+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:49:22.387239+0000","scrub_duration":0.001699959,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":389,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.033462+0000","last_change":"2023-09-30T09:56:16.049380+0000","last_active":"2023-09-30T09:56:18.033462+0000","last_peered":"2023-09-30T09:56:18.033462+0000","last_clean":"2023-09-30T09:56:18.033462+0000","last_became_active":"2023-09-30T09:56:16.039822+0000","last_became_peered":"2023-09-30T09:56:16.039822+0000","last_unstale":"2023-09-30T09:56:18.033462+0000","last_undegraded":"2023-09-30T09:56:18.033462+0000","last_fullsized":"2023-09-30T09:56:18.033462+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:13:02.626062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024746+0000","last_change":"2023-09-30T09:51:57.167306+0000","last_active":"2023-09-30T09:56:18.024746+0000","last_peered":"2023-09-30T09:56:18.024746+0000","last_clean":"2023-09-30T09:56:18.024746+0000","last_became_active":"2023-09-30T09:51:57.167040+0000","last_became_peered":"2023-09-30T09:51:57.167040+0000","last_unstale":"2023-09-30T09:56:18.024746+0000","last_undegraded":"2023-09-30T09:56:18.024746+0000","last_fullsized":"2023-09-30T09:56:18.024746+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:27:26.793530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":1132,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.024359+0000","last_change":"2023-09-30T09:55:06.180049+0000","last_active":"2023-09-30T09:56:19.024359+0000","last_peered":"2023-09-30T09:56:19.024359+0000","last_clean":"2023-09-30T09:56:19.024359+0000","last_became_active":"2023-09-30T09:55:06.179542+0000","last_became_peered":"2023-09-30T09:55:06.179542+0000","last_unstale":"2023-09-30T09:56:19.024359+0000","last_undegraded":"2023-09-30T09:56:19.024359+0000","last_fullsized":"2023-09-30T09:56:19.024359+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:15.182022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:15.182022+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:28:32.391624+0000","scrub_duration":0.0026830959999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":359,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032917+0000","last_change":"2023-09-30T09:55:35.060477+0000","last_active":"2023-09-30T09:56:18.032917+0000","last_peered":"2023-09-30T09:56:18.032917+0000","last_clean":"2023-09-30T09:56:18.032917+0000","last_became_active":"2023-09-30T09:55:35.052062+0000","last_became_peered":"2023-09-30T09:55:35.052062+0000","last_unstale":"2023-09-30T09:56:18.032917+0000","last_undegraded":"2023-09-30T09:56:18.032917+0000","last_fullsized":"2023-09-30T09:56:18.032917+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:50:59.125368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":744,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029064+0000","last_change":"2023-09-30T09:50:26.903683+0000","last_active":"2023-09-30T09:56:18.029064+0000","last_peered":"2023-09-30T09:56:18.029064+0000","last_clean":"2023-09-30T09:56:18.029064+0000","last_became_active":"2023-09-30T09:50:26.903262+0000","last_became_peered":"2023-09-30T09:50:26.903262+0000","last_unstale":"2023-09-30T09:56:18.029064+0000","last_undegraded":"2023-09-30T09:56:18.029064+0000","last_fullsized":"2023-09-30T09:56:18.029064+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:02:40.724403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":640,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031492+0000","last_change":"2023-09-30T09:55:49.124176+0000","last_active":"2023-09-30T09:56:18.031492+0000","last_peered":"2023-09-30T09:56:18.031492+0000","last_clean":"2023-09-30T09:56:18.031492+0000","last_became_active":"2023-09-30T09:55:49.123676+0000","last_became_peered":"2023-09-30T09:55:49.123676+0000","last_unstale":"2023-09-30T09:56:18.031492+0000","last_undegraded":"2023-09-30T09:56:18.031492+0000","last_fullsized":"2023-09-30T09:56:18.031492+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:49:16.116795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:49:16.116795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:25:26.020663+0000","scrub_duration":0.045979904000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":454,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034153+0000","last_change":"2023-09-30T09:51:57.192521+0000","last_active":"2023-09-30T09:56:18.034153+0000","last_peered":"2023-09-30T09:56:18.034153+0000","last_clean":"2023-09-30T09:56:18.034153+0000","last_became_active":"2023-09-30T09:51:57.170876+0000","last_became_peered":"2023-09-30T09:51:57.170876+0000","last_unstale":"2023-09-30T09:56:18.034153+0000","last_undegraded":"2023-09-30T09:56:18.034153+0000","last_fullsized":"2023-09-30T09:56:18.034153+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:44:49.877225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":463,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029192+0000","last_change":"2023-09-30T09:55:49.121505+0000","last_active":"2023-09-30T09:56:18.029192+0000","last_peered":"2023-09-30T09:56:18.029192+0000","last_clean":"2023-09-30T09:56:18.029192+0000","last_became_active":"2023-09-30T09:55:49.121169+0000","last_became_peered":"2023-09-30T09:55:49.121169+0000","last_unstale":"2023-09-30T09:56:18.029192+0000","last_undegraded":"2023-09-30T09:56:18.029192+0000","last_fullsized":"2023-09-30T09:56:18.029192+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:10:34.361732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":586,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029945+0000","last_change":"2023-09-30T09:55:38.064129+0000","last_active":"2023-09-30T09:56:18.029945+0000","last_peered":"2023-09-30T09:56:18.029945+0000","last_clean":"2023-09-30T09:56:18.029945+0000","last_became_active":"2023-09-30T09:55:38.063727+0000","last_became_peered":"2023-09-30T09:55:38.063727+0000","last_unstale":"2023-09-30T09:56:18.029945+0000","last_undegraded":"2023-09-30T09:56:18.029945+0000","last_fullsized":"2023-09-30T09:56:18.029945+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:37:53.710495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:37:53.710495+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:54:17.971207+0000","scrub_duration":0.002088267,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":640,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025606+0000","last_change":"2023-09-30T09:49:31.301460+0000","last_active":"2023-09-30T09:56:18.025606+0000","last_peered":"2023-09-30T09:56:18.025606+0000","last_clean":"2023-09-30T09:56:18.025606+0000","last_became_active":"2023-09-30T09:49:31.301114+0000","last_became_peered":"2023-09-30T09:49:31.301114+0000","last_unstale":"2023-09-30T09:56:18.025606+0000","last_undegraded":"2023-09-30T09:56:18.025606+0000","last_fullsized":"2023-09-30T09:56:18.025606+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:42:01.066974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":228,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028067+0000","last_change":"2023-09-30T09:55:06.212302+0000","last_active":"2023-09-30T09:56:18.028067+0000","last_peered":"2023-09-30T09:56:18.028067+0000","last_clean":"2023-09-30T09:56:18.028067+0000","last_became_active":"2023-09-30T09:55:06.211890+0000","last_became_peered":"2023-09-30T09:55:06.211890+0000","last_unstale":"2023-09-30T09:56:18.028067+0000","last_undegraded":"2023-09-30T09:56:18.028067+0000","last_fullsized":"2023-09-30T09:56:18.028067+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:38:14.958331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1247,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034392+0000","last_change":"2023-09-30T09:55:20.013357+0000","last_active":"2023-09-30T09:56:18.034392+0000","last_peered":"2023-09-30T09:56:18.034392+0000","last_clean":"2023-09-30T09:56:18.034392+0000","last_became_active":"2023-09-30T09:55:20.013015+0000","last_became_peered":"2023-09-30T09:55:20.013015+0000","last_unstale":"2023-09-30T09:56:18.034392+0000","last_undegraded":"2023-09-30T09:56:18.034392+0000","last_fullsized":"2023-09-30T09:56:18.034392+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:38:05.154696+0000","last_clean_scrub_stamp":"2023-09-30T09:38:05.154696+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:30:47.162310+0000","scrub_duration":0.001634232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":84,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024035+0000","last_change":"2023-09-30T09:56:16.047280+0000","last_active":"2023-09-30T09:56:18.024035+0000","last_peered":"2023-09-30T09:56:18.024035+0000","last_clean":"2023-09-30T09:56:18.024035+0000","last_became_active":"2023-09-30T09:56:16.033154+0000","last_became_peered":"2023-09-30T09:56:16.033154+0000","last_unstale":"2023-09-30T09:56:18.024035+0000","last_undegraded":"2023-09-30T09:56:18.024035+0000","last_fullsized":"2023-09-30T09:56:18.024035+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:52:00.373555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":701,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028177+0000","last_change":"2023-09-30T09:49:31.300551+0000","last_active":"2023-09-30T09:56:18.028177+0000","last_peered":"2023-09-30T09:56:18.028177+0000","last_clean":"2023-09-30T09:56:18.028177+0000","last_became_active":"2023-09-30T09:49:31.300231+0000","last_became_peered":"2023-09-30T09:49:31.300231+0000","last_unstale":"2023-09-30T09:56:18.028177+0000","last_undegraded":"2023-09-30T09:56:18.028177+0000","last_fullsized":"2023-09-30T09:56:18.028177+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:45.832564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":975,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024759+0000","last_change":"2023-09-30T09:56:12.198931+0000","last_active":"2023-09-30T09:56:18.024759+0000","last_peered":"2023-09-30T09:56:18.024759+0000","last_clean":"2023-09-30T09:56:18.024759+0000","last_became_active":"2023-09-30T09:56:12.198479+0000","last_became_peered":"2023-09-30T09:56:12.198479+0000","last_unstale":"2023-09-30T09:56:18.024759+0000","last_undegraded":"2023-09-30T09:56:18.024759+0000","last_fullsized":"2023-09-30T09:56:18.024759+0000","mapping_epoch":679,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.728757+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.728757+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:40:05.730878+0000","scrub_duration":0.0019271430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":420,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029273+0000","last_change":"2023-09-30T09:55:49.125117+0000","last_active":"2023-09-30T09:56:18.029273+0000","last_peered":"2023-09-30T09:56:18.029273+0000","last_clean":"2023-09-30T09:56:18.029273+0000","last_became_active":"2023-09-30T09:55:49.124787+0000","last_became_peered":"2023-09-30T09:55:49.124787+0000","last_unstale":"2023-09-30T09:56:18.029273+0000","last_undegraded":"2023-09-30T09:56:18.029273+0000","last_fullsized":"2023-09-30T09:56:18.029273+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:51:02.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028406+0000","last_change":"2023-09-30T09:55:35.053238+0000","last_active":"2023-09-30T09:56:18.028406+0000","last_peered":"2023-09-30T09:56:18.028406+0000","last_clean":"2023-09-30T09:56:18.028406+0000","last_became_active":"2023-09-30T09:55:35.052746+0000","last_became_peered":"2023-09-30T09:55:35.052746+0000","last_unstale":"2023-09-30T09:56:18.028406+0000","last_undegraded":"2023-09-30T09:56:18.028406+0000","last_fullsized":"2023-09-30T09:56:18.028406+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:51:23.636286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":769,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.029857+0000","last_change":"2023-09-30T09:56:19.029857+0000","last_active":"2023-09-30T09:56:19.029857+0000","last_peered":"2023-09-30T09:56:19.029857+0000","last_clean":"2023-09-30T09:56:19.029857+0000","last_became_active":"2023-09-30T09:56:19.029372+0000","last_became_peered":"2023-09-30T09:56:19.029372+0000","last_unstale":"2023-09-30T09:56:19.029857+0000","last_undegraded":"2023-09-30T09:56:19.029857+0000","last_fullsized":"2023-09-30T09:56:19.029857+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":251,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:09.139024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:09.139024+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:43:41.661080+0000","scrub_duration":0.0014649859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028982+0000","last_change":"2023-09-30T09:55:35.051277+0000","last_active":"2023-09-30T09:56:18.028982+0000","last_peered":"2023-09-30T09:56:18.028982+0000","last_clean":"2023-09-30T09:56:18.028982+0000","last_became_active":"2023-09-30T09:55:35.050773+0000","last_became_peered":"2023-09-30T09:55:35.050773+0000","last_unstale":"2023-09-30T09:56:18.028982+0000","last_undegraded":"2023-09-30T09:56:18.028982+0000","last_fullsized":"2023-09-30T09:56:18.028982+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:39:07.565803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":715,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030184+0000","last_change":"2023-09-30T09:49:31.311818+0000","last_active":"2023-09-30T09:56:18.030184+0000","last_peered":"2023-09-30T09:56:18.030184+0000","last_clean":"2023-09-30T09:56:18.030184+0000","last_became_active":"2023-09-30T09:49:31.311481+0000","last_became_peered":"2023-09-30T09:49:31.311481+0000","last_unstale":"2023-09-30T09:56:18.030184+0000","last_undegraded":"2023-09-30T09:56:18.030184+0000","last_fullsized":"2023-09-30T09:56:18.030184+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:16:54.941072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":611,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025406+0000","last_change":"2023-09-30T09:56:16.692506+0000","last_active":"2023-09-30T09:56:18.025406+0000","last_peered":"2023-09-30T09:56:18.025406+0000","last_clean":"2023-09-30T09:56:18.025406+0000","last_became_active":"2023-09-30T09:56:12.198024+0000","last_became_peered":"2023-09-30T09:56:12.198024+0000","last_unstale":"2023-09-30T09:56:18.025406+0000","last_undegraded":"2023-09-30T09:56:18.025406+0000","last_fullsized":"2023-09-30T09:56:18.025406+0000","mapping_epoch":679,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:56:16.692428+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:56:16.692428+0000","last_clean_scrub_stamp":"2023-09-30T09:56:16.692428+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:04:40.230998+0000","scrub_duration":0.0016202269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":254,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022275+0000","last_change":"2023-09-30T09:42:48.677629+0000","last_active":"2023-09-30T09:56:18.022275+0000","last_peered":"2023-09-30T09:56:18.022275+0000","last_clean":"2023-09-30T09:56:18.022275+0000","last_became_active":"2023-09-30T09:42:48.677172+0000","last_became_peered":"2023-09-30T09:42:48.677172+0000","last_unstale":"2023-09-30T09:56:18.022275+0000","last_undegraded":"2023-09-30T09:56:18.022275+0000","last_fullsized":"2023-09-30T09:56:18.022275+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:57:33.040678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":237,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030718+0000","last_change":"2023-09-30T09:56:04.200329+0000","last_active":"2023-09-30T09:56:18.030718+0000","last_peered":"2023-09-30T09:56:18.030718+0000","last_clean":"2023-09-30T09:56:18.030718+0000","last_became_active":"2023-09-30T09:56:04.200012+0000","last_became_peered":"2023-09-30T09:56:04.200012+0000","last_unstale":"2023-09-30T09:56:18.030718+0000","last_undegraded":"2023-09-30T09:56:18.030718+0000","last_fullsized":"2023-09-30T09:56:18.030718+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:01:44.617024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":747,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031400+0000","last_change":"2023-09-30T09:55:49.124036+0000","last_active":"2023-09-30T09:56:18.031400+0000","last_peered":"2023-09-30T09:56:18.031400+0000","last_clean":"2023-09-30T09:56:18.031400+0000","last_became_active":"2023-09-30T09:55:49.122904+0000","last_became_peered":"2023-09-30T09:55:49.122904+0000","last_unstale":"2023-09-30T09:56:18.031400+0000","last_undegraded":"2023-09-30T09:56:18.031400+0000","last_fullsized":"2023-09-30T09:56:18.031400+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:56.428698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:56.428698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:33:29.088907+0000","scrub_duration":0.001333784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"644'215","reported_seq":908,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029584+0000","last_change":"2023-09-30T09:56:04.199590+0000","last_active":"2023-09-30T09:56:18.029584+0000","last_peered":"2023-09-30T09:56:18.029584+0000","last_clean":"2023-09-30T09:56:18.029584+0000","last_became_active":"2023-09-30T09:56:04.198968+0000","last_became_peered":"2023-09-30T09:56:04.198968+0000","last_unstale":"2023-09-30T09:56:18.029584+0000","last_undegraded":"2023-09-30T09:56:18.029584+0000","last_fullsized":"2023-09-30T09:56:18.029584+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":215,"log_dups_size":0,"ondisk_log_size":215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:46:16.237810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000366991,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":735,"num_read_kb":2210,"num_write":243,"num_write_kb":1428,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":633,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025372+0000","last_change":"2023-09-30T09:50:41.396047+0000","last_active":"2023-09-30T09:56:18.025372+0000","last_peered":"2023-09-30T09:56:18.025372+0000","last_clean":"2023-09-30T09:56:18.025372+0000","last_became_active":"2023-09-30T09:50:41.395627+0000","last_became_peered":"2023-09-30T09:50:41.395627+0000","last_unstale":"2023-09-30T09:56:18.025372+0000","last_undegraded":"2023-09-30T09:56:18.025372+0000","last_fullsized":"2023-09-30T09:56:18.025372+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:07:16.050977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032902+0000","last_change":"2023-09-30T09:56:07.684173+0000","last_active":"2023-09-30T09:56:18.032902+0000","last_peered":"2023-09-30T09:56:18.032902+0000","last_clean":"2023-09-30T09:56:18.032902+0000","last_became_active":"2023-09-30T09:56:07.683845+0000","last_became_peered":"2023-09-30T09:56:07.683845+0000","last_unstale":"2023-09-30T09:56:18.032902+0000","last_undegraded":"2023-09-30T09:56:18.032902+0000","last_fullsized":"2023-09-30T09:56:18.032902+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:15:39.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":1206,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.029693+0000","last_change":"2023-09-30T09:56:19.029693+0000","last_active":"2023-09-30T09:56:19.029693+0000","last_peered":"2023-09-30T09:56:19.029693+0000","last_clean":"2023-09-30T09:56:19.029693+0000","last_became_active":"2023-09-30T09:56:19.029203+0000","last_became_peered":"2023-09-30T09:56:19.029203+0000","last_unstale":"2023-09-30T09:56:19.029693+0000","last_undegraded":"2023-09-30T09:56:19.029693+0000","last_fullsized":"2023-09-30T09:56:19.029693+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.422431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.422431+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:21:07.406569+0000","scrub_duration":0.0023456140000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"644'42","reported_seq":857,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025009+0000","last_change":"2023-09-30T09:56:12.197041+0000","last_active":"2023-09-30T09:56:18.025009+0000","last_peered":"2023-09-30T09:56:18.025009+0000","last_clean":"2023-09-30T09:56:18.025009+0000","last_became_active":"2023-09-30T09:56:12.196550+0000","last_became_peered":"2023-09-30T09:56:12.196550+0000","last_unstale":"2023-09-30T09:56:18.025009+0000","last_undegraded":"2023-09-30T09:56:18.025009+0000","last_fullsized":"2023-09-30T09:56:18.025009+0000","mapping_epoch":679,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":42,"log_dups_size":0,"ondisk_log_size":42,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:27:54.946400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079558799999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":665,"num_read_kb":3718,"num_write":32,"num_write_kb":3092,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":742,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030539+0000","last_change":"2023-09-30T09:56:16.031416+0000","last_active":"2023-09-30T09:56:18.030539+0000","last_peered":"2023-09-30T09:56:18.030539+0000","last_clean":"2023-09-30T09:56:18.030539+0000","last_became_active":"2023-09-30T09:56:16.030763+0000","last_became_peered":"2023-09-30T09:56:16.030763+0000","last_unstale":"2023-09-30T09:56:18.030539+0000","last_undegraded":"2023-09-30T09:56:18.030539+0000","last_fullsized":"2023-09-30T09:56:18.030539+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:51:47.913934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":338,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025322+0000","last_change":"2023-09-30T09:56:12.198890+0000","last_active":"2023-09-30T09:56:18.025322+0000","last_peered":"2023-09-30T09:56:18.025322+0000","last_clean":"2023-09-30T09:56:18.025322+0000","last_became_active":"2023-09-30T09:56:12.198494+0000","last_became_peered":"2023-09-30T09:56:12.198494+0000","last_unstale":"2023-09-30T09:56:18.025322+0000","last_undegraded":"2023-09-30T09:56:18.025322+0000","last_fullsized":"2023-09-30T09:56:18.025322+0000","mapping_epoch":679,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:17:12.142363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":1201,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032503+0000","last_change":"2023-09-30T09:55:38.065400+0000","last_active":"2023-09-30T09:56:18.032503+0000","last_peered":"2023-09-30T09:56:18.032503+0000","last_clean":"2023-09-30T09:56:18.032503+0000","last_became_active":"2023-09-30T09:55:38.065017+0000","last_became_peered":"2023-09-30T09:55:38.065017+0000","last_unstale":"2023-09-30T09:56:18.032503+0000","last_undegraded":"2023-09-30T09:56:18.032503+0000","last_fullsized":"2023-09-30T09:56:18.032503+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:01.203661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:38:01.203661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:35:09.877057+0000","scrub_duration":0.002254565,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"644'264","reported_seq":1527,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032435+0000","last_change":"2023-09-30T09:55:20.009001+0000","last_active":"2023-09-30T09:56:18.032435+0000","last_peered":"2023-09-30T09:56:18.032435+0000","last_clean":"2023-09-30T09:56:18.032435+0000","last_became_active":"2023-09-30T09:55:20.008454+0000","last_became_peered":"2023-09-30T09:55:20.008454+0000","last_unstale":"2023-09-30T09:56:18.032435+0000","last_undegraded":"2023-09-30T09:56:18.032435+0000","last_fullsized":"2023-09-30T09:56:18.032435+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:59:08.571424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018706759999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1813,"num_read_kb":2868,"num_write":229,"num_write_kb":1359,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":342,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028463+0000","last_change":"2023-09-30T09:56:04.198792+0000","last_active":"2023-09-30T09:56:18.028463+0000","last_peered":"2023-09-30T09:56:18.028463+0000","last_clean":"2023-09-30T09:56:18.028463+0000","last_became_active":"2023-09-30T09:56:04.198468+0000","last_became_peered":"2023-09-30T09:56:04.198468+0000","last_unstale":"2023-09-30T09:56:18.028463+0000","last_undegraded":"2023-09-30T09:56:18.028463+0000","last_fullsized":"2023-09-30T09:56:18.028463+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:41:37.457966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":469,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030639+0000","last_change":"2023-09-30T09:56:16.028292+0000","last_active":"2023-09-30T09:56:18.030639+0000","last_peered":"2023-09-30T09:56:18.030639+0000","last_clean":"2023-09-30T09:56:18.030639+0000","last_became_active":"2023-09-30T09:56:16.027938+0000","last_became_peered":"2023-09-30T09:56:16.027938+0000","last_unstale":"2023-09-30T09:56:18.030639+0000","last_undegraded":"2023-09-30T09:56:18.030639+0000","last_fullsized":"2023-09-30T09:56:18.030639+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:46:54.999972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"644'350","reported_seq":1662,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.026084+0000","last_change":"2023-09-30T09:54:40.236180+0000","last_active":"2023-09-30T09:56:18.026084+0000","last_peered":"2023-09-30T09:56:18.026084+0000","last_clean":"2023-09-30T09:56:18.026084+0000","last_became_active":"2023-09-30T09:50:42.387121+0000","last_became_peered":"2023-09-30T09:50:42.387121+0000","last_unstale":"2023-09-30T09:56:18.026084+0000","last_undegraded":"2023-09-30T09:56:18.026084+0000","last_fullsized":"2023-09-30T09:56:18.026084+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":350,"log_dups_size":0,"ondisk_log_size":350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T10:34:05.192090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00090046199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1079,"num_read_kb":2584,"num_write":339,"num_write_kb":1551,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1127,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.034046+0000","last_change":"2023-09-30T09:56:19.034046+0000","last_active":"2023-09-30T09:56:19.034046+0000","last_peered":"2023-09-30T09:56:19.034046+0000","last_clean":"2023-09-30T09:56:19.034046+0000","last_became_active":"2023-09-30T09:56:19.033659+0000","last_became_peered":"2023-09-30T09:56:19.033659+0000","last_unstale":"2023-09-30T09:56:19.034046+0000","last_undegraded":"2023-09-30T09:56:19.034046+0000","last_fullsized":"2023-09-30T09:56:19.034046+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:38:08.145499+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-30T09:27:54.119176+0000","last_clean_scrub_stamp":"2023-09-30T09:38:08.145499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:32:25.427476+0000","scrub_duration":0.0066457280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":172,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022412+0000","last_change":"2023-09-30T09:55:31.023445+0000","last_active":"2023-09-30T09:56:18.022412+0000","last_peered":"2023-09-30T09:56:18.022412+0000","last_clean":"2023-09-30T09:56:18.022412+0000","last_became_active":"2023-09-30T09:55:31.023061+0000","last_became_peered":"2023-09-30T09:55:31.023061+0000","last_unstale":"2023-09-30T09:56:18.022412+0000","last_undegraded":"2023-09-30T09:56:18.022412+0000","last_fullsized":"2023-09-30T09:56:18.022412+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:49:21.250674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":542,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028617+0000","last_change":"2023-09-30T09:55:49.123422+0000","last_active":"2023-09-30T09:56:18.028617+0000","last_peered":"2023-09-30T09:56:18.028617+0000","last_clean":"2023-09-30T09:56:18.028617+0000","last_became_active":"2023-09-30T09:55:49.123110+0000","last_became_peered":"2023-09-30T09:55:49.123110+0000","last_unstale":"2023-09-30T09:56:18.028617+0000","last_undegraded":"2023-09-30T09:56:18.028617+0000","last_fullsized":"2023-09-30T09:56:18.028617+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:10:40.848966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":832,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.033740+0000","last_change":"2023-09-30T09:56:04.203765+0000","last_active":"2023-09-30T09:56:18.033740+0000","last_peered":"2023-09-30T09:56:18.033740+0000","last_clean":"2023-09-30T09:56:18.033740+0000","last_became_active":"2023-09-30T09:56:04.203571+0000","last_became_peered":"2023-09-30T09:56:04.203571+0000","last_unstale":"2023-09-30T09:56:18.033740+0000","last_undegraded":"2023-09-30T09:56:18.033740+0000","last_fullsized":"2023-09-30T09:56:18.033740+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:48:39.230227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:48:39.230227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:28:51.856227+0000","scrub_duration":0.0020492750000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"644'256","reported_seq":1742,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.034945+0000","last_change":"2023-09-30T09:55:09.210709+0000","last_active":"2023-09-30T09:56:18.034945+0000","last_peered":"2023-09-30T09:56:18.034945+0000","last_clean":"2023-09-30T09:56:18.034945+0000","last_became_active":"2023-09-30T09:55:09.208764+0000","last_became_peered":"2023-09-30T09:55:09.208764+0000","last_unstale":"2023-09-30T09:56:18.034945+0000","last_undegraded":"2023-09-30T09:56:18.034945+0000","last_fullsized":"2023-09-30T09:56:18.034945+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":256,"log_dups_size":0,"ondisk_log_size":256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:19:55.206358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001530228,"stat_sum":{"num_bytes":8388608,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2028,"num_read_kb":2464,"num_write":188,"num_write_kb":8612,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":269,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029117+0000","last_change":"2023-09-30T09:55:35.050416+0000","last_active":"2023-09-30T09:56:18.029117+0000","last_peered":"2023-09-30T09:56:18.029117+0000","last_clean":"2023-09-30T09:56:18.029117+0000","last_became_active":"2023-09-30T09:55:35.050060+0000","last_became_peered":"2023-09-30T09:55:35.050060+0000","last_unstale":"2023-09-30T09:56:18.029117+0000","last_undegraded":"2023-09-30T09:56:18.029117+0000","last_fullsized":"2023-09-30T09:56:18.029117+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T15:20:03.272812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026873+0000","last_change":"2023-09-30T09:56:16.024455+0000","last_active":"2023-09-30T09:56:19.026873+0000","last_peered":"2023-09-30T09:56:19.026873+0000","last_clean":"2023-09-30T09:56:19.026873+0000","last_became_active":"2023-09-30T09:56:16.024013+0000","last_became_peered":"2023-09-30T09:56:16.024013+0000","last_unstale":"2023-09-30T09:56:19.026873+0000","last_undegraded":"2023-09-30T09:56:19.026873+0000","last_fullsized":"2023-09-30T09:56:19.026873+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:10:11.515103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":802,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.033910+0000","last_change":"2023-09-30T09:56:19.033910+0000","last_active":"2023-09-30T09:56:19.033910+0000","last_peered":"2023-09-30T09:56:19.033910+0000","last_clean":"2023-09-30T09:56:19.033910+0000","last_became_active":"2023-09-30T09:56:19.033507+0000","last_became_peered":"2023-09-30T09:56:19.033507+0000","last_unstale":"2023-09-30T09:56:19.033910+0000","last_undegraded":"2023-09-30T09:56:19.033910+0000","last_fullsized":"2023-09-30T09:56:19.033910+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:55.431671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:55.431671+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:31:02.045496+0000","scrub_duration":0.001577902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"644'253","reported_seq":1737,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.036312+0000","last_change":"2023-09-30T09:56:19.036312+0000","last_active":"2023-09-30T09:56:19.036312+0000","last_peered":"2023-09-30T09:56:19.036312+0000","last_clean":"2023-09-30T09:56:19.036312+0000","last_became_active":"2023-09-30T09:56:19.035567+0000","last_became_peered":"2023-09-30T09:56:19.035567+0000","last_unstale":"2023-09-30T09:56:19.036312+0000","last_undegraded":"2023-09-30T09:56:19.036312+0000","last_fullsized":"2023-09-30T09:56:19.036312+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":253,"log_dups_size":0,"ondisk_log_size":253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T18:32:48.808343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00122043,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1727,"num_read_kb":3650,"num_write":210,"num_write_kb":2245,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":256,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.029344+0000","last_change":"2023-09-30T09:56:04.189586+0000","last_active":"2023-09-30T09:56:18.029344+0000","last_peered":"2023-09-30T09:56:18.029344+0000","last_clean":"2023-09-30T09:56:18.029344+0000","last_became_active":"2023-09-30T09:56:04.189213+0000","last_became_peered":"2023-09-30T09:56:04.189213+0000","last_unstale":"2023-09-30T09:56:18.029344+0000","last_undegraded":"2023-09-30T09:56:18.029344+0000","last_fullsized":"2023-09-30T09:56:18.029344+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:19:41.345044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":277,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028266+0000","last_change":"2023-09-30T09:55:06.172278+0000","last_active":"2023-09-30T09:56:18.028266+0000","last_peered":"2023-09-30T09:56:18.028266+0000","last_clean":"2023-09-30T09:56:18.028266+0000","last_became_active":"2023-09-30T09:55:06.171955+0000","last_became_peered":"2023-09-30T09:55:06.171955+0000","last_unstale":"2023-09-30T09:56:18.028266+0000","last_undegraded":"2023-09-30T09:56:18.028266+0000","last_fullsized":"2023-09-30T09:56:18.028266+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:47:33.173930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":1252,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028572+0000","last_change":"2023-09-30T09:55:38.068371+0000","last_active":"2023-09-30T09:56:18.028572+0000","last_peered":"2023-09-30T09:56:18.028572+0000","last_clean":"2023-09-30T09:56:18.028572+0000","last_became_active":"2023-09-30T09:55:38.067940+0000","last_became_peered":"2023-09-30T09:55:38.067940+0000","last_unstale":"2023-09-30T09:56:18.028572+0000","last_undegraded":"2023-09-30T09:56:18.028572+0000","last_fullsized":"2023-09-30T09:56:18.028572+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:31:53.832706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:31:53.832706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:28:09.245875+0000","scrub_duration":0.0013764249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":72,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.023420+0000","last_change":"2023-09-30T09:55:31.024573+0000","last_active":"2023-09-30T09:56:18.023420+0000","last_peered":"2023-09-30T09:56:18.023420+0000","last_clean":"2023-09-30T09:56:18.023420+0000","last_became_active":"2023-09-30T09:55:31.024173+0000","last_became_peered":"2023-09-30T09:55:31.024173+0000","last_unstale":"2023-09-30T09:56:18.023420+0000","last_undegraded":"2023-09-30T09:56:18.023420+0000","last_fullsized":"2023-09-30T09:56:18.023420+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:41:47.224041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"644'380","reported_seq":1846,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031849+0000","last_change":"2023-09-30T09:56:04.200828+0000","last_active":"2023-09-30T09:56:18.031849+0000","last_peered":"2023-09-30T09:56:18.031849+0000","last_clean":"2023-09-30T09:56:18.031849+0000","last_became_active":"2023-09-30T09:56:04.200034+0000","last_became_peered":"2023-09-30T09:56:04.200034+0000","last_unstale":"2023-09-30T09:56:18.031849+0000","last_undegraded":"2023-09-30T09:56:18.031849+0000","last_fullsized":"2023-09-30T09:56:18.031849+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":380,"log_dups_size":0,"ondisk_log_size":380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:48:07.235547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001189673,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2651,"num_read_kb":5320,"num_write":283,"num_write_kb":4761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":669,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022132+0000","last_change":"2023-09-30T09:56:16.029529+0000","last_active":"2023-09-30T09:56:18.022132+0000","last_peered":"2023-09-30T09:56:18.022132+0000","last_clean":"2023-09-30T09:56:18.022132+0000","last_became_active":"2023-09-30T09:56:16.029162+0000","last_became_peered":"2023-09-30T09:56:16.029162+0000","last_unstale":"2023-09-30T09:56:18.022132+0000","last_undegraded":"2023-09-30T09:56:18.022132+0000","last_fullsized":"2023-09-30T09:56:18.022132+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T14:04:54.078754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":1114,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.022698+0000","last_change":"2023-09-30T09:51:59.873401+0000","last_active":"2023-09-30T09:56:18.022698+0000","last_peered":"2023-09-30T09:56:18.022698+0000","last_clean":"2023-09-30T09:56:18.022698+0000","last_became_active":"2023-09-30T09:51:59.873066+0000","last_became_peered":"2023-09-30T09:51:59.873066+0000","last_unstale":"2023-09-30T09:56:18.022698+0000","last_undegraded":"2023-09-30T09:56:18.022698+0000","last_fullsized":"2023-09-30T09:56:18.022698+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:03.613861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:03.613861+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:48:11.565720+0000","scrub_duration":0.006177096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"644'190","reported_seq":1414,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.031601+0000","last_change":"2023-09-30T09:55:49.123303+0000","last_active":"2023-09-30T09:56:18.031601+0000","last_peered":"2023-09-30T09:56:18.031601+0000","last_clean":"2023-09-30T09:56:18.031601+0000","last_became_active":"2023-09-30T09:55:49.122113+0000","last_became_peered":"2023-09-30T09:55:49.122113+0000","last_unstale":"2023-09-30T09:56:18.031601+0000","last_undegraded":"2023-09-30T09:56:18.031601+0000","last_fullsized":"2023-09-30T09:56:18.031601+0000","mapping_epoch":670,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":671,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:11:07.896775+0000","last_clean_scrub_stamp":"2023-09-30T09:11:07.896775+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T11:05:29.519408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094274099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":3608,"num_write":166,"num_write_kb":2342,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":18,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026024+0000","last_change":"2023-09-30T09:56:16.051225+0000","last_active":"2023-09-30T09:56:19.026024+0000","last_peered":"2023-09-30T09:56:19.026024+0000","last_clean":"2023-09-30T09:56:19.026024+0000","last_became_active":"2023-09-30T09:56:16.031575+0000","last_became_peered":"2023-09-30T09:56:16.031575+0000","last_unstale":"2023-09-30T09:56:19.026024+0000","last_undegraded":"2023-09-30T09:56:19.026024+0000","last_fullsized":"2023-09-30T09:56:19.026024+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:20:02.019985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028903+0000","last_change":"2023-09-30T09:55:35.053458+0000","last_active":"2023-09-30T09:56:18.028903+0000","last_peered":"2023-09-30T09:56:18.028903+0000","last_clean":"2023-09-30T09:56:18.028903+0000","last_became_active":"2023-09-30T09:55:35.053063+0000","last_became_peered":"2023-09-30T09:55:35.053063+0000","last_unstale":"2023-09-30T09:56:18.028903+0000","last_undegraded":"2023-09-30T09:56:18.028903+0000","last_fullsized":"2023-09-30T09:56:18.028903+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T16:34:44.710137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":870,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.033963+0000","last_change":"2023-09-30T09:56:19.033963+0000","last_active":"2023-09-30T09:56:19.033963+0000","last_peered":"2023-09-30T09:56:19.033963+0000","last_clean":"2023-09-30T09:56:19.033963+0000","last_became_active":"2023-09-30T09:56:19.033614+0000","last_became_peered":"2023-09-30T09:56:19.033614+0000","last_unstale":"2023-09-30T09:56:19.033963+0000","last_undegraded":"2023-09-30T09:56:19.033963+0000","last_fullsized":"2023-09-30T09:56:19.033963+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:32:07.806917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:32:07.806917+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:13:51.692768+0000","scrub_duration":0.00677642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.025897+0000","last_change":"2023-09-30T09:56:16.025946+0000","last_active":"2023-09-30T09:56:19.025897+0000","last_peered":"2023-09-30T09:56:19.025897+0000","last_clean":"2023-09-30T09:56:19.025897+0000","last_became_active":"2023-09-30T09:56:16.025593+0000","last_became_peered":"2023-09-30T09:56:16.025593+0000","last_unstale":"2023-09-30T09:56:19.025897+0000","last_undegraded":"2023-09-30T09:56:19.025897+0000","last_fullsized":"2023-09-30T09:56:19.025897+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:56:59.679977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":41,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032702+0000","last_change":"2023-09-30T09:55:17.024355+0000","last_active":"2023-09-30T09:56:18.032702+0000","last_peered":"2023-09-30T09:56:18.032702+0000","last_clean":"2023-09-30T09:56:18.032702+0000","last_became_active":"2023-09-30T09:55:17.023998+0000","last_became_peered":"2023-09-30T09:55:17.023998+0000","last_unstale":"2023-09-30T09:56:18.032702+0000","last_undegraded":"2023-09-30T09:56:18.032702+0000","last_fullsized":"2023-09-30T09:56:18.032702+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:22:42.172168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":677,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.023860+0000","last_change":"2023-09-30T09:54:02.680525+0000","last_active":"2023-09-30T09:56:18.023860+0000","last_peered":"2023-09-30T09:56:18.023860+0000","last_clean":"2023-09-30T09:56:18.023860+0000","last_became_active":"2023-09-30T09:54:02.200078+0000","last_became_peered":"2023-09-30T09:54:02.200078+0000","last_unstale":"2023-09-30T09:56:18.023860+0000","last_undegraded":"2023-09-30T09:56:18.023860+0000","last_fullsized":"2023-09-30T09:56:18.023860+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:54:02.680294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:54:02.680294+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T20:00:33.759005+0000","scrub_duration":0.0025974169999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":613,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025209+0000","last_change":"2023-09-30T09:44:26.065631+0000","last_active":"2023-09-30T09:56:18.025209+0000","last_peered":"2023-09-30T09:56:18.025209+0000","last_clean":"2023-09-30T09:56:18.025209+0000","last_became_active":"2023-09-30T09:44:26.058612+0000","last_became_peered":"2023-09-30T09:44:26.058612+0000","last_unstale":"2023-09-30T09:56:18.025209+0000","last_undegraded":"2023-09-30T09:56:18.025209+0000","last_fullsized":"2023-09-30T09:56:18.025209+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T13:47:29.584787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030212+0000","last_change":"2023-09-30T09:55:35.052986+0000","last_active":"2023-09-30T09:56:18.030212+0000","last_peered":"2023-09-30T09:56:18.030212+0000","last_clean":"2023-09-30T09:56:18.030212+0000","last_became_active":"2023-09-30T09:55:35.052418+0000","last_became_peered":"2023-09-30T09:55:35.052418+0000","last_unstale":"2023-09-30T09:56:18.030212+0000","last_undegraded":"2023-09-30T09:56:18.030212+0000","last_fullsized":"2023-09-30T09:56:18.030212+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T19:05:47.726885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":583,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.025555+0000","last_change":"2023-09-30T09:49:31.313229+0000","last_active":"2023-09-30T09:56:18.025555+0000","last_peered":"2023-09-30T09:56:18.025555+0000","last_clean":"2023-09-30T09:56:18.025555+0000","last_became_active":"2023-09-30T09:49:31.301137+0000","last_became_peered":"2023-09-30T09:49:31.301137+0000","last_unstale":"2023-09-30T09:56:18.025555+0000","last_undegraded":"2023-09-30T09:56:18.025555+0000","last_fullsized":"2023-09-30T09:56:18.025555+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:59:57.362379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.028001+0000","last_change":"2023-09-30T09:55:35.051703+0000","last_active":"2023-09-30T09:56:18.028001+0000","last_peered":"2023-09-30T09:56:18.028001+0000","last_clean":"2023-09-30T09:56:18.028001+0000","last_became_active":"2023-09-30T09:55:35.051349+0000","last_became_peered":"2023-09-30T09:55:35.051349+0000","last_unstale":"2023-09-30T09:56:18.028001+0000","last_undegraded":"2023-09-30T09:56:18.028001+0000","last_fullsized":"2023-09-30T09:56:18.028001+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T17:13:27.965290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":861,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.032900+0000","last_change":"2023-09-30T09:54:03.339280+0000","last_active":"2023-09-30T09:56:18.032900+0000","last_peered":"2023-09-30T09:56:18.032900+0000","last_clean":"2023-09-30T09:56:18.032900+0000","last_became_active":"2023-09-30T09:50:29.883457+0000","last_became_peered":"2023-09-30T09:50:29.883457+0000","last_unstale":"2023-09-30T09:56:18.032900+0000","last_undegraded":"2023-09-30T09:56:18.032900+0000","last_fullsized":"2023-09-30T09:56:18.032900+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:54:03.339198+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:54:03.339198+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:05.265754+0000","scrub_duration":0.0015674910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":160,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.024208+0000","last_change":"2023-09-30T09:55:31.023790+0000","last_active":"2023-09-30T09:56:19.024208+0000","last_peered":"2023-09-30T09:56:19.024208+0000","last_clean":"2023-09-30T09:56:19.024208+0000","last_became_active":"2023-09-30T09:55:31.023424+0000","last_became_peered":"2023-09-30T09:55:31.023424+0000","last_unstale":"2023-09-30T09:56:19.024208+0000","last_undegraded":"2023-09-30T09:56:19.024208+0000","last_fullsized":"2023-09-30T09:56:19.024208+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:48:10.845081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":450,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.024720+0000","last_change":"2023-09-30T09:55:31.016117+0000","last_active":"2023-09-30T09:56:18.024720+0000","last_peered":"2023-09-30T09:56:18.024720+0000","last_clean":"2023-09-30T09:56:18.024720+0000","last_became_active":"2023-09-30T09:55:31.015808+0000","last_became_peered":"2023-09-30T09:55:31.015808+0000","last_unstale":"2023-09-30T09:56:18.024720+0000","last_undegraded":"2023-09-30T09:56:18.024720+0000","last_fullsized":"2023-09-30T09:56:18.024720+0000","mapping_epoch":661,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":662,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:54:52.567443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":775,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.030316+0000","last_change":"2023-09-30T09:56:04.203349+0000","last_active":"2023-09-30T09:56:18.030316+0000","last_peered":"2023-09-30T09:56:18.030316+0000","last_clean":"2023-09-30T09:56:18.030316+0000","last_became_active":"2023-09-30T09:56:04.203000+0000","last_became_peered":"2023-09-30T09:56:04.203000+0000","last_unstale":"2023-09-30T09:56:18.030316+0000","last_undegraded":"2023-09-30T09:56:18.030316+0000","last_fullsized":"2023-09-30T09:56:18.030316+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-30T09:44:14.528327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:08:36.918738+0000","last_clean_scrub_stamp":"2023-09-30T09:44:14.528327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:52:28.264439+0000","scrub_duration":0.004695744,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":20,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-30T09:56:19.026378+0000","last_change":"2023-09-30T09:56:16.048318+0000","last_active":"2023-09-30T09:56:19.026378+0000","last_peered":"2023-09-30T09:56:19.026378+0000","last_clean":"2023-09-30T09:56:19.026378+0000","last_became_active":"2023-09-30T09:56:16.028434+0000","last_became_peered":"2023-09-30T09:56:16.028434+0000","last_unstale":"2023-09-30T09:56:19.026378+0000","last_undegraded":"2023-09-30T09:56:19.026378+0000","last_fullsized":"2023-09-30T09:56:19.026378+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:06.927306+0000","last_clean_scrub_stamp":"2023-09-30T09:12:06.927306+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T12:28:50.558440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":728,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-30T09:56:18.027642+0000","last_change":"2023-09-30T09:50:26.905427+0000","last_active":"2023-09-30T09:56:18.027642+0000","last_peered":"2023-09-30T09:56:18.027642+0000","last_clean":"2023-09-30T09:56:18.027642+0000","last_became_active":"2023-09-30T09:50:26.905100+0000","last_became_peered":"2023-09-30T09:50:26.905100+0000","last_unstale":"2023-09-30T09:56:18.027642+0000","last_undegraded":"2023-09-30T09:56:18.027642+0000","last_fullsized":"2023-09-30T09:56:18.027642+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-30T09:12:07.928984+0000","last_clean_scrub_stamp":"2023-09-30T09:12:07.928984+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-01T09:56:38.036208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5492,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":123,"num_bytes_recovered":14254,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33222,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11978,"num_read_kb":26422,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":314,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234544,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":643488,"kb_used_data":15832,"kb_used_omap":0,"kb_used_meta":627648,"kb_avail":93101664,"statfs":{"total":95995035648,"available":95336103936,"internally_reserved":0,"allocated":16211968,"data_stored":14109791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":642711552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.96599999999999997,"15min":3.4100000000000001},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.79800000000000004,"5min":6.3479999999999999,"15min":157.578},"last":0.371},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.97299999999999998,"15min":3.4159999999999999},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.83699999999999997,"5min":7.2919999999999998,"15min":157.589},"last":0.54500000000000004}]},{"osd":2,"last update":"Sat Sep 30 09:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":1.1220000000000001,"15min":3.008},"min":{"1min":0.38900000000000001,"5min":0.29899999999999999,"15min":0.28100000000000003},"max":{"1min":0.71199999999999997,"5min":11.467000000000001,"15min":187.74600000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.0069999999999999,"15min":3.0529999999999999},"min":{"1min":0.38,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.73799999999999999,"5min":7.0720000000000001,"15min":187.583},"last":0.60099999999999998}]},{"osd":3,"last update":"Sat Sep 30 09:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":2.036,"15min":4.0430000000000001},"min":{"1min":0.39300000000000002,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":0.82599999999999996,"5min":102.139,"15min":154.06700000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.8839999999999999,"15min":4.202},"min":{"1min":0.46400000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.86799999999999999,"5min":99.713999999999999,"15min":154.24799999999999},"last":0.68899999999999995}]},{"osd":4,"last update":"Sat Sep 30 09:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.028,"15min":2.0350000000000001},"min":{"1min":0.315,"5min":0.22,"15min":0.22},"max":{"1min":0.70199999999999996,"5min":7.1660000000000004,"15min":238.511},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.3200000000000001,"15min":2.2000000000000002},"min":{"1min":0.35999999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.83999999999999997,"5min":9.6709999999999994,"15min":238.387},"last":0.65700000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:56:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.327,"15min":5.0209999999999999},"min":{"1min":0.38,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.80600000000000005,"5min":16.314,"15min":197.75700000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.1679999999999999,"15min":4.9199999999999999},"min":{"1min":0.441,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.86699999999999999,"5min":11.863,"15min":197.63300000000001},"last":0.625}]},{"osd":6,"last update":"Sat Sep 30 09:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.157,"15min":1.1479999999999999},"min":{"1min":0.40200000000000002,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":0.85499999999999998,"5min":9.4730000000000008,"15min":9.4730000000000008},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.0920000000000001,"15min":1.1830000000000001},"min":{"1min":0.52600000000000002,"5min":0.34599999999999997,"15min":0.29599999999999999},"max":{"1min":0.873,"5min":11.960000000000001,"15min":12.058},"last":0.76900000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496182,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670348,"kb_used_data":7684,"kb_used_omap":0,"kb_used_meta":662656,"kb_avail":93074804,"statfs":{"total":95995035648,"available":95308599296,"internally_reserved":0,"allocated":7868416,"data_stored":5785460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":678559744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.622}]},{"osd":1,"last update":"Sat Sep 30 09:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":1.278,"15min":1.8839999999999999},"min":{"1min":0.502,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.85099999999999998,"5min":10.954000000000001,"15min":103.038},"last":0.5},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.4059999999999999,"15min":1.972},"min":{"1min":0.379,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.92800000000000005,"5min":25.446999999999999,"15min":103.05},"last":0.439}]},{"osd":2,"last update":"Sat Sep 30 09:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":1.1279999999999999,"15min":2.9670000000000001},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":0.879,"5min":12.044,"15min":207.02500000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.286,"15min":2.9460000000000002},"min":{"1min":0.43099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.76700000000000002,"5min":10.923999999999999,"15min":207.114},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Sep 30 09:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.284,"15min":2.5379999999999998},"min":{"1min":0.47299999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.79300000000000004,"5min":10.894,"15min":107.11799999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":1.403,"15min":2.673},"min":{"1min":0.51800000000000002,"5min":0.33500000000000002,"15min":0.33000000000000002},"max":{"1min":0.94299999999999995,"5min":10.625999999999999,"15min":106.982},"last":0.48599999999999999}]},{"osd":4,"last update":"Sat Sep 30 09:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.048,"15min":1.657},"min":{"1min":0.36499999999999999,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":0.80400000000000005,"5min":7.2320000000000002,"15min":63.798000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.625,"5min":0.997,"15min":1.53},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":0.89800000000000002,"5min":10.724,"15min":63.133000000000003},"last":0.63600000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.266,"15min":1.679},"min":{"1min":0.36199999999999999,"5min":0.27800000000000002,"15min":0.27100000000000002},"max":{"1min":0.76700000000000002,"5min":11.972,"15min":19.212},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.627,"5min":1.2689999999999999,"15min":1.738},"min":{"1min":0.40400000000000003,"5min":0.31,"15min":0.26900000000000002},"max":{"1min":0.85599999999999998,"5min":18.135999999999999,"15min":18.135999999999999},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 30 09:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.833,"15min":1.417},"min":{"1min":0.44700000000000001,"5min":0.32800000000000001,"15min":0.26500000000000001},"max":{"1min":0.91500000000000004,"5min":39.868000000000002,"15min":39.868000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.3140000000000001,"15min":1.2749999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.27500000000000002},"max":{"1min":0.96099999999999997,"5min":10.843,"15min":12.625},"last":0.45600000000000002}]}]},{"osd":1,"up_from":301,"seq":1292785156380,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":771112,"kb_used_data":17432,"kb_used_omap":9,"kb_used_meta":753654,"kb_avail":92974040,"statfs":{"total":95995035648,"available":95205416960,"internally_reserved":0,"allocated":17850368,"data_stored":14093852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":771741714},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:09 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.44},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.318}]},{"osd":2,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.89800000000000002,"15min":3.1030000000000002},"min":{"1min":0.316,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":0.52000000000000002,"5min":16.443000000000001,"15min":189.14500000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.94799999999999995,"15min":3.1160000000000001},"min":{"1min":0.32500000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":0.53700000000000003,"5min":11.973000000000001,"15min":189.333},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":1.4850000000000001,"15min":3.5129999999999999},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.23400000000000001},"max":{"1min":0.443,"5min":31.151,"15min":199.41},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.2909999999999999,"15min":3.262},"min":{"1min":0.31900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.60199999999999998,"5min":31.727,"15min":199.58099999999999},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.359,"15min":1.5449999999999999},"min":{"1min":0.42699999999999999,"5min":0.29499999999999998,"15min":0.26800000000000002},"max":{"1min":0.91000000000000003,"5min":47.332000000000001,"15min":48.604999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.831,"15min":1.839},"min":{"1min":0.44900000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":0.86099999999999999,"5min":47.606999999999999,"15min":47.606999999999999},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:56:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":2.714,"15min":2.407},"min":{"1min":0.51700000000000002,"5min":0.35399999999999998,"15min":0.32800000000000001},"max":{"1min":0.82499999999999996,"5min":165.88300000000001,"15min":165.88300000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.64700000000000002,"5min":2.734,"15min":1.944},"min":{"1min":0.50900000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84099999999999997,"5min":165.863,"15min":165.863},"last":0.57399999999999995}]},{"osd":6,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":2.2999999999999998,"15min":1.851},"min":{"1min":0.45900000000000002,"5min":0.33800000000000002,"15min":0.29899999999999999},"max":{"1min":0.91500000000000004,"5min":97.281999999999996,"15min":97.281999999999996},"last":0.497},{"interface":"front","average":{"1min":0.71699999999999997,"5min":2.5329999999999999,"15min":1.956},"min":{"1min":0.45800000000000002,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":0.94799999999999995,"5min":97.242000000000004,"15min":97.242000000000004},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.109,"15min":1.393},"min":{"1min":0.48799999999999999,"5min":0.36899999999999999,"15min":0.28299999999999997},"max":{"1min":0.83899999999999997,"5min":9.8390000000000004,"15min":9.8390000000000004},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.1459999999999999,"15min":1.4299999999999999},"min":{"1min":0.53600000000000003,"5min":0.35699999999999998,"15min":0.316},"max":{"1min":0.89300000000000002,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":0.53300000000000003}]}]},{"osd":0,"up_from":649,"seq":2787433775129,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":98536,"kb_used_data":48032,"kb_used_omap":10,"kb_used_meta":50485,"kb_avail":93646616,"statfs":{"total":95995035648,"available":95894134784,"internally_reserved":0,"allocated":49184768,"data_stored":45192966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10881,"internal_metadata":51697023},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.377},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.433}]},{"osd":3,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.48099999999999998}]},{"osd":4,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.54700000000000004}]},{"osd":5,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.89000000000000001}]},{"osd":6,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.876}]},{"osd":7,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.84099999999999997}]}]},{"osd":2,"up_from":104,"seq":446676599217,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":938032,"kb_used_data":9436,"kb_used_omap":5,"kb_used_meta":928570,"kb_avail":92807120,"statfs":{"total":95995035648,"available":95034490880,"internally_reserved":0,"allocated":9662464,"data_stored":5960119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5491,"internal_metadata":950856333},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.40500000000000003}]},{"osd":1,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":1.9299999999999999,"15min":3.7530000000000001},"min":{"1min":0.26000000000000001,"5min":0.20499999999999999,"15min":0.20399999999999999},"max":{"1min":0.58699999999999997,"5min":80.828999999999994,"15min":202.369},"last":0.376},{"interface":"front","average":{"1min":0.46400000000000002,"5min":2,"15min":3.8570000000000002},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.21099999999999999},"max":{"1min":0.66500000000000004,"5min":83.114999999999995,"15min":202.54499999999999},"last":0.315}]},{"osd":3,"last update":"Sat Sep 30 09:56:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":3.6760000000000002,"15min":4.3570000000000002},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.23699999999999999},"max":{"1min":0.66400000000000003,"5min":231.00700000000001,"15min":231.00700000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":3.6000000000000001,"15min":4.2670000000000003},"min":{"1min":0.35999999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.64900000000000002,"5min":230.91499999999999,"15min":230.91499999999999},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.97099999999999997,"15min":2.8300000000000001},"min":{"1min":0.42199999999999999,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":0.90200000000000002,"5min":4.96,"15min":247.655},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.86899999999999999,"15min":2.7330000000000001},"min":{"1min":0.40500000000000003,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":0.86199999999999999,"5min":5.7770000000000001,"15min":248.012},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 30 09:56:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":1.1259999999999999,"15min":2.9449999999999998},"min":{"1min":0.44500000000000001,"5min":0.36399999999999999,"15min":0.30099999999999999},"max":{"1min":0.879,"5min":10.762,"15min":243.86500000000001},"last":0.433},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.4079999999999999,"15min":3.194},"min":{"1min":0.49099999999999999,"5min":0.36199999999999999,"15min":0.32100000000000001},"max":{"1min":0.94799999999999995,"5min":15.164999999999999,"15min":244.02199999999999},"last":0.53500000000000003}]},{"osd":6,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.6080000000000001,"15min":2.141},"min":{"1min":0.442,"5min":0.34699999999999998,"15min":0.29399999999999998},"max":{"1min":0.82699999999999996,"5min":46.732999999999997,"15min":202.70599999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.68600000000000005,"5min":1.946,"15min":2.4569999999999999},"min":{"1min":0.52600000000000002,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":0.84399999999999997,"5min":46.68,"15min":202.916},"last":0.502}]},{"osd":7,"last update":"Sat Sep 30 09:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.2190000000000001,"15min":3.407},"min":{"1min":0.51300000000000001,"5min":0.39500000000000002,"15min":0.32000000000000001},"max":{"1min":0.83499999999999996,"5min":9.1950000000000003,"15min":244.911},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.70999999999999996,"5min":1.6200000000000001,"15min":3.7000000000000002},"min":{"1min":0.55600000000000005,"5min":0.40999999999999998,"15min":0.36599999999999999},"max":{"1min":0.88500000000000001,"5min":10.782,"15min":244.66},"last":0.621}]}]},{"osd":3,"up_from":109,"seq":468151435692,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1020828,"kb_used_data":9676,"kb_used_omap":2,"kb_used_meta":1011133,"kb_avail":92724324,"statfs":{"total":95995035648,"available":94949707776,"internally_reserved":0,"allocated":9908224,"data_stored":6044551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":1035400551},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.45100000000000001}]},{"osd":1,"last update":"Sat Sep 30 09:56:13 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":1.6699999999999999,"15min":2.5179999999999998},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.19400000000000001},"max":{"1min":0.54000000000000004,"5min":63.744999999999997,"15min":142.434},"last":0.378},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.708,"15min":2.4449999999999998},"min":{"1min":0.33100000000000002,"5min":0.24399999999999999,"15min":0.20699999999999999},"max":{"1min":0.55500000000000005,"5min":63.762,"15min":142.44900000000001},"last":0.41999999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.91700000000000004,"15min":3.085},"min":{"1min":0.28499999999999998,"5min":0.218,"15min":0.218},"max":{"1min":0.79900000000000004,"5min":8.7040000000000006,"15min":147.87},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.98599999999999999,"15min":3.1499999999999999},"min":{"1min":0.29299999999999998,"5min":0.22,"15min":0.22},"max":{"1min":0.82699999999999996,"5min":13.064,"15min":148.04900000000001},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.1830000000000001,"15min":2.3300000000000001},"min":{"1min":0.44600000000000001,"5min":0.28499999999999998,"15min":0.26800000000000002},"max":{"1min":5.6829999999999998,"5min":7.4729999999999999,"15min":172.298},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.629,"5min":1.462,"15min":2.6080000000000001},"min":{"1min":0.436,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":0.95399999999999996,"5min":9.8409999999999993,"15min":172.45099999999999},"last":0.66200000000000003}]},{"osd":5,"last update":"Sat Sep 30 09:56:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":2.2629999999999999,"15min":1.958},"min":{"1min":0.40500000000000003,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":1.0309999999999999,"5min":61.426000000000002,"15min":61.426000000000002},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.67000000000000004,"5min":2.4980000000000002,"15min":2.052},"min":{"1min":0.39600000000000002,"5min":0.373,"15min":0.311},"max":{"1min":1.0529999999999999,"5min":61.469000000000001,"15min":61.469000000000001},"last":0.72199999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":2.4489999999999998,"15min":1.9330000000000001},"min":{"1min":0.44700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":5.7839999999999998,"5min":75.781000000000006,"15min":75.781000000000006},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":2.5019999999999998,"15min":2.173},"min":{"1min":0.47699999999999998,"5min":0.40899999999999997,"15min":0.35899999999999999},"max":{"1min":1.016,"5min":75.754000000000005,"15min":75.754000000000005},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 30 09:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.6080000000000001,"15min":2.645},"min":{"1min":0.437,"5min":0.36299999999999999,"15min":0.32900000000000001},"max":{"1min":0.92800000000000005,"5min":90.855999999999995,"15min":213.548},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.92200000000000004,"5min":2.641,"15min":2.649},"min":{"1min":0.51100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.7460000000000004,"5min":91.108000000000004,"15min":213.583},"last":0.79000000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019455,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":627088,"kb_used_data":7368,"kb_used_omap":0,"kb_used_meta":619712,"kb_avail":93118064,"statfs":{"total":95995035648,"available":95352897536,"internally_reserved":0,"allocated":7544832,"data_stored":5457895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":634585088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.747}]},{"osd":1,"last update":"Sat Sep 30 09:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.522,"15min":2.3119999999999998},"min":{"1min":0.436,"5min":0.224,"15min":0.224},"max":{"1min":0.752,"5min":32.536000000000001,"15min":108.551},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.5349999999999999,"15min":2.4420000000000002},"min":{"1min":0.36299999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.83199999999999996,"5min":32.637,"15min":108.884},"last":0.71299999999999997}]},{"osd":2,"last update":"Sat Sep 30 09:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":4.8680000000000003,"15min":3.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71699999999999997,"5min":139.53899999999999,"15min":142.88999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.65400000000000003,"5min":4.4050000000000002,"15min":3.8399999999999999},"min":{"1min":0.49399999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.82899999999999996,"5min":139.56399999999999,"15min":142.90700000000001},"last":0.65700000000000003}]},{"osd":3,"last update":"Sat Sep 30 09:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.5960000000000001,"15min":2.415},"min":{"1min":0.46200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":0.81499999999999995,"5min":29.675999999999998,"15min":105.142},"last":0.622},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.4259999999999999,"15min":2.2879999999999998},"min":{"1min":0.441,"5min":0.309,"15min":0.29299999999999998},"max":{"1min":0.76800000000000002,"5min":29.664999999999999,"15min":105.157},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 30 09:56:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":1.0600000000000001,"15min":4.0300000000000002},"min":{"1min":0.38900000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":0.64500000000000002,"5min":8.4329999999999998,"15min":140.11099999999999},"last":0.438},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.5920000000000001,"15min":4.109},"min":{"1min":0.35599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.70199999999999996,"5min":22.248000000000001,"15min":140.065},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 30 09:56:41 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.732,"15min":1.3140000000000001},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":0.79000000000000004,"5min":71.864999999999995,"15min":71.864999999999995},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.9550000000000001,"15min":1.403},"min":{"1min":0.40899999999999997,"5min":0.29099999999999998,"15min":0.253},"max":{"1min":0.71999999999999997,"5min":72.018000000000001,"15min":72.018000000000001},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Sep 30 09:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.7949999999999999,"15min":1.9350000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27600000000000002},"max":{"1min":1.069,"5min":16.858000000000001,"15min":161.72999999999999},"last":0.498},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.2290000000000001,"15min":1.655},"min":{"1min":0.44500000000000001,"5min":0.32400000000000001,"15min":0.27000000000000002},"max":{"1min":1.117,"5min":14.018000000000001,"15min":162.60900000000001},"last":0.51100000000000001}]}]},{"osd":5,"up_from":489,"seq":2100239007897,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":262732,"kb_used_data":7620,"kb_used_omap":6,"kb_used_meta":255097,"kb_avail":93482420,"statfs":{"total":95995035648,"available":95725998080,"internally_reserved":0,"allocated":7802880,"data_stored":5716565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":261219423},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 30 09:56:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.442},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.1020000000000001,"15min":5.8470000000000004},"min":{"1min":0.35899999999999999,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":1.083,"5min":11.234999999999999,"15min":273.48200000000003},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.0620000000000001,"15min":5.5430000000000001},"min":{"1min":0.316,"5min":0.28899999999999998,"15min":0.255},"max":{"1min":0.80700000000000005,"5min":8.0820000000000007,"15min":273.50200000000001},"last":0.59899999999999998}]},{"osd":2,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.266,"15min":3.363},"min":{"1min":0.39700000000000002,"5min":0.28899999999999998,"15min":0.26400000000000001},"max":{"1min":1.071,"5min":13.625999999999999,"15min":194.14400000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.629,"5min":1.4430000000000001,"15min":3.0979999999999999},"min":{"1min":0.41499999999999998,"5min":0.313,"15min":0.27800000000000002},"max":{"1min":0.95499999999999996,"5min":11.337999999999999,"15min":198.083},"last":0.68500000000000005}]},{"osd":3,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.972,"15min":6.6200000000000001},"min":{"1min":0.44600000000000001,"5min":0.377,"15min":0.30299999999999999},"max":{"1min":1.0329999999999999,"5min":41.872,"15min":274.01799999999997},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.8069999999999999,"15min":3.0670000000000002},"min":{"1min":0.54900000000000004,"5min":0.29799999999999999,"15min":0.29499999999999998},"max":{"1min":1.139,"5min":41.829000000000001,"15min":130.07400000000001},"last":0.66500000000000004}]},{"osd":4,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.997,"15min":2.1509999999999998},"min":{"1min":0.30599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.68100000000000005,"5min":13.601000000000001,"15min":132.94399999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.0209999999999999,"15min":2.2749999999999999},"min":{"1min":0.41599999999999998,"5min":0.29399999999999998,"15min":0.28899999999999998},"max":{"1min":0.89000000000000001,"5min":14.223000000000001,"15min":136.083},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.5189999999999999,"15min":2.0270000000000001},"min":{"1min":0.41399999999999998,"5min":0.30399999999999999,"15min":0.30199999999999999},"max":{"1min":0.75,"5min":66.123000000000005,"15min":130.09800000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.7549999999999999,"15min":2.044},"min":{"1min":0.49099999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.999,"5min":66.102999999999994,"15min":130.03399999999999},"last":0.61899999999999999}]},{"osd":7,"last update":"Sat Sep 30 09:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":1.155,"15min":1.3899999999999999},"min":{"1min":0.51800000000000002,"5min":0.33800000000000002,"15min":0.26700000000000002},"max":{"1min":0.92600000000000005,"5min":32.578000000000003,"15min":32.578000000000003},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.4159999999999999,"15min":1.367},"min":{"1min":0.46700000000000003,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":0.86499999999999999,"5min":32.420999999999999,"15min":32.420999999999999},"last":0.67400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1918,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10881,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10222,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2333,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2713,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7073,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-30T09:57:03.139 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-30T09:57:03.139 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_1d10b71792f3be8887a7631e69851ac2df3585af/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-30T09:57:03.210 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=eb91c134950e42d4b2e29478c28c1dd0 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_1d10b71792f3be8887a7631e69851ac2df3585af/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-30T09:57:03.212 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-30T09:57:03.222 INFO:tasks.cephadm:Teardown begin 2023-09-30T09:57:03.222 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-30T09:57:03.277 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-30T09:57:03.347 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-30T09:57:03.347 DEBUG:teuthology.orchestra.run.smithi019:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-30T09:57:03.366 DEBUG:teuthology.orchestra.run.smithi167:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-30T09:57:03.385 INFO:tasks.cephadm:Stopping all daemons... 2023-09-30T09:57:03.385 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-30T09:57:03.385 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a 2023-09-30T09:57:03.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:03 smithi019 ceph-mon[216815]: pgmap v1827: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:57:03.431 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:03 smithi019 ceph-mon[216815]: from='client.47945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:57:03.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:03 smithi019 ceph-mon[216815]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:57:03.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:57:03 smithi019 ceph-mon[224043]: pgmap v1827: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:57:03.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:57:03 smithi019 ceph-mon[224043]: from='client.47945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:57:03.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:57:03 smithi019 ceph-mon[224043]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:57:03.521 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:03 smithi167 ceph-mon[192880]: pgmap v1827: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-30T09:57:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:03 smithi167 ceph-mon[192880]: from='client.47945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-30T09:57:03.522 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:03 smithi167 ceph-mon[192880]: from='mgr.25835 172.21.15.19:0/399515613' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-30T09:57:03.704 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:03 smithi019 systemd[1]: Stopping Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:04.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[216791]: 2023-09-30T09:57:03.702+0000 7faf5817c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:04.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[216791]: 2023-09-30T09:57:03.702+0000 7faf5817c700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-30T09:57:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[202899]: [30/Sep/2023:09:57:03] ENGINE Bus STOPPING 2023-09-30T09:57:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[202899]: [30/Sep/2023:09:57:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:57:04.085 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[202899]: [30/Sep/2023:09:57:03] ENGINE Bus STOPPED 2023-09-30T09:57:04.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[202899]: [30/Sep/2023:09:57:03] ENGINE Bus STARTING 2023-09-30T09:57:04.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[202899]: ::ffff:172.21.15.167 - - [30/Sep/2023:09:57:03] "GET /metrics HTTP/1.1" 200 34077 "" "Prometheus/2.43.0" 2023-09-30T09:57:04.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[202899]: [30/Sep/2023:09:57:03] ENGINE Serving on http://:::9283 2023-09-30T09:57:04.086 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:03 smithi019 conmon[202899]: [30/Sep/2023:09:57:03] ENGINE Bus STARTED 2023-09-30T09:57:04.645 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-30T09:57:04.712 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:04 smithi019 bash[721263]: c1584e4cc7f31463fd0eb275db2ada84011832d67c7f751cb1fee23ed1708808 2023-09-30T09:57:04.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:04 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-30T09:57:04.713 INFO:journalctl@ceph.mon.a.smithi019.stdout:Sep 30 09:57:04 smithi019 systemd[1]: Stopped Ceph mon.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:57:04.728 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:04.728 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-30T09:57:04.729 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-30T09:57:04.729 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.c 2023-09-30T09:57:05.070 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:57:04 smithi019 systemd[1]: Stopping Ceph mon.c for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:05.329 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:57:05 smithi019 conmon[224019]: 2023-09-30T09:57:05.068+0000 7feafde37700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:05.329 INFO:journalctl@ceph.mon.c.smithi019.stdout:Sep 30 09:57:05 smithi019 conmon[224019]: 2023-09-30T09:57:05.068+0000 7feafde37700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-30T09:57:05.330 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:05 smithi019 conmon[202899]: [30/Sep/2023:09:57:05] ENGINE Bus STOPPING 2023-09-30T09:57:05.330 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:05 smithi019 conmon[202899]: [30/Sep/2023:09:57:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-30T09:57:05.330 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:05 smithi019 conmon[202899]: [30/Sep/2023:09:57:05] ENGINE Bus STOPPED 2023-09-30T09:57:05.330 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:05 smithi019 conmon[202899]: [30/Sep/2023:09:57:05] ENGINE Bus STARTING 2023-09-30T09:57:05.330 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:05 smithi019 conmon[202899]: [30/Sep/2023:09:57:05] ENGINE Serving on http://:::9283 2023-09-30T09:57:05.330 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:05 smithi019 conmon[202899]: [30/Sep/2023:09:57:05] ENGINE Bus STARTED 2023-09-30T09:57:06.097 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-30T09:57:06.137 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:06.137 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-30T09:57:06.137 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-30T09:57:06.138 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.b 2023-09-30T09:57:06.434 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:06 smithi167 systemd[1]: Stopping Ceph mon.b for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:06.735 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:06 smithi167 conmon[192856]: 2023-09-30T09:57:06.431+0000 7f346a4cd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:06.735 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:06 smithi167 conmon[192856]: 2023-09-30T09:57:06.431+0000 7f346a4cd700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-30T09:57:07.022 INFO:journalctl@ceph.mon.b.smithi167.stdout:Sep 30 09:57:06 smithi167 bash[426194]: fd31a48646fc58ae18b1f0decc619094459267f8e567d306c9cfab2c5e991a9c 2023-09-30T09:57:07.428 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-30T09:57:07.467 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:07.467 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-30T09:57:07.467 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-30T09:57:07.468 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y 2023-09-30T09:57:07.778 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:07 smithi019 systemd[1]: Stopping Ceph mgr.y for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:08.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Sep 30 09:57:07 smithi019 conmon[202899]: 2023-09-30T09:57:07.776+0000 7f1dcd5c1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:08.570 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-30T09:57:08.610 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:08.611 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-30T09:57:08.611 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-30T09:57:08.611 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.x 2023-09-30T09:57:09.021 INFO:journalctl@ceph.mgr.x.smithi167.stdout:Sep 30 09:57:08 smithi167 systemd[1]: Stopping Ceph mgr.x for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:09.719 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-30T09:57:09.756 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:09.756 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-30T09:57:09.757 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-30T09:57:09.757 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.0 2023-09-30T09:57:10.077 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:57:09 smithi019 systemd[1]: Stopping Ceph osd.0 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:10.334 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:57:10 smithi019 conmon[699606]: 2023-09-30T09:57:10.076+0000 7f20be1f1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:10.335 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:57:10 smithi019 conmon[699606]: 2023-09-30T09:57:10.076+0000 7f20be1f1700 -1 osd.0 685 *** Got signal Terminated *** 2023-09-30T09:57:10.335 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:57:10 smithi019 conmon[699606]: 2023-09-30T09:57:10.076+0000 7f20be1f1700 -1 osd.0 685 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:57:16.084 INFO:journalctl@ceph.osd.0.smithi019.stdout:Sep 30 09:57:15 smithi019 bash[722175]: 8d4c99f376a7b9825399ee13a23699b9f59f15059280bc9472efcbb77a269c53 2023-09-30T09:57:17.868 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-30T09:57:17.913 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:17.913 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-30T09:57:17.913 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-30T09:57:17.914 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.1 2023-09-30T09:57:18.232 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:57:17 smithi019 systemd[1]: Stopping Ceph osd.1 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:18.584 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:57:18 smithi019 conmon[421886]: 2023-09-30T09:57:18.230+0000 7fcfa01a8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:18.585 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:57:18 smithi019 conmon[421886]: 2023-09-30T09:57:18.230+0000 7fcfa01a8700 -1 osd.1 685 *** Got signal Terminated *** 2023-09-30T09:57:18.585 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:57:18 smithi019 conmon[421886]: 2023-09-30T09:57:18.230+0000 7fcfa01a8700 -1 osd.1 685 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:57:24.835 INFO:journalctl@ceph.osd.1.smithi019.stdout:Sep 30 09:57:24 smithi019 bash[722762]: d081a274ac32bbf428309a04f1773bd0329c82acad6506c5036e5697017c8b38 2023-09-30T09:57:26.267 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-30T09:57:26.315 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:26.315 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-30T09:57:26.315 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-30T09:57:26.315 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.2 2023-09-30T09:57:26.584 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:57:26 smithi019 systemd[1]: Stopping Ceph osd.2 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:27.084 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:57:26 smithi019 conmon[253819]: 2023-09-30T09:57:26.659+0000 7fb31eb5f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:27.084 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:57:26 smithi019 conmon[253819]: 2023-09-30T09:57:26.659+0000 7fb31eb5f700 -1 osd.2 685 *** Got signal Terminated *** 2023-09-30T09:57:27.085 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:57:26 smithi019 conmon[253819]: 2023-09-30T09:57:26.659+0000 7fb31eb5f700 -1 osd.2 685 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:57:32.835 INFO:journalctl@ceph.osd.2.smithi019.stdout:Sep 30 09:57:32 smithi019 bash[723338]: 328715512fcdddec7a437219a7ac01e77c06f1cf7540a0f364ff1112e009e3ba 2023-09-30T09:57:34.558 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-30T09:57:34.605 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:34.605 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-30T09:57:34.605 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-30T09:57:34.606 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.3 2023-09-30T09:57:34.835 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:57:34 smithi019 systemd[1]: Stopping Ceph osd.3 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:35.334 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:57:34 smithi019 conmon[261441]: 2023-09-30T09:57:34.906+0000 7f8c2036b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:35.335 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:57:34 smithi019 conmon[261441]: 2023-09-30T09:57:34.906+0000 7f8c2036b700 -1 osd.3 685 *** Got signal Terminated *** 2023-09-30T09:57:35.335 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:57:34 smithi019 conmon[261441]: 2023-09-30T09:57:34.906+0000 7f8c2036b700 -1 osd.3 685 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:57:36.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:57:35 smithi167 conmon[177400]: ts=2023-09-30T09:57:35.673Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.19:8765: connect: connection refused" 2023-09-30T09:57:36.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:57:35 smithi167 conmon[177400]: ts=2023-09-30T09:57:35.675Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.19:8765: connect: connection refused" 2023-09-30T09:57:36.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:57:35 smithi167 conmon[177400]: ts=2023-09-30T09:57:35.675Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.19:8765: connect: connection refused" 2023-09-30T09:57:36.022 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:57:35 smithi167 conmon[177400]: ts=2023-09-30T09:57:35.677Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.19:8765: connect: connection refused" 2023-09-30T09:57:37.021 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:36 smithi167 conmon[130376]: 2023-09-30T09:57:36.582+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:37.771 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:37 smithi167 conmon[130376]: 2023-09-30T09:57:37.557+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:37.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:37 smithi167 conmon[139912]: 2023-09-30T09:57:37.406+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:38.272 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:37 smithi167 conmon[112464]: 2023-09-30T09:57:37.857+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:38.771 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:38 smithi167 conmon[130376]: 2023-09-30T09:57:38.581+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:38.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:38 smithi167 conmon[139912]: 2023-09-30T09:57:38.373+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:39.084 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:57:38 smithi019 conmon[261441]: 2023-09-30T09:57:38.768+0000 7f8c18959700 -1 osd.3 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.772911+0000 front 2023-09-30T09:57:12.773041+0000 (oldest deadline 2023-09-30T09:57:38.073249+0000) 2023-09-30T09:57:39.271 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:38 smithi167 conmon[112464]: 2023-09-30T09:57:38.852+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:39.771 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:39 smithi167 conmon[130376]: 2023-09-30T09:57:39.554+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:39.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:39 smithi167 conmon[139912]: 2023-09-30T09:57:39.322+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:39.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:39 smithi167 conmon[343682]: 2023-09-30T09:57:39.343+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:40.084 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:57:39 smithi019 conmon[261441]: 2023-09-30T09:57:39.769+0000 7f8c18959700 -1 osd.3 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.772911+0000 front 2023-09-30T09:57:12.773041+0000 (oldest deadline 2023-09-30T09:57:38.073249+0000) 2023-09-30T09:57:40.271 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:39 smithi167 conmon[112464]: 2023-09-30T09:57:39.817+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:40.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:40 smithi167 conmon[130376]: 2023-09-30T09:57:40.551+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:40.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:40 smithi167 conmon[139912]: 2023-09-30T09:57:40.357+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:40.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:40 smithi167 conmon[343682]: 2023-09-30T09:57:40.342+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:41.271 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:40 smithi167 conmon[112464]: 2023-09-30T09:57:40.862+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:41.584 INFO:journalctl@ceph.osd.3.smithi019.stdout:Sep 30 09:57:41 smithi019 bash[723926]: 1257456223c880652b760ebb4683b5062a3777de7df18bda0f7be5a07bbb92a0 2023-09-30T09:57:41.771 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:41 smithi167 conmon[130376]: 2023-09-30T09:57:41.558+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:41.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:41 smithi167 conmon[139912]: 2023-09-30T09:57:41.364+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:41.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:41 smithi167 conmon[343682]: 2023-09-30T09:57:41.361+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:42.271 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:41 smithi167 conmon[112464]: 2023-09-30T09:57:41.818+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:42.771 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:42 smithi167 conmon[130376]: 2023-09-30T09:57:42.582+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:42.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:42 smithi167 conmon[343682]: 2023-09-30T09:57:42.383+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:42.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:42 smithi167 conmon[139912]: 2023-09-30T09:57:42.399+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:43.272 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:42 smithi167 conmon[112464]: 2023-09-30T09:57:42.784+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:43.389 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-30T09:57:43.443 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:43.443 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-30T09:57:43.444 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-30T09:57:43.444 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.4 2023-09-30T09:57:43.737 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:43 smithi167 conmon[130376]: 2023-09-30T09:57:43.582+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:43.737 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:43 smithi167 conmon[139912]: 2023-09-30T09:57:43.408+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:43.738 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:43 smithi167 conmon[343682]: 2023-09-30T09:57:43.430+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:43.738 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:43 smithi167 systemd[1]: Stopping Ceph osd.4 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:44.021 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:43 smithi167 conmon[112464]: 2023-09-30T09:57:43.735+0000 7f7bdaeef700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:44.022 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:43 smithi167 conmon[112464]: 2023-09-30T09:57:43.735+0000 7f7bdaeef700 -1 osd.4 685 *** Got signal Terminated *** 2023-09-30T09:57:44.022 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:43 smithi167 conmon[112464]: 2023-09-30T09:57:43.735+0000 7f7bdaeef700 -1 osd.4 685 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:57:44.022 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:43 smithi167 conmon[112464]: 2023-09-30T09:57:43.781+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:44.771 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:44 smithi167 conmon[130376]: 2023-09-30T09:57:44.561+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:44.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:44 smithi167 conmon[139912]: 2023-09-30T09:57:44.365+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:44.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:44 smithi167 conmon[343682]: 2023-09-30T09:57:44.465+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:45.271 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:44 smithi167 conmon[112464]: 2023-09-30T09:57:44.806+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:45.761 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:45 smithi167 conmon[130376]: 2023-09-30T09:57:45.541+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:45.762 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:45 smithi167 conmon[139912]: 2023-09-30T09:57:45.331+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:45.762 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:45 smithi167 conmon[343682]: 2023-09-30T09:57:45.455+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:45.762 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:45 smithi167 conmon[343682]: 2023-09-30T09:57:45.455+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:46.021 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:45 smithi167 conmon[112464]: 2023-09-30T09:57:45.761+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:46.770 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:46 smithi167 conmon[130376]: 2023-09-30T09:57:46.493+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:46.771 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:46 smithi167 conmon[139912]: 2023-09-30T09:57:46.310+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:46.771 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:46 smithi167 conmon[343682]: 2023-09-30T09:57:46.412+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:46.771 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:46 smithi167 conmon[343682]: 2023-09-30T09:57:46.412+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:47.022 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:46 smithi167 conmon[112464]: 2023-09-30T09:57:46.770+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:47.771 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:47 smithi167 conmon[139912]: 2023-09-30T09:57:47.278+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:47.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:47 smithi167 conmon[130376]: 2023-09-30T09:57:47.480+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:47.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:47 smithi167 conmon[130376]: 2023-09-30T09:57:47.480+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:47.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:47 smithi167 conmon[343682]: 2023-09-30T09:57:47.399+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:47.773 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:47 smithi167 conmon[343682]: 2023-09-30T09:57:47.399+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:48.267 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:47 smithi167 conmon[112464]: 2023-09-30T09:57:47.771+0000 7f7bd6cec700 -1 osd.4 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:13.988301+0000 front 2023-09-30T09:57:13.988288+0000 (oldest deadline 2023-09-30T09:57:36.888133+0000) 2023-09-30T09:57:48.522 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:48 smithi167 conmon[139912]: 2023-09-30T09:57:48.267+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:48.522 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:48 smithi167 conmon[139912]: 2023-09-30T09:57:48.267+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:48.523 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:48 smithi167 conmon[130376]: 2023-09-30T09:57:48.503+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:48.523 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:48 smithi167 conmon[130376]: 2023-09-30T09:57:48.503+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:48.523 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:48 smithi167 conmon[343682]: 2023-09-30T09:57:48.356+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:48.523 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:48 smithi167 conmon[343682]: 2023-09-30T09:57:48.356+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:49.645 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:49 smithi167 conmon[139912]: 2023-09-30T09:57:49.310+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:49.645 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:49 smithi167 conmon[139912]: 2023-09-30T09:57:49.310+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:49.645 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:49 smithi167 conmon[343682]: 2023-09-30T09:57:49.390+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:49.645 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:49 smithi167 conmon[343682]: 2023-09-30T09:57:49.390+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:49.646 INFO:journalctl@ceph.osd.4.smithi167.stdout:Sep 30 09:57:49 smithi167 bash[426811]: 6557c10c6a0bf7b57ebe5ad13fd928f89d0b221793354cbc2d3d74adc408f78f 2023-09-30T09:57:49.646 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:49 smithi167 conmon[130376]: 2023-09-30T09:57:49.524+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:49.646 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:49 smithi167 conmon[130376]: 2023-09-30T09:57:49.524+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:50.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:50 smithi167 conmon[130376]: 2023-09-30T09:57:50.548+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:50.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:50 smithi167 conmon[130376]: 2023-09-30T09:57:50.548+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:50.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:50 smithi167 conmon[139912]: 2023-09-30T09:57:50.329+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:50.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:50 smithi167 conmon[139912]: 2023-09-30T09:57:50.329+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:50.773 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:50 smithi167 conmon[343682]: 2023-09-30T09:57:50.343+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:50.773 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:50 smithi167 conmon[343682]: 2023-09-30T09:57:50.343+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:51.522 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:51 smithi167 conmon[139912]: 2023-09-30T09:57:51.349+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:51.522 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:51 smithi167 conmon[139912]: 2023-09-30T09:57:51.349+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:51.523 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:51 smithi167 conmon[343682]: 2023-09-30T09:57:51.349+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:51.523 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:51 smithi167 conmon[343682]: 2023-09-30T09:57:51.349+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:51.802 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:51 smithi167 conmon[130376]: 2023-09-30T09:57:51.529+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:51.802 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:51 smithi167 conmon[130376]: 2023-09-30T09:57:51.529+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:51.810 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-30T09:57:51.865 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:51.865 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-30T09:57:51.865 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-30T09:57:51.865 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.5 2023-09-30T09:57:52.152 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:51 smithi167 systemd[1]: Stopping Ceph osd.5 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:57:52.506 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[139912]: 2023-09-30T09:57:52.380+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:52.507 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[139912]: 2023-09-30T09:57:52.380+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:52.507 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[343682]: 2023-09-30T09:57:52.150+0000 7f3255e76700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:57:52.507 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[343682]: 2023-09-30T09:57:52.150+0000 7f3255e76700 -1 osd.5 685 *** Got signal Terminated *** 2023-09-30T09:57:52.507 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[343682]: 2023-09-30T09:57:52.150+0000 7f3255e76700 -1 osd.5 685 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:57:52.507 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[343682]: 2023-09-30T09:57:52.355+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:52.508 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[343682]: 2023-09-30T09:57:52.355+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:52.771 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[130376]: 2023-09-30T09:57:52.506+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:52.771 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:52 smithi167 conmon[130376]: 2023-09-30T09:57:52.506+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:53.771 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:53 smithi167 conmon[343682]: 2023-09-30T09:57:53.353+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:53.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:53 smithi167 conmon[343682]: 2023-09-30T09:57:53.353+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:53.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:53 smithi167 conmon[130376]: 2023-09-30T09:57:53.544+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:53.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:53 smithi167 conmon[130376]: 2023-09-30T09:57:53.544+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:53.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:53 smithi167 conmon[139912]: 2023-09-30T09:57:53.429+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:53.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:53 smithi167 conmon[139912]: 2023-09-30T09:57:53.429+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:54.771 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:54 smithi167 conmon[343682]: 2023-09-30T09:57:54.322+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:54.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:54 smithi167 conmon[343682]: 2023-09-30T09:57:54.322+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:54.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:54 smithi167 conmon[130376]: 2023-09-30T09:57:54.527+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:54.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:54 smithi167 conmon[130376]: 2023-09-30T09:57:54.527+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:54.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:54 smithi167 conmon[130376]: 2023-09-30T09:57:54.527+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:57:54.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:54 smithi167 conmon[139912]: 2023-09-30T09:57:54.473+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:54.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:54 smithi167 conmon[139912]: 2023-09-30T09:57:54.473+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:55.771 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:55 smithi167 conmon[343682]: 2023-09-30T09:57:55.366+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:55.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:55 smithi167 conmon[343682]: 2023-09-30T09:57:55.366+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:55.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:55 smithi167 conmon[343682]: 2023-09-30T09:57:55.366+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:31.535233+0000 front 2023-09-30T09:57:31.535100+0000 (oldest deadline 2023-09-30T09:57:54.435117+0000) 2023-09-30T09:57:55.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:55 smithi167 conmon[130376]: 2023-09-30T09:57:55.498+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:55.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:55 smithi167 conmon[130376]: 2023-09-30T09:57:55.498+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:55.773 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:55 smithi167 conmon[130376]: 2023-09-30T09:57:55.498+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:57:55.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:55 smithi167 conmon[139912]: 2023-09-30T09:57:55.471+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:55.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:55 smithi167 conmon[139912]: 2023-09-30T09:57:55.471+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:56.771 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:56 smithi167 conmon[139912]: 2023-09-30T09:57:56.520+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:56.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:56 smithi167 conmon[139912]: 2023-09-30T09:57:56.520+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:56.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:56 smithi167 conmon[343682]: 2023-09-30T09:57:56.416+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:14.231213+0000 front 2023-09-30T09:57:14.231290+0000 (oldest deadline 2023-09-30T09:57:38.331168+0000) 2023-09-30T09:57:56.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:56 smithi167 conmon[343682]: 2023-09-30T09:57:56.416+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:21.232619+0000 front 2023-09-30T09:57:21.232602+0000 (oldest deadline 2023-09-30T09:57:44.633009+0000) 2023-09-30T09:57:56.772 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:56 smithi167 conmon[343682]: 2023-09-30T09:57:56.416+0000 7f324fc6f700 -1 osd.5 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:31.535233+0000 front 2023-09-30T09:57:31.535100+0000 (oldest deadline 2023-09-30T09:57:54.435117+0000) 2023-09-30T09:57:56.773 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:56 smithi167 conmon[130376]: 2023-09-30T09:57:56.488+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:56.773 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:56 smithi167 conmon[130376]: 2023-09-30T09:57:56.488+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:56.773 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:56 smithi167 conmon[130376]: 2023-09-30T09:57:56.488+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:57:57.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:57 smithi167 conmon[130376]: 2023-09-30T09:57:57.493+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:57.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:57 smithi167 conmon[130376]: 2023-09-30T09:57:57.493+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:57.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:57 smithi167 conmon[130376]: 2023-09-30T09:57:57.493+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:57:57.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:57 smithi167 conmon[139912]: 2023-09-30T09:57:57.521+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:57.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:57 smithi167 conmon[139912]: 2023-09-30T09:57:57.521+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:57.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:57 smithi167 conmon[139912]: 2023-09-30T09:57:57.521+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:57:58.521 INFO:journalctl@ceph.osd.5.smithi167.stdout:Sep 30 09:57:58 smithi167 bash[427392]: 3ae29cf3e4041c22113679517653d49d03b88cc9abb72f3b98181d7e8e8d68cb 2023-09-30T09:57:58.522 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:58 smithi167 conmon[130376]: 2023-09-30T09:57:58.469+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:58.522 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:58 smithi167 conmon[130376]: 2023-09-30T09:57:58.469+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:58.522 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:58 smithi167 conmon[130376]: 2023-09-30T09:57:58.469+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:57:58.842 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:58 smithi167 conmon[139912]: 2023-09-30T09:57:58.570+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:58.842 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:58 smithi167 conmon[139912]: 2023-09-30T09:57:58.570+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:58.842 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:58 smithi167 conmon[139912]: 2023-09-30T09:57:58.570+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:57:59.773 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:59 smithi167 conmon[130376]: 2023-09-30T09:57:59.498+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:57:59.774 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:59 smithi167 conmon[130376]: 2023-09-30T09:57:59.498+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:57:59.774 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:59 smithi167 conmon[130376]: 2023-09-30T09:57:59.498+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:57:59.774 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:59 smithi167 conmon[139912]: 2023-09-30T09:57:59.586+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:57:59.774 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:59 smithi167 conmon[139912]: 2023-09-30T09:57:59.586+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:57:59.775 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:57:59 smithi167 conmon[139912]: 2023-09-30T09:57:59.586+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:57:59.791 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-30T09:57:59.830 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:57:59.831 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-30T09:57:59.831 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-30T09:57:59.831 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.6 2023-09-30T09:58:00.143 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:57:59 smithi167 systemd[1]: Stopping Ceph osd.6 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:58:00.143 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[130376]: 2023-09-30T09:58:00.142+0000 7fac2a961700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:58:00.521 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[130376]: 2023-09-30T09:58:00.142+0000 7fac2a961700 -1 osd.6 685 *** Got signal Terminated *** 2023-09-30T09:58:00.521 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[130376]: 2023-09-30T09:58:00.142+0000 7fac2a961700 -1 osd.6 685 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:58:01.022 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[130376]: 2023-09-30T09:58:00.523+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:58:01.022 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[130376]: 2023-09-30T09:58:00.523+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:58:01.022 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[130376]: 2023-09-30T09:58:00.523+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:58:01.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[139912]: 2023-09-30T09:58:00.593+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:01.023 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[139912]: 2023-09-30T09:58:00.593+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:01.023 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:00 smithi167 conmon[139912]: 2023-09-30T09:58:00.593+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:02.021 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:01 smithi167 conmon[139912]: 2023-09-30T09:58:01.562+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:02.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:01 smithi167 conmon[139912]: 2023-09-30T09:58:01.562+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:02.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:01 smithi167 conmon[139912]: 2023-09-30T09:58:01.562+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:02.022 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:01 smithi167 conmon[130376]: 2023-09-30T09:58:01.554+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:58:02.022 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:01 smithi167 conmon[130376]: 2023-09-30T09:58:01.554+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:58:02.022 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:01 smithi167 conmon[130376]: 2023-09-30T09:58:01.554+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:58:02.023 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:01 smithi167 conmon[130376]: 2023-09-30T09:58:01.554+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:39.797075+0000 front 2023-09-30T09:57:39.797031+0000 (oldest deadline 2023-09-30T09:58:01.497016+0000) 2023-09-30T09:58:02.771 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:02 smithi167 conmon[139912]: 2023-09-30T09:58:02.610+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:02.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:02 smithi167 conmon[139912]: 2023-09-30T09:58:02.610+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:02.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:02 smithi167 conmon[139912]: 2023-09-30T09:58:02.610+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:02.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:02 smithi167 conmon[130376]: 2023-09-30T09:58:02.506+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:58:02.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:02 smithi167 conmon[130376]: 2023-09-30T09:58:02.506+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:58:02.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:02 smithi167 conmon[130376]: 2023-09-30T09:58:02.506+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:58:02.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:02 smithi167 conmon[130376]: 2023-09-30T09:58:02.506+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:39.797075+0000 front 2023-09-30T09:57:39.797031+0000 (oldest deadline 2023-09-30T09:58:01.497016+0000) 2023-09-30T09:58:03.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:03 smithi167 conmon[130376]: 2023-09-30T09:58:03.475+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:58:03.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:03 smithi167 conmon[130376]: 2023-09-30T09:58:03.475+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:58:03.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:03 smithi167 conmon[130376]: 2023-09-30T09:58:03.475+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:58:03.773 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:03 smithi167 conmon[130376]: 2023-09-30T09:58:03.475+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:39.797075+0000 front 2023-09-30T09:57:39.797031+0000 (oldest deadline 2023-09-30T09:58:01.497016+0000) 2023-09-30T09:58:03.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:03 smithi167 conmon[139912]: 2023-09-30T09:58:03.602+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:03.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:03 smithi167 conmon[139912]: 2023-09-30T09:58:03.602+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:03.774 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:03 smithi167 conmon[139912]: 2023-09-30T09:58:03.602+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:04.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:04 smithi167 conmon[130376]: 2023-09-30T09:58:04.475+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.090810+0000 front 2023-09-30T09:57:12.090792+0000 (oldest deadline 2023-09-30T09:57:35.590778+0000) 2023-09-30T09:58:04.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:04 smithi167 conmon[130376]: 2023-09-30T09:58:04.475+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:20.693322+0000 front 2023-09-30T09:57:20.693309+0000 (oldest deadline 2023-09-30T09:57:46.593252+0000) 2023-09-30T09:58:04.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:04 smithi167 conmon[130376]: 2023-09-30T09:58:04.475+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:28.894479+0000 front 2023-09-30T09:57:28.894548+0000 (oldest deadline 2023-09-30T09:57:53.594360+0000) 2023-09-30T09:58:04.772 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:04 smithi167 conmon[130376]: 2023-09-30T09:58:04.475+0000 7fac26f5f700 -1 osd.6 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:39.797075+0000 front 2023-09-30T09:57:39.797031+0000 (oldest deadline 2023-09-30T09:58:01.497016+0000) 2023-09-30T09:58:04.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:04 smithi167 conmon[139912]: 2023-09-30T09:58:04.583+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:04.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:04 smithi167 conmon[139912]: 2023-09-30T09:58:04.583+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:04.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:04 smithi167 conmon[139912]: 2023-09-30T09:58:04.583+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:04.773 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:04 smithi167 conmon[139912]: 2023-09-30T09:58:04.583+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:05.861 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:05 smithi167 conmon[139912]: 2023-09-30T09:58:05.565+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:05.861 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:05 smithi167 conmon[139912]: 2023-09-30T09:58:05.565+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:05.861 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:05 smithi167 conmon[139912]: 2023-09-30T09:58:05.565+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:05.861 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:05 smithi167 conmon[139912]: 2023-09-30T09:58:05.565+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:06.272 INFO:journalctl@ceph.osd.6.smithi167.stdout:Sep 30 09:58:05 smithi167 bash[427966]: 4a5f3bfc86e40a0217e0b893f113b5c93707d1a16223c551162621200d5266ae 2023-09-30T09:58:06.896 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:06 smithi167 conmon[139912]: 2023-09-30T09:58:06.561+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:06.896 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:06 smithi167 conmon[139912]: 2023-09-30T09:58:06.561+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:06.896 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:06 smithi167 conmon[139912]: 2023-09-30T09:58:06.561+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:06.897 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:06 smithi167 conmon[139912]: 2023-09-30T09:58:06.561+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:07.942 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:07 smithi167 conmon[139912]: 2023-09-30T09:58:07.557+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:07.942 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:07 smithi167 conmon[139912]: 2023-09-30T09:58:07.557+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:07.942 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:07 smithi167 conmon[139912]: 2023-09-30T09:58:07.557+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:07.942 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:07 smithi167 conmon[139912]: 2023-09-30T09:58:07.557+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:07.949 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-30T09:58:07.991 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:58:07.992 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-30T09:58:07.992 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-30T09:58:07.992 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.7 2023-09-30T09:58:08.272 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:08 smithi167 systemd[1]: Stopping Ceph osd.7 for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:58:08.771 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:08 smithi167 conmon[139912]: 2023-09-30T09:58:08.298+0000 7f24371d3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-30T09:58:08.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:08 smithi167 conmon[139912]: 2023-09-30T09:58:08.298+0000 7f24371d3700 -1 osd.7 685 *** Got signal Terminated *** 2023-09-30T09:58:08.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:08 smithi167 conmon[139912]: 2023-09-30T09:58:08.298+0000 7f24371d3700 -1 osd.7 685 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-30T09:58:08.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:08 smithi167 conmon[139912]: 2023-09-30T09:58:08.507+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:08.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:08 smithi167 conmon[139912]: 2023-09-30T09:58:08.507+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:08.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:08 smithi167 conmon[139912]: 2023-09-30T09:58:08.507+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:08.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:08 smithi167 conmon[139912]: 2023-09-30T09:58:08.507+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:10.021 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:09 smithi167 conmon[139912]: 2023-09-30T09:58:09.554+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:10.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:09 smithi167 conmon[139912]: 2023-09-30T09:58:09.554+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:10.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:09 smithi167 conmon[139912]: 2023-09-30T09:58:09.554+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:10.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:09 smithi167 conmon[139912]: 2023-09-30T09:58:09.554+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:11.021 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:10 smithi167 conmon[139912]: 2023-09-30T09:58:10.523+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:11.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:10 smithi167 conmon[139912]: 2023-09-30T09:58:10.523+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:11.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:10 smithi167 conmon[139912]: 2023-09-30T09:58:10.523+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:11.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:10 smithi167 conmon[139912]: 2023-09-30T09:58:10.523+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:11.771 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:11 smithi167 conmon[139912]: 2023-09-30T09:58:11.512+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:11.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:11 smithi167 conmon[139912]: 2023-09-30T09:58:11.512+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:11.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:11 smithi167 conmon[139912]: 2023-09-30T09:58:11.512+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:11.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:11 smithi167 conmon[139912]: 2023-09-30T09:58:11.512+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:11.772 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:11 smithi167 conmon[139912]: 2023-09-30T09:58:11.512+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.167:6804 osd.4 since back 2023-09-30T09:57:47.194065+0000 front 2023-09-30T09:57:47.194174+0000 (oldest deadline 2023-09-30T09:58:11.294178+0000) 2023-09-30T09:58:13.021 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:12 smithi167 conmon[139912]: 2023-09-30T09:58:12.524+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2023-09-30T09:57:12.690061+0000 front 2023-09-30T09:57:12.690038+0000 (oldest deadline 2023-09-30T09:57:36.789845+0000) 2023-09-30T09:58:13.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:12 smithi167 conmon[139912]: 2023-09-30T09:58:12.524+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2023-09-30T09:57:22.691070+0000 front 2023-09-30T09:57:22.691017+0000 (oldest deadline 2023-09-30T09:57:47.291360+0000) 2023-09-30T09:58:13.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:12 smithi167 conmon[139912]: 2023-09-30T09:58:12.524+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2023-09-30T09:57:27.292039+0000 front 2023-09-30T09:57:27.292085+0000 (oldest deadline 2023-09-30T09:57:57.292454+0000) 2023-09-30T09:58:13.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:12 smithi167 conmon[139912]: 2023-09-30T09:58:12.524+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2023-09-30T09:57:38.393488+0000 front 2023-09-30T09:57:38.393548+0000 (oldest deadline 2023-09-30T09:58:04.293338+0000) 2023-09-30T09:58:13.022 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:12 smithi167 conmon[139912]: 2023-09-30T09:58:12.524+0000 7f2431fce700 -1 osd.7 685 heartbeat_check: no reply from 172.21.15.167:6804 osd.4 since back 2023-09-30T09:57:47.194065+0000 front 2023-09-30T09:57:47.194174+0000 (oldest deadline 2023-09-30T09:58:11.294178+0000) 2023-09-30T09:58:14.521 INFO:journalctl@ceph.osd.7.smithi167.stdout:Sep 30 09:58:14 smithi167 bash[428551]: 92e8410b51909808c2ec91b8a742824dde4d3834dcb7ac795dd924ba95f3fe74 2023-09-30T09:58:15.781 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-30T09:58:15.821 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:58:15.822 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-30T09:58:15.822 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-30T09:58:15.822 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-30T09:58:16.123 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:15 smithi167 systemd[1]: Stopping Ceph prometheus.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:58:16.521 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-30T09:58:16.522 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-30T09:58:16.522 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-30T09:58:16.522 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-30T09:58:16.522 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-30T09:58:16.522 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-30T09:58:16.522 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-30T09:58:16.523 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-30T09:58:16.523 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.123Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-30T09:58:16.523 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.125Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-30T09:58:16.523 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.125Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-30T09:58:16.523 INFO:journalctl@ceph.prometheus.a.smithi167.stdout:Sep 30 09:58:16 smithi167 conmon[177400]: ts=2023-09-30T09:58:16.125Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-30T09:58:16.790 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-30T09:58:16.828 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-30T09:58:16.828 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-30T09:58:16.828 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-30T09:58:18.302 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:58:18 smithi019 systemd[1]: Stopping Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:58:18.569 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:58:18 smithi019 conmon[201097]: ts=2023-09-30T09:58:18.302Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-30T09:58:18.834 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:58:18 smithi019 bash[724820]: b16b2fb388da6510e25ce057c250c9fa7f5d7e82ec53d1b96a8573f420b97bc1 2023-09-30T09:58:19.584 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:58:19 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-30T09:58:19.585 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Sep 30 09:58:19 smithi019 systemd[1]: Stopped Ceph alertmanager.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:58:19.585 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:58:19 smithi019 systemd[1]: Stopping Ceph node-exporter.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:58:20.317 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:58:20 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:58:20.317 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:58:20 smithi019 bash[725106]: 63ac922663db64aba5fa0bcb9b17921ae749c25ff950ab0599a3051ca205d79f 2023-09-30T09:58:20.318 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:58:20 smithi019 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-30T09:58:20.318 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Sep 30 09:58:20 smithi019 systemd[1]: Stopped Ceph node-exporter.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:58:29.325 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-30T09:58:30.772 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:58:30 smithi167 systemd[1]: Stopping Ceph grafana.a for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:58:31.521 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:58:31 smithi167 bash[429740]: f7c9f74b80886c83d45e9a4ce841b90929cdca4f5133696113b2831ed06b1d11 2023-09-30T09:58:31.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:58:31 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:58:31.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:58:31 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-30T09:58:31.522 INFO:journalctl@ceph.grafana.a.smithi167.stdout:Sep 30 09:58:31 smithi167 systemd[1]: Stopped Ceph grafana.a for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:58:31.854 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:58:31 smithi167 systemd[1]: Stopping Ceph node-exporter.b for 608495ac-5f70-11ee-8db4-212e2dc638e7... 2023-09-30T09:58:32.518 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:58:32 smithi167 bash[430023]: d6c59fa1a46188b98f96b42a4a61170a66ba6b368023c8b05460e73c82932884 2023-09-30T09:58:32.518 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:58:32 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-30T09:58:32.518 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:58:32 smithi167 systemd[1]: ceph-608495ac-5f70-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-30T09:58:32.518 INFO:journalctl@ceph.node-exporter.b.smithi167.stdout:Sep 30 09:58:32 smithi167 systemd[1]: Stopped Ceph node-exporter.b for 608495ac-5f70-11ee-8db4-212e2dc638e7. 2023-09-30T09:58:36.213 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-30T09:58:36.252 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-30T09:58:36.282 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-30T09:58:36.285 DEBUG:teuthology.misc:Transferring archived files from smithi019:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221/remote/smithi019/crash 2023-09-30T09:58:36.286 DEBUG:teuthology.orchestra.run.smithi019:> sudo tar c -f - -C /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/crash -- . 2023-09-30T09:58:36.341 INFO:teuthology.orchestra.run.smithi019.stderr:tar: /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-30T09:58:36.342 INFO:teuthology.orchestra.run.smithi019.stderr:tar: Error is not recoverable: exiting now 2023-09-30T09:58:36.344 DEBUG:teuthology.misc:Transferring archived files from smithi167:/var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221/remote/smithi167/crash 2023-09-30T09:58:36.344 DEBUG:teuthology.orchestra.run.smithi167:> sudo tar c -f - -C /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/crash -- . 2023-09-30T09:58:36.399 INFO:teuthology.orchestra.run.smithi167.stderr:tar: /var/lib/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-30T09:58:36.399 INFO:teuthology.orchestra.run.smithi167.stderr:tar: Error is not recoverable: exiting now 2023-09-30T09:58:36.400 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-30T09:58:36.400 DEBUG:teuthology.orchestra.run.smithi019:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-30T09:58:36.431 INFO:teuthology.orchestra.run.smithi019.stderr:grep: /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-30T09:58:36.433 INFO:tasks.cephadm:Compressing logs... 2023-09-30T09:58:36.433 DEBUG:teuthology.orchestra.run.smithi019:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-09-30T09:58:36.476 DEBUG:teuthology.orchestra.run.smithi167:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-09-30T09:58:36.515 INFO:teuthology.orchestra.run.smithi167.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-09-30T09:58:36.515 INFO:teuthology.orchestra.run.smithi167.stderr:: No such file or directory 2023-09-30T09:58:36.521 INFO:teuthology.orchestra.run.smithi019.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-30T09:58:36.527 INFO:teuthology.orchestra.run.smithi019.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-09-30T09:58:36.538 INFO:teuthology.orchestra.run.smithi019.stderr:gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.2.log 2023-09-30T09:58:36.538 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mds.foo.smithi019.kcgktg.log 2023-09-30T09:58:36.538 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.0.log 2023-09-30T09:58:36.538 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mds.foo.smithi019.kcgktg.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.3.log 2023-09-30T09:58:36.540 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.1.log 2023-09-30T09:58:36.540 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.c.log 2023-09-30T09:58:36.540 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-volume.log 2023-09-30T09:58:36.541 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.a.log 2023-09-30T09:58:36.541 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mgr.y.log 2023-09-30T09:58:36.541 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.a.log: 81.8% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mds.foo.smithi019.kcgktg.log.gz 2023-09-30T09:58:36.548 INFO:teuthology.orchestra.run.smithi167.stderr:gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mds.foo.smithi167.bnuarp.log 2023-09-30T09:58:36.549 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/cephadm.log: gzip /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mds.foo.smithi167.bnuarp.log: -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.5.log 2023-09-30T09:58:36.549 INFO:teuthology.orchestra.run.smithi167.stderr: 62.7% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mds.foo.smithi167.bnuarp.log.gz 2023-09-30T09:58:36.550 INFO:teuthology.orchestra.run.smithi167.stderr:gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.6.log 2023-09-30T09:58:36.551 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.4.log 2023-09-30T09:58:36.552 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mgr.x.log 2023-09-30T09:58:36.552 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-volume.log 2023-09-30T09:58:36.552 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.7.log 2023-09-30T09:58:36.552 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.b.log 2023-09-30T09:58:36.760 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.7.log: /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.b.log: 91.4% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-volume.log.gz 2023-09-30T09:58:37.144 INFO:teuthology.orchestra.run.smithi167.stderr: 91.6% -- replaced with /var/log/ceph/cephadm.log.gz 2023-09-30T09:58:37.189 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mgr.y.log: 91.0% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-volume.log.gz 2023-09-30T09:58:37.323 INFO:teuthology.orchestra.run.smithi167.stderr: 89.5% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mgr.x.log.gz 2023-09-30T09:58:38.506 INFO:teuthology.orchestra.run.smithi019.stderr: 89.5% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mgr.y.log.gz 2023-09-30T09:58:39.225 INFO:teuthology.orchestra.run.smithi019.stderr: 95.4% -- replaced with /var/log/ceph/cephadm.log.gz 2023-09-30T09:58:40.224 INFO:teuthology.orchestra.run.smithi167.stderr: 92.0% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.b.log.gz 2023-09-30T09:58:41.122 INFO:teuthology.orchestra.run.smithi019.stderr: 91.5% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.c.log.gz 2023-09-30T09:58:43.856 INFO:teuthology.orchestra.run.smithi019.stderr: 90.3% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-mon.a.log.gz 2023-09-30T10:02:40.039 INFO:teuthology.orchestra.run.smithi167.stderr: 93.0% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.6.log.gz 2023-09-30T10:03:08.958 INFO:teuthology.orchestra.run.smithi019.stderr: 91.9% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.1.log.gz 2023-09-30T10:03:39.208 INFO:teuthology.orchestra.run.smithi167.stderr: 92.3% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.7.log.gz 2023-09-30T10:03:56.685 INFO:teuthology.orchestra.run.smithi167.stderr: 92.0% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.5.log.gz 2023-09-30T10:04:03.858 INFO:teuthology.orchestra.run.smithi167.stderr: 91.9% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.4.log.gz 2023-09-30T10:04:03.861 INFO:teuthology.orchestra.run.smithi167.stderr: 2023-09-30T10:04:03.861 INFO:teuthology.orchestra.run.smithi167.stderr:real 5m27.369s 2023-09-30T10:04:03.861 INFO:teuthology.orchestra.run.smithi167.stderr:user 7m4.204s 2023-09-30T10:04:03.861 INFO:teuthology.orchestra.run.smithi167.stderr:sys 0m17.511s 2023-09-30T10:04:37.057 INFO:teuthology.orchestra.run.smithi019.stderr: 93.0% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.3.log.gz 2023-09-30T10:04:52.194 INFO:teuthology.orchestra.run.smithi019.stderr: 92.1% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.2.log.gz 2023-09-30T10:04:57.857 INFO:teuthology.orchestra.run.smithi019.stderr: 91.8% -- replaced with /var/log/ceph/608495ac-5f70-11ee-8db4-212e2dc638e7/ceph-osd.0.log.gz 2023-09-30T10:04:57.869 INFO:teuthology.orchestra.run.smithi019.stderr: 2023-09-30T10:04:57.869 INFO:teuthology.orchestra.run.smithi019.stderr:real 6m21.377s 2023-09-30T10:04:57.869 INFO:teuthology.orchestra.run.smithi019.stderr:user 8m42.571s 2023-09-30T10:04:57.869 INFO:teuthology.orchestra.run.smithi019.stderr:sys 0m19.640s 2023-09-30T10:04:57.870 INFO:tasks.cephadm:Archiving logs... 2023-09-30T10:04:57.870 DEBUG:teuthology.misc:Transferring archived files from smithi019:/var/log/ceph to /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221/remote/smithi019/log 2023-09-30T10:04:57.871 DEBUG:teuthology.orchestra.run.smithi019:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-30T10:06:23.491 DEBUG:teuthology.misc:Transferring archived files from smithi167:/var/log/ceph to /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221/remote/smithi167/log 2023-09-30T10:06:23.492 DEBUG:teuthology.orchestra.run.smithi167:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-30T10:07:36.095 INFO:tasks.cephadm:Removing cluster... 2023-09-30T10:07:36.096 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 --force 2023-09-30T10:07:37.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 608495ac-5f70-11ee-8db4-212e2dc638e7 --force 2023-09-30T10:07:38.815 INFO:tasks.cephadm:Removing cephadm ... 2023-09-30T10:07:38.816 DEBUG:teuthology.orchestra.run.smithi019:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-30T10:07:38.838 DEBUG:teuthology.orchestra.run.smithi167:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-30T10:07:38.862 INFO:tasks.cephadm:Teardown complete 2023-09-30T10:07:38.862 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-30T10:07:38.875 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-30T10:07:38.876 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-30T10:07:38.881 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-30T10:07:39.016 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-30T10:07:39.028 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-30T10:07:39.029 DEBUG:teuthology.orchestra.run.smithi019:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-30T10:07:39.032 DEBUG:teuthology.orchestra.run.smithi167:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-30T10:07:39.047 INFO:teuthology.orchestra.run.smithi167.stderr:bash: ntpq: command not found 2023-09-30T10:07:39.048 INFO:teuthology.orchestra.run.smithi019.stderr:bash: ntpq: command not found 2023-09-30T10:07:39.067 INFO:teuthology.orchestra.run.smithi167.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-30T10:07:39.067 INFO:teuthology.orchestra.run.smithi167.stdout:=============================================================================== 2023-09-30T10:07:39.067 INFO:teuthology.orchestra.run.smithi167.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 40 -1039us[-1039us] +/- 84ms 2023-09-30T10:07:39.067 INFO:teuthology.orchestra.run.smithi167.stdout:^* hv02.front.sepia.ceph.com 3 6 377 41 -764us[ -765us] +/- 77ms 2023-09-30T10:07:39.068 INFO:teuthology.orchestra.run.smithi167.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 41 +1975us[+1975us] +/- 85ms 2023-09-30T10:07:39.068 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T10:07:39.079 INFO:teuthology.orchestra.run.smithi019.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-30T10:07:39.079 INFO:teuthology.orchestra.run.smithi019.stdout:=============================================================================== 2023-09-30T10:07:39.079 INFO:teuthology.orchestra.run.smithi019.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 47 -1030us[-1030us] +/- 84ms 2023-09-30T10:07:39.079 INFO:teuthology.orchestra.run.smithi019.stdout:^* hv02.front.sepia.ceph.com 3 6 377 48 -762us[ -797us] +/- 77ms 2023-09-30T10:07:39.079 INFO:teuthology.orchestra.run.smithi019.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 41 +1979us[+1979us] +/- 85ms 2023-09-30T10:07:39.080 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-30T10:07:39.081 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-30T10:07:39.093 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-30T10:07:39.094 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-30T10:07:39.110 DEBUG:teuthology.orchestra.run.smithi019:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-30T10:07:39.302 DEBUG:teuthology.orchestra.run.smithi167:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-30T10:07:39.432 DEBUG:teuthology.orchestra.run.smithi019:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-30T10:07:39.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T10:07:39.466 DEBUG:teuthology.orchestra.run.smithi167:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-30T10:07:39.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T10:07:39.497 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-30T10:07:39.508 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-30T10:07:39.526 INFO:teuthology.task.internal:Duration was 4670.028794 seconds 2023-09-30T10:07:39.527 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-30T10:07:39.542 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-30T10:07:39.542 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-30T10:07:39.545 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-30T10:07:39.618 INFO:teuthology.orchestra.run.smithi167.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-30T10:07:39.631 INFO:teuthology.orchestra.run.smithi019.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-30T10:07:39.878 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-30T10:07:39.879 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi019.front.sepia.ceph.com 2023-09-30T10:07:39.879 DEBUG:teuthology.orchestra.run.smithi019:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-30T10:07:39.902 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi167.front.sepia.ceph.com 2023-09-30T10:07:39.902 DEBUG:teuthology.orchestra.run.smithi167:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-30T10:07:39.934 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-30T10:07:39.934 DEBUG:teuthology.orchestra.run.smithi019:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-30T10:07:39.945 DEBUG:teuthology.orchestra.run.smithi167:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-30T10:07:40.121 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-30T10:07:40.122 DEBUG:teuthology.orchestra.run.smithi019:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-30T10:07:40.124 DEBUG:teuthology.orchestra.run.smithi167:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-30T10:07:40.504 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-30T10:07:40.515 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-30T10:07:40.516 DEBUG:teuthology.orchestra.run.smithi019:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-30T10:07:40.546 DEBUG:teuthology.orchestra.run.smithi167:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-30T10:07:40.587 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-30T10:07:40.598 DEBUG:teuthology.orchestra.run.smithi019:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-30T10:07:40.600 DEBUG:teuthology.orchestra.run.smithi167:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-30T10:07:40.638 INFO:teuthology.orchestra.run.smithi019.stdout:kernel.core_pattern = core 2023-09-30T10:07:40.688 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern = core 2023-09-30T10:07:40.704 DEBUG:teuthology.orchestra.run.smithi019:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-30T10:07:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T10:07:40.736 DEBUG:teuthology.orchestra.run.smithi167:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-30T10:07:40.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T10:07:40.753 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-30T10:07:40.765 INFO:teuthology.task.internal:Transferring archived files... 2023-09-30T10:07:40.766 DEBUG:teuthology.misc:Transferring archived files from smithi019:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221/remote/smithi019 2023-09-30T10:07:40.767 DEBUG:teuthology.orchestra.run.smithi019:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-30T10:07:40.840 DEBUG:teuthology.misc:Transferring archived files from smithi167:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221/remote/smithi167 2023-09-30T10:07:40.840 DEBUG:teuthology.orchestra.run.smithi167:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-30T10:07:40.890 INFO:teuthology.task.internal:Removing archive directory... 2023-09-30T10:07:40.890 DEBUG:teuthology.orchestra.run.smithi019:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-30T10:07:40.893 DEBUG:teuthology.orchestra.run.smithi167:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-30T10:07:40.935 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-30T10:07:40.948 INFO:teuthology.task.internal:Not uploading archives. 2023-09-30T10:07:40.949 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-30T10:07:40.961 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-30T10:07:40.961 DEBUG:teuthology.orchestra.run.smithi019:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-30T10:07:40.963 DEBUG:teuthology.orchestra.run.smithi167:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-30T10:07:40.978 INFO:teuthology.orchestra.run.smithi019.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 30 10:07 /home/ubuntu/cephtest 2023-09-30T10:07:40.978 INFO:teuthology.orchestra.run.smithi019.stdout: 4456457 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 30 09:54 /home/ubuntu/cephtest/mnt.0 2023-09-30T10:07:40.978 INFO:teuthology.orchestra.run.smithi019.stdout: 4456458 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 30 09:54 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-30T10:07:40.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-30T10:07:40.992 INFO:teuthology.orchestra.run.smithi019.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-30T10:07:40.992 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi019 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-30T10:07:40.999 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-30T10:07:41.014 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-30T10:07:41.026 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi019 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-30T10:07:41.026 INFO:teuthology.nuke:Checking targets against current locks 2023-09-30T10:07:41.033 INFO:teuthology.orchestra.run.smithi167.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 30 10:07 /home/ubuntu/cephtest 2023-09-30T10:07:41.065 DEBUG:teuthology.nuke:shortname: smithi019 2023-09-30T10:07:41.065 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-30T10:07:41.089 DEBUG:teuthology.nuke:shortname: smithi167 2023-09-30T10:07:41.089 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-30T10:07:41.094 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi019.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-30 08:44:44.310717', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-30T10:07:41.116 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi167.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-09-29_19:44:19-upgrade:quincy-x-main-distro-default-smithi/7407221', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-30 08:44:44.313552', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-30T10:07:41.150 INFO:teuthology.orchestra.console.smithi019:Power off 2023-09-30T10:07:41.150 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-30T10:07:41.172 DEBUG:teuthology.orchestra.console.smithi019:power off output: Chassis Power Control: Down/Off 2023-09-30T10:07:41.173 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-30T10:07:41.189 INFO:teuthology.orchestra.console.smithi167:Power off 2023-09-30T10:07:41.189 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-30T10:07:41.200 DEBUG:teuthology.orchestra.console.smithi019:check power output: Chassis Power is on 2023-09-30T10:07:41.213 DEBUG:teuthology.orchestra.console.smithi167:power off output: Chassis Power Control: Down/Off 2023-09-30T10:07:41.213 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-30T10:07:41.237 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is on 2023-09-30T10:07:45.202 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-30T10:07:45.239 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-30T10:07:45.315 DEBUG:teuthology.orchestra.console.smithi019:check power output: Chassis Power is on 2023-09-30T10:07:45.349 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is on 2023-09-30T10:07:49.318 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-30T10:07:49.350 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-30T10:07:49.430 DEBUG:teuthology.orchestra.console.smithi019:check power output: Chassis Power is off 2023-09-30T10:07:49.461 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is off 2023-09-30T10:07:49.531 INFO:teuthology.orchestra.console.smithi019:Power off completed 2023-09-30T10:07:49.562 INFO:teuthology.orchestra.console.smithi167:Power off completed 2023-09-30T10:07:49.669 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 4670.028793811798 failure_reason: '' flavor: default owner: scheduled_yuriw@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=eb91c134950e42d4b2e29478c28c1dd0 status: fail success: false 2023-09-30T10:07:49.669 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-30T10:07:49.749 INFO:teuthology.run:FAIL